From a15dfd2461666fc3a5e964c853f2a115b282cdf7 Mon Sep 17 00:00:00 2001 From: ameshf Date: Sun, 9 Nov 2025 03:52:49 +1100 Subject: [PATCH 1/3] feat: Add auto-scaling --- README.md | 2 +- api/v1beta1/temporalcluster_types.go | 58 +- api/v1beta1/zz_generated.deepcopy.go | 43 + .../bases/temporal.io_temporalclusters.yaml | 3260 ++++++++++++++++- config/rbac/role.yaml | 11 + controllers/temporalcluster_controller.go | 120 +- docs/api/v1beta1.md | 93 + internal/resource/base/deployment_builder.go | 2 +- internal/resource/base/hpa_builder.go | 139 + 9 files changed, 3604 insertions(+), 124 deletions(-) create mode 100644 internal/resource/base/hpa_builder.go diff --git a/README.md b/README.md index ea40c757..add8610e 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,7 @@ Please note this table only reports end-to-end tests suite coverage, others vers - [x] Cluster monitoring. - [x] Complete end2end test suite. - [x] Archival. -- [ ] Auto scaling. +- [x] Auto scaling. - [ ] Multi cluster replication. ## Contributing diff --git a/api/v1beta1/temporalcluster_types.go b/api/v1beta1/temporalcluster_types.go index a0677e64..620cd689 100644 --- a/api/v1beta1/temporalcluster_types.go +++ b/api/v1beta1/temporalcluster_types.go @@ -28,10 +28,12 @@ import ( monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" "go.temporal.io/server/common/primitives" "golang.org/x/exp/slices" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" ) // LogSpec contains the temporal logging configuration. @@ -62,6 +64,32 @@ type LogSpec struct { Development bool `json:"development"` } +// AutoscalingSpec defines the configuration for Horizontal Pod Autoscaling. +type AutoscalingSpec struct { + // MinReplicas is the lower limit for the number of replicas to which the autoscaler + // can scale down. It defaults to 1 pod. + // +optional + // +kubebuilder:validation:Minimum=1 + MinReplicas *int32 `json:"minReplicas,omitempty"` + + // MaxReplicas is the upper limit for the number of replicas to which the autoscaler can scale up. + // +kubebuilder:validation:Minimum=1 + // +optional + MaxReplicas *int32 `json:"maxReplicas,omitempty"` + + // Metrics contains the specifications for which to use to calculate the + // desired replica count (the maximum replica count across all metrics will + // be used). If not set, defaults to 80% CPU and 70% memory utilization. + // +optional + Metrics []autoscalingv2.MetricSpec `json:"metrics,omitempty"` + + // Behavior configures the scaling behavior of the target + // in both Up and Down directions (scaleUp and scaleDown fields respectively). + // If not set, the default HPAScalingRules for scale up and scale down are used. + // +optional + Behavior *autoscalingv2.HorizontalPodAutoscalerBehavior `json:"behavior,omitempty"` +} + // ServiceSpec contains a temporal service specifications. type ServiceSpec struct { // Port defines a custom gRPC port for the service. @@ -88,7 +116,7 @@ type ServiceSpec struct { // Number of desired replicas for the service. Default to 1. // +kubebuilder:validation:Minimum=1 // +optional - Replicas *int32 `json:"replicas"` + Replicas *int32 `json:"replicas,omitempty"` // Compute Resources required by this service. // More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ // +optional @@ -100,6 +128,10 @@ type ServiceSpec struct { // InitContainers adds a list of init containers to the service's deployment. // +optional InitContainers []corev1.Container `json:"initContainers,omitempty"` + // Autoscaling enables horizontal pod autoscaling for the service. + // When enabled, the controller will bypass the replicas field and create an HPA resource instead. + // +optional + Autoscaling *AutoscalingSpec `json:"autoscaling,omitempty"` // ServiceAccountOverride } @@ -116,6 +148,30 @@ func (s *InternalFrontendServiceSpec) IsEnabled() bool { return s != nil && s.Enabled } +// IsAutoscalingEnabled returns true if autoscaling is enabled for the service. +func (s *ServiceSpec) IsAutoscalingEnabled() bool { + return s != nil && s.Autoscaling != nil +} + +// GetEffectiveReplicas returns the replica count to use for the deployment. +// If autoscaling is enabled, returns the current replicas value which HPA may have +// updated via the coordination logic. Otherwise, returns the configured replicas value. +func (s *ServiceSpec) GetEffectiveReplicas() *int32 { + // If autoscaling is configured, respect the current Replicas value which may be + // updated by HPA coordination logic, but ensure it doesn't go below MinReplicas + if s.IsAutoscalingEnabled() && s.Autoscaling.MinReplicas != nil { + if s.Replicas != nil { + replicas := max(*s.Autoscaling.MinReplicas, *s.Replicas) + return &replicas + } + return s.Autoscaling.MinReplicas + } + if s.Replicas != nil { + return s.Replicas + } + return ptr.To[int32](1) +} + // ServicesSpec contains all temporal services specifications. type ServicesSpec struct { // Frontend service custom specifications. diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index ce8d20cb..1cb107e4 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -25,6 +25,7 @@ import ( "github.com/alexandrevilain/temporal-operator/pkg/version" "github.com/gocql/gocql" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + "k8s.io/api/autoscaling/v2" "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" @@ -118,6 +119,43 @@ func (in *AuthorizationSpecJWTKeyProvider) DeepCopy() *AuthorizationSpecJWTKeyPr return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AutoscalingSpec) DeepCopyInto(out *AutoscalingSpec) { + *out = *in + if in.MinReplicas != nil { + in, out := &in.MinReplicas, &out.MinReplicas + *out = new(int32) + **out = **in + } + if in.MaxReplicas != nil { + in, out := &in.MaxReplicas, &out.MaxReplicas + *out = new(int32) + **out = **in + } + if in.Metrics != nil { + in, out := &in.Metrics, &out.Metrics + *out = make([]v2.MetricSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Behavior != nil { + in, out := &in.Behavior, &out.Behavior + *out = new(v2.HorizontalPodAutoscalerBehavior) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AutoscalingSpec. +func (in *AutoscalingSpec) DeepCopy() *AutoscalingSpec { + if in == nil { + return nil + } + out := new(AutoscalingSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CassandraConsistencySpec) DeepCopyInto(out *CassandraConsistencySpec) { *out = *in @@ -1352,6 +1390,11 @@ func (in *ServiceSpec) DeepCopyInto(out *ServiceSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.Autoscaling != nil { + in, out := &in.Autoscaling, &out.Autoscaling + *out = new(AutoscalingSpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceSpec. diff --git a/config/crd/bases/temporal.io_temporalclusters.yaml b/config/crd/bases/temporal.io_temporalclusters.yaml index 7eebc60f..98b6b049 100644 --- a/config/crd/bases/temporal.io_temporalclusters.yaml +++ b/config/crd/bases/temporal.io_temporalclusters.yaml @@ -2965,6 +2965,610 @@ spec: frontend: description: Frontend service custom specifications. properties: + autoscaling: + description: |- + Autoscaling enables horizontal pod autoscaling for the service. + When enabled, the controller will bypass the replicas field and create an HPA resource instead. + properties: + behavior: + description: |- + Behavior configures the scaling behavior of the target + in both Up and Down directions (scaleUp and scaleDown fields respectively). + If not set, the default HPAScalingRules for scale up and scale down are used. + properties: + scaleDown: + description: |- + scaleDown is scaling policy for scaling Down. + If not set, the default value is to allow to scale down to minReplicas pods, with a + 300 second stabilization window (i.e., the highest recommendation for + the last 300sec is used). + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + If not set, use the default values: + - For scale up: allow doubling the number of pods, or an absolute change of 4 pods in a 15s window. + - For scale down: allow all pods to be removed in a 15s window. + items: + description: HPAScalingPolicy is a single policy which must hold true for a specified past interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify the scaling policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + type: integer + tolerance: + anyOf: + - type: integer + - type: string + description: |- + tolerance is the tolerance on the ratio between the current and desired + metric value under which no updates are made to the desired number of + replicas (e.g. 0.01 for 1%). Must be greater than or equal to zero. If not + set, the default cluster-wide tolerance is applied (by default 10%). + + For example, if autoscaling is configured with a memory consumption target of 100Mi, + and scale-down and scale-up tolerances of 5% and 1% respectively, scaling will be + triggered when the actual consumption falls below 95Mi or exceeds 101Mi. + + This is an alpha field and requires enabling the HPAConfigurableTolerance + feature gate. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + scaleUp: + description: |- + scaleUp is scaling policy for scaling Up. + If not set, the default value is the higher of: + * increase no more than 4 pods per 60 seconds + * double the number of pods per 60 seconds + No stabilization is used. + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + If not set, use the default values: + - For scale up: allow doubling the number of pods, or an absolute change of 4 pods in a 15s window. + - For scale down: allow all pods to be removed in a 15s window. + items: + description: HPAScalingPolicy is a single policy which must hold true for a specified past interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify the scaling policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + type: integer + tolerance: + anyOf: + - type: integer + - type: string + description: |- + tolerance is the tolerance on the ratio between the current and desired + metric value under which no updates are made to the desired number of + replicas (e.g. 0.01 for 1%). Must be greater than or equal to zero. If not + set, the default cluster-wide tolerance is applied (by default 10%). + + For example, if autoscaling is configured with a memory consumption target of 100Mi, + and scale-down and scale-up tolerances of 5% and 1% respectively, scaling will be + triggered when the actual consumption falls below 95Mi or exceeds 101Mi. + + This is an alpha field and requires enabling the HPAConfigurableTolerance + feature gate. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + maxReplicas: + description: MaxReplicas is the upper limit for the number of replicas to which the autoscaler can scale up. + format: int32 + minimum: 1 + type: integer + metrics: + description: |- + Metrics contains the specifications for which to use to calculate the + desired replica count (the maximum replica count across all metrics will + be used). If not set, defaults to 80% CPU and 70% memory utilization. + items: + description: |- + MetricSpec specifies how to scale based on a single metric + (only `type` and one other matching field should be set at once). + properties: + containerResource: + description: |- + containerResource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing a single container in + each pod of the current scale target (e.g. CPU or memory). Such metrics are + built in to Kubernetes, and have special scaling options on top of those + available to normal per-pod metrics using the "pods" source. + properties: + container: + description: container is the name of the container in the pods of the scaling target + type: string + name: + description: name is the name of the resource in question. + type: string + target: + description: target specifies the target value for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - container + - name + - target + type: object + external: + description: |- + external refers to a global metric that is not associated + with any Kubernetes object. It allows autoscaling based on information + coming from components running outside of cluster + (for example length of queue in cloud messaging service, or + QPS from loadbalancer running outside of cluster). + properties: + metric: + description: metric identifies the target metric by name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + object: + description: |- + object refers to a metric describing a single kubernetes object + (for example, hits-per-second on an Ingress object). + properties: + describedObject: + description: describedObject specifies the descriptions of a object,such as kind,name apiVersion + properties: + apiVersion: + description: apiVersion is the API version of the referent + type: string + kind: + description: 'kind is the kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'name is the name of the referent; More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + required: + - kind + - name + type: object + metric: + description: metric identifies the target metric by name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + description: |- + pods refers to a metric describing each pod in the current scale target + (for example, transactions-processed-per-second). The values will be + averaged together before being compared to the target value. + properties: + metric: + description: metric identifies the target metric by name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + resource: + description: |- + resource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing each pod in the + current scale target (e.g. CPU or memory). Such metrics are built in to + Kubernetes, and have special scaling options on top of those available + to normal per-pod metrics using the "pods" source. + properties: + name: + description: name is the name of the resource in question. + type: string + target: + description: target specifies the target value for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - name + - target + type: object + type: + description: |- + type is the type of metric source. It should be one of "ContainerResource", "External", + "Object", "Pods" or "Resource", each mapping to a matching field in the object. + type: string + required: + - type + type: object + type: array + minReplicas: + description: |- + MinReplicas is the lower limit for the number of replicas to which the autoscaler + can scale down. It defaults to 1 pod. + format: int32 + minimum: 1 + type: integer + type: object httpPort: description: |- HTTPPort defines a custom http port for the service. @@ -3132,61 +3736,665 @@ spec: history: description: History service custom specifications. properties: - httpPort: - description: |- - HTTPPort defines a custom http port for the service. - Default values are: - 7243 for Frontend service - format: int32 - type: integer - initContainers: - description: InitContainers adds a list of init containers to the service's deployment. - items: - description: A single application container that you want to run within a pod. - type: object - x-kubernetes-preserve-unknown-fields: true - type: array - membershipPort: - description: |- - MembershipPort defines a custom membership port for the service. - Default values are: - 6933 for Frontend service - 6934 for History service - 6935 for Matching service - 6939 for Worker service - format: int32 - type: integer - overrides: + autoscaling: description: |- - Overrides adds some overrides to the resources deployed for the service. - Those overrides takes precedence over spec.services.overrides. + Autoscaling enables horizontal pod autoscaling for the service. + When enabled, the controller will bypass the replicas field and create an HPA resource instead. properties: - deployment: - description: Override configuration for the temporal service Deployment. + behavior: + description: |- + Behavior configures the scaling behavior of the target + in both Up and Down directions (scaleUp and scaleDown fields respectively). + If not set, the default HPAScalingRules for scale up and scale down are used. properties: - jsonPatch: - x-kubernetes-preserve-unknown-fields: true - metadata: + scaleDown: description: |- - ObjectMetaOverride provides the ability to override an object metadata. - It's a subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta. + scaleDown is scaling policy for scaling Down. + If not set, the default value is to allow to scale down to minReplicas pods, with a + 300 second stabilization window (i.e., the highest recommendation for + the last 300sec is used). properties: - annotations: - additionalProperties: - type: string + policies: description: |- - Annotations is an unstructured key value map stored with a resource that may be - set by external tools to store and retrieve arbitrary metadata. - type: object - labels: - additionalProperties: - type: string + policies is a list of potential scaling polices which can be used during scaling. + If not set, use the default values: + - For scale up: allow doubling the number of pods, or an absolute change of 4 pods in a 15s window. + - For scale down: allow all pods to be removed in a 15s window. + items: + description: HPAScalingPolicy is a single policy which must hold true for a specified past interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify the scaling policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: description: |- - Map of string keys and values that can be used to organize and categorize - (scope and select) objects. - type: object - type: object - spec: + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + type: integer + tolerance: + anyOf: + - type: integer + - type: string + description: |- + tolerance is the tolerance on the ratio between the current and desired + metric value under which no updates are made to the desired number of + replicas (e.g. 0.01 for 1%). Must be greater than or equal to zero. If not + set, the default cluster-wide tolerance is applied (by default 10%). + + For example, if autoscaling is configured with a memory consumption target of 100Mi, + and scale-down and scale-up tolerances of 5% and 1% respectively, scaling will be + triggered when the actual consumption falls below 95Mi or exceeds 101Mi. + + This is an alpha field and requires enabling the HPAConfigurableTolerance + feature gate. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + scaleUp: + description: |- + scaleUp is scaling policy for scaling Up. + If not set, the default value is the higher of: + * increase no more than 4 pods per 60 seconds + * double the number of pods per 60 seconds + No stabilization is used. + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + If not set, use the default values: + - For scale up: allow doubling the number of pods, or an absolute change of 4 pods in a 15s window. + - For scale down: allow all pods to be removed in a 15s window. + items: + description: HPAScalingPolicy is a single policy which must hold true for a specified past interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify the scaling policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + type: integer + tolerance: + anyOf: + - type: integer + - type: string + description: |- + tolerance is the tolerance on the ratio between the current and desired + metric value under which no updates are made to the desired number of + replicas (e.g. 0.01 for 1%). Must be greater than or equal to zero. If not + set, the default cluster-wide tolerance is applied (by default 10%). + + For example, if autoscaling is configured with a memory consumption target of 100Mi, + and scale-down and scale-up tolerances of 5% and 1% respectively, scaling will be + triggered when the actual consumption falls below 95Mi or exceeds 101Mi. + + This is an alpha field and requires enabling the HPAConfigurableTolerance + feature gate. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + maxReplicas: + description: MaxReplicas is the upper limit for the number of replicas to which the autoscaler can scale up. + format: int32 + minimum: 1 + type: integer + metrics: + description: |- + Metrics contains the specifications for which to use to calculate the + desired replica count (the maximum replica count across all metrics will + be used). If not set, defaults to 80% CPU and 70% memory utilization. + items: + description: |- + MetricSpec specifies how to scale based on a single metric + (only `type` and one other matching field should be set at once). + properties: + containerResource: + description: |- + containerResource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing a single container in + each pod of the current scale target (e.g. CPU or memory). Such metrics are + built in to Kubernetes, and have special scaling options on top of those + available to normal per-pod metrics using the "pods" source. + properties: + container: + description: container is the name of the container in the pods of the scaling target + type: string + name: + description: name is the name of the resource in question. + type: string + target: + description: target specifies the target value for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - container + - name + - target + type: object + external: + description: |- + external refers to a global metric that is not associated + with any Kubernetes object. It allows autoscaling based on information + coming from components running outside of cluster + (for example length of queue in cloud messaging service, or + QPS from loadbalancer running outside of cluster). + properties: + metric: + description: metric identifies the target metric by name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + object: + description: |- + object refers to a metric describing a single kubernetes object + (for example, hits-per-second on an Ingress object). + properties: + describedObject: + description: describedObject specifies the descriptions of a object,such as kind,name apiVersion + properties: + apiVersion: + description: apiVersion is the API version of the referent + type: string + kind: + description: 'kind is the kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'name is the name of the referent; More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + required: + - kind + - name + type: object + metric: + description: metric identifies the target metric by name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + description: |- + pods refers to a metric describing each pod in the current scale target + (for example, transactions-processed-per-second). The values will be + averaged together before being compared to the target value. + properties: + metric: + description: metric identifies the target metric by name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + resource: + description: |- + resource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing each pod in the + current scale target (e.g. CPU or memory). Such metrics are built in to + Kubernetes, and have special scaling options on top of those available + to normal per-pod metrics using the "pods" source. + properties: + name: + description: name is the name of the resource in question. + type: string + target: + description: target specifies the target value for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - name + - target + type: object + type: + description: |- + type is the type of metric source. It should be one of "ContainerResource", "External", + "Object", "Pods" or "Resource", each mapping to a matching field in the object. + type: string + required: + - type + type: object + type: array + minReplicas: + description: |- + MinReplicas is the lower limit for the number of replicas to which the autoscaler + can scale down. It defaults to 1 pod. + format: int32 + minimum: 1 + type: integer + type: object + httpPort: + description: |- + HTTPPort defines a custom http port for the service. + Default values are: + 7243 for Frontend service + format: int32 + type: integer + initContainers: + description: InitContainers adds a list of init containers to the service's deployment. + items: + description: A single application container that you want to run within a pod. + type: object + x-kubernetes-preserve-unknown-fields: true + type: array + membershipPort: + description: |- + MembershipPort defines a custom membership port for the service. + Default values are: + 6933 for Frontend service + 6934 for History service + 6935 for Matching service + 6939 for Worker service + format: int32 + type: integer + overrides: + description: |- + Overrides adds some overrides to the resources deployed for the service. + Those overrides takes precedence over spec.services.overrides. + properties: + deployment: + description: Override configuration for the temporal service Deployment. + properties: + jsonPatch: + x-kubernetes-preserve-unknown-fields: true + metadata: + description: |- + ObjectMetaOverride provides the ability to override an object metadata. + It's a subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta. + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. + type: object + labels: + additionalProperties: + type: string + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. + type: object + type: object + spec: description: Specification of the desired behavior of the Deployment. properties: template: @@ -3264,43 +4472,647 @@ spec: only the result of this request. type: string required: - - name + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + type: object + internalFrontend: + description: |- + Internal Frontend service custom specifications. + Only compatible with temporal >= 1.20.0 + properties: + autoscaling: + description: |- + Autoscaling enables horizontal pod autoscaling for the service. + When enabled, the controller will bypass the replicas field and create an HPA resource instead. + properties: + behavior: + description: |- + Behavior configures the scaling behavior of the target + in both Up and Down directions (scaleUp and scaleDown fields respectively). + If not set, the default HPAScalingRules for scale up and scale down are used. + properties: + scaleDown: + description: |- + scaleDown is scaling policy for scaling Down. + If not set, the default value is to allow to scale down to minReplicas pods, with a + 300 second stabilization window (i.e., the highest recommendation for + the last 300sec is used). + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + If not set, use the default values: + - For scale up: allow doubling the number of pods, or an absolute change of 4 pods in a 15s window. + - For scale down: allow all pods to be removed in a 15s window. + items: + description: HPAScalingPolicy is a single policy which must hold true for a specified past interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify the scaling policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + type: integer + tolerance: + anyOf: + - type: integer + - type: string + description: |- + tolerance is the tolerance on the ratio between the current and desired + metric value under which no updates are made to the desired number of + replicas (e.g. 0.01 for 1%). Must be greater than or equal to zero. If not + set, the default cluster-wide tolerance is applied (by default 10%). + + For example, if autoscaling is configured with a memory consumption target of 100Mi, + and scale-down and scale-up tolerances of 5% and 1% respectively, scaling will be + triggered when the actual consumption falls below 95Mi or exceeds 101Mi. + + This is an alpha field and requires enabling the HPAConfigurableTolerance + feature gate. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + scaleUp: + description: |- + scaleUp is scaling policy for scaling Up. + If not set, the default value is the higher of: + * increase no more than 4 pods per 60 seconds + * double the number of pods per 60 seconds + No stabilization is used. + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + If not set, use the default values: + - For scale up: allow doubling the number of pods, or an absolute change of 4 pods in a 15s window. + - For scale down: allow all pods to be removed in a 15s window. + items: + description: HPAScalingPolicy is a single policy which must hold true for a specified past interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify the scaling policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + type: integer + tolerance: + anyOf: + - type: integer + - type: string + description: |- + tolerance is the tolerance on the ratio between the current and desired + metric value under which no updates are made to the desired number of + replicas (e.g. 0.01 for 1%). Must be greater than or equal to zero. If not + set, the default cluster-wide tolerance is applied (by default 10%). + + For example, if autoscaling is configured with a memory consumption target of 100Mi, + and scale-down and scale-up tolerances of 5% and 1% respectively, scaling will be + triggered when the actual consumption falls below 95Mi or exceeds 101Mi. + + This is an alpha field and requires enabling the HPAConfigurableTolerance + feature gate. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + maxReplicas: + description: MaxReplicas is the upper limit for the number of replicas to which the autoscaler can scale up. + format: int32 + minimum: 1 + type: integer + metrics: + description: |- + Metrics contains the specifications for which to use to calculate the + desired replica count (the maximum replica count across all metrics will + be used). If not set, defaults to 80% CPU and 70% memory utilization. + items: + description: |- + MetricSpec specifies how to scale based on a single metric + (only `type` and one other matching field should be set at once). + properties: + containerResource: + description: |- + containerResource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing a single container in + each pod of the current scale target (e.g. CPU or memory). Such metrics are + built in to Kubernetes, and have special scaling options on top of those + available to normal per-pod metrics using the "pods" source. + properties: + container: + description: container is the name of the container in the pods of the scaling target + type: string + name: + description: name is the name of the resource in question. + type: string + target: + description: target specifies the target value for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - container + - name + - target + type: object + external: + description: |- + external refers to a global metric that is not associated + with any Kubernetes object. It allows autoscaling based on information + coming from components running outside of cluster + (for example length of queue in cloud messaging service, or + QPS from loadbalancer running outside of cluster). + properties: + metric: + description: metric identifies the target metric by name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + object: + description: |- + object refers to a metric describing a single kubernetes object + (for example, hits-per-second on an Ingress object). + properties: + describedObject: + description: describedObject specifies the descriptions of a object,such as kind,name apiVersion + properties: + apiVersion: + description: apiVersion is the API version of the referent + type: string + kind: + description: 'kind is the kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'name is the name of the referent; More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + required: + - kind + - name + type: object + metric: + description: metric identifies the target metric by name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + description: |- + pods refers to a metric describing each pod in the current scale target + (for example, transactions-processed-per-second). The values will be + averaged together before being compared to the target value. + properties: + metric: + description: metric identifies the target metric by name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + resource: + description: |- + resource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing each pod in the + current scale target (e.g. CPU or memory). Such metrics are built in to + Kubernetes, and have special scaling options on top of those available + to normal per-pod metrics using the "pods" source. + properties: + name: + description: name is the name of the resource in question. + type: string + target: + description: target specifies the target value for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - name + - target + type: object + type: + description: |- + type is the type of metric source. It should be one of "ContainerResource", "External", + "Object", "Pods" or "Resource", each mapping to a matching field in the object. + type: string + required: + - type type: object type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + minReplicas: description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object + MinReplicas is the lower limit for the number of replicas to which the autoscaler + can scale down. It defaults to 1 pod. + format: int32 + minimum: 1 + type: integer type: object - type: object - internalFrontend: - description: |- - Internal Frontend service custom specifications. - Only compatible with temporal >= 1.20.0 - properties: enabled: default: false description: Enabled defines if we want to spawn the internal frontend service. @@ -3426,52 +5238,656 @@ spec: properties: name: description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + type: object + matching: + description: Matching service custom specifications. + properties: + autoscaling: + description: |- + Autoscaling enables horizontal pod autoscaling for the service. + When enabled, the controller will bypass the replicas field and create an HPA resource instead. + properties: + behavior: + description: |- + Behavior configures the scaling behavior of the target + in both Up and Down directions (scaleUp and scaleDown fields respectively). + If not set, the default HPAScalingRules for scale up and scale down are used. + properties: + scaleDown: + description: |- + scaleDown is scaling policy for scaling Down. + If not set, the default value is to allow to scale down to minReplicas pods, with a + 300 second stabilization window (i.e., the highest recommendation for + the last 300sec is used). + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + If not set, use the default values: + - For scale up: allow doubling the number of pods, or an absolute change of 4 pods in a 15s window. + - For scale down: allow all pods to be removed in a 15s window. + items: + description: HPAScalingPolicy is a single policy which must hold true for a specified past interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify the scaling policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + type: integer + tolerance: + anyOf: + - type: integer + - type: string + description: |- + tolerance is the tolerance on the ratio between the current and desired + metric value under which no updates are made to the desired number of + replicas (e.g. 0.01 for 1%). Must be greater than or equal to zero. If not + set, the default cluster-wide tolerance is applied (by default 10%). + + For example, if autoscaling is configured with a memory consumption target of 100Mi, + and scale-down and scale-up tolerances of 5% and 1% respectively, scaling will be + triggered when the actual consumption falls below 95Mi or exceeds 101Mi. + + This is an alpha field and requires enabling the HPAConfigurableTolerance + feature gate. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + scaleUp: + description: |- + scaleUp is scaling policy for scaling Up. + If not set, the default value is the higher of: + * increase no more than 4 pods per 60 seconds + * double the number of pods per 60 seconds + No stabilization is used. + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + If not set, use the default values: + - For scale up: allow doubling the number of pods, or an absolute change of 4 pods in a 15s window. + - For scale down: allow all pods to be removed in a 15s window. + items: + description: HPAScalingPolicy is a single policy which must hold true for a specified past interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify the scaling policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + type: integer + tolerance: + anyOf: + - type: integer + - type: string + description: |- + tolerance is the tolerance on the ratio between the current and desired + metric value under which no updates are made to the desired number of + replicas (e.g. 0.01 for 1%). Must be greater than or equal to zero. If not + set, the default cluster-wide tolerance is applied (by default 10%). + + For example, if autoscaling is configured with a memory consumption target of 100Mi, + and scale-down and scale-up tolerances of 5% and 1% respectively, scaling will be + triggered when the actual consumption falls below 95Mi or exceeds 101Mi. + + This is an alpha field and requires enabling the HPAConfigurableTolerance + feature gate. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + maxReplicas: + description: MaxReplicas is the upper limit for the number of replicas to which the autoscaler can scale up. + format: int32 + minimum: 1 + type: integer + metrics: + description: |- + Metrics contains the specifications for which to use to calculate the + desired replica count (the maximum replica count across all metrics will + be used). If not set, defaults to 80% CPU and 70% memory utilization. + items: + description: |- + MetricSpec specifies how to scale based on a single metric + (only `type` and one other matching field should be set at once). + properties: + containerResource: + description: |- + containerResource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing a single container in + each pod of the current scale target (e.g. CPU or memory). Such metrics are + built in to Kubernetes, and have special scaling options on top of those + available to normal per-pod metrics using the "pods" source. + properties: + container: + description: container is the name of the container in the pods of the scaling target + type: string + name: + description: name is the name of the resource in question. + type: string + target: + description: target specifies the target value for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - container + - name + - target + type: object + external: + description: |- + external refers to a global metric that is not associated + with any Kubernetes object. It allows autoscaling based on information + coming from components running outside of cluster + (for example length of queue in cloud messaging service, or + QPS from loadbalancer running outside of cluster). + properties: + metric: + description: metric identifies the target metric by name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + object: + description: |- + object refers to a metric describing a single kubernetes object + (for example, hits-per-second on an Ingress object). + properties: + describedObject: + description: describedObject specifies the descriptions of a object,such as kind,name apiVersion + properties: + apiVersion: + description: apiVersion is the API version of the referent + type: string + kind: + description: 'kind is the kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'name is the name of the referent; More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + required: + - kind + - name + type: object + metric: + description: metric identifies the target metric by name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + description: |- + pods refers to a metric describing each pod in the current scale target + (for example, transactions-processed-per-second). The values will be + averaged together before being compared to the target value. + properties: + metric: + description: metric identifies the target metric by name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + resource: + description: |- + resource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing each pod in the + current scale target (e.g. CPU or memory). Such metrics are built in to + Kubernetes, and have special scaling options on top of those available + to normal per-pod metrics using the "pods" source. + properties: + name: + description: name is the name of the resource in question. + type: string + target: + description: target specifies the target value for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - name + - target + type: object + type: description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. + type is the type of metric source. It should be one of "ContainerResource", "External", + "Object", "Pods" or "Resource", each mapping to a matching field in the object. type: string required: - - name + - type type: object type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + minReplicas: description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object + MinReplicas is the lower limit for the number of replicas to which the autoscaler + can scale down. It defaults to 1 pod. + format: int32 + minimum: 1 + type: integer type: object - type: object - matching: - description: Matching service custom specifications. - properties: httpPort: description: |- HTTPPort defines a custom http port for the service. @@ -3702,6 +6118,610 @@ spec: worker: description: Worker service custom specifications. properties: + autoscaling: + description: |- + Autoscaling enables horizontal pod autoscaling for the service. + When enabled, the controller will bypass the replicas field and create an HPA resource instead. + properties: + behavior: + description: |- + Behavior configures the scaling behavior of the target + in both Up and Down directions (scaleUp and scaleDown fields respectively). + If not set, the default HPAScalingRules for scale up and scale down are used. + properties: + scaleDown: + description: |- + scaleDown is scaling policy for scaling Down. + If not set, the default value is to allow to scale down to minReplicas pods, with a + 300 second stabilization window (i.e., the highest recommendation for + the last 300sec is used). + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + If not set, use the default values: + - For scale up: allow doubling the number of pods, or an absolute change of 4 pods in a 15s window. + - For scale down: allow all pods to be removed in a 15s window. + items: + description: HPAScalingPolicy is a single policy which must hold true for a specified past interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify the scaling policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + type: integer + tolerance: + anyOf: + - type: integer + - type: string + description: |- + tolerance is the tolerance on the ratio between the current and desired + metric value under which no updates are made to the desired number of + replicas (e.g. 0.01 for 1%). Must be greater than or equal to zero. If not + set, the default cluster-wide tolerance is applied (by default 10%). + + For example, if autoscaling is configured with a memory consumption target of 100Mi, + and scale-down and scale-up tolerances of 5% and 1% respectively, scaling will be + triggered when the actual consumption falls below 95Mi or exceeds 101Mi. + + This is an alpha field and requires enabling the HPAConfigurableTolerance + feature gate. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + scaleUp: + description: |- + scaleUp is scaling policy for scaling Up. + If not set, the default value is the higher of: + * increase no more than 4 pods per 60 seconds + * double the number of pods per 60 seconds + No stabilization is used. + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + If not set, use the default values: + - For scale up: allow doubling the number of pods, or an absolute change of 4 pods in a 15s window. + - For scale down: allow all pods to be removed in a 15s window. + items: + description: HPAScalingPolicy is a single policy which must hold true for a specified past interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify the scaling policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + type: integer + tolerance: + anyOf: + - type: integer + - type: string + description: |- + tolerance is the tolerance on the ratio between the current and desired + metric value under which no updates are made to the desired number of + replicas (e.g. 0.01 for 1%). Must be greater than or equal to zero. If not + set, the default cluster-wide tolerance is applied (by default 10%). + + For example, if autoscaling is configured with a memory consumption target of 100Mi, + and scale-down and scale-up tolerances of 5% and 1% respectively, scaling will be + triggered when the actual consumption falls below 95Mi or exceeds 101Mi. + + This is an alpha field and requires enabling the HPAConfigurableTolerance + feature gate. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + maxReplicas: + description: MaxReplicas is the upper limit for the number of replicas to which the autoscaler can scale up. + format: int32 + minimum: 1 + type: integer + metrics: + description: |- + Metrics contains the specifications for which to use to calculate the + desired replica count (the maximum replica count across all metrics will + be used). If not set, defaults to 80% CPU and 70% memory utilization. + items: + description: |- + MetricSpec specifies how to scale based on a single metric + (only `type` and one other matching field should be set at once). + properties: + containerResource: + description: |- + containerResource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing a single container in + each pod of the current scale target (e.g. CPU or memory). Such metrics are + built in to Kubernetes, and have special scaling options on top of those + available to normal per-pod metrics using the "pods" source. + properties: + container: + description: container is the name of the container in the pods of the scaling target + type: string + name: + description: name is the name of the resource in question. + type: string + target: + description: target specifies the target value for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - container + - name + - target + type: object + external: + description: |- + external refers to a global metric that is not associated + with any Kubernetes object. It allows autoscaling based on information + coming from components running outside of cluster + (for example length of queue in cloud messaging service, or + QPS from loadbalancer running outside of cluster). + properties: + metric: + description: metric identifies the target metric by name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + object: + description: |- + object refers to a metric describing a single kubernetes object + (for example, hits-per-second on an Ingress object). + properties: + describedObject: + description: describedObject specifies the descriptions of a object,such as kind,name apiVersion + properties: + apiVersion: + description: apiVersion is the API version of the referent + type: string + kind: + description: 'kind is the kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'name is the name of the referent; More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + required: + - kind + - name + type: object + metric: + description: metric identifies the target metric by name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + description: |- + pods refers to a metric describing each pod in the current scale target + (for example, transactions-processed-per-second). The values will be + averaged together before being compared to the target value. + properties: + metric: + description: metric identifies the target metric by name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + resource: + description: |- + resource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing each pod in the + current scale target (e.g. CPU or memory). Such metrics are built in to + Kubernetes, and have special scaling options on top of those available + to normal per-pod metrics using the "pods" source. + properties: + name: + description: name is the name of the resource in question. + type: string + target: + description: target specifies the target value for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - name + - target + type: object + type: + description: |- + type is the type of metric source. It should be one of "ContainerResource", "External", + "Object", "Pods" or "Resource", each mapping to a matching field in the object. + type: string + required: + - type + type: object + type: array + minReplicas: + description: |- + MinReplicas is the lower limit for the number of replicas to which the autoscaler + can scale down. It defaults to 1 pod. + format: int32 + minimum: 1 + type: integer + type: object httpPort: description: |- HTTPPort defines a custom http port for the service. diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index bf4b433a..9ed4272f 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -46,6 +46,17 @@ rules: - list - update - watch +- apiGroups: + - autoscaling + resources: + - horizontalpodautoscalers + verbs: + - create + - delete + - get + - list + - update + - watch - apiGroups: - batch resources: diff --git a/controllers/temporalcluster_controller.go b/controllers/temporalcluster_controller.go index 44b4f5c1..be7118d1 100644 --- a/controllers/temporalcluster_controller.go +++ b/controllers/temporalcluster_controller.go @@ -41,6 +41,7 @@ import ( istionetworkingv1beta1 "istio.io/client-go/pkg/apis/networking/v1beta1" istiosecurityv1beta1 "istio.io/client-go/pkg/apis/security/v1beta1" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" @@ -76,6 +77,7 @@ type TemporalClusterReconciler struct { //+kubebuilder:rbac:groups="",resources=events,verbs=get;create;patch //+kubebuilder:rbac:groups="",resources=serviceaccounts,verbs=get;list;watch;create;update //+kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;delete +//+kubebuilder:rbac:groups=autoscaling,resources=horizontalpodautoscalers,verbs=get;list;watch;create;update;delete //+kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;update;delete //+kubebuilder:rbac:groups="networking.k8s.io",resources=ingresses,verbs=get;list;watch;create;update;delete //+kubebuilder:rbac:groups="cert-manager.io",resources=certificates;issuers,verbs=get;list;watch;create;update;delete @@ -167,6 +169,11 @@ func (r *TemporalClusterReconciler) reconcileResources(ctx context.Context, temp return fmt.Errorf("can't compute configmap hash: %w", err) } + // Sync replicas from current Deployments (HPA decisions) into the spec + if err := r.coordinateWithHPA(ctx, temporalCluster); err != nil { + return err + } + builders, err := r.resourceBuilders(temporalCluster, configHash) if err != nil { return err @@ -222,6 +229,7 @@ func (r *TemporalClusterReconciler) resourceBuilders(temporalCluster *v1beta1.Te builders = append(builders, base.NewServiceAccountBuilder(serviceName, temporalCluster, r.Scheme)) builders = append(builders, base.NewDeploymentBuilder(serviceName, temporalCluster, r.Scheme, specs, configHash)) + builders = append(builders, base.NewHPABuilder(serviceName, temporalCluster, r.Scheme, specs)) builders = append(builders, base.NewHeadlessServiceBuilder(serviceName, temporalCluster, r.Scheme, specs)) builders = append(builders, istio.NewPeerAuthenticationBuilder(serviceName, temporalCluster, r.Scheme, specs)) @@ -273,9 +281,117 @@ func (r *TemporalClusterReconciler) handleErrorWithRequeue(cluster *v1beta1.Temp return reconcile.Result{RequeueAfter: requeueAfter}, err } +// coordinateWithHPA synchronizes the TemporalCluster spec with HPA scaling decisions. +// When HPA scales a deployment, this method updates the corresponding ServiceSpec.Replicas +// to reflect the HPA's decision, preventing the controller from overriding HPA scaling. +func (r *TemporalClusterReconciler) coordinateWithHPA(ctx context.Context, cluster *v1beta1.TemporalCluster) error { + logger := log.FromContext(ctx) + needsUpdate := false + + // Map of service names to their deployment objects (live state) + deployments := make(map[string]*appsv1.Deployment) + deployList := &appsv1.DeploymentList{} + if err := r.List(ctx, deployList, + client.InNamespace(cluster.Namespace), + client.MatchingFields{ownerKey: cluster.Name}, + ); err != nil { + return fmt.Errorf("failed to list deployments for HPA coordination: %w", err) + } + for i := range deployList.Items { + deployment := &deployList.Items[i] + // Extract service name from deployment name (remove cluster prefix) + serviceName := deployment.Name + if clusterPrefix := cluster.Name + "-"; len(serviceName) > len(clusterPrefix) { + serviceName = serviceName[len(clusterPrefix):] + } + deployments[serviceName] = deployment + } + + // Check each service that has autoscaling enabled + services := []primitives.ServiceName{ + primitives.FrontendService, + primitives.HistoryService, + primitives.MatchingService, + primitives.WorkerService, + primitives.InternalFrontendService, + } + + for _, serviceName := range services { + serviceSpec, err := cluster.Spec.Services.GetServiceSpec(serviceName) + if err != nil || serviceSpec == nil || !serviceSpec.IsAutoscalingEnabled() { + continue // Skip services without autoscaling + } + + deployment, exists := deployments[string(serviceName)] + if !exists { + continue // Skip if deployment not found + } + + // Check if HPA has scaled the deployment + currentReplicas := deployment.Spec.Replicas + if currentReplicas == nil { + continue // Skip if deployment has no replica spec + } + + specReplicas := serviceSpec.Replicas + if specReplicas == nil || *specReplicas != *currentReplicas { + // HPA has changed the deployment replicas, sync the spec + // Update the service spec to match what HPA set + if err := r.updateServiceReplicas(cluster, serviceName, *currentReplicas); err != nil { + return fmt.Errorf("failed to update %s service replicas: %w", serviceName, err) + } + needsUpdate = true + } + } + + // Update the cluster spec if any replicas were changed + if needsUpdate { + if err := r.Update(ctx, cluster); err != nil { + return fmt.Errorf("failed to update cluster spec with HPA coordination: %w", err) + } + logger.Info("Updated TemporalCluster spec to coordinate with HPA scaling decisions") + } + + return nil +} + +// updateServiceReplicas updates the replica count for a specific service in the cluster spec. +func (r *TemporalClusterReconciler) updateServiceReplicas(cluster *v1beta1.TemporalCluster, serviceName primitives.ServiceName, replicas int32) error { + switch serviceName { + case primitives.FrontendService: + if cluster.Spec.Services.Frontend == nil { + cluster.Spec.Services.Frontend = &v1beta1.ServiceSpec{} + } + cluster.Spec.Services.Frontend.Replicas = &replicas + case primitives.HistoryService: + if cluster.Spec.Services.History == nil { + cluster.Spec.Services.History = &v1beta1.ServiceSpec{} + } + cluster.Spec.Services.History.Replicas = &replicas + case primitives.MatchingService: + if cluster.Spec.Services.Matching == nil { + cluster.Spec.Services.Matching = &v1beta1.ServiceSpec{} + } + cluster.Spec.Services.Matching.Replicas = &replicas + case primitives.WorkerService: + if cluster.Spec.Services.Worker == nil { + cluster.Spec.Services.Worker = &v1beta1.ServiceSpec{} + } + cluster.Spec.Services.Worker.Replicas = &replicas + case primitives.InternalFrontendService: + if cluster.Spec.Services.InternalFrontend == nil { + cluster.Spec.Services.InternalFrontend = &v1beta1.InternalFrontendServiceSpec{} + } + cluster.Spec.Services.InternalFrontend.Replicas = &replicas + default: + return fmt.Errorf("unknown service name: %s", serviceName) + } + return nil +} + // SetupWithManager sets up the controller with the Manager. func (r *TemporalClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { - for _, resource := range []client.Object{&appsv1.Deployment{}, &corev1.ConfigMap{}, &corev1.Service{}, &corev1.ServiceAccount{}, &networkingv1.Ingress{}, &batchv1.Job{}} { + for _, resource := range []client.Object{&appsv1.Deployment{}, &autoscalingv2.HorizontalPodAutoscaler{}, &corev1.ConfigMap{}, &corev1.Service{}, &corev1.ServiceAccount{}, &networkingv1.Ingress{}, &batchv1.Job{}} { if err := mgr.GetFieldIndexer().IndexField(context.Background(), resource, ownerKey, addResourceToIndex); err != nil { return err } @@ -288,6 +404,7 @@ func (r *TemporalClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { predicate.AnnotationChangedPredicate{}, ))). Owns(&appsv1.Deployment{}). + Owns(&autoscalingv2.HorizontalPodAutoscaler{}). Owns(&corev1.ConfigMap{}). Owns(&corev1.Service{}). Owns(&corev1.ServiceAccount{}). @@ -334,6 +451,7 @@ func (r *TemporalClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { func addResourceToIndex(rawObj client.Object) []string { switch resourceObject := rawObj.(type) { case *appsv1.Deployment, + *autoscalingv2.HorizontalPodAutoscaler, *corev1.ConfigMap, *corev1.Service, *corev1.ServiceAccount, diff --git a/docs/api/v1beta1.md b/docs/api/v1beta1.md index cec32b63..a59b749e 100644 --- a/docs/api/v1beta1.md +++ b/docs/api/v1beta1.md @@ -570,6 +570,84 @@ the specified URIs.

+

AutoscalingSpec +

+

+(Appears on: +ServiceSpec) +

+

AutoscalingSpec defines the configuration for Horizontal Pod Autoscaling.

+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+minReplicas
+ +int32 + +		 +(Optional) +

MinReplicas is the lower limit for the number of replicas to which the autoscaler +can scale down. It defaults to 1 pod.

+
+maxReplicas
+ +int32 + +		 +(Optional) +

MaxReplicas is the upper limit for the number of replicas to which the autoscaler can scale up.

+
+metrics
+ + +[]Kubernetes autoscaling/v2.MetricSpec + + +		 +(Optional) +

Metrics contains the specifications for which to use to calculate the +desired replica count (the maximum replica count across all metrics will +be used). If not set, defaults to 80% CPU and 70% memory utilization.

+
+behavior
+ + +Kubernetes autoscaling/v2.HorizontalPodAutoscalerBehavior + + +		 +(Optional) +

Behavior configures the scaling behavior of the target +in both Up and Down directions (scaleUp and scaleDown fields respectively). +If not set, the default HPAScalingRules for scale up and scale down are used.

+
+
+

CassandraConsistencySpec

@@ -4354,6 +4432,21 @@ Those overrides takes precedence over spec.services.overrides.

InitContainers adds a list of init containers to the service’s deployment.

+ + +autoscaling
+ + +AutoscalingSpec + + + + +(Optional) +

Autoscaling enables horizontal pod autoscaling for the service. +When enabled, the controller will bypass the replicas field and create an HPA resource instead.

+ + diff --git a/internal/resource/base/deployment_builder.go b/internal/resource/base/deployment_builder.go index 03a7bafe..14e882fd 100644 --- a/internal/resource/base/deployment_builder.go +++ b/internal/resource/base/deployment_builder.go @@ -334,7 +334,7 @@ func (b *DeploymentBuilder) Update(object client.Object) error { }) } - deployment.Spec.Replicas = b.service.Replicas + deployment.Spec.Replicas = b.service.GetEffectiveReplicas() deployment.Spec.Selector = &metav1.LabelSelector{ MatchLabels: metadata.LabelsSelector(b.instance, b.serviceName), diff --git a/internal/resource/base/hpa_builder.go b/internal/resource/base/hpa_builder.go new file mode 100644 index 00000000..b9b0dc3a --- /dev/null +++ b/internal/resource/base/hpa_builder.go @@ -0,0 +1,139 @@ +// Licensed to Amesh Fernando under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Amesh Fernando licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package base + +import ( + "fmt" + + "github.com/alexandrevilain/controller-tools/pkg/resource" + "github.com/alexandrevilain/temporal-operator/api/v1beta1" + "github.com/alexandrevilain/temporal-operator/internal/metadata" + autoscalingv2 "k8s.io/api/autoscaling/v2" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +var _ resource.Builder = (*HPABuilder)(nil) + +type HPABuilder struct { + serviceName string + instance *v1beta1.TemporalCluster + scheme *runtime.Scheme + service *v1beta1.ServiceSpec +} + +func NewHPABuilder(serviceName string, instance *v1beta1.TemporalCluster, scheme *runtime.Scheme, service *v1beta1.ServiceSpec) *HPABuilder { + return &HPABuilder{ + serviceName: serviceName, + instance: instance, + scheme: scheme, + service: service, + } +} + +func (b *HPABuilder) Build() client.Object { + return &autoscalingv2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: b.instance.ChildResourceName(b.serviceName), + Namespace: b.instance.Namespace, + Labels: metadata.GetLabels(b.instance, b.serviceName, b.instance.Spec.Version, b.instance.Labels), + Annotations: metadata.GetAnnotations(b.instance.Name, b.instance.Annotations), + }, + } +} + +func (b *HPABuilder) Enabled() bool { + return isBuilderEnabled(b.instance, b.serviceName) && b.service.IsAutoscalingEnabled() +} + +func (b *HPABuilder) Update(object client.Object) error { + hpa := object.(*autoscalingv2.HorizontalPodAutoscaler) + hpa.Labels = metadata.Merge( + object.GetLabels(), + metadata.GetLabels(b.instance, b.serviceName, b.instance.Spec.Version, b.instance.Labels), + ) + hpa.Annotations = metadata.Merge( + object.GetAnnotations(), + metadata.GetAnnotations(b.instance.Name, b.instance.Annotations), + ) + + autoscaling := b.service.Autoscaling + + hpa.Spec.ScaleTargetRef = autoscalingv2.CrossVersionObjectReference{ + Kind: "Deployment", + Name: b.instance.ChildResourceName(b.serviceName), + APIVersion: "apps/v1", + } + + // Set min/max replicas with defaults + if autoscaling.MinReplicas != nil { + hpa.Spec.MinReplicas = autoscaling.MinReplicas + } else { + hpa.Spec.MinReplicas = ptr.To[int32](1) + } + + if autoscaling.MaxReplicas != nil { + hpa.Spec.MaxReplicas = *autoscaling.MaxReplicas + } else { + hpa.Spec.MaxReplicas = 10 // Default max replicas + } + + // Set metrics + if len(autoscaling.Metrics) > 0 { + hpa.Spec.Metrics = autoscaling.Metrics + } else { + // Default metrics: 80% CPU and 70% memory utilization (industry standards) + hpa.Spec.Metrics = []autoscalingv2.MetricSpec{ + { + Type: autoscalingv2.ResourceMetricSourceType, + Resource: &autoscalingv2.ResourceMetricSource{ + Name: corev1.ResourceCPU, + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.UtilizationMetricType, + AverageUtilization: ptr.To[int32](80), + }, + }, + }, + { + Type: autoscalingv2.ResourceMetricSourceType, + Resource: &autoscalingv2.ResourceMetricSource{ + Name: corev1.ResourceMemory, + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.UtilizationMetricType, + AverageUtilization: ptr.To[int32](70), + }, + }, + }, + } + } + + // Set behavior if specified + if autoscaling.Behavior != nil { + hpa.Spec.Behavior = autoscaling.Behavior + } + + if err := controllerutil.SetControllerReference(b.instance, hpa, b.scheme); err != nil { + return fmt.Errorf("failed setting controller reference: %w", err) + } + + return nil +} From 05bf3eb8c53aeb774544d497f04e1944bcd399bd Mon Sep 17 00:00:00 2001 From: ameshf Date: Wed, 12 Nov 2025 13:54:27 +1100 Subject: [PATCH 2/3] Address comments. --- api/v1beta1/temporalcluster_types.go | 10 +- controllers/temporalcluster_controller.go | 149 ++++++++-------------- internal/resource/base/hpa_builder.go | 4 +- 3 files changed, 57 insertions(+), 106 deletions(-) diff --git a/api/v1beta1/temporalcluster_types.go b/api/v1beta1/temporalcluster_types.go index 620cd689..24266661 100644 --- a/api/v1beta1/temporalcluster_types.go +++ b/api/v1beta1/temporalcluster_types.go @@ -116,7 +116,7 @@ type ServiceSpec struct { // Number of desired replicas for the service. Default to 1. // +kubebuilder:validation:Minimum=1 // +optional - Replicas *int32 `json:"replicas,omitempty"` + Replicas *int32 `json:"replicas"` // Compute Resources required by this service. // More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ // +optional @@ -159,12 +159,8 @@ func (s *ServiceSpec) IsAutoscalingEnabled() bool { func (s *ServiceSpec) GetEffectiveReplicas() *int32 { // If autoscaling is configured, respect the current Replicas value which may be // updated by HPA coordination logic, but ensure it doesn't go below MinReplicas - if s.IsAutoscalingEnabled() && s.Autoscaling.MinReplicas != nil { - if s.Replicas != nil { - replicas := max(*s.Autoscaling.MinReplicas, *s.Replicas) - return &replicas - } - return s.Autoscaling.MinReplicas + if s.IsAutoscalingEnabled() { + return nil } if s.Replicas != nil { return s.Replicas diff --git a/controllers/temporalcluster_controller.go b/controllers/temporalcluster_controller.go index be7118d1..d839f99a 100644 --- a/controllers/temporalcluster_controller.go +++ b/controllers/temporalcluster_controller.go @@ -169,21 +169,38 @@ func (r *TemporalClusterReconciler) reconcileResources(ctx context.Context, temp return fmt.Errorf("can't compute configmap hash: %w", err) } - // Sync replicas from current Deployments (HPA decisions) into the spec - if err := r.coordinateWithHPA(ctx, temporalCluster); err != nil { + builders, err := r.resourceBuilders(temporalCluster, configHash) + if err != nil { return err } - builders, err := r.resourceBuilders(temporalCluster, configHash) + // Separate deployment builders from others to handle SSA for Deployments + var deploymentBuilders []resource.Builder + var otherBuilders []resource.Builder + + for _, builder := range builders { + if _, isDeployment := builder.(*base.DeploymentBuilder); isDeployment { + deploymentBuilders = append(deploymentBuilders, builder) + } else { + otherBuilders = append(otherBuilders, builder) + } + } + + // Reconcile non-deployment resources normally + objects, err := r.Reconciler.ReconcileBuilders(ctx, temporalCluster, otherBuilders) if err != nil { return err } - objects, err := r.Reconciler.ReconcileBuilders(ctx, temporalCluster, builders) + // Handle deployments with SSA + deploymentObjects, err := r.reconcileDeploymentsWithSSA(ctx, deploymentBuilders) if err != nil { return err } + // Combine all objects + objects = append(objects, deploymentObjects...) + statuses, err := status.ReconciledObjectsToServiceStatuses(temporalCluster, objects) if err != nil { return err @@ -281,110 +298,48 @@ func (r *TemporalClusterReconciler) handleErrorWithRequeue(cluster *v1beta1.Temp return reconcile.Result{RequeueAfter: requeueAfter}, err } -// coordinateWithHPA synchronizes the TemporalCluster spec with HPA scaling decisions. -// When HPA scales a deployment, this method updates the corresponding ServiceSpec.Replicas -// to reflect the HPA's decision, preventing the controller from overriding HPA scaling. -func (r *TemporalClusterReconciler) coordinateWithHPA(ctx context.Context, cluster *v1beta1.TemporalCluster) error { - logger := log.FromContext(ctx) - needsUpdate := false - - // Map of service names to their deployment objects (live state) - deployments := make(map[string]*appsv1.Deployment) - deployList := &appsv1.DeploymentList{} - if err := r.List(ctx, deployList, - client.InNamespace(cluster.Namespace), - client.MatchingFields{ownerKey: cluster.Name}, - ); err != nil { - return fmt.Errorf("failed to list deployments for HPA coordination: %w", err) - } - for i := range deployList.Items { - deployment := &deployList.Items[i] - // Extract service name from deployment name (remove cluster prefix) - serviceName := deployment.Name - if clusterPrefix := cluster.Name + "-"; len(serviceName) > len(clusterPrefix) { - serviceName = serviceName[len(clusterPrefix):] - } - deployments[serviceName] = deployment - } - - // Check each service that has autoscaling enabled - services := []primitives.ServiceName{ - primitives.FrontendService, - primitives.HistoryService, - primitives.MatchingService, - primitives.WorkerService, - primitives.InternalFrontendService, - } +func (r *TemporalClusterReconciler) reconcileDeploymentsWithSSA(ctx context.Context, builders []resource.Builder) ([]client.Object, error) { + objects := make([]client.Object, 0) - for _, serviceName := range services { - serviceSpec, err := cluster.Spec.Services.GetServiceSpec(serviceName) - if err != nil || serviceSpec == nil || !serviceSpec.IsAutoscalingEnabled() { - continue // Skip services without autoscaling - } + for _, builder := range builders { + deploymentBuilder := builder.(*base.DeploymentBuilder) - deployment, exists := deployments[string(serviceName)] - if !exists { - continue // Skip if deployment not found - } + // Build the desired deployment + desiredObj := deploymentBuilder.Build() + desired := desiredObj.(*appsv1.Deployment) - // Check if HPA has scaled the deployment - currentReplicas := deployment.Spec.Replicas - if currentReplicas == nil { - continue // Skip if deployment has no replica spec + // Update the desired deployment with the current configuration + if err := deploymentBuilder.Update(desired); err != nil { + return nil, fmt.Errorf("failed to update deployment configuration: %w", err) } - - specReplicas := serviceSpec.Replicas - if specReplicas == nil || *specReplicas != *currentReplicas { - // HPA has changed the deployment replicas, sync the spec - // Update the service spec to match what HPA set - if err := r.updateServiceReplicas(cluster, serviceName, *currentReplicas); err != nil { - return fmt.Errorf("failed to update %s service replicas: %w", serviceName, err) - } - needsUpdate = true + // Apply all Deployments with SSA for consistency + if err := r.applyDeploymentWithSSA(ctx, desired); err != nil { + return nil, fmt.Errorf("failed to apply deployment %s with SSA: %w", desired.Name, err) } - } - - // Update the cluster spec if any replicas were changed - if needsUpdate { - if err := r.Update(ctx, cluster); err != nil { - return fmt.Errorf("failed to update cluster spec with HPA coordination: %w", err) + // Fetch the current state of the deployment after apply + current := &appsv1.Deployment{} + if err := r.Get(ctx, client.ObjectKeyFromObject(desired), current); err != nil { + return nil, fmt.Errorf("failed to get deployment %s after SSA: %w", desired.Name, err) } - logger.Info("Updated TemporalCluster spec to coordinate with HPA scaling decisions") + objects = append(objects, current) } - return nil + return objects, nil } -// updateServiceReplicas updates the replica count for a specific service in the cluster spec. -func (r *TemporalClusterReconciler) updateServiceReplicas(cluster *v1beta1.TemporalCluster, serviceName primitives.ServiceName, replicas int32) error { - switch serviceName { - case primitives.FrontendService: - if cluster.Spec.Services.Frontend == nil { - cluster.Spec.Services.Frontend = &v1beta1.ServiceSpec{} - } - cluster.Spec.Services.Frontend.Replicas = &replicas - case primitives.HistoryService: - if cluster.Spec.Services.History == nil { - cluster.Spec.Services.History = &v1beta1.ServiceSpec{} - } - cluster.Spec.Services.History.Replicas = &replicas - case primitives.MatchingService: - if cluster.Spec.Services.Matching == nil { - cluster.Spec.Services.Matching = &v1beta1.ServiceSpec{} - } - cluster.Spec.Services.Matching.Replicas = &replicas - case primitives.WorkerService: - if cluster.Spec.Services.Worker == nil { - cluster.Spec.Services.Worker = &v1beta1.ServiceSpec{} - } - cluster.Spec.Services.Worker.Replicas = &replicas - case primitives.InternalFrontendService: - if cluster.Spec.Services.InternalFrontend == nil { - cluster.Spec.Services.InternalFrontend = &v1beta1.InternalFrontendServiceSpec{} +// applyDeploymentWithSSA applies a deployment using Server-Side Apply with field management. +func (r *TemporalClusterReconciler) applyDeploymentWithSSA(ctx context.Context, desired *appsv1.Deployment) error { + desired.TypeMeta = metav1.TypeMeta{APIVersion: "apps/v1", Kind: "Deployment"} + + // replicas must be omitted when HPA is enabled (builder should already do this) + // if autoscalingEnabled { desired.Spec.Replicas = nil } + + if err := r.Patch(ctx, desired, client.Apply, client.FieldOwner("temporal-operator")); err != nil { + if apierrors.IsConflict(err) { + // Only if you’re intentionally handing over ownership: + return r.Patch(ctx, desired, client.Apply, client.FieldOwner("temporal-operator"), client.ForceOwnership) } - cluster.Spec.Services.InternalFrontend.Replicas = &replicas - default: - return fmt.Errorf("unknown service name: %s", serviceName) + return err } return nil } diff --git a/internal/resource/base/hpa_builder.go b/internal/resource/base/hpa_builder.go index b9b0dc3a..1cdf2cd5 100644 --- a/internal/resource/base/hpa_builder.go +++ b/internal/resource/base/hpa_builder.go @@ -1,7 +1,7 @@ -// Licensed to Amesh Fernando under one or more contributor +// Licensed to Alexandre VILAIN under one or more contributor // license agreements. See the NOTICE file distributed with // this work for additional information regarding copyright -// ownership. Amesh Fernando licenses this file to you under +// ownership. Alexandre VILAIN licenses this file to you under // the Apache License, Version 2.0 (the "License"); you may // not use this file except in compliance with the License. // You may obtain a copy of the License at From 521fc559690bd21c344e7701c34381edffc4747d Mon Sep 17 00:00:00 2001 From: ameshf Date: Mon, 17 Nov 2025 15:38:30 +1100 Subject: [PATCH 3/3] ignore replicas when hpa is provided --- api/v1beta1/temporalcluster_types.go | 16 ----- controllers/temporalcluster_controller.go | 70 +------------------- internal/resource/base/deployment_builder.go | 4 +- 3 files changed, 4 insertions(+), 86 deletions(-) diff --git a/api/v1beta1/temporalcluster_types.go b/api/v1beta1/temporalcluster_types.go index 24266661..bc4c65c8 100644 --- a/api/v1beta1/temporalcluster_types.go +++ b/api/v1beta1/temporalcluster_types.go @@ -33,7 +33,6 @@ import ( networkingv1 "k8s.io/api/networking/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/ptr" ) // LogSpec contains the temporal logging configuration. @@ -153,21 +152,6 @@ func (s *ServiceSpec) IsAutoscalingEnabled() bool { return s != nil && s.Autoscaling != nil } -// GetEffectiveReplicas returns the replica count to use for the deployment. -// If autoscaling is enabled, returns the current replicas value which HPA may have -// updated via the coordination logic. Otherwise, returns the configured replicas value. -func (s *ServiceSpec) GetEffectiveReplicas() *int32 { - // If autoscaling is configured, respect the current Replicas value which may be - // updated by HPA coordination logic, but ensure it doesn't go below MinReplicas - if s.IsAutoscalingEnabled() { - return nil - } - if s.Replicas != nil { - return s.Replicas - } - return ptr.To[int32](1) -} - // ServicesSpec contains all temporal services specifications. type ServicesSpec struct { // Frontend service custom specifications. diff --git a/controllers/temporalcluster_controller.go b/controllers/temporalcluster_controller.go index d839f99a..039cde15 100644 --- a/controllers/temporalcluster_controller.go +++ b/controllers/temporalcluster_controller.go @@ -174,33 +174,11 @@ func (r *TemporalClusterReconciler) reconcileResources(ctx context.Context, temp return err } - // Separate deployment builders from others to handle SSA for Deployments - var deploymentBuilders []resource.Builder - var otherBuilders []resource.Builder - - for _, builder := range builders { - if _, isDeployment := builder.(*base.DeploymentBuilder); isDeployment { - deploymentBuilders = append(deploymentBuilders, builder) - } else { - otherBuilders = append(otherBuilders, builder) - } - } - - // Reconcile non-deployment resources normally - objects, err := r.Reconciler.ReconcileBuilders(ctx, temporalCluster, otherBuilders) - if err != nil { - return err - } - - // Handle deployments with SSA - deploymentObjects, err := r.reconcileDeploymentsWithSSA(ctx, deploymentBuilders) + objects, err := r.Reconciler.ReconcileBuilders(ctx, temporalCluster, builders) if err != nil { return err } - // Combine all objects - objects = append(objects, deploymentObjects...) - statuses, err := status.ReconciledObjectsToServiceStatuses(temporalCluster, objects) if err != nil { return err @@ -298,52 +276,6 @@ func (r *TemporalClusterReconciler) handleErrorWithRequeue(cluster *v1beta1.Temp return reconcile.Result{RequeueAfter: requeueAfter}, err } -func (r *TemporalClusterReconciler) reconcileDeploymentsWithSSA(ctx context.Context, builders []resource.Builder) ([]client.Object, error) { - objects := make([]client.Object, 0) - - for _, builder := range builders { - deploymentBuilder := builder.(*base.DeploymentBuilder) - - // Build the desired deployment - desiredObj := deploymentBuilder.Build() - desired := desiredObj.(*appsv1.Deployment) - - // Update the desired deployment with the current configuration - if err := deploymentBuilder.Update(desired); err != nil { - return nil, fmt.Errorf("failed to update deployment configuration: %w", err) - } - // Apply all Deployments with SSA for consistency - if err := r.applyDeploymentWithSSA(ctx, desired); err != nil { - return nil, fmt.Errorf("failed to apply deployment %s with SSA: %w", desired.Name, err) - } - // Fetch the current state of the deployment after apply - current := &appsv1.Deployment{} - if err := r.Get(ctx, client.ObjectKeyFromObject(desired), current); err != nil { - return nil, fmt.Errorf("failed to get deployment %s after SSA: %w", desired.Name, err) - } - objects = append(objects, current) - } - - return objects, nil -} - -// applyDeploymentWithSSA applies a deployment using Server-Side Apply with field management. -func (r *TemporalClusterReconciler) applyDeploymentWithSSA(ctx context.Context, desired *appsv1.Deployment) error { - desired.TypeMeta = metav1.TypeMeta{APIVersion: "apps/v1", Kind: "Deployment"} - - // replicas must be omitted when HPA is enabled (builder should already do this) - // if autoscalingEnabled { desired.Spec.Replicas = nil } - - if err := r.Patch(ctx, desired, client.Apply, client.FieldOwner("temporal-operator")); err != nil { - if apierrors.IsConflict(err) { - // Only if you’re intentionally handing over ownership: - return r.Patch(ctx, desired, client.Apply, client.FieldOwner("temporal-operator"), client.ForceOwnership) - } - return err - } - return nil -} - // SetupWithManager sets up the controller with the Manager. func (r *TemporalClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { for _, resource := range []client.Object{&appsv1.Deployment{}, &autoscalingv2.HorizontalPodAutoscaler{}, &corev1.ConfigMap{}, &corev1.Service{}, &corev1.ServiceAccount{}, &networkingv1.Ingress{}, &batchv1.Job{}} { diff --git a/internal/resource/base/deployment_builder.go b/internal/resource/base/deployment_builder.go index 14e882fd..bceae712 100644 --- a/internal/resource/base/deployment_builder.go +++ b/internal/resource/base/deployment_builder.go @@ -334,7 +334,9 @@ func (b *DeploymentBuilder) Update(object client.Object) error { }) } - deployment.Spec.Replicas = b.service.GetEffectiveReplicas() + if !b.service.IsAutoscalingEnabled() { + deployment.Spec.Replicas = b.service.Replicas + } deployment.Spec.Selector = &metav1.LabelSelector{ MatchLabels: metadata.LabelsSelector(b.instance, b.serviceName),