diff --git a/.github/workflows/linkinator.yaml b/.github/workflows/linkinator.yaml index 509d28c24..6975c9bfb 100644 --- a/.github/workflows/linkinator.yaml +++ b/.github/workflows/linkinator.yaml @@ -34,11 +34,7 @@ jobs: --retry-wait-time 2 --timeout 30 --github-token ${{ secrets.GITHUB_TOKEN }} - --exclude '^https://github\.com/kedacore/http-add-on/pkgs/container/http-add-on-interceptor$' - --exclude '^https://github\.com/kedacore/http-add-on/pkgs/container/http-add-on-operator$' - --exclude '^https://github\.com/kedacore/http-add-on/pkgs/container/http-add-on-scaler$' - --exclude '^http://opentelemetry-collector\.open-telemetry-system:4318$' - --exclude '^http://opentelemetry-collector\.open-telemetry-system:4318/$' - --exclude '^http://opentelemetry-collector\.open-telemetry-system:4318/v1/traces$' + --exclude '^https://github\.com/kedacore/http-add-on/pkgs/container/http-add-on-(interceptor|operator|scaler)(/.*)?$' + --exclude '^http://opentelemetry-collector\.open-telemetry-system:431[7-8](/.*)?$' --exclude '^https://www\.gnu\.org/software/make/$' "./**/*.md" diff --git a/CHANGELOG.md b/CHANGELOG.md index 13098cf0e..e5ccd661b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,7 +31,7 @@ This changelog keeps track of work items that have been completed and are ready ### Improvements -- **General**: TODO ([#TODO](https://github.com/kedacore/http-add-on/issues/TODO)) +- **General**: Add prometehus and otel instrumentation for the operator ([#965](https://github.com/kedacore/http-add-on/issues/965)) ### Fixes diff --git a/Makefile b/Makefile index bfbbfdd06..4e55e2ef2 100644 --- a/Makefile +++ b/Makefile @@ -230,6 +230,9 @@ deploy: manifests kustomize ## Deploy to the K8s cluster specified in ~/.kube/co cd config/operator && \ $(KUSTOMIZE) edit set image ghcr.io/kedacore/http-add-on-operator=${IMAGE_OPERATOR_VERSIONED_TAG} + cd config/operator && \ + $(KUSTOMIZE) edit add patch --path e2e-test/otel/deployment.yaml --group apps --kind Deployment --name operator --version v1 + $(KUSTOMIZE) build config/default | kubectl apply -f - undeploy: diff --git a/config/operator/e2e-test/otel/deployment.yaml b/config/operator/e2e-test/otel/deployment.yaml new file mode 100644 index 000000000..90ccec043 --- /dev/null +++ b/config/operator/e2e-test/otel/deployment.yaml @@ -0,0 +1,23 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: operator +spec: + replicas: 1 + template: + spec: + containers: + - name: operator + env: + - name: OTEL_PROM_EXPORTER_ENABLED + value: "true" + - name: OTEL_PROM_EXPORTER_PORT + value: "8080" + - name: OTEL_EXPORTER_OTLP_METRICS_ENABLED + value: "true" + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "http://opentelemetry-collector.open-telemetry-system:4318" + - name: OTEL_METRIC_EXPORT_INTERVAL + value: "1" + - name: OTEL_EXPORTER_OTLP_PROTOCOL + value: "http" diff --git a/config/operator/e2e-test/otel/kustomization.yaml b/config/operator/e2e-test/otel/kustomization.yaml new file mode 100644 index 000000000..42835f535 --- /dev/null +++ b/config/operator/e2e-test/otel/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- deployment.yaml diff --git a/config/operator/kustomization.yaml b/config/operator/kustomization.yaml index 08e075759..170190d9d 100644 --- a/config/operator/kustomization.yaml +++ b/config/operator/kustomization.yaml @@ -5,6 +5,7 @@ resources: - role.yaml - role_binding.yaml - service_account.yaml +- metrics.service.yaml labels: - includeSelectors: true includeTemplates: true diff --git a/config/operator/metrics.service.yaml b/config/operator/metrics.service.yaml new file mode 100644 index 000000000..cda28c6e5 --- /dev/null +++ b/config/operator/metrics.service.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: operator-metrics +spec: + type: ClusterIP + ports: + - name: metrics + protocol: TCP + port: 8080 + targetPort: metrics \ No newline at end of file diff --git a/docs/faq.md b/docs/faq.md index cb42d9541..badcac056 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -22,8 +22,8 @@ However, Osiris and KEDA-HTTP differ in several ways: Knative Serving and KEDA-HTTP both have core support for autoscaling, including scale-to-zero of compute workloads. KEDA-HTTP is focused solely on deploying production-grade autoscaling HTTP applications, while Knative builds in additional functionality: - Pure [event-based workloads](https://knative.dev/docs/eventing/). [KEDA core](https://github.com/kedacore/keda), without KEDA-HTTP, can support such workloads natively. -- Complex deployment strategies like [blue-green](https://knative.dev/docs/serving/samples/blue-green-deployment/). -- Supporting other autoscaling mechanisms beyond the built-in [HPA](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/), such as the [Knative Pod Autoscaler (KPA)](https://knative.dev/docs/serving/autoscaling/autoscaling-concepts/#knative-pod-autoscaler-kpa). +- Complex deployment strategies like [blue-green](https://knative.dev/docs/serving/traffic-management/#routing-and-managing-traffic-with-bluegreen-deployment). +- Supporting other autoscaling mechanisms beyond the built-in [HPA](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/), such as the [Knative Pod Autoscaler (KPA)](https://knative.dev/docs/serving/autoscaling/autoscaler-types/#knative-pod-autoscaler-kpa). Additionally, Knative supports a service mesh, while KEDA-HTTP does not out of the box (support for that is forthcoming). diff --git a/docs/install.md b/docs/install.md index 9ee6688e2..e21a27d58 100644 --- a/docs/install.md +++ b/docs/install.md @@ -52,7 +52,7 @@ There are a few values that you can pass to the above `helm install` command by helm install http-add-on kedacore/keda-add-ons-http --create-namespace --namespace ${NAMESPACE} --set images.tag=canary ``` -For an exhaustive list of configuration options, see the official HTTP Add-on chart [values.yaml file](https://github.com/kedacore/charts/blob/master/http-add-on/values.yaml). +For an exhaustive list of configuration options, see the official HTTP Add-on chart [values.yaml file](https://github.com/kedacore/charts/blob/main/http-add-on/values.yaml). ### A Note for Developers and Local Cluster Users diff --git a/docs/operate.md b/docs/operate.md index 266d4f7fe..e29b24824 100644 --- a/docs/operate.md +++ b/docs/operate.md @@ -9,7 +9,7 @@ There are currently 2 supported methods for exposing metrics from the intercepto ### Configuring the Prometheus compatible metrics endpoint When configured, the interceptor proxy can expose metrics on a Prometheus compatible endpoint. -This endpoint can be enabled by setting the `OTEL_PROM_EXPORTER_ENABLED` environment variable to `true` on the interceptor deployment (`true` by default) and by setting `OTEL_PROM_EXPORTER_PORT` to an unused port for the endpoint to be made avaialble on (`2223` by default). +This endpoint can be enabled by setting the `OTEL_PROM_EXPORTER_ENABLED` environment variable to `true` on the interceptor deployment (`true` by default) and by setting `OTEL_PROM_EXPORTER_PORT` to an unused port for the endpoint to be made available on (`2223` by default). ### Configuring the OTEL HTTP exporter When configured, the interceptor proxy can export metrics to a OTEL HTTP collector. @@ -71,3 +71,29 @@ Optional variables `OTEL_EXPORTER_OTLP_TRACES_TIMEOUT` - The batcher timeout in seconds to send batch of data points (`5` by default) ### Configuring Service Failover + +# Configuring metrics for the KEDA HTTP Add-on Operator + +### Exportable metrics: +* **keda_http_scaled_object_total** - the number of http_scaled_objects + +There are currently 2 supported methods for exposing metrics from the operator - via a Prometheus compatible metrics endpoint or by pushing metrics to a OTEL HTTP collector. + +### Configuring the Prometheus compatible metrics endpoint +When configured, the operator can expose metrics on a Prometheus compatible endpoint. + +This endpoint can be enabled by setting the `OTEL_PROM_EXPORTER_ENABLED` environment variable to `true` on the operator deployment (`true` by default) and by setting `OTEL_PROM_EXPORTER_PORT` to an unused port for the endpoint to be made available on (`8080` by default). + +### Configuring the OTEL HTTP exporter + +When configured, the operator can export metrics to a OTEL HTTP collector. + +The OTEL exporter can be enabled by setting the `OTEL_EXPORTER_OTLP_METRICS_ENABLED` environment variable to `true` on the operator deployment (`false` by default). When enabled, the `OTEL_EXPORTER_OTLP_ENDPOINT` environment variable must also be configured so the exporter knows what collector to send the metrics to (e.g. http://opentelemetry-collector.open-telemetry-system:4318). + +If you need to provide any headers such as authentication details in order to utilise your OTEL collector you can add them into the `OTEL_EXPORTER_OTLP_HEADERS` environment variable. The frequency at which the metrics are exported can be configured by setting `OTEL_METRIC_EXPORT_INTERVAL` to the number of seconds you require between each export interval (`30` by default). + +The `OTEL_EXPORTER_OTLP_PROTOCOL` defaults to `http` + +### Configuring the OTEL GRPC exporter + +Please note that using `OTEL_EXPORTER_OTLP_PROTOCOL` will allows you to set it up to `grpc` to connect to otel collector. Also `OTEL_EXPORTER_OTLP_ENDPOINT` should be set to the right endpoint (eg: http://opentelemetry-collector.open-telemetry-system:4317) diff --git a/go.mod b/go.mod index d8c687612..1a2634e73 100644 --- a/go.mod +++ b/go.mod @@ -36,6 +36,7 @@ require ( go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 go.opentelemetry.io/contrib/propagators/b3 v1.38.0 go.opentelemetry.io/otel v1.38.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.35.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.38.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 @@ -64,6 +65,7 @@ require ( github.com/Masterminds/semver/v3 v3.4.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect diff --git a/go.sum b/go.sum index 3b139f000..04c4078a6 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -207,6 +209,8 @@ go.opentelemetry.io/contrib/propagators/b3 v1.38.0 h1:uHsCCOSKl0kLrV2dLkFK+8Ywk9 go.opentelemetry.io/contrib/propagators/b3 v1.38.0/go.mod h1:wMRSZJZcY8ya9mApLLhwIMjqmApy2o/Ml+62lhvxyHU= go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.35.0 h1:QcFwRrZLc82r8wODjvyCbP7Ifp3UANaBSmhDSFjnqSc= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.35.0/go.mod h1:CXIWhUomyWBG/oY2/r/kLp6K/cmx9e/7DLpBuuGdLCA= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.38.0 h1:Oe2z/BCg5q7k4iXC3cqJxKYg0ieRiOqF0cecFYdPTwk= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.38.0/go.mod h1:ZQM5lAJpOsKnYagGg/zV2krVqTtaVdYdDkhMoX6Oalg= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24= diff --git a/operator/controllers/http/config/config.go b/operator/controllers/http/config/config.go index 667121670..4ddf32626 100644 --- a/operator/controllers/http/config/config.go +++ b/operator/controllers/http/config/config.go @@ -90,3 +90,23 @@ func NewExternalScalerFromEnv() (*ExternalScaler, error) { Port: port, }, nil } + +// Metrics is the configuration for configuring metrics in the operator. +type Metrics struct { + // Sets whether or not to enable the Prometheus metrics exporter + OtelPrometheusExporterEnabled bool `envconfig:"OTEL_PROM_EXPORTER_ENABLED" default:"true"` + // Sets the port which the Prometheus compatible metrics endpoint should be served on + OtelPrometheusExporterPort int `envconfig:"OTEL_PROM_EXPORTER_PORT" default:"8080"` + // Sets whether or not to enable the OTEL metrics exporter + OtelHTTPExporterEnabled bool `envconfig:"OTEL_EXPORTER_OTLP_METRICS_ENABLED" default:"false"` + // Sets OTEL metrics exporter protocol + OtelExporterProtocol string `envconfig:"OTEL_EXPORTER_OTLP_PROTOCOL" default:"http"` +} + +// Parse parses standard configs using envconfig and returns a pointer to the +// newly created config. Returns nil and a non-nil error if parsing failed +func MustParseMetrics() *Metrics { + ret := new(Metrics) + envconfig.MustProcess("", ret) + return ret +} diff --git a/operator/controllers/http/httpscaledobject_controller.go b/operator/controllers/http/httpscaledobject_controller.go index 4eb5e8934..e8881b9f2 100644 --- a/operator/controllers/http/httpscaledobject_controller.go +++ b/operator/controllers/http/httpscaledobject_controller.go @@ -19,6 +19,7 @@ package http import ( "context" "fmt" + "sync" "time" kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" @@ -34,6 +35,16 @@ import ( httpv1alpha1 "github.com/kedacore/http-add-on/operator/apis/http/v1alpha1" "github.com/kedacore/http-add-on/operator/controllers/http/config" "github.com/kedacore/http-add-on/operator/controllers/util" + "github.com/kedacore/http-add-on/operator/metrics" +) + +type httpScaledObjectMetricsData struct { + namespace string +} + +var ( + httpScaledObjectPromMetricsMap map[string]httpScaledObjectMetricsData + httpScaledObjectPromMetricsLock *sync.Mutex ) // HTTPScaledObjectReconciler reconciles a HTTPScaledObject object @@ -48,6 +59,11 @@ type HTTPScaledObjectReconciler struct { BaseConfig config.Base } +func init() { + httpScaledObjectPromMetricsMap = make(map[string]httpScaledObjectMetricsData) + httpScaledObjectPromMetricsLock = &sync.Mutex{} +} + // +kubebuilder:rbac:groups=http.keda.sh,resources=httpscaledobjects,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=http.keda.sh,resources=httpscaledobjects/status,verbs=get;update;patch // +kubebuilder:rbac:groups=http.keda.sh,resources=httpscaledobjects/finalizers,verbs=update @@ -78,6 +94,7 @@ func (r *HTTPScaledObjectReconciler) Reconcile(ctx context.Context, req ctrl.Req } if httpso.GetDeletionTimestamp() != nil { + r.updatePromMetricsOnDelete(ctx, httpso) return ctrl.Result{}, finalizeScaledObject(ctx, logger, r.Client, httpso) } @@ -139,6 +156,7 @@ func (r *HTTPScaledObjectReconciler) Reconcile(ctx context.Context, req ctrl.Req ) // success reconciling + r.updatePromMetrics(ctx, httpso) logger.Info("Reconcile success") return ctrl.Result{}, nil } @@ -162,3 +180,35 @@ func (r *HTTPScaledObjectReconciler) SetupWithManager(mgr ctrl.Manager) error { ))). Complete(r) } + +func (r *HTTPScaledObjectReconciler) updatePromMetrics(ctx context.Context, scaledObject *httpv1alpha1.HTTPScaledObject) { + httpScaledObjectPromMetricsLock.Lock() + defer httpScaledObjectPromMetricsLock.Unlock() + + namespacedName := client.ObjectKeyFromObject(scaledObject).String() + metricsData, ok := httpScaledObjectPromMetricsMap[namespacedName] + if ok { + metrics.RecordDeleteHTTPScaledObjectCount(scaledObject.Namespace) + } + metricsData.namespace = scaledObject.Namespace + + logger := log.FromContext(ctx, "updatePromMetrics", namespacedName) + logger.Info("updatePromMetrics") + metrics.RecordHTTPScaledObjectCount(scaledObject.Namespace) + + httpScaledObjectPromMetricsMap[namespacedName] = metricsData +} + +func (r *HTTPScaledObjectReconciler) updatePromMetricsOnDelete(ctx context.Context, scaledObject *httpv1alpha1.HTTPScaledObject) { + httpScaledObjectPromMetricsLock.Lock() + defer httpScaledObjectPromMetricsLock.Unlock() + + namespacedName := scaledObject.Name + scaledObject.Namespace + logger := log.FromContext(ctx, "updatePromMetricsOnDelete", namespacedName) + logger.Info("updatePromMetricsOnDelete") + + if _, ok := httpScaledObjectPromMetricsMap[namespacedName]; ok { + metrics.RecordDeleteHTTPScaledObjectCount(scaledObject.Namespace) + } + delete(httpScaledObjectPromMetricsMap, namespacedName) +} diff --git a/operator/main.go b/operator/main.go index 4844a268a..72391db31 100644 --- a/operator/main.go +++ b/operator/main.go @@ -20,6 +20,11 @@ import ( "flag" "os" + httpv1alpha1 "github.com/kedacore/http-add-on/operator/apis/http/v1alpha1" + httpcontrollers "github.com/kedacore/http-add-on/operator/controllers/http" + "github.com/kedacore/http-add-on/operator/controllers/http/config" + "github.com/kedacore/http-add-on/operator/metrics" + "github.com/kedacore/http-add-on/pkg/util" kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -30,10 +35,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics/server" - - httpv1alpha1 "github.com/kedacore/http-add-on/operator/apis/http/v1alpha1" - httpcontrollers "github.com/kedacore/http-add-on/operator/controllers/http" - "github.com/kedacore/http-add-on/operator/controllers/http/config" // +kubebuilder:scaffold:imports ) @@ -69,7 +70,9 @@ func main() { } opts.BindFlags(flag.CommandLine) flag.Parse() - + metricsCfg := config.MustParseMetrics() + // setup the configured metrics collectors + metrics.NewMetricsCollectors(metricsCfg) ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) externalScalerCfg, err := config.NewExternalScalerFromEnv() @@ -112,6 +115,8 @@ func main() { os.Exit(1) } + ctx := ctrl.SetupSignalHandler() + ctx = util.ContextWithLogger(ctx, ctrl.Log) if err = (&httpcontrollers.HTTPScaledObjectReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), @@ -134,7 +139,7 @@ func main() { } setupLog.Info("starting manager") - if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { + if err := mgr.Start(ctx); err != nil { setupLog.Error(err, "problem running manager") os.Exit(1) } diff --git a/operator/metrics/metricscollector.go b/operator/metrics/metricscollector.go new file mode 100644 index 000000000..6929c1590 --- /dev/null +++ b/operator/metrics/metricscollector.go @@ -0,0 +1,44 @@ +package metrics + +import ( + "go.opentelemetry.io/otel/exporters/prometheus" + ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" + + "github.com/kedacore/http-add-on/operator/controllers/http/config" +) + +var ( + collectors []Collector +) + +const meterName = "keda-http-add-on-operator" + +type Collector interface { + RecordHTTPScaledObjectCount(namespace string) + RecordDeleteHTTPScaledObjectCount(namespace string) +} + +func NewMetricsCollectors(metricsConfig *config.Metrics) { + if metricsConfig.OtelPrometheusExporterEnabled { + options := prometheus.WithRegisterer(ctrlmetrics.Registry) + promometrics := NewPrometheusMetrics(options) + collectors = append(collectors, promometrics) + } + + if metricsConfig.OtelHTTPExporterEnabled { + otelhttpmetrics := NewOtelMetrics() + collectors = append(collectors, otelhttpmetrics) + } +} + +func RecordHTTPScaledObjectCount(namespace string) { + for _, collector := range collectors { + collector.RecordHTTPScaledObjectCount(namespace) + } +} + +func RecordDeleteHTTPScaledObjectCount(namespace string) { + for _, collector := range collectors { + collector.RecordDeleteHTTPScaledObjectCount(namespace) + } +} diff --git a/operator/metrics/otelmetrics.go b/operator/metrics/otelmetrics.go new file mode 100644 index 000000000..0eebfdcfe --- /dev/null +++ b/operator/metrics/otelmetrics.go @@ -0,0 +1,90 @@ +package metrics + +import ( + "context" + "log" + "os" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" + api "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/resource" + semconv "go.opentelemetry.io/otel/semconv/v1.4.0" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + "github.com/kedacore/http-add-on/pkg/build" +) + +var otLog = logf.Log.WithName("otel_collector") + +type OtelMetrics struct { + meter api.Meter + httpScaledObjectCounter api.Int64UpDownCounter +} + +func NewOtelMetrics(options ...metric.Option) *OtelMetrics { + if options == nil { + protocol := os.Getenv("OTEL_EXPORTER_OTLP_PROTOCOL") + + var exporter metric.Exporter + var err error + switch protocol { + case "grpc": + otLog.V(1).Info("start OTEL grpc client") + exporter, err = otlpmetricgrpc.New(context.Background()) + default: + otLog.V(1).Info("start OTEL http client") + exporter, err = otlpmetrichttp.New(context.Background()) + } + + if err != nil { + log.Fatalf("could not create otelmetrichttp exporter: %v", err) + return nil + } + res := resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceNameKey.String("http-add-on-operator"), + semconv.ServiceVersionKey.String(build.Version()), + ) + + options = []metric.Option{ + metric.WithReader(metric.NewPeriodicReader(exporter)), + metric.WithResource(res), + } + } + + provider := metric.NewMeterProvider(options...) + meter := provider.Meter(meterName) + + httpScaledObjectCounter, err := meter.Int64UpDownCounter("keda.http.scaled.object.total", api.WithDescription("a counter of http_scaled_objects processed by the operator")) + if err != nil { + log.Fatalf("could not create new otelhttpmetric request counter: %v", err) + } + + return &OtelMetrics{ + meter: meter, + httpScaledObjectCounter: httpScaledObjectCounter, + } +} + +func (om *OtelMetrics) RecordHTTPScaledObjectCount(namespace string) { + ctx := context.Background() + opt := api.WithAttributeSet( + attribute.NewSet( + attribute.Key("namespace").String(namespace), + ), + ) + om.httpScaledObjectCounter.Add(ctx, 1, opt) +} + +func (om *OtelMetrics) RecordDeleteHTTPScaledObjectCount(namespace string) { + ctx := context.Background() + opt := api.WithAttributeSet( + attribute.NewSet( + attribute.Key("namespace").String(namespace), + ), + ) + om.httpScaledObjectCounter.Add(ctx, -1, opt) +} diff --git a/operator/metrics/otelmetrics_test.go b/operator/metrics/otelmetrics_test.go new file mode 100644 index 000000000..96d41240d --- /dev/null +++ b/operator/metrics/otelmetrics_test.go @@ -0,0 +1,44 @@ +package metrics + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" +) + +var ( + testOtel *OtelMetrics + testReader metric.Reader +) + +func init() { + testReader = metric.NewManualReader() + options := metric.WithReader(testReader) + testOtel = NewOtelMetrics(options) +} + +func TestHTTPScaledObjectCount(t *testing.T) { + testOtel.RecordHTTPScaledObjectCount("test-namespace") + got := metricdata.ResourceMetrics{} + err := testReader.Collect(context.Background(), &got) + + assert.Nil(t, err) + scopeMetrics := got.ScopeMetrics[0] + assert.NotEqual(t, len(scopeMetrics.Metrics), 0) + + metricInfo := retrieveMetric(scopeMetrics.Metrics, "keda.http.scaled.object.total") + data := metricInfo.Data.(metricdata.Sum[int64]).DataPoints[0] + assert.Equal(t, data.Value, int64(1)) +} + +func retrieveMetric(metrics []metricdata.Metrics, metricname string) *metricdata.Metrics { + for _, m := range metrics { + if m.Name == metricname { + return &m + } + } + return nil +} diff --git a/operator/metrics/prommetrics.go b/operator/metrics/prommetrics.go new file mode 100644 index 000000000..911763152 --- /dev/null +++ b/operator/metrics/prommetrics.go @@ -0,0 +1,75 @@ +package metrics + +import ( + "context" + "log" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/prometheus" + api "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/resource" + semconv "go.opentelemetry.io/otel/semconv/v1.4.0" + + "github.com/kedacore/http-add-on/pkg/build" +) + +type PrometheusMetrics struct { + meter api.Meter + httpScaledObjectCounter api.Int64UpDownCounter +} + +func NewPrometheusMetrics(options ...prometheus.Option) *PrometheusMetrics { + var exporter *prometheus.Exporter + var err error + if options == nil { + exporter, err = prometheus.New() + } else { + exporter, err = prometheus.New(options...) + } + if err != nil { + log.Fatalf("could not create Prometheus exporter: %v", err) + } + + res := resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceNameKey.String("http-add-on-operator"), + semconv.ServiceVersionKey.String(build.Version()), + ) + + provider := metric.NewMeterProvider( + metric.WithReader(exporter), + metric.WithResource(res), + ) + meter := provider.Meter(meterName) + + httpScaledObjectCounter, err := meter.Int64UpDownCounter("keda_http_scaled_object_total", api.WithDescription("a counter of http_scaled_objects processed by the operator")) + if err != nil { + log.Fatalf("could not create new Prometheus request counter: %v", err) + } + + return &PrometheusMetrics{ + meter: meter, + httpScaledObjectCounter: httpScaledObjectCounter, + } +} + +func (p *PrometheusMetrics) RecordHTTPScaledObjectCount(namespace string) { + ctx := context.Background() + opt := api.WithAttributeSet( + attribute.NewSet( + attribute.Key("namespace").String(namespace), + ), + ) + p.httpScaledObjectCounter.Add(ctx, 1, opt) +} + +func (p *PrometheusMetrics) RecordDeleteHTTPScaledObjectCount(namespace string) { + ctx := context.Background() + opt := api.WithAttributeSet( + attribute.NewSet( + attribute.Key("namespace").String(namespace), + ), + ) + p.httpScaledObjectCounter.Add(ctx, -1, opt) +} diff --git a/operator/metrics/prommetrics_test.go b/operator/metrics/prommetrics_test.go new file mode 100644 index 000000000..bba08c316 --- /dev/null +++ b/operator/metrics/prommetrics_test.go @@ -0,0 +1,35 @@ +package metrics + +import ( + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/assert" + promexporter "go.opentelemetry.io/otel/exporters/prometheus" +) + +func TestPromRequestCountMetric(t *testing.T) { + testRegistry := prometheus.NewRegistry() + options := []promexporter.Option{promexporter.WithRegisterer(testRegistry)} + testPrometheus := NewPrometheusMetrics(options...) + expectedOutput := ` + # HELP keda_http_scaled_object_total a counter of http_scaled_objects processed by the operator + # TYPE keda_http_scaled_object_total gauge + keda_http_scaled_object_total{namespace="test-namespace",otel_scope_name="keda-http-add-on-operator",otel_scope_version=""} 0 + keda_http_scaled_object_total{namespace="other-test-namespace",otel_scope_name="keda-http-add-on-operator",otel_scope_version=""} 1 + # HELP otel_scope_info Instrumentation Scope metadata + # TYPE otel_scope_info gauge + otel_scope_info{otel_scope_name="keda-http-add-on-operator",otel_scope_version=""} 1 + # HELP target_info Target metadata + # TYPE target_info gauge + target_info{"service.name"="http-add-on-operator","service.version"="main"} 1 + ` + expectedOutputReader := strings.NewReader(expectedOutput) + testPrometheus.RecordHTTPScaledObjectCount("test-namespace") + testPrometheus.RecordDeleteHTTPScaledObjectCount("test-namespace") + testPrometheus.RecordHTTPScaledObjectCount("other-test-namespace") + err := testutil.CollectAndCompare(testRegistry, expectedOutputReader) + assert.Nil(t, err) +} diff --git a/tests/checks/operator_otel_metrics/operator_otel_metrics_test.go b/tests/checks/operator_otel_metrics/operator_otel_metrics_test.go new file mode 100644 index 000000000..7161a19d8 --- /dev/null +++ b/tests/checks/operator_otel_metrics/operator_otel_metrics_test.go @@ -0,0 +1,349 @@ +//go:build e2e +// +build e2e + +package operator_otel_metrics_test + +import ( + "context" + "fmt" + "slices" + "strings" + "testing" + "time" + + prommodel "github.com/prometheus/client_model/go" + "github.com/prometheus/common/expfmt" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + + . "github.com/kedacore/http-add-on/tests/helper" +) + +const ( + testName = "operator-otel-metrics-test" +) + +var ( + testNamespace = fmt.Sprintf("%s-ns", testName) + deploymentName = fmt.Sprintf("%s-deployment", testName) + serviceName = fmt.Sprintf("%s-service", testName) + clientName = fmt.Sprintf("%s-client", testName) + httpScaledObjectName = fmt.Sprintf("%s-http-so", testName) + host = testName + minReplicaCount = 0 + maxReplicaCount = 1 + otelCollectorPromURL = "http://opentelemetry-collector.open-telemetry-system:8889/metrics" + otlpGrpcClientEndpoint = "http://opentelemetry-collector.open-telemetry-system:4317" + otlpHTTPClientEndpoint = "http://opentelemetry-collector.open-telemetry-system:4318" +) + +type templateData struct { + TestNamespace string + DeploymentName string + ServiceName string + ClientName string + HTTPScaledObjectName string + Host string + MinReplicas int + MaxReplicas int +} + +const ( + serviceTemplate = ` +apiVersion: v1 +kind: Service +metadata: + name: {{.ServiceName}} + namespace: {{.TestNamespace}} + labels: + app: {{.DeploymentName}} +spec: + ports: + - port: 8080 + targetPort: http + protocol: TCP + name: http + selector: + app: {{.DeploymentName}} +` + + deploymentTemplate = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{.DeploymentName}} + namespace: {{.TestNamespace}} + labels: + app: {{.DeploymentName}} +spec: + replicas: 0 + selector: + matchLabels: + app: {{.DeploymentName}} + template: + metadata: + labels: + app: {{.DeploymentName}} + spec: + containers: + - name: {{.DeploymentName}} + image: registry.k8s.io/e2e-test-images/agnhost:2.45 + args: + - netexec + ports: + - name: http + containerPort: 8080 + protocol: TCP + readinessProbe: + httpGet: + path: / + port: http +` + + loadJobTemplate = ` +apiVersion: batch/v1 +kind: Job +metadata: + name: generate-request + namespace: {{.TestNamespace}} +spec: + template: + spec: + containers: + - name: curl-client + image: curlimages/curl + imagePullPolicy: Always + command: ["curl", "-H", "Host: {{.Host}}", "keda-add-ons-http-interceptor-proxy.keda:8080"] + restartPolicy: Never + activeDeadlineSeconds: 600 + backoffLimit: 5 +` + + httpScaledObjectTemplate = ` +kind: HTTPScaledObject +apiVersion: http.keda.sh/v1alpha1 +metadata: + name: {{.HTTPScaledObjectName}} + namespace: {{.TestNamespace}} +spec: + hosts: + - {{.Host}} + targetPendingRequests: 100 + scaledownPeriod: 10 + scaleTargetRef: + name: {{.DeploymentName}} + service: {{.ServiceName}} + port: 8080 + replicas: + min: {{ .MinReplicas }} + max: {{ .MaxReplicas }} +` + + clientTemplate = ` +apiVersion: v1 +kind: Pod +metadata: + name: {{.ClientName}} + namespace: {{.TestNamespace}} +spec: + containers: + - name: {{.ClientName}} + image: curlimages/curl + command: + - sh + - -c + - "exec tail -f /dev/null"` +) + +func TestMetricGeneration(t *testing.T) { + // setup + t.Log("--- setting up ---") + // Create kubernetes resources + kc := GetKubernetesClient(t) + data, templates := getTemplateData() + CreateKubernetesResources(t, kc, testNamespace, data, templates) + + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, minReplicaCount, 6, 10), + "replica count should be %d after 1 minutes", minReplicaCount) + + // Send a test request to the interceptor + sendLoad(t, kc, data) + + // Fetch metrics and validate them + family := fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure %s", otelCollectorPromURL)) + val, ok := family["keda_http_scaled_object_total"] + // If the metric is not found first time around then retry with a delay. + if !ok { + // Add a small sleep to allow metrics to be pushed from the exporter to the collector + time.Sleep(10 * time.Second) + // Fetch metrics and validate them + family := fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure %s", otelCollectorPromURL)) + val, ok = family["keda_http_scaled_object_total"] + } + assert.True(t, ok, "keda_http_scaled_object_total is available") + + httpSacaledObjectCount := getMetricsValue(val) + assert.GreaterOrEqual(t, httpSacaledObjectCount, float64(1)) + + // Set the operator to comunicate over GRPC and test functionality + changeOtlpProtocolInOperator(t, kc, "keda-add-ons-http-operator", "keda") + CreateManyHttpScaledObjecs(t, 10) + time.Sleep(time.Second * 10) + family = fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure %s", otelCollectorPromURL)) + val, ok = family["keda_http_scaled_object_total"] + assert.True(t, ok, "keda_http_scaled_object_total is available") + + httpSacaledObjectCountGrpc := getMetricsValue(val) + assert.GreaterOrEqual(t, httpSacaledObjectCountGrpc, float64(10)) + + DeleteManyHttpScaledObjecs(t, 10) + time.Sleep(time.Second * 10) + // Fetch metrics and validate them after deleting httpscaledobjects + family = fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure %s", otelCollectorPromURL)) + val, ok = family["keda_http_scaled_object_total"] + assert.True(t, ok, "keda_http_scaled_object_total is available") + + httpSacaledObjectCountAfterCleanUp := getMetricsValue(val) + assert.Equal(t, float64(1), httpSacaledObjectCountAfterCleanUp) + + // cleanup + fallbackHTTPProtocolInOperator(t, kc, "keda-add-ons-http-operator", "keda") + DeleteKubernetesResources(t, testNamespace, data, templates) +} + +func sendLoad(t *testing.T, kc *kubernetes.Clientset, data templateData) { + t.Log("--- sending load ---") + + KubectlApplyWithTemplate(t, data, "loadJobTemplate", loadJobTemplate) + + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, maxReplicaCount, 6, 10), + "replica count should be %d after 1 minutes", maxReplicaCount) +} + +func fetchAndParsePrometheusMetrics(t *testing.T, cmd string) map[string]*prommodel.MetricFamily { + out, _, err := ExecCommandOnSpecificPod(t, clientName, testNamespace, cmd) + assert.NoErrorf(t, err, "cannot execute command - %s", err) + + parser := expfmt.TextParser{} + // Ensure EOL + reader := strings.NewReader(strings.ReplaceAll(out, "\r\n", "\n")) + families, err := parser.TextToMetricFamilies(reader) + assert.NoErrorf(t, err, "cannot parse metrics - %s", err) + + return families +} + +func getMetricsValue(val *prommodel.MetricFamily) float64 { + if val.GetName() == "keda_http_scaled_object_total" { + metrics := val.GetMetric() + for _, metric := range metrics { + labels := metric.GetLabel() + for _, label := range labels { + if *label.Name == "namespace" && *label.Value == testNamespace { + return metric.GetGauge().GetValue() + } + } + } + } + return 0 +} + +func getTemplateData() (templateData, []Template) { + return templateData{ + TestNamespace: testNamespace, + DeploymentName: deploymentName, + ServiceName: serviceName, + ClientName: clientName, + HTTPScaledObjectName: httpScaledObjectName, + Host: host, + MinReplicas: minReplicaCount, + MaxReplicas: maxReplicaCount, + }, []Template{ + {Name: "deploymentTemplate", Config: deploymentTemplate}, + {Name: "serviceNameTemplate", Config: serviceTemplate}, + {Name: "clientTemplate", Config: clientTemplate}, + {Name: "httpScaledObjectTemplate", Config: httpScaledObjectTemplate}, + } +} + +func changeOtlpProtocolInOperator(t *testing.T, kc *kubernetes.Clientset, name string, namespace string) { + operator, _ := kc.AppsV1().Deployments(namespace).Get(context.TODO(), name, metav1.GetOptions{}) + // Modify the environment variables + t.Log("changeOtlpProtocolInOperator") + for i, container := range operator.Spec.Template.Spec.Containers { + if container.Name == name { + container.Env = slices.DeleteFunc(container.Env, func(n corev1.EnvVar) bool { + return n.Name == "OTEL_EXPORTER_OTLP_ENDPOINT" + }) + + container.Env = append(container.Env, corev1.EnvVar{Name: "OTEL_EXPORTER_OTLP_PROTOCOL", Value: "grpc"}) + container.Env = append(container.Env, corev1.EnvVar{Name: "OTEL_EXPORTER_OTLP_ENDPOINT", Value: otlpGrpcClientEndpoint}) + operator.Spec.Template.Spec.Containers[i].Env = container.Env + } + } + + _, err := kc.AppsV1().Deployments(namespace).Update(context.TODO(), operator, metav1.UpdateOptions{}) + + require.NoErrorf(t, err, "error changing keda http addon operator - %s", err) + WaitForDeploymentReplicaReadyCount(t, kc, operator.Name, "keda", 1, 60, 2) +} + +func fallbackHTTPProtocolInOperator(t *testing.T, kc *kubernetes.Clientset, name string, namespace string) { + t.Log("fallback HTTP OTLP protocol in operator") + + operator, _ := kc.AppsV1().Deployments(namespace).Get(context.TODO(), name, metav1.GetOptions{}) + // Modify the environment variables + for i, container := range operator.Spec.Template.Spec.Containers { + if container.Name == name { + container.Env = slices.DeleteFunc(container.Env, func(n corev1.EnvVar) bool { + if n.Name == "OTEL_EXPORTER_OTLP_ENDPOINT" || n.Name == "OTEL_EXPORTER_OTLP_PROTOCOL" { + return true + } + return false + }) + container.Env = append(container.Env, corev1.EnvVar{Name: "OTEL_EXPORTER_OTLP_ENDPOINT", Value: otlpHTTPClientEndpoint}) + operator.Spec.Template.Spec.Containers[i].Env = container.Env + } + } + + _, err := kc.AppsV1().Deployments(namespace).Update(context.TODO(), operator, metav1.UpdateOptions{}) + + require.NoErrorf(t, err, "error changing keda http addon operator - %s", err) + WaitForDeploymentReplicaReadyCount(t, kc, operator.Name, "keda", 1, 60, 2) +} + +func getTemplateHTTPScaledObjecData(httpScaledObjecID string) (templateData, []Template) { + deploymentCustomTemplateName := fmt.Sprintf("deploymentTemplate-%s", httpScaledObjecID) + deploymentCustom := fmt.Sprintf("other-deployment-%s", httpScaledObjecID) + httpScaledObjectCustom := fmt.Sprintf("other-http-scaled-object-name-%s", httpScaledObjecID) + templateName := fmt.Sprintf("otherHttpScaledObjectName-%s", httpScaledObjecID) + return templateData{ + TestNamespace: testNamespace, + DeploymentName: deploymentCustom, + ServiceName: serviceName, + ClientName: clientName, + HTTPScaledObjectName: httpScaledObjectCustom, + Host: host, + MinReplicas: minReplicaCount, + MaxReplicas: maxReplicaCount, + }, []Template{ + {Name: templateName, Config: httpScaledObjectTemplate}, + {Name: deploymentCustomTemplateName, Config: deploymentTemplate}, + } +} + +func CreateManyHttpScaledObjecs(t *testing.T, objectsCount int) { + for i := 0; i < objectsCount; i++ { + httpScaledObjecData, httpScaledObjecDataTemplates := getTemplateHTTPScaledObjecData(fmt.Sprintf("%d", i)) + KubectlApplyMultipleWithTemplate(t, httpScaledObjecData, httpScaledObjecDataTemplates) + } +} +func DeleteManyHttpScaledObjecs(t *testing.T, objectsCount int) { + for i := 0; i < objectsCount; i++ { + httpScaledObjecData, httpScaledObjecDataTemplates := getTemplateHTTPScaledObjecData(fmt.Sprintf("%d", i)) + KubectlDeleteMultipleWithTemplate(t, httpScaledObjecData, httpScaledObjecDataTemplates) + } +} diff --git a/tests/checks/operator_prometheus_metrics/operator_prometheus_metrics_test.go b/tests/checks/operator_prometheus_metrics/operator_prometheus_metrics_test.go new file mode 100644 index 000000000..5846d657a --- /dev/null +++ b/tests/checks/operator_prometheus_metrics/operator_prometheus_metrics_test.go @@ -0,0 +1,279 @@ +//go:build e2e +// +build e2e + +package operator_prometheus_metrics_test + +import ( + "fmt" + "strings" + "testing" + "time" + + prommodel "github.com/prometheus/client_model/go" + "github.com/prometheus/common/expfmt" + "github.com/stretchr/testify/assert" + "k8s.io/client-go/kubernetes" + + . "github.com/kedacore/http-add-on/tests/helper" +) + +const ( + testName = "operator-prom-metrics-test" +) + +var ( + testNamespace = fmt.Sprintf("%s-ns", testName) + deploymentName = fmt.Sprintf("%s-deployment", testName) + serviceName = fmt.Sprintf("%s-service", testName) + clientName = fmt.Sprintf("%s-client", testName) + httpScaledObjectName = fmt.Sprintf("%s-http-so", testName) + host = testName + minReplicaCount = 0 + maxReplicaCount = 1 + kedaOperatorPrometheusURL = "http://keda-add-ons-http-operator-metrics.keda:8080/metrics" +) + +type templateData struct { + TestNamespace string + DeploymentName string + ServiceName string + ClientName string + HTTPScaledObjectName string + Host string + MinReplicas int + MaxReplicas int +} + +const ( + serviceTemplate = ` +apiVersion: v1 +kind: Service +metadata: + name: {{.ServiceName}} + namespace: {{.TestNamespace}} + labels: + app: {{.DeploymentName}} +spec: + ports: + - port: 8080 + targetPort: http + protocol: TCP + name: http + selector: + app: {{.DeploymentName}} +` + + deploymentTemplate = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{.DeploymentName}} + namespace: {{.TestNamespace}} + labels: + app: {{.DeploymentName}} +spec: + replicas: 0 + selector: + matchLabels: + app: {{.DeploymentName}} + template: + metadata: + labels: + app: {{.DeploymentName}} + spec: + containers: + - name: {{.DeploymentName}} + image: registry.k8s.io/e2e-test-images/agnhost:2.45 + args: + - netexec + ports: + - name: http + containerPort: 8080 + protocol: TCP + readinessProbe: + httpGet: + path: / + port: http +` + + loadJobTemplate = ` +apiVersion: batch/v1 +kind: Job +metadata: + name: generate-request + namespace: {{.TestNamespace}} +spec: + template: + spec: + containers: + - name: curl-client + image: curlimages/curl + imagePullPolicy: Always + command: ["curl", "-H", "Host: {{.Host}}", "keda-add-ons-http-interceptor-proxy.keda:8080"] + restartPolicy: Never + activeDeadlineSeconds: 600 + backoffLimit: 5 +` + + httpScaledObjectTemplate = ` +kind: HTTPScaledObject +apiVersion: http.keda.sh/v1alpha1 +metadata: + name: {{.HTTPScaledObjectName}} + namespace: {{.TestNamespace}} +spec: + hosts: + - {{.Host}} + targetPendingRequests: 100 + scaledownPeriod: 10 + scaleTargetRef: + name: {{.DeploymentName}} + service: {{.ServiceName}} + port: 8080 + replicas: + min: {{ .MinReplicas }} + max: {{ .MaxReplicas }} +` + + clientTemplate = ` +apiVersion: v1 +kind: Pod +metadata: + name: {{.ClientName}} + namespace: {{.TestNamespace}} +spec: + containers: + - name: {{.ClientName}} + image: curlimages/curl + command: + - sh + - -c + - "exec tail -f /dev/null"` +) + +func TestMetricGeneration(t *testing.T) { + // setup + t.Log("--- setting up ---") + // Create kubernetes resources + kc := GetKubernetesClient(t) + data, templates := getTemplateData() + CreateKubernetesResources(t, kc, testNamespace, data, templates) + + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, minReplicaCount, 6, 10), + "replica count should be %d after 1 minutes", minReplicaCount) + t.Log("--- ASSERT replicas ---") + + // Send a test request to the interceptor + sendLoad(t, kc, data) + + CreateManyHttpScaledObjecs(t, 10) + time.Sleep(time.Second * 10) + + // Fetch metrics and validate them + family := fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure %s", kedaOperatorPrometheusURL)) + val, ok := family["keda_http_scaled_object_total"] + assert.True(t, ok, "keda_http_scaled_object_total is available") + httpSacaledObjectCount := getMetricsValue(val) + assert.GreaterOrEqual(t, httpSacaledObjectCount, float64(10)) + + DeleteManyHttpScaledObjecs(t, 10) + time.Sleep(time.Second * 10) + // Fetch metrics and validate them after deleting httpscaledobjects + family = fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure %s", kedaOperatorPrometheusURL)) + val, ok = family["keda_http_scaled_object_total"] + assert.True(t, ok, "keda_http_scaled_object_total is available") + httpSacaledObjectCountAfterCleanUp := getMetricsValue(val) + assert.Equal(t, float64(1), httpSacaledObjectCountAfterCleanUp) + + // cleanup + DeleteKubernetesResources(t, testNamespace, data, templates) +} + +func sendLoad(t *testing.T, kc *kubernetes.Clientset, data templateData) { + t.Log("--- sending load ---") + + KubectlApplyWithTemplate(t, data, "loadJobTemplate", loadJobTemplate) + + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, maxReplicaCount, 12, 10), + "replica count should be %d after 2 minutes", maxReplicaCount) +} + +func fetchAndParsePrometheusMetrics(t *testing.T, cmd string) map[string]*prommodel.MetricFamily { + out, _, err := ExecCommandOnSpecificPod(t, clientName, testNamespace, cmd) + assert.NoErrorf(t, err, "cannot execute command - %s", err) + t.Logf("OUTPUT - %s", out) + + parser := expfmt.TextParser{} + // Ensure EOL + reader := strings.NewReader(strings.ReplaceAll(out, "\r\n", "\n")) + families, err := parser.TextToMetricFamilies(reader) + assert.NoErrorf(t, err, "cannot parse metrics - %s", err) + + return families +} + +func getMetricsValue(val *prommodel.MetricFamily) float64 { + if val.GetName() == "keda_http_scaled_object_total" { + metrics := val.GetMetric() + for _, metric := range metrics { + labels := metric.GetLabel() + for _, label := range labels { + if *label.Name == "namespace" && *label.Value == testNamespace { + return metric.GetGauge().GetValue() + } + } + } + } + return 0 +} + +func getTemplateData() (templateData, []Template) { + return templateData{ + TestNamespace: testNamespace, + DeploymentName: deploymentName, + ServiceName: serviceName, + ClientName: clientName, + HTTPScaledObjectName: httpScaledObjectName, + Host: host, + MinReplicas: minReplicaCount, + MaxReplicas: maxReplicaCount, + }, []Template{ + {Name: "deploymentTemplate", Config: deploymentTemplate}, + {Name: "serviceNameTemplate", Config: serviceTemplate}, + {Name: "clientTemplate", Config: clientTemplate}, + {Name: "httpScaledObjectTemplate", Config: httpScaledObjectTemplate}, + } +} + +func getTemplateHTTPScaledObjecData(httpScaledObjecID string) (templateData, []Template) { + deploymentCustomTemplateName := fmt.Sprintf("deploymentTemplate-%s", httpScaledObjecID) + deploymentCustom := fmt.Sprintf("other-deployment-%s", httpScaledObjecID) + httpScaledObjectCustom := fmt.Sprintf("other-http-scaled-object-name-%s", httpScaledObjecID) + templateName := fmt.Sprintf("otherHttpScaledObjectName-%s", httpScaledObjecID) + return templateData{ + TestNamespace: testNamespace, + DeploymentName: deploymentCustom, + ServiceName: serviceName, + ClientName: clientName, + HTTPScaledObjectName: httpScaledObjectCustom, + Host: host, + MinReplicas: minReplicaCount, + MaxReplicas: maxReplicaCount, + }, []Template{ + {Name: templateName, Config: httpScaledObjectTemplate}, + {Name: deploymentCustomTemplateName, Config: deploymentTemplate}, + } +} + +func CreateManyHttpScaledObjecs(t *testing.T, objectsCount int) { + for i := 0; i < objectsCount; i++ { + httpScaledObjecData, httpScaledObjecDataTemplates := getTemplateHTTPScaledObjecData(fmt.Sprintf("%d", i)) + KubectlApplyMultipleWithTemplate(t, httpScaledObjecData, httpScaledObjecDataTemplates) + } +} +func DeleteManyHttpScaledObjecs(t *testing.T, objectsCount int) { + for i := 0; i < objectsCount; i++ { + httpScaledObjecData, httpScaledObjecDataTemplates := getTemplateHTTPScaledObjecData(fmt.Sprintf("%d", i)) + KubectlDeleteMultipleWithTemplate(t, httpScaledObjecData, httpScaledObjecDataTemplates) + } +}