diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 45a8b2d674..4a02581af5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -147,7 +147,7 @@ jobs: uses: actions/checkout@v5 - name: Execute dockerlinter - uses: hadolint/hadolint-action@v3.1.0 + uses: hadolint/hadolint-action@v3.3.0 with: dockerfile: Dockerfile ignore: DL3007,DL3018 diff --git a/.github/workflows/pr-semantics.yaml b/.github/workflows/pr-semantics.yaml index fd83483796..ddb118d0bb 100644 --- a/.github/workflows/pr-semantics.yaml +++ b/.github/workflows/pr-semantics.yaml @@ -68,6 +68,7 @@ jobs: replication sentinel cluster + metrics ignoreLabels: | bot ignore-semantic-pull-request diff --git a/.github/workflows/publish-charts.yaml b/.github/workflows/publish-charts.yaml index e130db64f9..19084e07f0 100644 --- a/.github/workflows/publish-charts.yaml +++ b/.github/workflows/publish-charts.yaml @@ -22,7 +22,7 @@ jobs: with: version: v3.5.4 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: '3.9' check-latest: true diff --git a/.github/workflows/stale.yaml b/.github/workflows/stale.yaml index 92202fa67c..1e142cf546 100644 --- a/.github/workflows/stale.yaml +++ b/.github/workflows/stale.yaml @@ -17,7 +17,7 @@ jobs: issues: write pull-requests: write steps: - - uses: actions/stale@v9 + - uses: actions/stale@v10 with: repo-token: ${{ secrets.GITHUB_TOKEN }} ascending: true diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 47ff0e2512..8ddcd46f9a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -14,7 +14,7 @@ For development and testing of operator on local system, we need to set up a [Minikube](https://minikube.sigs.k8s.io/docs/start/) or local Kubernetes cluster. -Minikube is a single node Kubernetes cluster that generally gets used for the development and testing on Kubernetes. For creating a Minkube cluster we need to simply run: +Minikube is a single node Kubernetes cluster that generally gets used for the development and testing on Kubernetes. For creating a Minikube cluster we need to simply run: ```shell $ minikube start --vm-driver virtualbox diff --git a/Makefile b/Makefile index c029edbde5..86725aed3b 100644 --- a/Makefile +++ b/Makefile @@ -291,3 +291,4 @@ GOBIN=$(LOCALBIN) go install $${package} ;\ mv "$$(echo "$(1)" | sed "s/-$(3)$$//")" $(1) ;\ } endef + diff --git a/README.md b/README.md index d0a6dc69ca..c05fe45f98 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ This operator only supports versions of Redis `>=6`. ## Architecture
- +
## Purpose diff --git a/api/common/v1beta2/common_types.go b/api/common/v1beta2/common_types.go index ac9ef29b25..4819d892f8 100644 --- a/api/common/v1beta2/common_types.go +++ b/api/common/v1beta2/common_types.go @@ -160,7 +160,7 @@ type RedisConfig struct { AdditionalRedisConfig *string `json:"additionalRedisConfig,omitempty"` } -// Storage is the inteface to add pvc and pv support in redis +// Storage is the interface to add pvc and pv support in redis // +k8s:deepcopy-gen=true type Storage struct { KeepAfterDelete bool `json:"keepAfterDelete,omitempty"` @@ -178,7 +178,7 @@ type AdditionalVolume struct { // TLS Configuration for redis instances // +k8s:deepcopy-gen=true type TLSConfig struct { - CaKeyFile string `json:"ca,omitempty"` + CaCertFile string `json:"ca,omitempty"` CertKeyFile string `json:"cert,omitempty"` KeyFile string `json:"key,omitempty"` // Reference to secret which contains the certificates @@ -288,7 +288,8 @@ type ACLConfig struct { Secret *corev1.SecretVolumeSource `json:"secret,omitempty"` // PersistentVolumeClaim-based ACL configuration // Specify the PVC name to mount ACL file from persistent storage - // The operator will automatically mount /etc/redis/user.acl from the PVC + // The operator mounts the PVC at /data/redis so Redis can read and update /data/redis/user.acl + // This feature requires the GenerateConfigInInitContainer feature gate to be enabled. PersistentVolumeClaim *string `json:"persistentVolumeClaim,omitempty"` } diff --git a/charts/redis-cluster/README.md b/charts/redis-cluster/README.md index 8485d44641..62f367d655 100644 --- a/charts/redis-cluster/README.md +++ b/charts/redis-cluster/README.md @@ -46,7 +46,7 @@ helm delete --namespace | Key | Type | Default | Description | |-----|------|---------|-------------| -| TLS.ca | string | `"ca.key"` | | +| TLS.ca | string | `"ca.crt"` | | | TLS.cert | string | `"tls.crt"` | | | TLS.key | string | `"tls.key"` | | | TLS.secret.secretName | string | `""` | | @@ -70,7 +70,7 @@ helm delete --namespace | priorityClassName | string | `""` | | | redisCluster.clusterSize | int | `3` | Default number of replicas for both leader and follower when not explicitly set | | redisCluster.clusterVersion | string | `"v7"` | | -| redisCluster.enableMasterSlaveAntiAffinity | bool | `false` | Enable pod anti-affinity between leader and follower pods by adding the appropriate label. Notice that this requires the operator to have its mutating webhook enabled, otherwise it will only add an annotation to the RedisCluster CR. Default is false. | +| redisCluster.enableMasterSlaveAntiAffinity | bool | `false` | Enable pod anti-affinity between leader and follower pods by adding the appropriate label. Notice that this requires the operator to have its mutating webhook enabled, otherwise it will only add an annotation to the RedisCluster CR. Default is false. | | redisCluster.follower.affinity | string | `nil` | | | redisCluster.follower.livenessProbe | object | `{}` | | | redisCluster.follower.nodeSelector | string | `nil` | | @@ -103,7 +103,7 @@ helm delete --namespace | redisCluster.name | string | `""` | | | redisCluster.persistenceEnabled | bool | `true` | | | redisCluster.persistentVolumeClaimRetentionPolicy | object | `{}` | | -| redisCluster.recreateStatefulSetOnUpdateInvalid | bool | `false` | Some fields of statefulset are immutable, such as volumeClaimTemplates. When set to true, the operator will delete the statefulset and recreate it. Default is false. | +| redisCluster.recreateStatefulSetOnUpdateInvalid | bool | `false` | Some fields of statefulset are immutable, such as volumeClaimTemplates. When set to true, the operator will delete the statefulset and recreate it. Default is false. | | redisCluster.redisSecret.secretKey | string | `""` | | | redisCluster.redisSecret.secretName | string | `""` | | | redisCluster.resources | object | `{}` | | diff --git a/charts/redis-cluster/templates/follower-service.yaml b/charts/redis-cluster/templates/follower-service.yaml index 6fa6455291..26b6c28071 100644 --- a/charts/redis-cluster/templates/follower-service.yaml +++ b/charts/redis-cluster/templates/follower-service.yaml @@ -1,4 +1,5 @@ -{{- if and (gt (int .Values.redisCluster.follower.replicas) 0) (eq .Values.externalService.enabled true) }} +{{- $followerReplicas := .Values.redisCluster.follower.replicas | default .Values.redisCluster.clusterSize }} +{{- if and (gt (int $followerReplicas) 0) (eq .Values.externalService.enabled true) }} --- apiVersion: v1 kind: Service diff --git a/charts/redis-cluster/templates/follower-sm.yaml b/charts/redis-cluster/templates/follower-sm.yaml index c7bc21bc8d..b5137d50e6 100644 --- a/charts/redis-cluster/templates/follower-sm.yaml +++ b/charts/redis-cluster/templates/follower-sm.yaml @@ -1,4 +1,5 @@ -{{- if and (eq .Values.serviceMonitor.enabled true) (gt (int .Values.redisCluster.follower.replicas) 0) }} +{{- $followerReplicas := .Values.redisCluster.follower.replicas | default .Values.redisCluster.clusterSize }} +{{- if and (eq .Values.serviceMonitor.enabled true) (eq .Values.redisExporter.enabled true) (gt (int $followerReplicas) 0) }} --- apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor @@ -19,6 +20,7 @@ spec: selector: matchLabels: app: {{ .Values.redisCluster.name | default .Release.Name }}-follower + app.kubernetes.io/component: metrics redis_setup_type: cluster role: follower endpoints: diff --git a/charts/redis-cluster/templates/leader-service.yaml b/charts/redis-cluster/templates/leader-service.yaml index e7cedd864e..cc65e6be3c 100644 --- a/charts/redis-cluster/templates/leader-service.yaml +++ b/charts/redis-cluster/templates/leader-service.yaml @@ -1,4 +1,5 @@ -{{- if and (gt (int .Values.redisCluster.leader.replicas) 0) (eq .Values.externalService.enabled true) }} +{{- $leaderReplicas := .Values.redisCluster.leader.replicas | default .Values.redisCluster.clusterSize }} +{{- if and (gt (int $leaderReplicas) 0) (eq .Values.externalService.enabled true) }} --- apiVersion: v1 kind: Service diff --git a/charts/redis-cluster/templates/leader-sm.yaml b/charts/redis-cluster/templates/leader-sm.yaml index d8af1d6df2..385e8f0619 100644 --- a/charts/redis-cluster/templates/leader-sm.yaml +++ b/charts/redis-cluster/templates/leader-sm.yaml @@ -1,4 +1,5 @@ -{{- if and (eq .Values.serviceMonitor.enabled true) (gt (int .Values.redisCluster.leader.replicas) 0) }} +{{- $leaderReplicas := .Values.redisCluster.leader.replicas | default .Values.redisCluster.clusterSize }} +{{- if and (eq .Values.serviceMonitor.enabled true) (eq .Values.redisExporter.enabled true) (gt (int $leaderReplicas) 0) }} --- apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor @@ -19,6 +20,7 @@ spec: selector: matchLabels: app: {{ .Values.redisCluster.name | default .Release.Name }}-leader + app.kubernetes.io/component: metrics redis_setup_type: cluster role: leader endpoints: diff --git a/charts/redis-cluster/templates/redis-cluster.yaml b/charts/redis-cluster/templates/redis-cluster.yaml index 8dcbb52204..c59741bfb7 100644 --- a/charts/redis-cluster/templates/redis-cluster.yaml +++ b/charts/redis-cluster/templates/redis-cluster.yaml @@ -16,14 +16,14 @@ spec: persistenceEnabled: {{ .Values.redisCluster.persistenceEnabled }} clusterVersion: {{ .Values.redisCluster.clusterVersion }} redisLeader: {{- include "redis.role" .Values.redisCluster.leader | nindent 4 }} - replicas: {{ .Values.redisCluster.leader.replicas }} + replicas: {{ .Values.redisCluster.leader.replicas | default .Values.redisCluster.clusterSize }} {{- if .Values.externalConfig.enabled }} redisConfig: additionalRedisConfig: "{{ .Values.redisCluster.name | default .Release.Name }}-ext-config" {{- end }} redisFollower: {{- include "redis.role" .Values.redisCluster.follower | nindent 4 }} - replicas: {{ .Values.redisCluster.follower.replicas }} + replicas: {{ .Values.redisCluster.follower.replicas | default .Values.redisCluster.clusterSize }} {{- if .Values.externalConfig.enabled }} redisConfig: additionalRedisConfig: "{{ .Values.redisCluster.name | default .Release.Name }}-ext-config" diff --git a/charts/redis-cluster/values.yaml b/charts/redis-cluster/values.yaml index 8190a7d40a..559b4bc592 100644 --- a/charts/redis-cluster/values.yaml +++ b/charts/redis-cluster/values.yaml @@ -23,7 +23,7 @@ redisCluster: # memory: 128Mi minReadySeconds: 0 # -- Some fields of statefulset are immutable, such as volumeClaimTemplates. - # When set to true, the operator will delete the statefulset and recreate it. + # When set to true, the operator will delete the statefulset and recreate it. # Default is false. recreateStatefulSetOnUpdateInvalid: false # -- MaxMemoryPercentOfLimit is the percentage of the Redis container memory limit to be used as maxmemory. @@ -35,7 +35,7 @@ redisCluster: # whenScaled: Retain # -- Enable pod anti-affinity between leader and follower pods by adding the appropriate label. # Notice that this requires the operator to have its mutating webhook enabled, - # otherwise it will only add an annotation to the RedisCluster CR. + # otherwise it will only add an annotation to the RedisCluster CR. # Default is false. enableMasterSlaveAntiAffinity: false leader: @@ -83,7 +83,7 @@ redisCluster: # successThreshold: 1 # failureThreshold: 4 # initialDelaySeconds: 15 - + follower: # -- Number of Redis follower (slave) nodes. If not set, uses clusterSize value replicas: 3 @@ -238,7 +238,7 @@ podSecurityContext: # serviceAccountName: redis-sa TLS: - ca: ca.key + ca: ca.crt cert: tls.crt key: tls.key secret: diff --git a/charts/redis-operator/crds/crds.yaml b/charts/redis-operator/crds/crds.yaml index d01f1b010a..eeb1e287b5 100644 --- a/charts/redis-operator/crds/crds.yaml +++ b/charts/redis-operator/crds/crds.yaml @@ -117,7 +117,8 @@ spec: description: |- PersistentVolumeClaim-based ACL configuration Specify the PVC name to mount ACL file from persistent storage - The operator will automatically mount /etc/redis/user.acl from the PVC + The operator mounts the PVC at /data/redis so Redis can read and update /data/redis/user.acl + This feature requires the GenerateConfigInInitContainer feature gate to be enabled. type: string secret: description: |- @@ -3219,7 +3220,7 @@ spec: type: object type: array storage: - description: Storage is the inteface to add pvc and pv support in + description: Storage is the interface to add pvc and pv support in redis properties: keepAfterDelete: @@ -5544,7 +5545,8 @@ spec: description: |- PersistentVolumeClaim-based ACL configuration Specify the PVC name to mount ACL file from persistent storage - The operator will automatically mount /etc/redis/user.acl from the PVC + The operator mounts the PVC at /data/redis so Redis can read and update /data/redis/user.acl + This feature requires the GenerateConfigInInitContainer feature gate to be enabled. type: string secret: description: |- @@ -13393,7 +13395,8 @@ spec: description: |- PersistentVolumeClaim-based ACL configuration Specify the PVC name to mount ACL file from persistent storage - The operator will automatically mount /etc/redis/user.acl from the PVC + The operator mounts the PVC at /data/redis so Redis can read and update /data/redis/user.acl + This feature requires the GenerateConfigInInitContainer feature gate to be enabled. type: string secret: description: |- @@ -16757,7 +16760,7 @@ spec: type: object type: array storage: - description: Storage is the inteface to add pvc and pv support in + description: Storage is the interface to add pvc and pv support in redis properties: keepAfterDelete: diff --git a/charts/redis-operator/templates/role.yaml b/charts/redis-operator/templates/role.yaml index 61b89710b2..73dbe8c9cf 100644 --- a/charts/redis-operator/templates/role.yaml +++ b/charts/redis-operator/templates/role.yaml @@ -12,6 +12,7 @@ metadata: app.kubernetes.io/version : {{ .Chart.AppVersion }} app.kubernetes.io/component: role app.kubernetes.io/part-of : {{ .Release.Name }} + rbac.authorization.k8s.io/aggregate-to-admin: "true" rules: - apiGroups: - redis.redis.opstreelabs.in @@ -80,7 +81,7 @@ rules: - configmaps - events - persistentvolumeclaims - - namespace + - namespaces verbs: - create - delete diff --git a/charts/redis-replication/README.md b/charts/redis-replication/README.md index 8b82119857..77cf57c0fe 100644 --- a/charts/redis-replication/README.md +++ b/charts/redis-replication/README.md @@ -44,7 +44,7 @@ helm delete --namespace | Key | Type | Default | Description | |-----|------|---------|-------------| -| TLS.ca | string | `"ca.key"` | | +| TLS.ca | string | `"ca.crt"` | | | TLS.cert | string | `"tls.crt"` | | | TLS.key | string | `"tls.key"` | | | TLS.secret.secretName | string | `""` | | @@ -96,6 +96,12 @@ helm delete --namespace | redisReplication.serviceType | string | `"ClusterIP"` | | | redisReplication.tag | string | `"v7.0.15"` | | | securityContext | object | `{}` | | +| sentinel | object | `{"announceHostnames":"no","downAfterMilliseconds":"5000","enabled":false,"failoverTimeout":"10000","ignoreAnnotations":[],"image":"quay.io/opstree/redis-sentinel","imagePullPolicy":"IfNotPresent","minReadySeconds":0,"parallelSyncs":"1","persistentVolumeClaimRetentionPolicy":{},"resolveHostnames":"no","resources":{},"size":3,"tag":"v7.0.15"}` | Sentinel configuration for automatic failover. When enabled, the operator creates a Sentinel StatefulSet alongside the replication pods. The operator queries Sentinel for the current master instead of forcing master-by-ordinal. | +| sentinel.announceHostnames | string | `"no"` | Whether Sentinel announces hostnames instead of IPs to clients | +| sentinel.downAfterMilliseconds | string | `"5000"` | Time in milliseconds before master is considered down | +| sentinel.failoverTimeout | string | `"10000"` | Failover timeout in milliseconds | +| sentinel.parallelSyncs | string | `"1"` | Number of replicas to reconfigure in parallel during failover | +| sentinel.resolveHostnames | string | `"no"` | Use hostnames instead of IPs for Sentinel monitoring. WARNING: the operator does not pass RESOLVE_HOSTNAMES env var to sentinel pods, so setting this to "yes" will cause SENTINEL MONITOR to fail. Keep as "no". | | serviceAccountName | string | `""` | | | serviceMonitor.enabled | bool | `false` | | | serviceMonitor.extraLabels | object | `{}` | extraLabels are added to the servicemonitor when enabled set to true | diff --git a/charts/redis-replication/templates/redis-replication.yaml b/charts/redis-replication/templates/redis-replication.yaml index 21c4a62d9c..812bc3f77c 100644 --- a/charts/redis-replication/templates/redis-replication.yaml +++ b/charts/redis-replication/templates/redis-replication.yaml @@ -123,4 +123,39 @@ spec: minAvailable: {{ .Values.pdb.minAvailable }} maxUnavailable: {{ .Values.pdb.maxUnavailable }} {{- end }} - + {{- if .Values.sentinel.enabled }} + sentinel: + image: "{{ .Values.sentinel.image }}:{{ .Values.sentinel.tag }}" + imagePullPolicy: "{{ .Values.sentinel.imagePullPolicy }}" + size: {{ .Values.sentinel.size }} + {{- if .Values.sentinel.resources }} + resources: {{ toYaml .Values.sentinel.resources | nindent 6 }} + {{- end }} + {{- if .Values.sentinel.ignoreAnnotations }} + ignoreAnnotations: {{ toYaml .Values.sentinel.ignoreAnnotations | nindent 6 }} + {{- end }} + {{- if .Values.sentinel.minReadySeconds }} + minReadySeconds: {{ .Values.sentinel.minReadySeconds }} + {{- end }} + {{- if .Values.sentinel.persistentVolumeClaimRetentionPolicy }} + persistentVolumeClaimRetentionPolicy: {{ toYaml .Values.sentinel.persistentVolumeClaimRetentionPolicy | nindent 6 }} + {{- end }} + {{- if .Values.sentinel.parallelSyncs }} + parallelSyncs: {{ .Values.sentinel.parallelSyncs | quote }} + {{- end }} + {{- if .Values.sentinel.failoverTimeout }} + failoverTimeout: {{ .Values.sentinel.failoverTimeout | quote }} + {{- end }} + {{- if .Values.sentinel.downAfterMilliseconds }} + downAfterMilliseconds: {{ .Values.sentinel.downAfterMilliseconds | quote }} + {{- end }} + {{- if .Values.sentinel.resolveHostnames }} + resolveHostnames: {{ .Values.sentinel.resolveHostnames | quote }} + {{- end }} + {{- if .Values.sentinel.announceHostnames }} + announceHostnames: {{ .Values.sentinel.announceHostnames | quote }} + {{- end }} + {{- if .Values.sentinel.additionalSentinelConfig }} + additionalSentinelConfig: {{ .Values.sentinel.additionalSentinelConfig | quote }} + {{- end }} + {{- end }} diff --git a/charts/redis-replication/templates/servicemonitor.yaml b/charts/redis-replication/templates/servicemonitor.yaml index e4cdc589e6..7768b16425 100644 --- a/charts/redis-replication/templates/servicemonitor.yaml +++ b/charts/redis-replication/templates/servicemonitor.yaml @@ -1,4 +1,4 @@ -{{- if eq .Values.serviceMonitor.enabled true }} +{{- if and (eq .Values.serviceMonitor.enabled true) (eq .Values.redisExporter.enabled true) }} --- apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor @@ -19,6 +19,7 @@ spec: selector: matchLabels: app: {{ .Values.redisReplication.name | default .Release.Name }} + app.kubernetes.io/component: metrics redis_setup_type: replication role: replication endpoints: diff --git a/charts/redis-replication/values.yaml b/charts/redis-replication/values.yaml index f14f086df8..11348aae95 100644 --- a/charts/redis-replication/values.yaml +++ b/charts/redis-replication/values.yaml @@ -177,7 +177,7 @@ topologySpreadConstraints: [] serviceAccountName: "" TLS: - ca: ca.key + ca: ca.crt cert: tls.crt key: tls.key secret: @@ -199,3 +199,41 @@ pdb: enabled: false minAvailable: 1 maxUnavailable: null + +# -- Sentinel configuration for automatic failover. +# When enabled, the operator creates a Sentinel StatefulSet alongside the replication pods. +# The operator queries Sentinel for the current master instead of forcing master-by-ordinal. +sentinel: + enabled: false + image: quay.io/opstree/redis-sentinel + tag: v7.0.15 + imagePullPolicy: IfNotPresent + size: 3 + resources: {} + # requests: + # cpu: 100m + # memory: 128Mi + # limits: + # cpu: 100m + # memory: 128Mi + ignoreAnnotations: [] + # - "redis.opstreelabs.in/ignore" + minReadySeconds: 0 + persistentVolumeClaimRetentionPolicy: {} + # whenDeleted: Delete + # whenScaled: Retain + # -- Number of replicas to reconfigure in parallel during failover + parallelSyncs: "1" + # -- Failover timeout in milliseconds + failoverTimeout: "10000" + # -- Time in milliseconds before master is considered down + downAfterMilliseconds: "5000" + # -- Use hostnames instead of IPs for Sentinel monitoring. + # WARNING: the operator does not pass RESOLVE_HOSTNAMES env var to sentinel pods, + # so setting this to "yes" will cause SENTINEL MONITOR to fail. Keep as "no". + resolveHostnames: "no" + # -- Whether Sentinel announces hostnames instead of IPs to clients + announceHostnames: "no" + # -- Additional raw sentinel.conf lines + # additionalSentinelConfig: | + # sentinel auth-pass mymaster password123 diff --git a/charts/redis-sentinel/README.md b/charts/redis-sentinel/README.md index 4a0fa322c0..aee298cd10 100644 --- a/charts/redis-sentinel/README.md +++ b/charts/redis-sentinel/README.md @@ -44,7 +44,7 @@ helm delete --namespace | Key | Type | Default | Description | |-----|------|---------|-------------| -| TLS.ca | string | `"ca.key"` | | +| TLS.ca | string | `"ca.crt"` | | | TLS.cert | string | `"tls.crt"` | | | TLS.key | string | `"tls.key"` | | | TLS.secret.secretName | string | `""` | | diff --git a/charts/redis-sentinel/templates/servicemonitor.yaml b/charts/redis-sentinel/templates/servicemonitor.yaml index 2351fd5236..7d968b4e66 100644 --- a/charts/redis-sentinel/templates/servicemonitor.yaml +++ b/charts/redis-sentinel/templates/servicemonitor.yaml @@ -1,4 +1,4 @@ -{{- if eq .Values.serviceMonitor.enabled true }} +{{- if and (eq .Values.serviceMonitor.enabled true) (eq .Values.redisExporter.enabled true) }} --- apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor @@ -18,7 +18,8 @@ metadata: spec: selector: matchLabels: - app: {{ .Values.redisSentinel.name | default .Release.Name }} + app: {{ .Values.redisSentinel.name | default .Release.Name }}-sentinel + app.kubernetes.io/component: metrics redis_setup_type: sentinel role: sentinel endpoints: diff --git a/charts/redis-sentinel/values.yaml b/charts/redis-sentinel/values.yaml index f0176b6212..cb09dfc9da 100644 --- a/charts/redis-sentinel/values.yaml +++ b/charts/redis-sentinel/values.yaml @@ -165,7 +165,7 @@ topologySpreadConstraints: [] serviceAccountName: "" TLS: - ca: ca.key + ca: ca.crt cert: tls.crt key: tls.key secret: diff --git a/charts/redis/README.md b/charts/redis/README.md index fdfee22404..bbe2fec0a7 100644 --- a/charts/redis/README.md +++ b/charts/redis/README.md @@ -43,7 +43,7 @@ helm delete --namespace | Key | Type | Default | Description | |-----|------|---------|-------------| -| TLS.ca | string | `"ca.key"` | | +| TLS.ca | string | `"ca.crt"` | | | TLS.cert | string | `"tls.crt"` | | | TLS.key | string | `"tls.key"` | | | TLS.secret.secretName | string | `""` | | diff --git a/charts/redis/templates/servicemonitor.yaml b/charts/redis/templates/servicemonitor.yaml index 0bccc1f8b2..7bf42d0f5f 100644 --- a/charts/redis/templates/servicemonitor.yaml +++ b/charts/redis/templates/servicemonitor.yaml @@ -1,4 +1,4 @@ -{{- if eq .Values.serviceMonitor.enabled true }} +{{- if and (eq .Values.serviceMonitor.enabled true) (eq .Values.redisExporter.enabled true) }} --- apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor @@ -19,6 +19,7 @@ spec: selector: matchLabels: app: {{ .Values.redisStandalone.name | default .Release.Name }} + app.kubernetes.io/component: metrics redis_setup_type: standalone role: standalone endpoints: diff --git a/charts/redis/values.yaml b/charts/redis/values.yaml index 186a55f85b..b40ae25bc3 100644 --- a/charts/redis/values.yaml +++ b/charts/redis/values.yaml @@ -160,7 +160,7 @@ topologySpreadConstraints: [] serviceAccountName: "" TLS: - ca: ca.key + ca: ca.crt cert: tls.crt key: tls.key secret: diff --git a/config/crd/bases/redis.redis.opstreelabs.in_redis.yaml b/config/crd/bases/redis.redis.opstreelabs.in_redis.yaml index a797eef580..5d131f3d07 100644 --- a/config/crd/bases/redis.redis.opstreelabs.in_redis.yaml +++ b/config/crd/bases/redis.redis.opstreelabs.in_redis.yaml @@ -118,7 +118,8 @@ spec: description: |- PersistentVolumeClaim-based ACL configuration Specify the PVC name to mount ACL file from persistent storage - The operator will automatically mount /etc/redis/user.acl from the PVC + The operator mounts the PVC at /data/redis so Redis can read and update /data/redis/user.acl + This feature requires the GenerateConfigInInitContainer feature gate to be enabled. type: string secret: description: |- @@ -3220,7 +3221,7 @@ spec: type: object type: array storage: - description: Storage is the inteface to add pvc and pv support in + description: Storage is the interface to add pvc and pv support in redis properties: keepAfterDelete: diff --git a/config/crd/bases/redis.redis.opstreelabs.in_redisclusters.yaml b/config/crd/bases/redis.redis.opstreelabs.in_redisclusters.yaml index 0ec9292dc4..d6e0bfd4f2 100644 --- a/config/crd/bases/redis.redis.opstreelabs.in_redisclusters.yaml +++ b/config/crd/bases/redis.redis.opstreelabs.in_redisclusters.yaml @@ -146,7 +146,8 @@ spec: description: |- PersistentVolumeClaim-based ACL configuration Specify the PVC name to mount ACL file from persistent storage - The operator will automatically mount /etc/redis/user.acl from the PVC + The operator mounts the PVC at /data/redis so Redis can read and update /data/redis/user.acl + This feature requires the GenerateConfigInInitContainer feature gate to be enabled. type: string secret: description: |- diff --git a/config/crd/bases/redis.redis.opstreelabs.in_redisreplications.yaml b/config/crd/bases/redis.redis.opstreelabs.in_redisreplications.yaml index 38e0bdee7a..9020a029f5 100644 --- a/config/crd/bases/redis.redis.opstreelabs.in_redisreplications.yaml +++ b/config/crd/bases/redis.redis.opstreelabs.in_redisreplications.yaml @@ -124,7 +124,8 @@ spec: description: |- PersistentVolumeClaim-based ACL configuration Specify the PVC name to mount ACL file from persistent storage - The operator will automatically mount /etc/redis/user.acl from the PVC + The operator mounts the PVC at /data/redis so Redis can read and update /data/redis/user.acl + This feature requires the GenerateConfigInInitContainer feature gate to be enabled. type: string secret: description: |- @@ -3488,7 +3489,7 @@ spec: type: object type: array storage: - description: Storage is the inteface to add pvc and pv support in + description: Storage is the interface to add pvc and pv support in redis properties: keepAfterDelete: diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 54c9aa2d27..8cd329f0f2 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -18,3 +18,15 @@ resources: # default, aiding admins in cluster management. Those roles are # not used by the {{ .ProjectName }} itself. You can comment the following lines # if you do not want those helpers be installed with your Project. + +patches: + - target: + group: rbac.authorization.k8s.io + version: v1 + kind: ClusterRole + name: manager-role + patch: | + - op: add + path: /metadata/labels + value: + rbac.authorization.k8s.io/aggregate-to-admin: "true" diff --git a/docs/content/en/docs/CRD Reference/API Reference/_index.md b/docs/content/en/docs/CRD Reference/API Reference/_index.md index da02d86642..58dace4f43 100644 --- a/docs/content/en/docs/CRD Reference/API Reference/_index.md +++ b/docs/content/en/docs/CRD Reference/API Reference/_index.md @@ -43,7 +43,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `secret` _[SecretVolumeSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#secretvolumesource-v1-core)_ | Secret-based ACL configuration.
Adapts a Secret into a volume containing ACL rules.
The contents of the target Secret's Data field will be presented in a volume
as files using the keys in the Data field as the file names.
Secret volumes support ownership management and SELinux relabeling. | | | -| `persistentVolumeClaim` _string_ | PersistentVolumeClaim-based ACL configuration
Specify the PVC name to mount ACL file from persistent storage
The operator will automatically mount /etc/redis/user.acl from the PVC | | | +| `persistentVolumeClaim` _string_ | PersistentVolumeClaim-based ACL configuration
Specify the PVC name to mount ACL file from persistent storage
The operator mounts the PVC at /data/redis so Redis can read and update /data/redis/user.acl
This feature requires the GenerateConfigInInitContainer feature gate to be enabled. | | | #### AdditionalVolume @@ -661,7 +661,7 @@ _Appears in:_ -Storage is the inteface to add pvc and pv support in redis +Storage is the interface to add pvc and pv support in redis diff --git a/docs/content/en/docs/Configuration/Redis/_index.md b/docs/content/en/docs/Configuration/Redis/_index.md index f5cd13bae0..8e566ae99e 100644 --- a/docs/content/en/docs/Configuration/Redis/_index.md +++ b/docs/content/en/docs/Configuration/Redis/_index.md @@ -13,7 +13,7 @@ Redis standalone configuration can be customized by [values.yaml](https://github | Key | Type | Default | Description | |-----------------------------------------------------------------|--------|--------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| TLS.ca | string | `"ca.key"` | | +| TLS.ca | string | `"ca.crt"` | | | TLS.cert | string | `"tls.crt"` | | | TLS.key | string | `"tls.key"` | | | TLS.secret.secretName | string | `""` | | diff --git a/docs/content/en/docs/Configuration/RedisCluster/_index.md b/docs/content/en/docs/Configuration/RedisCluster/_index.md index 66bb9b8896..7782527c07 100644 --- a/docs/content/en/docs/Configuration/RedisCluster/_index.md +++ b/docs/content/en/docs/Configuration/RedisCluster/_index.md @@ -13,7 +13,7 @@ Redis cluster can be customized by [values.yaml](https://github.com/OT-CONTAINER | Key | Type | Default | Description | |-------------------------------------------------------------------------|--------|--------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| TLS.ca | string | `"ca.key"` | | +| TLS.ca | string | `"ca.crt"` | | | TLS.cert | string | `"tls.crt"` | | | TLS.key | string | `"tls.key"` | | | TLS.secret.secretName | string | `""` | | diff --git a/docs/content/en/docs/Configuration/RedisReplication/_index.md b/docs/content/en/docs/Configuration/RedisReplication/_index.md index 3894e27fc6..fbd622faf4 100644 --- a/docs/content/en/docs/Configuration/RedisReplication/_index.md +++ b/docs/content/en/docs/Configuration/RedisReplication/_index.md @@ -13,7 +13,7 @@ Redis replication configuration can be customized by [values.yaml](https://githu | Key | Type | Default | Description | |-----|------|---------|-------------| -| TLS.ca | string | `"ca.key"` | | +| TLS.ca | string | `"ca.crt"` | | | TLS.cert | string | `"tls.crt"` | | | TLS.key | string | `"tls.key"` | | | TLS.secret.secretName | string | `""` | | diff --git a/docs/content/en/docs/Contribute/_index.md b/docs/content/en/docs/Contribute/_index.md index 85a5e87029..d9d39966c1 100644 --- a/docs/content/en/docs/Contribute/_index.md +++ b/docs/content/en/docs/Contribute/_index.md @@ -21,7 +21,7 @@ description: > For development and testing of operator on local system, we need to set up a [Minikube](https://minikube.sigs.k8s.io/docs/start/) or local Kubernetes cluster. -Minikube is a single node Kubernetes cluster that generally gets used for the development and testing on Kubernetes. For creating a Minkube cluster we need to simply run: +Minikube is a single node Kubernetes cluster that generally gets used for the development and testing on Kubernetes. For creating a Minikube cluster we need to simply run: ```shell $ minikube start --vm-driver virtualbox diff --git a/docs/content/en/docs/Getting Started/Cluster/_index.md b/docs/content/en/docs/Getting Started/Cluster/_index.md index fa92bde5eb..2b8c1184a7 100644 --- a/docs/content/en/docs/Getting Started/Cluster/_index.md +++ b/docs/content/en/docs/Getting Started/Cluster/_index.md @@ -105,6 +105,13 @@ spec: resources: requests: storage: 1Gi + nodeConfVolume: true + nodeConfVolumeClaimTemplate: + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi ``` The yaml manifest can easily get applied by using `kubectl`. diff --git a/example/v1beta2/acl-pvc/cluster.yaml b/example/v1beta2/acl-pvc/cluster.yaml index f85a5bf6b7..f417aeebb5 100644 --- a/example/v1beta2/acl-pvc/cluster.yaml +++ b/example/v1beta2/acl-pvc/cluster.yaml @@ -10,7 +10,7 @@ spec: image: quay.io/opstree/redis:latest imagePullPolicy: IfNotPresent # ACL configuration from PVC - # The operator will mount /etc/redis/user.acl from the PVC + # The operator mounts the PVC at /data/redis, so Redis reads /data/redis/user.acl # Make sure the PVC contains a file named "user.acl" with Redis ACL rules acl: persistentVolumeClaim: "redis-acl-pvc" diff --git a/example/v1beta2/acl-pvc/replication.yaml b/example/v1beta2/acl-pvc/replication.yaml index a61b624e06..230d01100b 100644 --- a/example/v1beta2/acl-pvc/replication.yaml +++ b/example/v1beta2/acl-pvc/replication.yaml @@ -10,7 +10,7 @@ spec: image: quay.io/opstree/redis:latest imagePullPolicy: IfNotPresent # ACL configuration from PVC - # The operator will mount /etc/redis/user.acl from the PVC + # The operator mounts the PVC at /data/redis, so Redis reads /data/redis/user.acl # Make sure the PVC contains a file named "user.acl" with Redis ACL rules acl: persistentVolumeClaim: "redis-acl-pvc" diff --git a/example/v1beta2/acl-pvc/standalone.yaml b/example/v1beta2/acl-pvc/standalone.yaml index fab429e18a..b6fe224388 100644 --- a/example/v1beta2/acl-pvc/standalone.yaml +++ b/example/v1beta2/acl-pvc/standalone.yaml @@ -9,7 +9,7 @@ spec: image: quay.io/opstree/redis:latest imagePullPolicy: IfNotPresent # ACL configuration from PVC - # The operator will mount /etc/redis/user.acl from the PVC + # The operator mounts the PVC at /data/redis, so Redis reads /data/redis/user.acl # Make sure the PVC contains a file named "user.acl" with Redis ACL rules acl: persistentVolumeClaim: "redis-acl-pvc" diff --git a/example/v1beta2/acl_config/cluster.yaml b/example/v1beta2/acl_config/cluster.yaml index fc058b36fa..5b38c2bad9 100644 --- a/example/v1beta2/acl_config/cluster.yaml +++ b/example/v1beta2/acl_config/cluster.yaml @@ -20,11 +20,11 @@ spec: limits: cpu: 101m memory: 128Mi - # redisSecret: - # name: redis-secret - # key: password - # imagePullSecrets: - # - name: regcred + # redisSecret: + # name: redis-secret + # key: password + # imagePullSecrets: + # - name: regcred acl: secret: secretName: acl-secret diff --git a/example/v1beta2/acl_config/replication.yaml b/example/v1beta2/acl_config/replication.yaml index a1bed6997c..7679a6ea5d 100644 --- a/example/v1beta2/acl_config/replication.yaml +++ b/example/v1beta2/acl_config/replication.yaml @@ -20,11 +20,11 @@ spec: limits: cpu: 101m memory: 128Mi - # redisSecret: - # name: redis-secret - # key: password - # imagePullSecrets: - # - name: regcred + # redisSecret: + # name: redis-secret + # key: password + # imagePullSecrets: + # - name: regcred acl: secret: secretName: acl-secret diff --git a/example/v1beta2/acl_config/standalone.yaml b/example/v1beta2/acl_config/standalone.yaml index 8b910f3eee..2a1f595a9d 100644 --- a/example/v1beta2/acl_config/standalone.yaml +++ b/example/v1beta2/acl_config/standalone.yaml @@ -16,11 +16,11 @@ spec: limits: cpu: 101m memory: 128Mi - # redisSecret: - # name: redis-secret - # key: password - # imagePullSecrets: - # - name: regcred + # redisSecret: + # name: redis-secret + # key: password + # imagePullSecrets: + # - name: regcred acl: secret: secretName: acl-secret diff --git a/example/v1beta2/backup_restore/restore/redis-cluster.yaml b/example/v1beta2/backup_restore/restore/redis-cluster.yaml index a6c95ad78d..357d78156c 100644 --- a/example/v1beta2/backup_restore/restore/redis-cluster.yaml +++ b/example/v1beta2/backup_restore/restore/redis-cluster.yaml @@ -20,11 +20,11 @@ spec: limits: cpu: 101m memory: 128Mi - # redisSecret: - # name: redis-secret - # key: password - # imagePullSecrets: - # - name: regcred + # redisSecret: + # name: redis-secret + # key: password + # imagePullSecrets: + # - name: regcred initContainer: enabled: true image: quay.io/opstree/redis-operator-restore:latest diff --git a/example/v1beta2/env_vars/redis-cluster.yaml b/example/v1beta2/env_vars/redis-cluster.yaml index 1b1c30297f..13a7a8f56b 100644 --- a/example/v1beta2/env_vars/redis-cluster.yaml +++ b/example/v1beta2/env_vars/redis-cluster.yaml @@ -25,11 +25,11 @@ spec: value: "custom_value_1" - name: CUSTOM_ENV_VAR_2 value: "custom_value_2" - # redisSecret: - # name: redis-secret - # key: password - # imagePullSecrets: - # - name: regcred + # redisSecret: + # name: redis-secret + # key: password + # imagePullSecrets: + # - name: regcred storage: volumeClaimTemplate: spec: diff --git a/example/v1beta2/env_vars/redis-replication.yaml b/example/v1beta2/env_vars/redis-replication.yaml index 1ffdfb22aa..c7827d653c 100644 --- a/example/v1beta2/env_vars/redis-replication.yaml +++ b/example/v1beta2/env_vars/redis-replication.yaml @@ -23,8 +23,8 @@ spec: redisSecret: name: redis-secret key: password - # imagePullSecrets: - # - name: regcred + # imagePullSecrets: + # - name: regcred env: - name: CUSTOM_ENV_VAR_1 value: "custom_value_1" diff --git a/example/v1beta2/env_vars/redis-standalone.yaml b/example/v1beta2/env_vars/redis-standalone.yaml index 2979c78ace..cebbaec9e2 100644 --- a/example/v1beta2/env_vars/redis-standalone.yaml +++ b/example/v1beta2/env_vars/redis-standalone.yaml @@ -19,11 +19,11 @@ spec: limits: cpu: 101m memory: 128Mi - # redisSecret: - # name: redis-secret - # key: password - # imagePullSecrets: - # - name: regcred + # redisSecret: + # name: redis-secret + # key: password + # imagePullSecrets: + # - name: regcred redisExporter: enabled: false image: quay.io/opstree/redis-exporter:latest diff --git a/example/v1beta2/redis-cluster.yaml b/example/v1beta2/redis-cluster.yaml index 41d2728f1e..9e15e8ff82 100644 --- a/example/v1beta2/redis-cluster.yaml +++ b/example/v1beta2/redis-cluster.yaml @@ -20,11 +20,11 @@ spec: limits: cpu: 101m memory: 128Mi - # redisSecret: - # name: redis-secret - # key: password - # imagePullSecrets: - # - name: regcred + # redisSecret: + # name: redis-secret + # key: password + # imagePullSecrets: + # - name: regcred redisExporter: enabled: false image: quay.io/opstree/redis-exporter:latest diff --git a/example/v1beta2/redis-replication.yaml b/example/v1beta2/redis-replication.yaml index 7e945c1985..c94ac409cc 100644 --- a/example/v1beta2/redis-replication.yaml +++ b/example/v1beta2/redis-replication.yaml @@ -20,11 +20,11 @@ spec: limits: cpu: 101m memory: 128Mi -# redisSecret: -# name: redis-secret -# key: password - # imagePullSecrets: - # - name: regcred + # redisSecret: + # name: redis-secret + # key: password + # imagePullSecrets: + # - name: regcred redisExporter: enabled: false image: quay.io/opstree/redis-exporter:latest diff --git a/example/v1beta2/redis-standalone.yaml b/example/v1beta2/redis-standalone.yaml index dc016de8b3..99e1d2aaa5 100644 --- a/example/v1beta2/redis-standalone.yaml +++ b/example/v1beta2/redis-standalone.yaml @@ -19,11 +19,11 @@ spec: limits: cpu: 101m memory: 128Mi - # redisSecret: - # name: redis-secret - # key: password - # imagePullSecrets: - # - name: regcred + # redisSecret: + # name: redis-secret + # key: password + # imagePullSecrets: + # - name: regcred redisExporter: enabled: false image: quay.io/opstree/redis-exporter:latest diff --git a/example/v1beta2/sidecar_features/sidecar.yaml b/example/v1beta2/sidecar_features/sidecar.yaml index 9e721d2589..4bfe52feb7 100644 --- a/example/v1beta2/sidecar_features/sidecar.yaml +++ b/example/v1beta2/sidecar_features/sidecar.yaml @@ -16,11 +16,11 @@ spec: limits: cpu: 101m memory: 128Mi - # redisSecret: - # name: redis-secret - # key: password - # imagePullSecrets: - # - name: regcred + # redisSecret: + # name: redis-secret + # key: password + # imagePullSecrets: + # - name: regcred redisExporter: enabled: false image: quay.io/opstree/redis-exporter:latest diff --git a/example/v1beta2/tls_enabled/redis-cluster.yaml b/example/v1beta2/tls_enabled/redis-cluster.yaml index eaaaf6f6a5..21cf809704 100644 --- a/example/v1beta2/tls_enabled/redis-cluster.yaml +++ b/example/v1beta2/tls_enabled/redis-cluster.yaml @@ -7,7 +7,7 @@ spec: clusterSize: 3 TLS: - ca: ca.key + ca: ca.crt cert: tls.crt key: tls.key secret: diff --git a/example/v1beta2/tls_enabled/redis-replication.yaml b/example/v1beta2/tls_enabled/redis-replication.yaml index f5009686c7..7c07f6e08e 100644 --- a/example/v1beta2/tls_enabled/redis-replication.yaml +++ b/example/v1beta2/tls_enabled/redis-replication.yaml @@ -8,7 +8,7 @@ metadata: spec: clusterSize: 3 TLS: - ca: ca.key + ca: ca.crt cert: tls.crt key: tls.key secret: diff --git a/example/v1beta2/tls_enabled/redis-standalone.yaml b/example/v1beta2/tls_enabled/redis-standalone.yaml index 39f156e1ab..cd9ae404c4 100644 --- a/example/v1beta2/tls_enabled/redis-standalone.yaml +++ b/example/v1beta2/tls_enabled/redis-standalone.yaml @@ -7,7 +7,7 @@ metadata: name: redis-standalone spec: TLS: - ca: ca.key + ca: ca.crt cert: tls.crt key: tls.key secret: diff --git a/example/v1beta2/topology_spread_constraints/redis-cluster.yaml b/example/v1beta2/topology_spread_constraints/redis-cluster.yaml index 21bd089014..bb26537f2d 100644 --- a/example/v1beta2/topology_spread_constraints/redis-cluster.yaml +++ b/example/v1beta2/topology_spread_constraints/redis-cluster.yaml @@ -22,9 +22,9 @@ spec: limits: cpu: 101m memory: 128Mi - # redisSecret: - # name: redis-secret - # key: password + # redisSecret: + # name: redis-secret + # key: password redisLeader: topologySpreadConstraints: - maxSkew: 1 diff --git a/example/v1beta2/volume_mount/redis-cluster.yaml b/example/v1beta2/volume_mount/redis-cluster.yaml index ffe6afde77..28df3a60f1 100644 --- a/example/v1beta2/volume_mount/redis-cluster.yaml +++ b/example/v1beta2/volume_mount/redis-cluster.yaml @@ -23,8 +23,8 @@ spec: redisSecret: name: redis-secret key: password - # imagePullSecrets: - # - name: regcred + # imagePullSecrets: + # - name: regcred redisExporter: enabled: false image: quay.io/opstree/redis-exporter:latest diff --git a/example/v1beta2/volume_mount/redis-replication.yaml b/example/v1beta2/volume_mount/redis-replication.yaml index 453c3363d1..c8011b27cd 100644 --- a/example/v1beta2/volume_mount/redis-replication.yaml +++ b/example/v1beta2/volume_mount/redis-replication.yaml @@ -17,11 +17,11 @@ spec: limits: cpu: 101m memory: 128Mi - # redisSecret: - # name: redis-secret - # key: password - # imagePullSecrets: - # - name: regcred + # redisSecret: + # name: redis-secret + # key: password + # imagePullSecrets: + # - name: regcred redisExporter: enabled: false image: quay.io/opstree/redis-exporter:latest diff --git a/example/v1beta2/volume_mount/redis-standalone.yaml b/example/v1beta2/volume_mount/redis-standalone.yaml index 086fd39dd1..6b86bf57b5 100644 --- a/example/v1beta2/volume_mount/redis-standalone.yaml +++ b/example/v1beta2/volume_mount/redis-standalone.yaml @@ -16,11 +16,11 @@ spec: limits: cpu: 101m memory: 128Mi - # redisSecret: - # name: redis-secret - # key: password - # imagePullSecrets: - # - name: regcred + # redisSecret: + # name: redis-secret + # key: password + # imagePullSecrets: + # - name: regcred redisExporter: enabled: false image: quay.io/opstree/redis-exporter:latest diff --git a/internal/agent/bootstrap/redis/config.go b/internal/agent/bootstrap/redis/config.go index 54a582728b..43a8467d5d 100644 --- a/internal/agent/bootstrap/redis/config.go +++ b/internal/agent/bootstrap/redis/config.go @@ -41,6 +41,8 @@ func GenerateConfig() error { nodeport = util.CoalesceEnv1("NODEPORT", "false") tlsMode = util.CoalesceEnv1("TLS_MODE", "false") clusterMode = util.CoalesceEnv1("SETUP_MODE", "standalone") + aclMode = util.CoalesceEnv1("ACL_MODE", "") + aclFilePath = util.CoalesceEnv1("ACL_FILE_PATH", "/etc/redis/user.acl") ) if val, ok := util.CoalesceEnv("REDIS_PASSWORD", ""); ok && val != "" { @@ -98,7 +100,7 @@ func GenerateConfig() error { if tlsMode == "true" { cfg.Append("tls-cert-file", util.CoalesceEnv1("REDIS_TLS_CERT", "")) cfg.Append("tls-key-file", util.CoalesceEnv1("REDIS_TLS_CERT_KEY", "")) - cfg.Append("tls-ca-cert-file", util.CoalesceEnv1("REDIS_TLS_CA_KEY", "")) + cfg.Append("tls-ca-cert-file", util.CoalesceEnv1("REDIS_TLS_CA_CERT", "")) cfg.Append("tls-auth-clients", "optional") cfg.Append("tls-replication", "yes") @@ -112,8 +114,9 @@ func GenerateConfig() error { fmt.Println("Running without TLS mode") } - if aclMode := util.CoalesceEnv1("ACL_MODE", ""); aclMode == "true" { - cfg.Append("aclfile", "/etc/redis/user.acl") + if aclMode == "true" { + fmt.Println("ACL_MODE is true, modifying ACL file path to", aclFilePath) + cfg.Append("aclfile", aclFilePath) } else { fmt.Println("ACL_MODE is not true, skipping ACL file modification") } diff --git a/internal/agent/bootstrap/sentinel/config.go b/internal/agent/bootstrap/sentinel/config.go index 52f3f8da6d..e23e1f9694 100644 --- a/internal/agent/bootstrap/sentinel/config.go +++ b/internal/agent/bootstrap/sentinel/config.go @@ -88,8 +88,11 @@ func GenerateConfig() error { // acl_setup { - if aclMode, ok := util.CoalesceEnv("ACL_MODE", ""); ok && aclMode == "true" { - cfg.Append("aclfile", "/etc/redis/user.acl") + aclMode, _ := util.CoalesceEnv("ACL_MODE", "") + aclFilePath, _ := util.CoalesceEnv("ACL_FILE_PATH", "/etc/redis/user.acl") + if aclMode == "true" { + fmt.Println("ACL_MODE is true, modifying ACL file path to", aclFilePath) + cfg.Append("aclfile", aclFilePath) } else { fmt.Println("ACL_MODE is not true, skipping ACL file modification") } @@ -100,13 +103,13 @@ func GenerateConfig() error { if tlsMode, ok := util.CoalesceEnv("TLS_MODE", ""); ok && tlsMode == "true" { redisTLSCert, _ := util.CoalesceEnv("REDIS_TLS_CERT", "") redisTLSCertKey, _ := util.CoalesceEnv("REDIS_TLS_CERT_KEY", "") - redisTLSCAKey, _ := util.CoalesceEnv("REDIS_TLS_CA_KEY", "") + redisTLSCACert, _ := util.CoalesceEnv("REDIS_TLS_CA_CERT", "") cfg.Append("port", "0") cfg.Append("tls-port", "26379") cfg.Append("tls-cert-file", redisTLSCert) cfg.Append("tls-key-file", redisTLSCertKey) - cfg.Append("tls-ca-cert-file", redisTLSCAKey) + cfg.Append("tls-ca-cert-file", redisTLSCACert) cfg.Append("tls-auth-clients", "optional") // Sentinel should use tls for replication connection. cfg.Append("tls-replication", "yes") diff --git a/internal/controller/rediscluster/rediscluster_controller.go b/internal/controller/rediscluster/rediscluster_controller.go index 48b3b9a37c..d8d0fbd9a0 100644 --- a/internal/controller/rediscluster/rediscluster_controller.go +++ b/internal/controller/rediscluster/rediscluster_controller.go @@ -90,6 +90,23 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu if masterCount := k8sutils.CheckRedisNodeCount(ctx, r.K8sClient, instance, "leader"); masterCount == leaderCount { r.Recorder.Event(instance, corev1.EventTypeNormal, events.EventReasonRedisClusterDownscale, "Redis cluster is downscaling...") logger.Info("Redis cluster is downscaling...", "Current.LeaderReplicas", leaderCount, "Desired.LeaderReplicas", leaderReplicas) + + // Before resharding, ensure all remaining leader pods (the transfer targets) are masters. + // After scale-out, a failover may have converted some leader pods to slaves, which causes + // reshard to fail with "The specified node is not known or not a master". + // We handle one failover per reconcile cycle and requeue — the loop will converge + // over successive reconciliations until all target pods are masters. + for i := int32(0); i < leaderReplicas; i++ { + if !(k8sutils.VerifyLeaderPod(ctx, r.K8sClient, instance, i)) { + logger.Info("Transfer target leader pod is not a master, initiating failover before scale-down", "Pod.Index", i) + if err = k8sutils.ClusterFailover(ctx, r.K8sClient, instance, i); err != nil { + logger.Error(err, "Failed to initiate cluster failover for transfer target") + return intctrlutil.RequeueE(ctx, err, "") + } + return intctrlutil.RequeueAfter(ctx, time.Second*10, "Waiting for failover to complete before scale-down") + } + } + for shardIdx := leaderCount - 1; shardIdx >= leaderReplicas; shardIdx-- { logger.Info("Remove the shard", "Shard.Index", shardIdx) // Imp if the last index of leader sts is not leader make it then @@ -238,14 +255,34 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu } else { if leaderCount < leaderReplicas { // Scale up the cluster + // Step 1 : Fix any open slots from previous interrupted operations + if err := k8sutils.FixRedisCluster(ctx, r.K8sClient, instance); err != nil { + logger.Error(err, "Failed to fix redis cluster slots, proceeding with scale-up") + } // Step 2 : Add Redis Node k8sutils.AddRedisNodeToCluster(ctx, r.K8sClient, instance) monitoring.RedisClusterAddingNodeAttempt.WithLabelValues(instance.Namespace, instance.Name).Inc() - // Step 3 Rebalance the cluster using the empty masters - k8sutils.RebalanceRedisClusterEmptyMasters(ctx, r.K8sClient, instance) + + return intctrlutil.RequeueAfter(ctx, 10*time.Second, "added node, waiting for cluster convergence before rebalancing") } } } else { + stable, err := k8sutils.ClusterStableNoOpenSlots(ctx, r.K8sClient, instance) + if err != nil { + return ctrl.Result{}, err + } + if !stable { + return ctrl.Result{RequeueAfter: 10 * time.Second}, nil + } + + empty, err := k8sutils.ClusterHasEmptyMasters(ctx, r.K8sClient, instance) + if err != nil { + return ctrl.Result{}, err + } + if empty { + k8sutils.RebalanceRedisClusterEmptyMasters(ctx, r.K8sClient, instance) + } + if followerReplicas > 0 { logger.Info("All leader are part of the cluster, adding follower/replicas", "Leaders.Count", leaderCount, "Instance.Size", leaderReplicas, "Follower.Replicas", followerReplicas) k8sutils.ExecuteRedisReplicationCommand(ctx, r.K8sClient, instance) diff --git a/internal/k8sutils/cluster-scaling.go b/internal/k8sutils/cluster-scaling.go index c7fc8ca75d..d1d5afe18f 100644 --- a/internal/k8sutils/cluster-scaling.go +++ b/internal/k8sutils/cluster-scaling.go @@ -6,6 +6,7 @@ import ( "net" "strconv" "strings" + "time" rcvb2 "github.com/OT-CONTAINER-KIT/redis-operator/api/rediscluster/v1beta2" redis "github.com/redis/go-redis/v9" @@ -13,6 +14,220 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" ) +// ClusterHasEmptyMasters returns true if the cluster contains at least one master +// with no slot ranges assigned. +// This is useful to decide if `redis-cli --cluster rebalance --cluster-use-empty-masters` +// should be attempted. +func ClusterHasEmptyMasters(ctx context.Context, client kubernetes.Interface, cr *rcvb2.RedisCluster) (bool, error) { + seedPod := fmt.Sprintf("%s-leader-0", cr.Name) + redisClient := configureRedisClient(ctx, client, cr, seedPod) + defer redisClient.Close() + + nodes, err := clusterNodes(ctx, redisClient) + if err != nil { + return false, err + } + + return clusterHasEmptyMasters(nodes), nil +} + +func clusterHasEmptyMasters(nodes []clusterNodesResponse) bool { + for _, fields := range nodes { + // CLUSTER NODES format: + // 0:id 1:addr 2:flags 3:master 4:ping 5:pong 6:epoch 7:link [8+:slots...] + if len(fields) < 8 { + // malformed line; ignore rather than failing hard + continue + } + + flags := fields[2] + linkState := fields[7] + + if !hasFlag(flags, "master") { + continue + } + + // Ignore masters that are clearly not healthy participants + if hasAnyFlag(flags, "fail", "handshake", "noaddr") || linkState != "connected" { + continue + } + + // Slots are everything after index 7. + // If none exist, this master is "empty". + if len(fields) == 8 { + return true + } + + hasSlotToken := false + for _, tok := range fields[8:] { + if looksLikeSlotToken(tok) { + hasSlotToken = true + break + } + } + if !hasSlotToken { + return true + } + } + + return false +} + +func hasAnyFlag(flags string, wanted ...string) bool { + for _, w := range wanted { + if hasFlag(flags, w) { + return true + } + } + return false +} + +func hasFlag(flags string, flag string) bool { + // flags are comma-separated: "master", "myself,master", "slave", ... + for _, f := range strings.Split(flags, ",") { + if f == flag { + return true + } + } + return false +} + +func looksLikeSlotToken(tok string) bool { + // Very permissive on purpose. + // We only need to distinguish "some slot-related token exists" from "none". + if tok == "" { + return false + } + // migration markers are also slot-related + if strings.Contains(tok, "->-") || strings.Contains(tok, "-<-") { + return true + } + // common cases: "0-5460" or "5461" + if tok[0] >= '0' && tok[0] <= '9' { + return true + } + // bracketed forms: "[0-5460]" or "[5461->-...]" + if strings.HasPrefix(tok, "[") && len(tok) > 1 && tok[1] >= '0' && tok[1] <= '9' { + return true + } + return false +} + +// ClusterStableNoOpenSlots returns true if the cluster appears safe for +// disruptive operations like rebalance/reshard: +// - cluster_state == ok +// - no migrating/importing slot markers in CLUSTER NODES ("->-" / "-<-") +// - no nodes in handshake/fail/noaddr +// - (optional) cluster_slot_migration_active_tasks == 0 if present in CLUSTER INFO +func ClusterStableNoOpenSlots(ctx context.Context, client kubernetes.Interface, cr *rcvb2.RedisCluster) (bool, error) { + seedPod := fmt.Sprintf("%s-leader-0", cr.Name) + redisClient := configureRedisClient(ctx, client, cr, seedPod) + defer redisClient.Close() + + info, err := redisClient.ClusterInfo(ctx).Result() + if err != nil { + return false, err + } + + nodes, err := clusterNodes(ctx, redisClient) + if err != nil { + return false, err + } + + return clusterStableNoOpenSlots(info, nodes), nil +} + +func clusterStableNoOpenSlots(clusterInfo string, nodes []clusterNodesResponse) bool { + // 1) cluster_state gate + kv := parseClusterInfo(clusterInfo) + if kv["cluster_state"] != "ok" { + return false + } + + // 2) migration-task gate (Redis 7 exposes these fields in your output) + if v, ok := kv["cluster_slot_migration_active_tasks"]; ok { + n, convErr := strconv.Atoi(strings.TrimSpace(v)) + if convErr == nil && n > 0 { + return false + } + } + + // 3) open slot + node health gate from CLUSTER NODES + for _, fields := range nodes { + if len(fields) < 8 { + // malformed line -> treat as not stable (conservative) + return false + } + + flags := fields[2] + linkState := fields[7] + + // If nodes are not fully established, don't rebalance. + if hasAnyFlag(flags, "handshake", "fail", "noaddr") || linkState != "connected" { + return false + } + } + + return !clusterHasOpenSlots(nodes) +} + +func parseClusterInfo(info string) map[string]string { + out := make(map[string]string) + for _, line := range strings.Split(info, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + parts := strings.SplitN(line, ":", 2) + if len(parts) != 2 { + continue + } + out[parts[0]] = strings.TrimSpace(parts[1]) + } + return out +} + +func clusterHasOpenSlots(nodes []clusterNodesResponse) bool { + for _, fields := range nodes { + // Open slot markers appear in the slot tokens: + // "[5491->-...]" migrating / "[5491-<-...]" importing + for _, tok := range fields[8:] { + if strings.Contains(tok, "->-") || strings.Contains(tok, "-<-") { + return true + } + } + } + return false +} + +func waitForClusterNoOpenSlots(ctx context.Context, client kubernetes.Interface, cr *rcvb2.RedisCluster, timeout time.Duration) error { + redisClient := configureRedisClient(ctx, client, cr, cr.Name+"-leader-0") + defer redisClient.Close() + + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + info, err := redisClient.ClusterInfo(ctx).Result() + if err != nil || !strings.Contains(info, "cluster_state:ok") { + time.Sleep(2 * time.Second) + continue + } + + nodes, err := clusterNodes(ctx, redisClient) + if err != nil { + time.Sleep(2 * time.Second) + continue + } + + if clusterHasOpenSlots(nodes) { + time.Sleep(2 * time.Second) + continue + } + + return nil + } + return fmt.Errorf("cluster still has open slots or is not converged after %s", timeout) +} + // ReshardRedisCluster transfer the slots from the last node to the provided transfer node. // // NOTE: when all slot been transferred, the node become slave of the transfer node. @@ -127,8 +342,38 @@ func getRedisNodeID(ctx context.Context, client kubernetes.Interface, cr *rcvb2. return output } +// FixRedisCluster runs `redis-cli --cluster fix` to resolve any open/stuck slots +// (e.g., slots left in migrating/importing state from a previous interrupted rebalance). +// This must be called before add-node or rebalance when the cluster may have open slots. +func FixRedisCluster(ctx context.Context, client kubernetes.Interface, cr *rcvb2.RedisCluster) error { + pod := RedisDetails{ + PodName: cr.Name + "-leader-0", + Namespace: cr.Namespace, + } + cmd := []string{"redis-cli", "--cluster", "fix"} + cmd = append(cmd, getEndpoint(ctx, client, cr, pod)) + if cr.Spec.KubernetesConfig.ExistingPasswordSecret != nil { + pass, err := getRedisPassword(ctx, client, cr.Namespace, *cr.Spec.KubernetesConfig.ExistingPasswordSecret.Name, *cr.Spec.KubernetesConfig.ExistingPasswordSecret.Key) + if err != nil { + log.FromContext(ctx).Error(err, "Error in getting redis password") + } + cmd = append(cmd, "-a") + cmd = append(cmd, pass) + } + cmd = append(cmd, "--cluster-yes") + cmd = append(cmd, getRedisTLSArgs(cr.Spec.TLS, cr.Name+"-leader-0")...) + + _, err := executeCommand1(ctx, client, cr, cmd, cr.Name+"-leader-0") + return err +} + // Rebalance the Redis CLuster using the Empty Master Nodes func RebalanceRedisClusterEmptyMasters(ctx context.Context, client kubernetes.Interface, cr *rcvb2.RedisCluster) { + if err := waitForClusterNoOpenSlots(ctx, client, cr, 2*time.Minute); err != nil { + log.FromContext(ctx).Info("Skipping rebalance: cluster not ready", "reason", err.Error()) + return + } + // cmd = redis-cli --cluster rebalance : --cluster-use-empty-masters -a var cmd []string pod := RedisDetails{ @@ -197,6 +442,21 @@ func RebalanceRedisCluster(ctx context.Context, client kubernetes.Interface, cr executeCommand(ctx, client, cr, cmd, cr.Name+"-leader-1") } +func waitForNodePresence(ctx context.Context, client kubernetes.Interface, cr *rcvb2.RedisCluster, nodeIP string, timeout time.Duration) error { + redisClient := configureRedisClient(ctx, client, cr, cr.Name+"-leader-0") + defer redisClient.Close() + + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + nodes, err := clusterNodes(ctx, redisClient) + if err == nil && checkRedisNodePresence(ctx, nodes, nodeIP) { + return nil + } + time.Sleep(2 * time.Second) + } + return fmt.Errorf("node %s did not appear in cluster nodes after %s", nodeIP, timeout) +} + // Add redis cluster node would add a node to the existing redis cluster using redis-cli func AddRedisNodeToCluster(ctx context.Context, client kubernetes.Interface, cr *rcvb2.RedisCluster) { cmd := []string{"redis-cli", "--cluster", "add-node"} @@ -223,6 +483,10 @@ func AddRedisNodeToCluster(ctx context.Context, client kubernetes.Interface, cr cmd = append(cmd, getRedisTLSArgs(cr.Spec.TLS, cr.Name+"-leader-0")...) executeCommand(ctx, client, cr, cmd, cr.Name+"-leader-0") + + if err := waitForNodePresence(ctx, client, cr, getRedisServerIP(ctx, client, newPod), 90*time.Second); err != nil { + log.FromContext(ctx).Error(err, "node added but not converged yet: %w") + } } // getAttachedFollowerNodeIDs would return a slice of redis followers attached to a redis leader diff --git a/internal/k8sutils/cluster-scaling_test.go b/internal/k8sutils/cluster-scaling_test.go index 69fa1fba00..c55fcad561 100644 --- a/internal/k8sutils/cluster-scaling_test.go +++ b/internal/k8sutils/cluster-scaling_test.go @@ -10,6 +10,493 @@ import ( "github.com/stretchr/testify/assert" ) +func Test_clusterHasEmptyMasters_WhenConnectedMasterHasNoSlots_ReturnsTrue(t *testing.T) { + nodes := []clusterNodesResponse{ + // master with slots assigned + { + "id-master-with-slots", + "10.0.0.10:6379@16379", + "myself,master", + "-", + "0", + "0", + "1", + "connected", + "0-5460", + }, + // empty master (no slots) + { + "id-empty-master", + "10.0.0.11:6379@16379", + "master", + "-", + "0", + "0", + "2", + "connected", + // NOTE: no slot tokens => len(fields) == 8 + }, + // a replica (should be ignored) + { + "id-replica", + "10.0.0.12:6379@16379", + "slave", + "id-master-with-slots", + "0", + "0", + "3", + "connected", + }, + } + + // Act: + got := clusterHasEmptyMasters(nodes) + + // Assert: + if !got { + t.Fatalf("expected true, got false") + } +} + +func Test_clusterHasEmptyMasters_skipMalformedNode(t *testing.T) { + nodes := []clusterNodesResponse{ + // master with slot token + { + "id-empty-master", + "10.0.0.11:6379@16379", + "master", + "-", + "0", + "0", + "2", + // NOTE: missing linkState => len(fields) < 8 + }, + } + + // Act: + got := clusterHasEmptyMasters(nodes) + + // Assert: + if got { + t.Fatalf("expected false, got true") + } +} + +func Test_clusterHasEmptyMasters_skipFlags_ReturnsFalse(t *testing.T) { + tests := []struct { + name string + flags string + linkState string + }{ + { + name: "with non master flag", + flags: "bla", + linkState: "connected", + }, + { + name: "with master,fail flag", + flags: "master,fail", + linkState: "connected", + }, + { + name: "with master,handshake flag", + flags: "master,handshake", + linkState: "connected", + }, + { + name: "with master,noaddr flag", + flags: "master,noaddr", + linkState: "connected", + }, + { + name: "with non connected linkState", + flags: "master", + linkState: "disconnected", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + nodes := []clusterNodesResponse{ + // master with slot token + { + "id-empty-master", + "10.0.0.11:6379@16379", + tt.flags, + "-", + "0", + "0", + "2", + tt.linkState, + }, + } + + // Act: + got := clusterHasEmptyMasters(nodes) + + // Assert: + if got { + t.Fatalf("expected false, got true") + } + }) + } +} + +func Test_clusterHasEmptyMasters_ReturnsTrue(t *testing.T) { + tests := []struct { + name string + secondMasterSlots string + }{ + { + name: "with empty string", + secondMasterSlots: "", + }, + { + name: "with non slot value", + secondMasterSlots: "abc", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + nodes := []clusterNodesResponse{ + // master with slots assigned + { + "id-master-with-slots", + "10.0.0.10:6379@16379", + "myself,master", + "-", + "0", + "0", + "1", + "connected", + "0-5460", + }, + // second master with slot token + { + "id-empty-master", + "10.0.0.11:6379@16379", + "master", + "-", + "0", + "0", + "2", + "connected", + tt.secondMasterSlots, + }, + // a replica (should be ignored) + { + "id-replica", + "10.0.0.12:6379@16379", + "slave", + "id-master-with-slots", + "0", + "0", + "3", + "connected", + }, + } + + // Act: + got := clusterHasEmptyMasters(nodes) + + // Assert: + if !got { + t.Fatalf("expected true, got false") + } + }) + } +} + +func Test_clusterHasEmptyMasters_ReturnsFalse(t *testing.T) { + tests := []struct { + name string + expectedBool bool + secondMasterSlots string + }{ + { + name: "with slot (without brackets)", + secondMasterSlots: "5461", + }, { + name: "with slot (with brackets)", + secondMasterSlots: "[5461]", + }, { + name: "with migration markers", + secondMasterSlots: "[5461->-nodeid]", + }, { + name: "with migration markers", + secondMasterSlots: "[5461->-nodeid]", + }, { + name: "with import markers", + secondMasterSlots: "[5461-<-nodeid]", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + nodes := []clusterNodesResponse{ + // master with slots assigned + { + "id-master-with-slots", + "10.0.0.10:6379@16379", + "myself,master", + "-", + "0", + "0", + "1", + "connected", + "0-5460", + }, + // second master with slot token + { + "id-empty-master", + "10.0.0.11:6379@16379", + "master", + "-", + "0", + "0", + "2", + "connected", + tt.secondMasterSlots, + }, + // a replica (should be ignored) + { + "id-replica", + "10.0.0.12:6379@16379", + "slave", + "id-master-with-slots", + "0", + "0", + "3", + "connected", + }, + } + + // Act: + got := clusterHasEmptyMasters(nodes) + + // Assert: + if got { + t.Fatalf("expected false, got true") + } + }) + } +} + +func Test_clusterStableNoOpenSlots_ReturnsTrue(t *testing.T) { + info := "cluster_state:ok\ncluster_slot_migration_active_tasks:0\n" + nodes := []clusterNodesResponse{ + { + "id-1", + "10.0.0.10:6379@16379", + "myself,master", + "-", + "0", + "0", + "1", + "connected", + "0-5460", + }, + { + "id-2", + "10.0.0.11:6379@16379", + "master", + "-", + "0", + "0", + "2", + "connected", + "5461-10922", + }, + { + "id-3", + "10.0.0.12:6379@16379", + "slave", + "id-1", + "0", + "0", + "3", + "connected", + }, + } + + if got := clusterStableNoOpenSlots(info, nodes); !got { + t.Fatalf("expected true, got false") + } +} + +// For redis 6 compatibility +func Test_clusterStableNoOpenSlots_missingMigrationActiveTask_ReturnsTrue(t *testing.T) { + info := "cluster_state:ok\n" + nodes := []clusterNodesResponse{ + { + "id-1", + "10.0.0.10:6379@16379", + "myself,master", + "-", + "0", + "0", + "1", + "connected", + "0-5460", + }, + { + "id-2", + "10.0.0.11:6379@16379", + "master", + "-", + "0", + "0", + "2", + "connected", + "5461-10922", + }, + { + "id-3", + "10.0.0.12:6379@16379", + "slave", + "id-1", + "0", + "0", + "3", + "connected", + }, + } + + if got := clusterStableNoOpenSlots(info, nodes); !got { + t.Fatalf("expected true, got false") + } +} + +func Test_clusterStableNoOpenSlots_malformedNode_ReturnsFalse(t *testing.T) { + info := "cluster_state:ok\ncluster_slot_migration_active_tasks:0\n" + nodes := []clusterNodesResponse{ + { + "id-1", + "10.0.0.10:6379@16379", + "myself,master", + "-", + "0", + "0", + "1", + }, + } + + if got := clusterStableNoOpenSlots(info, nodes); got { + t.Fatalf("expected false, got true") + } +} + +func Test_clusterStableNoOpenSlots_ReturnsFalse(t *testing.T) { + tests := []struct { + name string + clusterState string + clusterSlotMigrationActiveTasks string + nodeFields string + linkState string + slots string + }{ + { + name: "cluster state no ok", + clusterState: "bad", + clusterSlotMigrationActiveTasks: "0", + nodeFields: "master", + linkState: "connected", + slots: "5461", + }, + { + name: "active migration tasks", + clusterState: "ok", + clusterSlotMigrationActiveTasks: "2", + nodeFields: "master", + linkState: "connected", + slots: "5461", + }, + { + name: "handshake flag", + clusterState: "ok", + clusterSlotMigrationActiveTasks: "0", + nodeFields: "master,handshake", + linkState: "connected", + slots: "5461", + }, + { + name: "fail flag", + clusterState: "ok", + clusterSlotMigrationActiveTasks: "0", + nodeFields: "master,fail", + linkState: "connected", + slots: "5461", + }, + { + name: "noaddr flag", + clusterState: "ok", + clusterSlotMigrationActiveTasks: "0", + nodeFields: "master,noaddr", + linkState: "connected", + slots: "5461", + }, + { + name: "linkState not connected", + clusterState: "ok", + clusterSlotMigrationActiveTasks: "0", + nodeFields: "master", + linkState: "disconnected", + slots: "5461", + }, + { + name: "migrating slots", + clusterState: "ok", + clusterSlotMigrationActiveTasks: "0", + nodeFields: "master", + linkState: "connected", + slots: "[5491->-6000]", + }, + { + name: "importing slots", + clusterState: "ok", + clusterSlotMigrationActiveTasks: "0", + nodeFields: "master", + linkState: "connected", + slots: "[5491-<-6000]", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + info := "cluster_state:" + tt.clusterState + "\ncluster_slot_migration_active_tasks:" + tt.clusterSlotMigrationActiveTasks + "\n" + + nodes := []clusterNodesResponse{ + { + "id-1", + "10.0.0.10:6379@16379", + "master", + "-", + "0", + "0", + "1", + "connected", + "0-5460", + }, + { + "id-2", + "10.0.0.11:6379@16379", + tt.nodeFields, + "-", + "0", + "0", + "2", + tt.linkState, + tt.slots, + }, + } + if got := clusterStableNoOpenSlots(info, nodes); got { + t.Fatalf("expected false, got true") + } + }) + } +} + func Test_verifyLeaderPodInfo(t *testing.T) { tests := []struct { name string diff --git a/internal/k8sutils/pvc.go b/internal/k8sutils/pvc.go index 39358061dd..28c4102da6 100644 --- a/internal/k8sutils/pvc.go +++ b/internal/k8sutils/pvc.go @@ -65,6 +65,15 @@ func HandlePVCResizing(ctx context.Context, storedStateful, newStateful *appsv1. return nil } + // If the desired capacity is less than the stored capacity, skip the resize entirely. + // Kubernetes does not support PVC shrinking. + if desiredCapacity < storedCapacity { + log.FromContext(ctx).Info(fmt.Sprintf( + "sts:%s skipping PVC resize: desired capacity %d is less than current %d (Kubernetes does not support PVC shrinking)", + storedStateful.Name, desiredCapacity, storedCapacity)) + return nil + } + // Create a label selector to list all related PVCs. labelSelector := labels.FormatLabels(map[string]string{ "app": storedStateful.Name, @@ -96,8 +105,9 @@ func HandlePVCResizing(ctx context.Context, storedStateful, newStateful *appsv1. if _, err := cl.CoreV1().PersistentVolumeClaims(storedStateful.Namespace).Update(context.Background(), pvc, metav1.UpdateOptions{}); err != nil { updateFailed = true log.FromContext(ctx).Error(fmt.Errorf("sts:%s resize pvc [%s] failed: %s", storedStateful.Name, pvc.Name, err.Error()), "") + } else { + log.FromContext(ctx).Info(fmt.Sprintf("sts:%s resized pvc [%s] from %d to %d", storedStateful.Name, pvc.Name, currentCapacity, desiredCapacity)) } - log.FromContext(ctx).Info(fmt.Sprintf("sts:%s resized pvc [%s] from %d to %d", storedStateful.Name, pvc.Name, currentCapacity, desiredCapacity)) } } diff --git a/internal/k8sutils/pvc_test.go b/internal/k8sutils/pvc_test.go index 0c385b4395..e1f5138dac 100644 --- a/internal/k8sutils/pvc_test.go +++ b/internal/k8sutils/pvc_test.go @@ -178,6 +178,102 @@ func TestHandlePVCResizing_UpdatePVC(t *testing.T) { } } +// TestHandlePVCResizing_ShrinkSkipped verifies that when the desired capacity is smaller than +// the current PVC size, the resize is skipped and no update is attempted. +func TestHandlePVCResizing_ShrinkSkipped(t *testing.T) { + ctx := context.Background() + + // Stored PVC spec with 10Gi and new spec with 4Gi for the target (redis-data). + storedQuantity := resource.MustParse("10Gi") + desiredQuantity := resource.MustParse("4Gi") + storedPVCSpec := corev1.PersistentVolumeClaimSpec{ + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: storedQuantity, + }, + }, + } + newPVCSpec := corev1.PersistentVolumeClaimSpec{ + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: desiredQuantity, + }, + }, + } + + storedTemplates := []corev1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{Name: "node-conf"}, + Spec: storedPVCSpec, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "redis-data"}, + Spec: storedPVCSpec, + }, + } + + annotations := map[string]string{ + "storageCapacity": strconv.FormatInt(storedQuantity.Value(), 10), + } + + storedStateful := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "redis", + Namespace: "default", + Annotations: annotations, + }, + Spec: appsv1.StatefulSetSpec{ + VolumeClaimTemplates: storedTemplates, + }, + } + + newTemplates := []corev1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{Name: "node-conf"}, + Spec: storedPVCSpec, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "redis-data"}, + Spec: newPVCSpec, + }, + } + newStateful := storedStateful.DeepCopy() + newStateful.Spec.VolumeClaimTemplates = newTemplates + + existingPVC := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "redis-data-0", + Namespace: "default", + Labels: map[string]string{ + "app": "redis", + "app.kubernetes.io/component": "middleware", + }, + }, + Spec: storedPVCSpec, + } + + cl := fake.NewSimpleClientset(existingPVC) + + err := HandlePVCResizing(ctx, storedStateful, newStateful, cl) + if err != nil { + t.Fatalf("Expected no error, got: %v", err) + } + + // Ensure no PVC list or update actions occurred (early return before PVC listing). + actions := cl.Actions() + for _, action := range actions { + if action.GetVerb() == "list" || action.GetVerb() == "update" { + t.Errorf("Unexpected action %q on shrink skip: %#v", action.GetVerb(), action) + } + } + + // Annotation should remain at the original stored capacity (not updated on shrink). + expectedAnnotation := strconv.FormatInt(storedQuantity.Value(), 10) + if storedStateful.Annotations["storageCapacity"] != expectedAnnotation { + t.Errorf("Expected annotation storageCapacity to remain %s, got %s", expectedAnnotation, storedStateful.Annotations["storageCapacity"]) + } +} + // TestHandlePVCResizing_UpdateFailure simulates a failure during PVC update and verifies that an error is returned. func TestHandlePVCResizing_UpdateFailure(t *testing.T) { ctx := context.Background() diff --git a/internal/k8sutils/redis-cluster.go b/internal/k8sutils/redis-cluster.go index 3a79ee6ae5..68260c39f5 100644 --- a/internal/k8sutils/redis-cluster.go +++ b/internal/k8sutils/redis-cluster.go @@ -103,7 +103,7 @@ func generateRedisClusterInitContainerParams(cr *rcvb2.RedisCluster) initContain initcontainerProp.AdditionalVolume = cr.Spec.Storage.VolumeMount.Volume initcontainerProp.AdditionalMountPath = cr.Spec.Storage.VolumeMount.MountPath } - if cr.Spec.Storage != nil { + if cr.Spec.Storage != nil && cr.Spec.PersistenceEnabled != nil && *cr.Spec.PersistenceEnabled { initcontainerProp.PersistenceEnabled = &trueProperty } } @@ -402,6 +402,17 @@ func (service RedisClusterService) CreateRedisClusterService(ctx context.Context log.FromContext(ctx).Error(err, "Cannot create master service for Redis", "Setup.Type", service.RedisServiceRole) return err } + + if cr.Spec.RedisExporter != nil && cr.Spec.RedisExporter.Enabled { + defaultP := ptr.To(common.RedisExporterPort) + exporterPort := *util.Coalesce(cr.Spec.RedisExporter.Port, defaultP) + selectorLabels := getRedisStableLabels(serviceName, string(cluster), service.RedisServiceRole) + err = CreateOrUpdateMetricsService(ctx, cr.Namespace, serviceName+"-metrics", selectorLabels, redisClusterAsOwner(cr), exporterPort, cl) + if err != nil { + log.FromContext(ctx).Error(err, "Cannot create metrics service for Redis", "Setup.Type", service.RedisServiceRole) + return err + } + } return nil } diff --git a/internal/k8sutils/redis-cluster_test.go b/internal/k8sutils/redis-cluster_test.go index 3c14acfebd..5a160ca728 100644 --- a/internal/k8sutils/redis-cluster_test.go +++ b/internal/k8sutils/redis-cluster_test.go @@ -265,7 +265,7 @@ func Test_generateRedisClusterContainerParams(t *testing.T) { SecretKey: ptr.To("password"), PersistenceEnabled: ptr.To(true), TLSConfig: &common.TLSConfig{ - CaKeyFile: "ca.key", + CaCertFile: "ca.crt", CertKeyFile: "tls.crt", KeyFile: "tls.key", Secret: corev1.SecretVolumeSource{ @@ -376,7 +376,7 @@ func Test_generateRedisClusterContainerParams(t *testing.T) { SecretKey: ptr.To("password"), PersistenceEnabled: ptr.To(true), TLSConfig: &common.TLSConfig{ - CaKeyFile: "ca.key", + CaCertFile: "ca.crt", CertKeyFile: "tls.crt", KeyFile: "tls.key", Secret: corev1.SecretVolumeSource{ @@ -524,3 +524,24 @@ func Test_generateRedisClusterInitContainerParams(t *testing.T) { actual := generateRedisClusterInitContainerParams(input) assert.EqualValues(t, expected, actual, "Expected %+v, got %+v", expected, actual) } + +func Test_generateRedisClusterInitContainerParams_PersistenceDisabled(t *testing.T) { + enabled := true + persistenceEnabled := false + + input := &rcvb2.RedisCluster{ + Spec: rcvb2.RedisClusterSpec{ + PersistenceEnabled: &persistenceEnabled, + Storage: &rcvb2.ClusterStorage{}, + InitContainer: &common.InitContainer{ + Enabled: &enabled, + Image: "busybox:latest", + }, + }, + } + + actual := generateRedisClusterInitContainerParams(input) + if actual.PersistenceEnabled != nil { + t.Fatalf("Expected PersistenceEnabled to be nil when persistence is disabled, got %v", *actual.PersistenceEnabled) + } +} diff --git a/internal/k8sutils/redis-replication.go b/internal/k8sutils/redis-replication.go index fbe1815f0a..86b07dce24 100644 --- a/internal/k8sutils/redis-replication.go +++ b/internal/k8sutils/redis-replication.go @@ -59,6 +59,14 @@ func CreateReplicationService(ctx context.Context, cr *rrvb2.RedisReplication, c log.FromContext(ctx).Error(err, "Cannot create replica service for Redis") return err } + if cr.Spec.RedisExporter != nil && cr.Spec.RedisExporter.Enabled { + exporterPort := *util.Coalesce(cr.Spec.RedisExporter.Port, ptr.To(common.RedisExporterPort)) + selectorLabels := getRedisStableLabels(cr.Name, string(replication), "replication") + if err := CreateOrUpdateMetricsService(ctx, cr.Namespace, cr.Name+"-metrics", selectorLabels, redisReplicationAsOwner(cr), exporterPort, cl); err != nil { + log.FromContext(ctx).Error(err, "Cannot create metrics service for Redis Replication") + return err + } + } return nil } @@ -181,7 +189,7 @@ func generateRedisReplicationContainerParams(cr *rrvb2.RedisReplication) contain if cr.Spec.LivenessProbe != nil { containerProp.LivenessProbe = cr.Spec.LivenessProbe } - if cr.Spec.Storage != nil { + if storageHasVolumeClaimTemplate(cr.Spec.Storage) { containerProp.PersistenceEnabled = &trueProperty } if cr.Spec.TLS != nil { @@ -217,7 +225,7 @@ func generateRedisReplicationInitContainerParams(cr *rrvb2.RedisReplication) ini initcontainerProp.AdditionalVolume = cr.Spec.Storage.VolumeMount.Volume initcontainerProp.AdditionalMountPath = cr.Spec.Storage.VolumeMount.MountPath } - if cr.Spec.Storage != nil { + if storageHasVolumeClaimTemplate(cr.Spec.Storage) { initcontainerProp.PersistenceEnabled = &trueProperty } } diff --git a/internal/k8sutils/redis-replication_test.go b/internal/k8sutils/redis-replication_test.go index 0a2607e8af..631945c0c9 100644 --- a/internal/k8sutils/redis-replication_test.go +++ b/internal/k8sutils/redis-replication_test.go @@ -177,7 +177,7 @@ func Test_generateRedisReplicationContainerParams(t *testing.T) { SecretKey: ptr.To("password"), PersistenceEnabled: ptr.To(true), TLSConfig: &common.TLSConfig{ - CaKeyFile: "ca.key", + CaCertFile: "ca.crt", CertKeyFile: "tls.crt", KeyFile: "tls.key", Secret: corev1.SecretVolumeSource{ diff --git a/internal/k8sutils/redis-sentinel.go b/internal/k8sutils/redis-sentinel.go index 336de4106c..24bb6db488 100644 --- a/internal/k8sutils/redis-sentinel.go +++ b/internal/k8sutils/redis-sentinel.go @@ -254,6 +254,15 @@ func (service RedisSentinelService) CreateRedisSentinelService(ctx context.Conte log.FromContext(ctx).Error(err, "Cannot create additional service for Redis", "Setup.Type", service.RedisServiceRole) return err } + if cr.Spec.RedisExporter != nil && cr.Spec.RedisExporter.Enabled { + exporterPort := *util.Coalesce(cr.Spec.RedisExporter.Port, ptr.To(common.RedisExporterPort)) + selectorLabels := getRedisStableLabels(serviceName, string(sentinel), service.RedisServiceRole) + err = CreateOrUpdateMetricsService(ctx, cr.Namespace, serviceName+"-metrics", selectorLabels, redisSentinelAsOwner(cr), exporterPort, cl) + if err != nil { + log.FromContext(ctx).Error(err, "Cannot create metrics service for Redis", "Setup.Type", service.RedisServiceRole) + return err + } + } return nil } diff --git a/internal/k8sutils/redis-sentinel_test.go b/internal/k8sutils/redis-sentinel_test.go index c97c83035a..b985569740 100644 --- a/internal/k8sutils/redis-sentinel_test.go +++ b/internal/k8sutils/redis-sentinel_test.go @@ -167,7 +167,7 @@ func Test_generateRedisSentinelContainerParams(t *testing.T) { SecretName: ptr.To("redis-secret"), SecretKey: ptr.To("password"), TLSConfig: &common.TLSConfig{ - CaKeyFile: "ca.key", + CaCertFile: "ca.crt", CertKeyFile: "tls.crt", KeyFile: "tls.key", Secret: corev1.SecretVolumeSource{ diff --git a/internal/k8sutils/redis-standalone.go b/internal/k8sutils/redis-standalone.go index b4d1c5857d..36fd34f63c 100644 --- a/internal/k8sutils/redis-standalone.go +++ b/internal/k8sutils/redis-standalone.go @@ -68,6 +68,15 @@ func CreateStandaloneService(ctx context.Context, cr *rvb2.Redis, cl kubernetes. return err } } + if cr.Spec.RedisExporter != nil && cr.Spec.RedisExporter.Enabled { + exporterPort := *util.Coalesce(cr.Spec.RedisExporter.Port, ptr.To(common.RedisExporterPort)) + selectorLabels := getRedisStableLabels(cr.Name, string(standalone), "standalone") + err = CreateOrUpdateMetricsService(ctx, cr.Namespace, cr.Name+"-metrics", selectorLabels, redisAsOwner(cr), exporterPort, cl) + if err != nil { + log.FromContext(ctx).Error(err, "Cannot create metrics service for Redis") + return err + } + } return nil } @@ -188,7 +197,7 @@ func generateRedisStandaloneContainerParams(cr *rvb2.Redis) containerParameters if cr.Spec.LivenessProbe != nil { containerProp.LivenessProbe = cr.Spec.LivenessProbe } - if cr.Spec.Storage != nil { + if storageHasVolumeClaimTemplate(cr.Spec.Storage) { containerProp.PersistenceEnabled = &trueProperty } if cr.Spec.TLS != nil { @@ -224,7 +233,7 @@ func generateRedisStandaloneInitContainerParams(cr *rvb2.Redis) initContainerPar initcontainerProp.AdditionalVolume = cr.Spec.Storage.VolumeMount.Volume initcontainerProp.AdditionalMountPath = cr.Spec.Storage.VolumeMount.MountPath } - if cr.Spec.Storage != nil { + if storageHasVolumeClaimTemplate(cr.Spec.Storage) { initcontainerProp.PersistenceEnabled = &trueProperty } } diff --git a/internal/k8sutils/redis-standalone_test.go b/internal/k8sutils/redis-standalone_test.go index 6616862b64..99645aaa72 100644 --- a/internal/k8sutils/redis-standalone_test.go +++ b/internal/k8sutils/redis-standalone_test.go @@ -172,7 +172,7 @@ func Test_generateRedisStandaloneContainerParams(t *testing.T) { SecretKey: ptr.To("password"), PersistenceEnabled: ptr.To(true), TLSConfig: &common.TLSConfig{ - CaKeyFile: "ca.key", + CaCertFile: "ca.crt", CertKeyFile: "tls.crt", KeyFile: "tls.key", Secret: corev1.SecretVolumeSource{ diff --git a/internal/k8sutils/secrets_test.go b/internal/k8sutils/secrets_test.go index cdc98c99fd..7f236f7537 100644 --- a/internal/k8sutils/secrets_test.go +++ b/internal/k8sutils/secrets_test.go @@ -129,7 +129,7 @@ func Test_getRedisTLSConfig(t *testing.T) { }, Spec: rcvb2.RedisClusterSpec{ TLS: &common.TLSConfig{ - CaKeyFile: "ca.crt", + CaCertFile: "ca.crt", CertKeyFile: "tls.crt", KeyFile: "tls.key", Secret: corev1.SecretVolumeSource{ @@ -157,7 +157,7 @@ func Test_getRedisTLSConfig(t *testing.T) { }, Spec: rcvb2.RedisClusterSpec{ TLS: &common.TLSConfig{ - CaKeyFile: "ca.crt", + CaCertFile: "ca.crt", CertKeyFile: "tls.crt", KeyFile: "tls.key", Secret: corev1.SecretVolumeSource{ @@ -195,7 +195,7 @@ func Test_getRedisTLSConfig(t *testing.T) { }, Spec: rcvb2.RedisClusterSpec{ TLS: &common.TLSConfig{ - CaKeyFile: "ca.crt", + CaCertFile: "ca.crt", CertKeyFile: "tls.crt", KeyFile: "tls.key", Secret: corev1.SecretVolumeSource{ diff --git a/internal/k8sutils/services.go b/internal/k8sutils/services.go index d64b467e8b..42e4860d57 100644 --- a/internal/k8sutils/services.go +++ b/internal/k8sutils/services.go @@ -2,6 +2,7 @@ package k8sutils import ( "context" + "strconv" "github.com/OT-CONTAINER-KIT/redis-operator/internal/controller/common" "github.com/OT-CONTAINER-KIT/redis-operator/internal/util/maps" @@ -14,8 +15,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" ) -var serviceType corev1.ServiceType - // exporterPortProvider return the exporter port if bool is true type exporterPortProvider func() (port int, enable bool) @@ -75,6 +74,7 @@ func enableMetricsPort(port int) *corev1.ServicePort { // generateServiceType generates service type func generateServiceType(k8sServiceType string) corev1.ServiceType { + var serviceType corev1.ServiceType switch k8sServiceType { case "LoadBalancer": serviceType = corev1.ServiceTypeLoadBalancer @@ -176,3 +176,41 @@ func patchService(ctx context.Context, storedService *corev1.Service, newService log.FromContext(ctx).V(1).Info("Redis service is already in-sync") return nil } + +// CreateOrUpdateMetricsService creates a dedicated ClusterIP service exposing only the +// redis-exporter port. This prevents monitoring tools from accidentally scraping the +// Redis data port and triggering false "security attack" logs. +func CreateOrUpdateMetricsService(ctx context.Context, namespace string, serviceName string, selectorLabels map[string]string, ownerDef metav1.OwnerReference, exporterPort int, cl kubernetes.Interface) error { + serviceLabels := maps.Copy(selectorLabels) + serviceLabels["app.kubernetes.io/component"] = "metrics" + + annotations := map[string]string{ + "prometheus.io/scrape": "true", + "prometheus.io/port": strconv.Itoa(exporterPort), + } + objectMeta := generateObjectMetaInformation(serviceName, namespace, serviceLabels, annotations) + + metricsPort := enableMetricsPort(exporterPort) + service := &corev1.Service{ + TypeMeta: generateMetaInformation("Service", "v1"), + ObjectMeta: objectMeta, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + Selector: selectorLabels, + Ports: []corev1.ServicePort{*metricsPort}, + }, + } + AddOwnerRefToObject(service, ownerDef) + + storedService, err := getService(ctx, cl, namespace, serviceName) + if err != nil { + if errors.IsNotFound(err) { + if err := patch.DefaultAnnotator.SetLastAppliedAnnotation(service); err != nil { //nolint:gocritic + log.FromContext(ctx).Error(err, "Unable to patch metrics service with compare annotations") + } + return createService(ctx, cl, namespace, service) + } + return err + } + return patchService(ctx, storedService, service, namespace, cl) +} diff --git a/internal/k8sutils/statefulset.go b/internal/k8sutils/statefulset.go index ef07df1b1f..ca60f40ff0 100644 --- a/internal/k8sutils/statefulset.go +++ b/internal/k8sutils/statefulset.go @@ -250,6 +250,17 @@ func syncManagedFields(stored, new *appsv1.StatefulSet) { new.ManagedFields = stored.ManagedFields } +// storageHasVolumeClaimTemplate checks if the Storage has a meaningful VolumeClaimTemplate defined +// (i.e., not just the zero value). This distinguishes between storage configured with only +// volumeMount (e.g. emptyDir) vs. storage that actually requests a PersistentVolumeClaim. +func storageHasVolumeClaimTemplate(storage *commonapi.Storage) bool { + if storage == nil { + return false + } + vct := storage.VolumeClaimTemplate + return len(vct.Spec.AccessModes) > 0 || vct.Spec.Resources.Requests != nil || vct.Spec.StorageClassName != nil || vct.Spec.VolumeName != "" +} + // hasVolumeClaimTemplates checks if the StatefulSet has VolumeClaimTemplates and if their counts match. func hasVolumeClaimTemplates(new, stored *appsv1.StatefulSet) bool { return len(new.Spec.VolumeClaimTemplates) >= 1 && len(new.Spec.VolumeClaimTemplates) == len(stored.Spec.VolumeClaimTemplates) @@ -549,7 +560,7 @@ func GenerateAuthAndTLSArgs(enableAuth, enableTLS bool) (string, string) { authArgs = " -a \"${REDIS_PASSWORD}\"" } if enableTLS { - tlsArgs = " --tls --cert \"${REDIS_TLS_CERT}\" --key \"${REDIS_TLS_CERT_KEY}\" --cacert \"${REDIS_TLS_CA_KEY}\"" + tlsArgs = " --tls --cert \"${REDIS_TLS_CERT}\" --key \"${REDIS_TLS_CERT_KEY}\" --cacert \"${REDIS_TLS_CA_CERT}\"" } return authArgs, tlsArgs } @@ -659,8 +670,8 @@ func GenerateTLSEnvironmentVariables(tlsconfig *commonapi.TLSConfig) []corev1.En tlsCert := "tls.crt" tlsCertKey := "tls.key" - if tlsconfig.CaKeyFile != "" { - caCert = tlsconfig.CaKeyFile + if tlsconfig.CaCertFile != "" { + caCert = tlsconfig.CaCertFile } if tlsconfig.CertKeyFile != "" { tlsCert = tlsconfig.CertKeyFile @@ -674,7 +685,7 @@ func GenerateTLSEnvironmentVariables(tlsconfig *commonapi.TLSConfig) []corev1.En Value: "true", }) envVars = append(envVars, corev1.EnvVar{ - Name: "REDIS_TLS_CA_KEY", + Name: "REDIS_TLS_CA_CERT", Value: path.Join(root, caCert), }) envVars = append(envVars, corev1.EnvVar{ @@ -800,15 +811,18 @@ func getVolumeMount(name string, persistenceEnabled *bool, clusterMode bool, nod } if aclConfig != nil { - volumeName := "acl-secret" if aclConfig.PersistentVolumeClaim != nil { - volumeName = "acl-pvc" + VolumeMounts = append(VolumeMounts, corev1.VolumeMount{ + Name: "acl-pvc", + MountPath: "/data/redis", + }) + } else { + VolumeMounts = append(VolumeMounts, corev1.VolumeMount{ + Name: "acl-secret", + MountPath: "/etc/redis/user.acl", + SubPath: "user.acl", + }) } - VolumeMounts = append(VolumeMounts, corev1.VolumeMount{ - Name: volumeName, - MountPath: "/etc/redis/user.acl", - SubPath: "user.acl", - }) } if externalConfig != nil { @@ -845,7 +859,7 @@ func getProbeInfo(probe *corev1.Probe, sentinel, enableTLS, enableAuth bool) *co redisHealthCheck = append(redisHealthCheck, "-a", "${REDIS_PASSWORD}") } if enableTLS { - redisHealthCheck = append(redisHealthCheck, "--tls", "--cert", "${REDIS_TLS_CERT}", "--key", "${REDIS_TLS_CERT_KEY}", "--cacert", "${REDIS_TLS_CA_KEY}") + redisHealthCheck = append(redisHealthCheck, "--tls", "--cert", "${REDIS_TLS_CERT}", "--key", "${REDIS_TLS_CERT_KEY}", "--cacert", "${REDIS_TLS_CA_CERT}") } redisHealthCheck = append(redisHealthCheck, "ping") @@ -917,6 +931,14 @@ func getEnvironmentVariables(role string, enabledPassword *bool, secretName *str Name: "ACL_MODE", Value: "true", }) + aclFilePath := "/etc/redis/user.acl" + if aclConfig.PersistentVolumeClaim != nil { + aclFilePath = "/data/redis/user.acl" + } + envVars = append(envVars, corev1.EnvVar{ + Name: "ACL_FILE_PATH", + Value: aclFilePath, + }) } envVars = append(envVars, corev1.EnvVar{ diff --git a/internal/k8sutils/statefulset_test.go b/internal/k8sutils/statefulset_test.go index 35d2b7fc5e..6fa75ff834 100644 --- a/internal/k8sutils/statefulset_test.go +++ b/internal/k8sutils/statefulset_test.go @@ -31,8 +31,8 @@ func TestGenerateAuthAndTLSArgs(t *testing.T) { }{ {"NoAuthNoTLS", false, false, "", ""}, {"AuthOnly", true, false, " -a \"${REDIS_PASSWORD}\"", ""}, - {"TLSOnly", false, true, "", " --tls --cert \"${REDIS_TLS_CERT}\" --key \"${REDIS_TLS_CERT_KEY}\" --cacert \"${REDIS_TLS_CA_KEY}\""}, - {"AuthAndTLS", true, true, " -a \"${REDIS_PASSWORD}\"", " --tls --cert \"${REDIS_TLS_CERT}\" --key \"${REDIS_TLS_CERT_KEY}\" --cacert \"${REDIS_TLS_CA_KEY}\""}, + {"TLSOnly", false, true, "", " --tls --cert \"${REDIS_TLS_CERT}\" --key \"${REDIS_TLS_CERT_KEY}\" --cacert \"${REDIS_TLS_CA_CERT}\""}, + {"AuthAndTLS", true, true, " -a \"${REDIS_PASSWORD}\"", " --tls --cert \"${REDIS_TLS_CERT}\" --key \"${REDIS_TLS_CERT_KEY}\" --cacert \"${REDIS_TLS_CA_CERT}\""}, } for _, tt := range tests { @@ -48,6 +48,62 @@ func TestGenerateAuthAndTLSArgs(t *testing.T) { } } +func TestStorageHasVolumeClaimTemplate(t *testing.T) { + tests := []struct { + name string + storage *common.Storage + want bool + }{ + {"nil storage", nil, false}, + {"empty VolumeClaimTemplate", &common.Storage{}, false}, + {"only volumeMount", &common.Storage{ + VolumeMount: common.AdditionalVolume{ + Volume: []corev1.Volume{{Name: "data", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}}, + MountPath: []corev1.VolumeMount{{Name: "data", MountPath: "/data"}}, + }, + }, false}, + {"with AccessModes", &common.Storage{ + VolumeClaimTemplate: corev1.PersistentVolumeClaim{ + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + }, + }, + }, true}, + {"with Resources.Requests", &common.Storage{ + VolumeClaimTemplate: corev1.PersistentVolumeClaim{ + Spec: corev1.PersistentVolumeClaimSpec{ + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + }, + }, + }, true}, + {"with StorageClassName", &common.Storage{ + VolumeClaimTemplate: corev1.PersistentVolumeClaim{ + Spec: corev1.PersistentVolumeClaimSpec{ + StorageClassName: ptr.To("standard"), + }, + }, + }, true}, + {"with VolumeName", &common.Storage{ + VolumeClaimTemplate: corev1.PersistentVolumeClaim{ + Spec: corev1.PersistentVolumeClaimSpec{ + VolumeName: "pv-data", + }, + }, + }, true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := storageHasVolumeClaimTemplate(tt.storage) + assert.Equal(t, tt.want, got) + }) + } +} + func TestGeneratePreStopCommand(t *testing.T) { tests := []struct { name string @@ -191,15 +247,34 @@ func TestGetVolumeMount(t *testing.T) { expectedMounts: []corev1.VolumeMount{{Name: "tls-certs", MountPath: "/tls", ReadOnly: true}}, }, { - name: "6. Only acl enabled", + name: "6. Only acl enabled (secret)", persistenceEnabled: nil, clusterMode: false, nodeConfVolume: false, externalConfig: nil, mountpath: []corev1.VolumeMount{}, tlsConfig: nil, - aclConfig: &common.ACLConfig{}, - expectedMounts: []corev1.VolumeMount{{Name: "acl-secret", MountPath: "/etc/redis/user.acl", SubPath: "user.acl"}}, + aclConfig: &common.ACLConfig{ + Secret: &corev1.SecretVolumeSource{SecretName: "acl-secret"}, + }, + expectedMounts: []corev1.VolumeMount{ + {Name: "acl-secret", MountPath: "/etc/redis/user.acl", SubPath: "user.acl"}, + }, + }, + { + name: "6b. Only acl enabled (PVC)", + persistenceEnabled: nil, + clusterMode: false, + nodeConfVolume: false, + externalConfig: nil, + mountpath: []corev1.VolumeMount{}, + tlsConfig: nil, + aclConfig: &common.ACLConfig{ + PersistentVolumeClaim: ptr.To("acl-pvc"), + }, + expectedMounts: []corev1.VolumeMount{ + {Name: "acl-pvc", MountPath: "/data/redis"}, + }, }, { name: "7. Everything enabled except externalConfig", @@ -214,7 +289,9 @@ func TestGetVolumeMount(t *testing.T) { }, }, tlsConfig: &common.TLSConfig{}, - aclConfig: &common.ACLConfig{}, + aclConfig: &common.ACLConfig{ + Secret: &corev1.SecretVolumeSource{SecretName: "acl-secret"}, + }, expectedMounts: []corev1.VolumeMount{ {Name: "persistent-volume", MountPath: "/data"}, {Name: "node-conf", MountPath: "/node-conf"}, @@ -242,7 +319,9 @@ func TestGetVolumeMount(t *testing.T) { externalConfig: nil, mountpath: []corev1.VolumeMount{}, tlsConfig: nil, - aclConfig: &common.ACLConfig{}, + aclConfig: &common.ACLConfig{ + Secret: &corev1.SecretVolumeSource{SecretName: "acl-secret"}, + }, expectedMounts: []corev1.VolumeMount{ {Name: "persistent-volume", MountPath: "/data"}, {Name: "node-conf", MountPath: "/node-conf"}, @@ -1431,7 +1510,7 @@ func TestGenerateInitContainerDefWithSecurityContext(t *testing.T) { func TestGenerateTLSEnvironmentVariables(t *testing.T) { tlsConfig := &common.TLSConfig{ - CaKeyFile: "test_ca.crt", + CaCertFile: "test_ca.crt", CertKeyFile: "test_tls.crt", KeyFile: "test_tls.key", } @@ -1444,7 +1523,7 @@ func TestGenerateTLSEnvironmentVariables(t *testing.T) { Value: "true", }, { - Name: "REDIS_TLS_CA_KEY", + Name: "REDIS_TLS_CA_CERT", Value: path.Join("/tls/", "test_ca.crt"), }, { @@ -1482,7 +1561,7 @@ func TestGetEnvironmentVariables(t *testing.T) { secretKey: ptr.To("test-key"), persistenceEnabled: ptr.To(true), tlsConfig: &common.TLSConfig{ - CaKeyFile: "test_ca.crt", + CaCertFile: "test_ca.crt", CertKeyFile: "test_tls.crt", KeyFile: "test_tls.key", Secret: corev1.SecretVolumeSource{ @@ -1499,11 +1578,12 @@ func TestGetEnvironmentVariables(t *testing.T) { }, clusterVersion: ptr.To("v6"), expectedEnvironment: []corev1.EnvVar{ + {Name: "ACL_FILE_PATH", Value: "/etc/redis/user.acl"}, {Name: "ACL_MODE", Value: "true"}, {Name: "PERSISTENCE_ENABLED", Value: "true"}, {Name: "REDIS_ADDR", Value: "redis://localhost:26379"}, {Name: "TLS_MODE", Value: "true"}, - {Name: "REDIS_TLS_CA_KEY", Value: path.Join("/tls/", "test_ca.crt")}, + {Name: "REDIS_TLS_CA_CERT", Value: path.Join("/tls/", "test_ca.crt")}, {Name: "REDIS_TLS_CERT", Value: path.Join("/tls/", "test_tls.crt")}, {Name: "REDIS_TLS_CERT_KEY", Value: path.Join("/tls/", "test_tls.key")}, {Name: "REDIS_PASSWORD", ValueFrom: &corev1.EnvVarSource{ @@ -1562,12 +1642,15 @@ func TestGetEnvironmentVariables(t *testing.T) { secretKey: ptr.To("test-key"), persistenceEnabled: ptr.To(true), tlsConfig: nil, - aclConfig: &common.ACLConfig{}, + aclConfig: &common.ACLConfig{ + Secret: &corev1.SecretVolumeSource{SecretName: "acl-secret"}, + }, envVar: &[]corev1.EnvVar{ {Name: "TEST_ENV", Value: "test-value"}, }, port: ptr.To(6380), expectedEnvironment: []corev1.EnvVar{ + {Name: "ACL_FILE_PATH", Value: "/etc/redis/user.acl"}, {Name: "ACL_MODE", Value: "true"}, {Name: "PERSISTENCE_ENABLED", Value: "true"}, {Name: "REDIS_ADDR", Value: "redis://localhost:6379"}, @@ -1585,6 +1668,37 @@ func TestGetEnvironmentVariables(t *testing.T) { {Name: "REDIS_PORT", Value: "6380"}, }, }, + { + name: "Test with cluster role and acl pvc", + role: "cluster", + enabledPassword: ptr.To(true), + secretName: ptr.To("test-secret"), + secretKey: ptr.To("test-key"), + persistenceEnabled: ptr.To(true), + tlsConfig: nil, + aclConfig: &common.ACLConfig{ + PersistentVolumeClaim: ptr.To("acl-pvc"), + }, + envVar: nil, + port: ptr.To(6381), + expectedEnvironment: []corev1.EnvVar{ + {Name: "ACL_FILE_PATH", Value: "/data/redis/user.acl"}, + {Name: "ACL_MODE", Value: "true"}, + {Name: "PERSISTENCE_ENABLED", Value: "true"}, + {Name: "REDIS_ADDR", Value: "redis://localhost:6379"}, + {Name: "REDIS_PASSWORD", ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test-secret", + }, + Key: "test-key", + }, + }}, + {Name: "REDIS_PORT", Value: "6381"}, + {Name: "SERVER_MODE", Value: "cluster"}, + {Name: "SETUP_MODE", Value: "cluster"}, + }, + }, { name: "Test with cluster role and only metrics enabled", role: "cluster", @@ -1625,7 +1739,7 @@ func Test_getExporterEnvironmentVariables(t *testing.T) { name: "Test with tls enabled and env var", params: containerParameters{ TLSConfig: &common.TLSConfig{ - CaKeyFile: "test_ca.crt", + CaCertFile: "test_ca.crt", CertKeyFile: "test_tls.crt", KeyFile: "test_tls.key", Secret: corev1.SecretVolumeSource{ @@ -1665,7 +1779,7 @@ func TestGenerateStatefulSetsDef(t *testing.T) { probeWithTLS := &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ Exec: &corev1.ExecAction{ - Command: []string{"sh", "-ec", "RESP=\"$(redis-cli -h $(hostname) -p ${REDIS_PORT} --tls --cert ${REDIS_TLS_CERT} --key ${REDIS_TLS_CERT_KEY} --cacert ${REDIS_TLS_CA_KEY} ping)\"\n[ \"$RESP\" = \"PONG\" ]"}, + Command: []string{"sh", "-ec", "RESP=\"$(redis-cli -h $(hostname) -p ${REDIS_PORT} --tls --cert ${REDIS_TLS_CERT} --key ${REDIS_TLS_CERT_KEY} --cacert ${REDIS_TLS_CA_CERT} ping)\"\n[ \"$RESP\" = \"PONG\" ]"}, }, }, } @@ -1756,6 +1870,10 @@ func TestGenerateStatefulSetsDef(t *testing.T) { Name: "test-sts", Image: "redis:latest", Env: []corev1.EnvVar{ + { + Name: "ACL_FILE_PATH", + Value: "/etc/redis/user.acl", + }, { Name: "ACL_MODE", Value: "true", @@ -1769,7 +1887,7 @@ func TestGenerateStatefulSetsDef(t *testing.T) { Value: "1.0", }, { - Name: "REDIS_TLS_CA_KEY", + Name: "REDIS_TLS_CA_CERT", Value: path.Join("/tls/", "ca.crt"), }, { diff --git a/internal/service/redis/client.go b/internal/service/redis/client.go index f3b72aa975..558a00e893 100644 --- a/internal/service/redis/client.go +++ b/internal/service/redis/client.go @@ -200,9 +200,11 @@ func (c *service) SentinelMonitor(ctx context.Context, master *ConnectionInfo, m } defer client.Close() + masterExists := false masterCheckCmd := rediscli.NewSliceCmd(ctx, "SENTINEL", "MASTER", masterGroupName) if err = client.Process(ctx, masterCheckCmd); err == nil { if err = masterCheckCmd.Err(); err == nil { + masterExists = true result, _ := masterCheckCmd.Result() var monitoredHost, monitoredPort string for i := 0; i+1 < len(result); i += 2 { @@ -232,13 +234,18 @@ func (c *service) SentinelMonitor(ctx context.Context, master *ConnectionInfo, m } } - cmd = rediscli.NewBoolCmd(ctx, "SENTINEL", "REMOVE", masterGroupName) - err = client.Process(ctx, cmd) - if err != nil { - return err - } - if err = cmd.Err(); err != nil { - return err + // Only remove if master was already being monitored (with wrong host/port). + // On first-time setup, SENTINEL REMOVE would fail with "ERR No such master + // with that name" and prevent SENTINEL MONITOR from ever being called. + if masterExists { + cmd = rediscli.NewBoolCmd(ctx, "SENTINEL", "REMOVE", masterGroupName) + err = client.Process(ctx, cmd) + if err != nil { + return err + } + if err = cmd.Err(); err != nil { + return err + } } cmd = rediscli.NewBoolCmd(ctx, "SENTINEL", "MONITOR", masterGroupName, master.Host, master.Port, quorum) diff --git a/static/updated-redis-operator-architecture-using-meshery.jpg b/static/updated-redis-operator-architecture-using-meshery.jpg new file mode 100644 index 0000000000..6f4c6d7ffb Binary files /dev/null and b/static/updated-redis-operator-architecture-using-meshery.jpg differ diff --git a/tests/e2e-chainsaw/v1beta2/acl-pvc/redis-cluster/chainsaw-test.yaml b/tests/e2e-chainsaw/v1beta2/acl-pvc/redis-cluster/chainsaw-test.yaml index f20d7ef783..c0200fd168 100644 --- a/tests/e2e-chainsaw/v1beta2/acl-pvc/redis-cluster/chainsaw-test.yaml +++ b/tests/e2e-chainsaw/v1beta2/acl-pvc/redis-cluster/chainsaw-test.yaml @@ -96,7 +96,7 @@ spec: timeout: 30s content: > kubectl exec --namespace ${NAMESPACE} --container redis-cluster-acl-pvc-leader redis-cluster-acl-pvc-leader-0 -- - cat /etc/redis/user.acl 2>&1 + cat /data/redis/user.acl 2>&1 check: (contains($stdout, 'user pvcuser')): true (contains($stdout, 'user readonly')): true @@ -113,7 +113,32 @@ spec: (contains($stdout, 'pvcuser')): true (contains($stdout, 'readonly')): true - # Step 11: Test cluster operations with ACL + # Step 11: Test ACL SAVE persists to PVC + - name: Verify ACL SAVE writes to PVC + try: + - script: + timeout: 30s + content: > + kubectl exec --namespace ${NAMESPACE} --container redis-cluster-acl-pvc-leader redis-cluster-acl-pvc-leader-0 -- + redis-cli -c -p 6379 --user admin --pass admin@secure456 ACL SETUSER saveduser on ~* &* +@all >saved@secure789 2>&1 + check: + (contains($stdout, 'OK')): true + - script: + timeout: 30s + content: > + kubectl exec --namespace ${NAMESPACE} --container redis-cluster-acl-pvc-leader redis-cluster-acl-pvc-leader-0 -- + redis-cli -c -p 6379 --user admin --pass admin@secure456 ACL SAVE 2>&1 + check: + (contains($stdout, 'OK')): true + - script: + timeout: 30s + content: > + kubectl exec --namespace ${NAMESPACE} --container redis-cluster-acl-pvc-leader redis-cluster-acl-pvc-leader-0 -- + cat /data/redis/user.acl 2>&1 + check: + (contains($stdout, 'user saveduser')): true + + # Step 12: Test cluster operations with ACL - name: Test cluster info with ACL user try: - script: @@ -124,13 +149,13 @@ spec: check: (contains($stdout, 'cluster_state:ok')): true - # Step 12: Verify PVC data persistence + # Step 13: Verify PVC data persistence - name: Verify ACL PVC volume mount try: - script: timeout: 30s content: > kubectl exec --namespace ${NAMESPACE} --container redis-cluster-acl-pvc-leader redis-cluster-acl-pvc-leader-0 -- - cat /proc/mounts | grep '/etc/redis/user.acl' 2>&1 + cat /proc/mounts | grep '/data/redis' 2>&1 check: - (contains($stdout, '/etc/redis/user.acl')): true + (contains($stdout, '/data/redis')): true diff --git a/tests/e2e-chainsaw/v1beta2/acl-pvc/redis-replication/chainsaw-test.yaml b/tests/e2e-chainsaw/v1beta2/acl-pvc/redis-replication/chainsaw-test.yaml index 427fe84988..1329db16c0 100644 --- a/tests/e2e-chainsaw/v1beta2/acl-pvc/redis-replication/chainsaw-test.yaml +++ b/tests/e2e-chainsaw/v1beta2/acl-pvc/redis-replication/chainsaw-test.yaml @@ -76,7 +76,7 @@ spec: timeout: 30s content: > kubectl exec --namespace ${NAMESPACE} redis-replication-acl-pvc-0 -- - cat /etc/redis/user.acl 2>&1 + cat /data/redis/user.acl 2>&1 check: (contains($stdout, 'user repluser')): true @@ -87,7 +87,7 @@ spec: timeout: 30s content: > kubectl exec --namespace ${NAMESPACE} redis-replication-acl-pvc-1 -- - cat /etc/redis/user.acl 2>&1 + cat /data/redis/user.acl 2>&1 check: (contains($stdout, 'user repluser')): true diff --git a/tests/e2e-chainsaw/v1beta2/acl-pvc/redis-standalone/chainsaw-test.yaml b/tests/e2e-chainsaw/v1beta2/acl-pvc/redis-standalone/chainsaw-test.yaml index b2f959869a..1b490f10aa 100644 --- a/tests/e2e-chainsaw/v1beta2/acl-pvc/redis-standalone/chainsaw-test.yaml +++ b/tests/e2e-chainsaw/v1beta2/acl-pvc/redis-standalone/chainsaw-test.yaml @@ -78,7 +78,7 @@ spec: timeout: 30s content: > kubectl exec --namespace ${NAMESPACE} redis-standalone-acl-pvc-0 -- - cat /etc/redis/user.acl 2>&1 + cat /data/redis/user.acl 2>&1 check: (contains($stdout, 'user standaloneuser')): true diff --git a/tests/e2e-chainsaw/v1beta2/metrics-service/redis-cluster/chainsaw-test.yaml b/tests/e2e-chainsaw/v1beta2/metrics-service/redis-cluster/chainsaw-test.yaml new file mode 100644 index 0000000000..be6b4bd5de --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/metrics-service/redis-cluster/chainsaw-test.yaml @@ -0,0 +1,108 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: metrics-service-redis-cluster +spec: + steps: + - name: Deploy Redis Cluster with exporter + try: + - apply: + file: secret.yaml + - apply: + file: cluster.yaml + - assert: + file: ready-cluster.yaml + + - name: Assert metrics services created for leader and follower + try: + - assert: + file: ready-metrics-svc.yaml + + - name: Verify leader metrics service only exposes exporter port + try: + - script: + timeout: 30s + content: | + #!/bin/bash + set -e + # Verify leader metrics service has exactly one port (9121) + PORTS=$(kubectl get svc -n ${NAMESPACE} redis-cluster-metrics-leader-metrics -o jsonpath='{.spec.ports[*].port}') + if [ "$PORTS" != "9121" ]; then + echo "Expected leader metrics service to only expose port 9121, got: $PORTS" + exit 1 + fi + # Verify follower metrics service has exactly one port (9121) + PORTS=$(kubectl get svc -n ${NAMESPACE} redis-cluster-metrics-follower-metrics -o jsonpath='{.spec.ports[*].port}') + if [ "$PORTS" != "9121" ]; then + echo "Expected follower metrics service to only expose port 9121, got: $PORTS" + exit 1 + fi + # Verify neither metrics service exposes Redis data port 6379 + for ROLE in leader follower; do + HAS_REDIS_PORT=$(kubectl get svc -n ${NAMESPACE} redis-cluster-metrics-${ROLE}-metrics -o jsonpath='{.spec.ports[?(@.port==6379)].port}') + if [ -n "$HAS_REDIS_PORT" ]; then + echo "${ROLE} metrics service should not expose Redis port 6379" + exit 1 + fi + HAS_BUS_PORT=$(kubectl get svc -n ${NAMESPACE} redis-cluster-metrics-${ROLE}-metrics -o jsonpath='{.spec.ports[?(@.port==16379)].port}') + if [ -n "$HAS_BUS_PORT" ]; then + echo "${ROLE} metrics service should not expose Redis bus port 16379" + exit 1 + fi + COMPONENT=$(kubectl get svc -n ${NAMESPACE} redis-cluster-metrics-${ROLE}-metrics -o jsonpath='{.metadata.labels.app\.kubernetes\.io/component}') + if [ "$COMPONENT" != "metrics" ]; then + echo "Expected app.kubernetes.io/component=metrics label on ${ROLE}, got: $COMPONENT" + exit 1 + fi + done + echo "Cluster metrics services validation passed" + check: + (contains($stdout, 'Cluster metrics services validation passed')): true + + - name: Verify metrics are fetchable via leader metrics service + try: + - script: + timeout: 60s + content: | + #!/bin/bash + set -e + OUTPUT=$(kubectl run -n ${NAMESPACE} metrics-check-leader --rm -i --restart=Never --image=busybox:1.36 -- \ + wget -qO- --timeout=10 http://redis-cluster-metrics-leader-metrics.${NAMESPACE}.svc.cluster.local:9121/metrics 2>&1) + echo "$OUTPUT" + if ! echo "$OUTPUT" | grep -q "redis_"; then + echo "ERROR: No redis_ metrics found in leader output" + exit 1 + fi + echo "Leader metrics fetch validation passed" + check: + (contains($stdout, 'redis_')): true + + - name: Verify metrics are fetchable via follower metrics service + try: + - script: + timeout: 60s + content: | + #!/bin/bash + set -e + OUTPUT=$(kubectl run -n ${NAMESPACE} metrics-check-follower --rm -i --restart=Never --image=busybox:1.36 -- \ + wget -qO- --timeout=10 http://redis-cluster-metrics-follower-metrics.${NAMESPACE}.svc.cluster.local:9121/metrics 2>&1) + echo "$OUTPUT" + if ! echo "$OUTPUT" | grep -q "redis_"; then + echo "ERROR: No redis_ metrics found in follower output" + exit 1 + fi + echo "Follower metrics fetch validation passed" + check: + (contains($stdout, 'redis_')): true + + - name: Cleanup + try: + - delete: + ref: + name: redis-cluster-metrics + kind: RedisCluster + apiVersion: redis.redis.opstreelabs.in/v1beta2 + - error: + file: ready-metrics-svc.yaml diff --git a/tests/e2e-chainsaw/v1beta2/metrics-service/redis-cluster/cluster.yaml b/tests/e2e-chainsaw/v1beta2/metrics-service/redis-cluster/cluster.yaml new file mode 100644 index 0000000000..28d4ff6e3a --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/metrics-service/redis-cluster/cluster.yaml @@ -0,0 +1,50 @@ +--- +apiVersion: redis.redis.opstreelabs.in/v1beta2 +kind: RedisCluster +metadata: + name: redis-cluster-metrics +spec: + clusterSize: 3 + clusterVersion: v7 + persistenceEnabled: true + podSecurityContext: + runAsUser: 1000 + fsGroup: 1000 + kubernetesConfig: + image: quay.io/opstree/redis:latest + imagePullPolicy: Always + resources: + requests: + cpu: 101m + memory: 128Mi + limits: + cpu: 101m + memory: 128Mi + redisSecret: + name: redis-secret + key: password + redisExporter: + enabled: true + image: quay.io/opstree/redis-exporter:v1.44.0 + imagePullPolicy: Always + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 100m + memory: 128Mi + storage: + volumeClaimTemplate: + spec: + accessModes: [ReadWriteOnce] + resources: + requests: + storage: 1Gi + nodeConfVolume: true + nodeConfVolumeClaimTemplate: + spec: + accessModes: [ReadWriteOnce] + resources: + requests: + storage: 1Gi diff --git a/tests/e2e-chainsaw/v1beta2/metrics-service/redis-cluster/ready-cluster.yaml b/tests/e2e-chainsaw/v1beta2/metrics-service/redis-cluster/ready-cluster.yaml new file mode 100644 index 0000000000..5acd6c5c50 --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/metrics-service/redis-cluster/ready-cluster.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: redis.redis.opstreelabs.in/v1beta2 +kind: RedisCluster +metadata: + name: redis-cluster-metrics +status: + readyFollowerReplicas: 3 + readyLeaderReplicas: 3 + state: Ready + reason: RedisCluster is ready diff --git a/tests/e2e-chainsaw/v1beta2/metrics-service/redis-cluster/ready-metrics-svc.yaml b/tests/e2e-chainsaw/v1beta2/metrics-service/redis-cluster/ready-metrics-svc.yaml new file mode 100644 index 0000000000..eb4fce3696 --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/metrics-service/redis-cluster/ready-metrics-svc.yaml @@ -0,0 +1,58 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: redis-cluster-metrics-leader-metrics + labels: + app: redis-cluster-metrics-leader + redis_setup_type: cluster + role: leader + app.kubernetes.io/component: metrics + annotations: + prometheus.io/port: "9121" + prometheus.io/scrape: "true" + ownerReferences: + - apiVersion: redis.redis.opstreelabs.in/v1beta2 + controller: true + kind: RedisCluster + name: redis-cluster-metrics +spec: + type: ClusterIP + ports: + - name: redis-exporter + port: 9121 + protocol: TCP + targetPort: 9121 + selector: + app: redis-cluster-metrics-leader + redis_setup_type: cluster + role: leader +--- +apiVersion: v1 +kind: Service +metadata: + name: redis-cluster-metrics-follower-metrics + labels: + app: redis-cluster-metrics-follower + redis_setup_type: cluster + role: follower + app.kubernetes.io/component: metrics + annotations: + prometheus.io/port: "9121" + prometheus.io/scrape: "true" + ownerReferences: + - apiVersion: redis.redis.opstreelabs.in/v1beta2 + controller: true + kind: RedisCluster + name: redis-cluster-metrics +spec: + type: ClusterIP + ports: + - name: redis-exporter + port: 9121 + protocol: TCP + targetPort: 9121 + selector: + app: redis-cluster-metrics-follower + redis_setup_type: cluster + role: follower diff --git a/tests/e2e-chainsaw/v1beta2/metrics-service/redis-cluster/secret.yaml b/tests/e2e-chainsaw/v1beta2/metrics-service/redis-cluster/secret.yaml new file mode 100644 index 0000000000..e9788c3f95 --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/metrics-service/redis-cluster/secret.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: redis-secret +type: Opaque +data: + password: T3BzdHJlZTEyMzQ= diff --git a/tests/e2e-chainsaw/v1beta2/metrics-service/redis-replication/chainsaw-test.yaml b/tests/e2e-chainsaw/v1beta2/metrics-service/redis-replication/chainsaw-test.yaml new file mode 100644 index 0000000000..af8a1932d1 --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/metrics-service/redis-replication/chainsaw-test.yaml @@ -0,0 +1,75 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: metrics-service-redis-replication +spec: + steps: + - name: Deploy Redis Replication with exporter + try: + - apply: + file: secret.yaml + - apply: + file: replication.yaml + - assert: + file: ready-sts.yaml + + - name: Assert metrics service created + try: + - assert: + file: ready-metrics-svc.yaml + + - name: Verify metrics service only exposes exporter port + try: + - script: + timeout: 30s + content: | + #!/bin/bash + set -e + PORTS=$(kubectl get svc -n ${NAMESPACE} redis-replication-metrics-metrics -o jsonpath='{.spec.ports[*].port}') + if [ "$PORTS" != "9121" ]; then + echo "Expected metrics service to only expose port 9121, got: $PORTS" + exit 1 + fi + HAS_REDIS_PORT=$(kubectl get svc -n ${NAMESPACE} redis-replication-metrics-metrics -o jsonpath='{.spec.ports[?(@.port==6379)].port}') + if [ -n "$HAS_REDIS_PORT" ]; then + echo "Metrics service should not expose Redis port 6379" + exit 1 + fi + COMPONENT=$(kubectl get svc -n ${NAMESPACE} redis-replication-metrics-metrics -o jsonpath='{.metadata.labels.app\.kubernetes\.io/component}') + if [ "$COMPONENT" != "metrics" ]; then + echo "Expected app.kubernetes.io/component=metrics label, got: $COMPONENT" + exit 1 + fi + echo "Replication metrics service validation passed" + check: + (contains($stdout, 'Replication metrics service validation passed')): true + + - name: Verify metrics are fetchable via metrics service + try: + - script: + timeout: 60s + content: | + #!/bin/bash + set -e + OUTPUT=$(kubectl run -n ${NAMESPACE} metrics-check --rm -i --restart=Never --image=busybox:1.36 -- \ + wget -qO- --timeout=10 http://redis-replication-metrics-metrics.${NAMESPACE}.svc.cluster.local:9121/metrics 2>&1) + echo "$OUTPUT" + if ! echo "$OUTPUT" | grep -q "redis_"; then + echo "ERROR: No redis_ metrics found in output" + exit 1 + fi + echo "Metrics fetch validation passed" + check: + (contains($stdout, 'redis_')): true + + - name: Cleanup + try: + - delete: + ref: + name: redis-replication-metrics + kind: RedisReplication + apiVersion: redis.redis.opstreelabs.in/v1beta2 + - error: + file: ready-metrics-svc.yaml diff --git a/tests/e2e-chainsaw/v1beta2/metrics-service/redis-replication/ready-metrics-svc.yaml b/tests/e2e-chainsaw/v1beta2/metrics-service/redis-replication/ready-metrics-svc.yaml new file mode 100644 index 0000000000..675269cd41 --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/metrics-service/redis-replication/ready-metrics-svc.yaml @@ -0,0 +1,29 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: redis-replication-metrics-metrics + labels: + app: redis-replication-metrics + redis_setup_type: replication + role: replication + app.kubernetes.io/component: metrics + annotations: + prometheus.io/port: "9121" + prometheus.io/scrape: "true" + ownerReferences: + - apiVersion: redis.redis.opstreelabs.in/v1beta2 + controller: true + kind: RedisReplication + name: redis-replication-metrics +spec: + type: ClusterIP + ports: + - name: redis-exporter + port: 9121 + protocol: TCP + targetPort: 9121 + selector: + app: redis-replication-metrics + redis_setup_type: replication + role: replication diff --git a/tests/e2e-chainsaw/v1beta2/metrics-service/redis-replication/ready-sts.yaml b/tests/e2e-chainsaw/v1beta2/metrics-service/redis-replication/ready-sts.yaml new file mode 100644 index 0000000000..64f9fd2d7d --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/metrics-service/redis-replication/ready-sts.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: redis-replication-metrics +status: + replicas: 3 + readyReplicas: 3 diff --git a/tests/e2e-chainsaw/v1beta2/metrics-service/redis-replication/replication.yaml b/tests/e2e-chainsaw/v1beta2/metrics-service/redis-replication/replication.yaml new file mode 100644 index 0000000000..0030686a6c --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/metrics-service/redis-replication/replication.yaml @@ -0,0 +1,41 @@ +--- +apiVersion: redis.redis.opstreelabs.in/v1beta2 +kind: RedisReplication +metadata: + name: redis-replication-metrics +spec: + clusterSize: 3 + podSecurityContext: + runAsUser: 1000 + fsGroup: 1000 + kubernetesConfig: + image: quay.io/opstree/redis:latest + imagePullPolicy: Always + resources: + requests: + cpu: 101m + memory: 128Mi + limits: + cpu: 101m + memory: 128Mi + redisSecret: + name: redis-secret + key: password + redisExporter: + enabled: true + image: quay.io/opstree/redis-exporter:v1.44.0 + imagePullPolicy: Always + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 100m + memory: 128Mi + storage: + volumeClaimTemplate: + spec: + accessModes: [ReadWriteOnce] + resources: + requests: + storage: 1Gi diff --git a/tests/e2e-chainsaw/v1beta2/metrics-service/redis-replication/secret.yaml b/tests/e2e-chainsaw/v1beta2/metrics-service/redis-replication/secret.yaml new file mode 100644 index 0000000000..e9788c3f95 --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/metrics-service/redis-replication/secret.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: redis-secret +type: Opaque +data: + password: T3BzdHJlZTEyMzQ= diff --git a/tests/e2e-chainsaw/v1beta2/metrics-service/redis-sentinel/chainsaw-test.yaml b/tests/e2e-chainsaw/v1beta2/metrics-service/redis-sentinel/chainsaw-test.yaml new file mode 100644 index 0000000000..06210a041b --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/metrics-service/redis-sentinel/chainsaw-test.yaml @@ -0,0 +1,88 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: metrics-service-redis-sentinel +spec: + steps: + - name: Deploy Redis Replication (prerequisite for Sentinel) + try: + - apply: + file: secret.yaml + - apply: + file: replication.yaml + - assert: + file: ready-replication-sts.yaml + + - name: Deploy Redis Sentinel with exporter + try: + - apply: + file: sentinel.yaml + - assert: + file: ready-sentinel-sts.yaml + + - name: Assert sentinel metrics service created + try: + - assert: + file: ready-metrics-svc.yaml + + - name: Verify sentinel metrics service only exposes exporter port + try: + - script: + timeout: 30s + content: | + #!/bin/bash + set -e + PORTS=$(kubectl get svc -n ${NAMESPACE} redis-sentinel-metrics-sentinel-metrics -o jsonpath='{.spec.ports[*].port}') + if [ "$PORTS" != "9121" ]; then + echo "Expected metrics service to only expose port 9121, got: $PORTS" + exit 1 + fi + # Verify it does NOT expose sentinel port 26379 + HAS_SENTINEL_PORT=$(kubectl get svc -n ${NAMESPACE} redis-sentinel-metrics-sentinel-metrics -o jsonpath='{.spec.ports[?(@.port==26379)].port}') + if [ -n "$HAS_SENTINEL_PORT" ]; then + echo "Metrics service should not expose Sentinel port 26379" + exit 1 + fi + COMPONENT=$(kubectl get svc -n ${NAMESPACE} redis-sentinel-metrics-sentinel-metrics -o jsonpath='{.metadata.labels.app\.kubernetes\.io/component}') + if [ "$COMPONENT" != "metrics" ]; then + echo "Expected app.kubernetes.io/component=metrics label, got: $COMPONENT" + exit 1 + fi + echo "Sentinel metrics service validation passed" + check: + (contains($stdout, 'Sentinel metrics service validation passed')): true + + - name: Verify metrics are fetchable via sentinel metrics service + try: + - script: + timeout: 60s + content: | + #!/bin/bash + set -e + OUTPUT=$(kubectl run -n ${NAMESPACE} metrics-check --rm -i --restart=Never --image=busybox:1.36 -- \ + wget -qO- --timeout=10 http://redis-sentinel-metrics-sentinel-metrics.${NAMESPACE}.svc.cluster.local:9121/metrics 2>&1) + echo "$OUTPUT" + if ! echo "$OUTPUT" | grep -q "redis_"; then + echo "ERROR: No redis_ metrics found in output" + exit 1 + fi + echo "Sentinel metrics fetch validation passed" + check: + (contains($stdout, 'redis_')): true + + - name: Cleanup + try: + - delete: + ref: + name: redis-sentinel-metrics + kind: RedisSentinel + apiVersion: redis.redis.opstreelabs.in/v1beta2 + - delete: + ref: + name: redis-replication-sentinel-metrics + kind: RedisReplication + apiVersion: redis.redis.opstreelabs.in/v1beta2 + - error: + file: ready-metrics-svc.yaml diff --git a/tests/e2e-chainsaw/v1beta2/metrics-service/redis-sentinel/ready-metrics-svc.yaml b/tests/e2e-chainsaw/v1beta2/metrics-service/redis-sentinel/ready-metrics-svc.yaml new file mode 100644 index 0000000000..4aed85c711 --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/metrics-service/redis-sentinel/ready-metrics-svc.yaml @@ -0,0 +1,29 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: redis-sentinel-metrics-sentinel-metrics + labels: + app: redis-sentinel-metrics-sentinel + redis_setup_type: sentinel + role: sentinel + app.kubernetes.io/component: metrics + annotations: + prometheus.io/port: "9121" + prometheus.io/scrape: "true" + ownerReferences: + - apiVersion: redis.redis.opstreelabs.in/v1beta2 + controller: true + kind: RedisSentinel + name: redis-sentinel-metrics +spec: + type: ClusterIP + ports: + - name: redis-exporter + port: 9121 + protocol: TCP + targetPort: 9121 + selector: + app: redis-sentinel-metrics-sentinel + redis_setup_type: sentinel + role: sentinel diff --git a/tests/e2e-chainsaw/v1beta2/metrics-service/redis-sentinel/ready-replication-sts.yaml b/tests/e2e-chainsaw/v1beta2/metrics-service/redis-sentinel/ready-replication-sts.yaml new file mode 100644 index 0000000000..f6576fe553 --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/metrics-service/redis-sentinel/ready-replication-sts.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: redis-replication-sentinel-metrics +status: + replicas: 3 + readyReplicas: 3 diff --git a/tests/e2e-chainsaw/v1beta2/metrics-service/redis-sentinel/ready-sentinel-sts.yaml b/tests/e2e-chainsaw/v1beta2/metrics-service/redis-sentinel/ready-sentinel-sts.yaml new file mode 100644 index 0000000000..9133ec1f23 --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/metrics-service/redis-sentinel/ready-sentinel-sts.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: redis-sentinel-metrics-sentinel +status: + replicas: 1 + readyReplicas: 1 diff --git a/tests/e2e-chainsaw/v1beta2/metrics-service/redis-sentinel/replication.yaml b/tests/e2e-chainsaw/v1beta2/metrics-service/redis-sentinel/replication.yaml new file mode 100644 index 0000000000..5d556b6821 --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/metrics-service/redis-sentinel/replication.yaml @@ -0,0 +1,30 @@ +--- +apiVersion: redis.redis.opstreelabs.in/v1beta2 +kind: RedisReplication +metadata: + name: redis-replication-sentinel-metrics +spec: + clusterSize: 3 + podSecurityContext: + runAsUser: 1000 + fsGroup: 1000 + kubernetesConfig: + image: quay.io/opstree/redis:latest + imagePullPolicy: Always + resources: + requests: + cpu: 101m + memory: 128Mi + limits: + cpu: 101m + memory: 128Mi + redisSecret: + name: redis-secret + key: password + storage: + volumeClaimTemplate: + spec: + accessModes: [ReadWriteOnce] + resources: + requests: + storage: 1Gi diff --git a/tests/e2e-chainsaw/v1beta2/metrics-service/redis-sentinel/secret.yaml b/tests/e2e-chainsaw/v1beta2/metrics-service/redis-sentinel/secret.yaml new file mode 100644 index 0000000000..e9788c3f95 --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/metrics-service/redis-sentinel/secret.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: redis-secret +type: Opaque +data: + password: T3BzdHJlZTEyMzQ= diff --git a/tests/e2e-chainsaw/v1beta2/metrics-service/redis-sentinel/sentinel.yaml b/tests/e2e-chainsaw/v1beta2/metrics-service/redis-sentinel/sentinel.yaml new file mode 100644 index 0000000000..0cc7df7e9a --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/metrics-service/redis-sentinel/sentinel.yaml @@ -0,0 +1,36 @@ +--- +apiVersion: redis.redis.opstreelabs.in/v1beta2 +kind: RedisSentinel +metadata: + name: redis-sentinel-metrics +spec: + clusterSize: 1 + podSecurityContext: + runAsUser: 1000 + fsGroup: 1000 + redisSentinelConfig: + redisReplicationName: redis-replication-sentinel-metrics + kubernetesConfig: + image: quay.io/opstree/redis-sentinel:latest + imagePullPolicy: Always + redisSecret: + name: redis-secret + key: password + resources: + requests: + cpu: 101m + memory: 128Mi + limits: + cpu: 101m + memory: 128Mi + redisExporter: + enabled: true + image: quay.io/opstree/redis-exporter:v1.44.0 + imagePullPolicy: Always + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 100m + memory: 128Mi diff --git a/tests/e2e-chainsaw/v1beta2/metrics-service/redis-standalone/chainsaw-test.yaml b/tests/e2e-chainsaw/v1beta2/metrics-service/redis-standalone/chainsaw-test.yaml new file mode 100644 index 0000000000..83c6208a5b --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/metrics-service/redis-standalone/chainsaw-test.yaml @@ -0,0 +1,76 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: metrics-service-redis-standalone +spec: + steps: + - name: Deploy Redis Standalone with exporter + try: + - apply: + file: standalone.yaml + - assert: + file: ready-sts.yaml + + - name: Assert metrics service created + try: + - assert: + file: ready-metrics-svc.yaml + + - name: Verify metrics service only exposes exporter port + try: + - script: + timeout: 30s + content: | + #!/bin/bash + set -e + # Verify the metrics service exists and has exactly one port (9121) + PORTS=$(kubectl get svc -n ${NAMESPACE} redis-standalone-metrics-metrics -o jsonpath='{.spec.ports[*].port}') + if [ "$PORTS" != "9121" ]; then + echo "Expected metrics service to only expose port 9121, got: $PORTS" + exit 1 + fi + # Verify it does NOT expose Redis data port 6379 + HAS_REDIS_PORT=$(kubectl get svc -n ${NAMESPACE} redis-standalone-metrics-metrics -o jsonpath='{.spec.ports[?(@.port==6379)].port}') + if [ -n "$HAS_REDIS_PORT" ]; then + echo "Metrics service should not expose Redis port 6379" + exit 1 + fi + # Verify the component label + COMPONENT=$(kubectl get svc -n ${NAMESPACE} redis-standalone-metrics-metrics -o jsonpath='{.metadata.labels.app\.kubernetes\.io/component}') + if [ "$COMPONENT" != "metrics" ]; then + echo "Expected app.kubernetes.io/component=metrics label, got: $COMPONENT" + exit 1 + fi + echo "Metrics service validation passed" + check: + (contains($stdout, 'Metrics service validation passed')): true + + - name: Verify metrics are fetchable via metrics service + try: + - script: + timeout: 60s + content: | + #!/bin/bash + set -e + OUTPUT=$(kubectl run -n ${NAMESPACE} metrics-check --rm -i --restart=Never --image=busybox:1.36 -- \ + wget -qO- --timeout=10 http://redis-standalone-metrics-metrics.${NAMESPACE}.svc.cluster.local:9121/metrics 2>&1) + echo "$OUTPUT" + if ! echo "$OUTPUT" | grep -q "redis_"; then + echo "ERROR: No redis_ metrics found in output" + exit 1 + fi + echo "Metrics fetch validation passed" + check: + (contains($stdout, 'redis_')): true + + - name: Cleanup + try: + - delete: + ref: + name: redis-standalone-metrics + kind: Redis + apiVersion: redis.redis.opstreelabs.in/v1beta2 + - error: + file: ready-metrics-svc.yaml diff --git a/tests/e2e-chainsaw/v1beta2/metrics-service/redis-standalone/ready-metrics-svc.yaml b/tests/e2e-chainsaw/v1beta2/metrics-service/redis-standalone/ready-metrics-svc.yaml new file mode 100644 index 0000000000..eed4525694 --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/metrics-service/redis-standalone/ready-metrics-svc.yaml @@ -0,0 +1,29 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: redis-standalone-metrics-metrics + labels: + app: redis-standalone-metrics + redis_setup_type: standalone + role: standalone + app.kubernetes.io/component: metrics + annotations: + prometheus.io/port: "9121" + prometheus.io/scrape: "true" + ownerReferences: + - apiVersion: redis.redis.opstreelabs.in/v1beta2 + controller: true + kind: Redis + name: redis-standalone-metrics +spec: + type: ClusterIP + ports: + - name: redis-exporter + port: 9121 + protocol: TCP + targetPort: 9121 + selector: + app: redis-standalone-metrics + redis_setup_type: standalone + role: standalone diff --git a/tests/e2e-chainsaw/v1beta2/metrics-service/redis-standalone/ready-sts.yaml b/tests/e2e-chainsaw/v1beta2/metrics-service/redis-standalone/ready-sts.yaml new file mode 100644 index 0000000000..6a16880209 --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/metrics-service/redis-standalone/ready-sts.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: redis-standalone-metrics +status: + replicas: 1 + readyReplicas: 1 diff --git a/tests/e2e-chainsaw/v1beta2/metrics-service/redis-standalone/standalone.yaml b/tests/e2e-chainsaw/v1beta2/metrics-service/redis-standalone/standalone.yaml new file mode 100644 index 0000000000..b98eb88eea --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/metrics-service/redis-standalone/standalone.yaml @@ -0,0 +1,37 @@ +--- +apiVersion: redis.redis.opstreelabs.in/v1beta2 +kind: Redis +metadata: + name: redis-standalone-metrics +spec: + podSecurityContext: + runAsUser: 1000 + fsGroup: 1000 + kubernetesConfig: + image: quay.io/opstree/redis:latest + imagePullPolicy: Always + resources: + requests: + cpu: 101m + memory: 128Mi + limits: + cpu: 101m + memory: 128Mi + redisExporter: + enabled: true + image: quay.io/opstree/redis-exporter:v1.44.0 + imagePullPolicy: Always + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 100m + memory: 128Mi + storage: + volumeClaimTemplate: + spec: + accessModes: [ReadWriteOnce] + resources: + requests: + storage: 1Gi diff --git a/tests/e2e-chainsaw/v1beta2/pvc-shrink-guard/redis-standalone/chainsaw-test.yaml b/tests/e2e-chainsaw/v1beta2/pvc-shrink-guard/redis-standalone/chainsaw-test.yaml new file mode 100644 index 0000000000..8c1b2e0d30 --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/pvc-shrink-guard/redis-standalone/chainsaw-test.yaml @@ -0,0 +1,83 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: pvc-shrink-guard-redis-standalone +spec: + steps: + - name: Deploy Redis standalone with 1Gi storage + try: + - apply: + file: standalone.yaml + - assert: + file: ready-sts.yaml + - assert: + file: ready-pvc.yaml + + - name: Attempt to shrink storage to 512Mi (should be blocked) + try: + - apply: + file: shrink-patch.yaml + # Wait for the operator to reconcile the change + - sleep: + duration: 30s + # PVC must remain at 1Gi - shrink should have been skipped + - assert: + file: pvc-unchanged.yaml + # StatefulSet should still be healthy + - assert: + file: ready-sts.yaml + + - name: Verify annotation not updated on shrink + try: + - script: + timeout: 30s + content: | + #!/bin/bash + set -e + ANNOTATION=$(kubectl get statefulset -n ${NAMESPACE} redis-standalone-shrink-test -o jsonpath='{.metadata.annotations.storageCapacity}') + CURRENT_PVC_SIZE=$(kubectl get pvc -n ${NAMESPACE} redis-standalone-shrink-test-redis-standalone-shrink-test-0 -o jsonpath='{.status.capacity.storage}') + echo "storageCapacity annotation: ${ANNOTATION}" + echo "Current PVC size: ${CURRENT_PVC_SIZE}" + if [ "${CURRENT_PVC_SIZE}" != "1Gi" ]; then + echo "FAIL: PVC was resized to ${CURRENT_PVC_SIZE}, expected 1Gi" + exit 1 + fi + # Annotation should either be empty/unset or reflect actual capacity, not the shrink target + if [ "${ANNOTATION}" == "536870912" ]; then + echo "FAIL: annotation was incorrectly updated to shrink target (512Mi = 536870912)" + exit 1 + fi + echo "PASS: PVC shrink was correctly blocked" + + - name: Expand storage to 2Gi (should succeed after prior shrink attempt) + try: + - apply: + file: expand-patch.yaml + - assert: + timeout: 5m + file: pvc-expanded.yaml + - assert: + file: ready-sts.yaml + + - name: Wait for operator to reconcile annotation + try: + - sleep: + duration: 30s + + - name: Verify annotation updated after successful expand + try: + - script: + timeout: 30s + content: | + #!/bin/bash + set -e + ANNOTATION=$(kubectl get statefulset -n ${NAMESPACE} redis-standalone-shrink-test -o jsonpath='{.metadata.annotations.storageCapacity}') + echo "storageCapacity annotation: ${ANNOTATION}" + # 2Gi = 2147483648 bytes + if [ "${ANNOTATION}" != "2147483648" ]; then + echo "FAIL: annotation should be 2147483648 (2Gi), got ${ANNOTATION}" + exit 1 + fi + echo "PASS: annotation correctly updated to 2Gi after expansion" diff --git a/tests/e2e-chainsaw/v1beta2/pvc-shrink-guard/redis-standalone/expand-patch.yaml b/tests/e2e-chainsaw/v1beta2/pvc-shrink-guard/redis-standalone/expand-patch.yaml new file mode 100644 index 0000000000..c113bfea7b --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/pvc-shrink-guard/redis-standalone/expand-patch.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: redis.redis.opstreelabs.in/v1beta2 +kind: Redis +metadata: + name: redis-standalone-shrink-test +spec: + storage: + volumeClaimTemplate: + spec: + accessModes: [ReadWriteOnce] + resources: + requests: + storage: 2Gi diff --git a/tests/e2e-chainsaw/v1beta2/pvc-shrink-guard/redis-standalone/pvc-capacity-expanded.yaml b/tests/e2e-chainsaw/v1beta2/pvc-shrink-guard/redis-standalone/pvc-capacity-expanded.yaml new file mode 100644 index 0000000000..e7cb52fc3e --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/pvc-shrink-guard/redis-standalone/pvc-capacity-expanded.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: redis-standalone-shrink-test-redis-standalone-shrink-test-0 +status: + capacity: + storage: 2Gi + phase: Bound diff --git a/tests/e2e-chainsaw/v1beta2/pvc-shrink-guard/redis-standalone/pvc-expanded.yaml b/tests/e2e-chainsaw/v1beta2/pvc-shrink-guard/redis-standalone/pvc-expanded.yaml new file mode 100644 index 0000000000..ebee53a5a5 --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/pvc-shrink-guard/redis-standalone/pvc-expanded.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: redis-standalone-shrink-test-redis-standalone-shrink-test-0 +spec: + resources: + requests: + storage: 2Gi +status: + phase: Bound diff --git a/tests/e2e-chainsaw/v1beta2/pvc-shrink-guard/redis-standalone/pvc-unchanged.yaml b/tests/e2e-chainsaw/v1beta2/pvc-shrink-guard/redis-standalone/pvc-unchanged.yaml new file mode 100644 index 0000000000..778ee9c6c7 --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/pvc-shrink-guard/redis-standalone/pvc-unchanged.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: redis-standalone-shrink-test-redis-standalone-shrink-test-0 +status: + capacity: + storage: 1Gi + phase: Bound diff --git a/tests/e2e-chainsaw/v1beta2/pvc-shrink-guard/redis-standalone/ready-pvc.yaml b/tests/e2e-chainsaw/v1beta2/pvc-shrink-guard/redis-standalone/ready-pvc.yaml new file mode 100644 index 0000000000..484f4decbd --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/pvc-shrink-guard/redis-standalone/ready-pvc.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: redis-standalone-shrink-test-redis-standalone-shrink-test-0 + labels: + app: redis-standalone-shrink-test + redis_setup_type: standalone + role: standalone +status: + accessModes: [ReadWriteOnce] + capacity: + storage: 1Gi + phase: Bound diff --git a/tests/e2e-chainsaw/v1beta2/pvc-shrink-guard/redis-standalone/ready-sts.yaml b/tests/e2e-chainsaw/v1beta2/pvc-shrink-guard/redis-standalone/ready-sts.yaml new file mode 100644 index 0000000000..12c30989e7 --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/pvc-shrink-guard/redis-standalone/ready-sts.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: redis-standalone-shrink-test + labels: + app: redis-standalone-shrink-test + redis_setup_type: standalone + role: standalone +status: + replicas: 1 + readyReplicas: 1 diff --git a/tests/e2e-chainsaw/v1beta2/pvc-shrink-guard/redis-standalone/shrink-patch.yaml b/tests/e2e-chainsaw/v1beta2/pvc-shrink-guard/redis-standalone/shrink-patch.yaml new file mode 100644 index 0000000000..cbead0d6f0 --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/pvc-shrink-guard/redis-standalone/shrink-patch.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: redis.redis.opstreelabs.in/v1beta2 +kind: Redis +metadata: + name: redis-standalone-shrink-test +spec: + storage: + volumeClaimTemplate: + spec: + accessModes: [ReadWriteOnce] + resources: + requests: + storage: 512Mi diff --git a/tests/e2e-chainsaw/v1beta2/pvc-shrink-guard/redis-standalone/standalone.yaml b/tests/e2e-chainsaw/v1beta2/pvc-shrink-guard/redis-standalone/standalone.yaml new file mode 100644 index 0000000000..cf51e22c8b --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/pvc-shrink-guard/redis-standalone/standalone.yaml @@ -0,0 +1,26 @@ +--- +apiVersion: redis.redis.opstreelabs.in/v1beta2 +kind: Redis +metadata: + name: redis-standalone-shrink-test +spec: + podSecurityContext: + runAsUser: 1000 + fsGroup: 1000 + kubernetesConfig: + image: quay.io/opstree/redis:latest + imagePullPolicy: Always + resources: + requests: + cpu: 101m + memory: 128Mi + limits: + cpu: 101m + memory: 128Mi + storage: + volumeClaimTemplate: + spec: + accessModes: [ReadWriteOnce] + resources: + requests: + storage: 1Gi diff --git a/tests/e2e-chainsaw/v1beta2/redis-cluster-restart/chainsaw-test.yaml b/tests/e2e-chainsaw/v1beta2/redis-cluster-restart/chainsaw-test.yaml index 7725612c90..b844d23786 100644 --- a/tests/e2e-chainsaw/v1beta2/redis-cluster-restart/chainsaw-test.yaml +++ b/tests/e2e-chainsaw/v1beta2/redis-cluster-restart/chainsaw-test.yaml @@ -17,7 +17,7 @@ spec: - name: Put data try: - script: - timeout: 30s + timeout: 90s content: > kubectl exec --namespace ${NAMESPACE} --container data-assert data-assert -- bash -c "cd /go/src/data-assert && go run main.go gen-redis-data --host redis-cluster-v1beta2-leader.${NAMESPACE}.svc.cluster.local:6379 --mode cluster" @@ -44,7 +44,7 @@ spec: - name: Assert data try: - script: - timeout: 30s + timeout: 60s content: > kubectl exec --namespace ${NAMESPACE} --container data-assert data-assert -- bash -c "cd /go/src/data-assert && go run main.go chk-redis-data --host redis-cluster-v1beta2-leader.${NAMESPACE}.svc.cluster.local:6379 --mode cluster" diff --git a/tests/e2e-chainsaw/v1beta2/setup/redis-cluster/chainsaw-test.yaml b/tests/e2e-chainsaw/v1beta2/setup/redis-cluster/chainsaw-test.yaml index 86154f8aee..2c82fcf967 100644 --- a/tests/e2e-chainsaw/v1beta2/setup/redis-cluster/chainsaw-test.yaml +++ b/tests/e2e-chainsaw/v1beta2/setup/redis-cluster/chainsaw-test.yaml @@ -121,7 +121,7 @@ spec: - name: Put data try: - script: - timeout: 30s + timeout: 90s content: > kubectl exec --namespace ${NAMESPACE} --container data-assert data-assert -- bash -c "cd /go/src/data-assert && go run main.go gen-redis-data --host redis-cluster-v1beta2-leader.${NAMESPACE}.svc.cluster.local:6379 --mode cluster --password Opstree1234 --tls" diff --git a/tests/e2e-chainsaw/v1beta2/setup/redis-cluster/ready-svc.yaml b/tests/e2e-chainsaw/v1beta2/setup/redis-cluster/ready-svc.yaml index 6f6bc90bc1..d7e727a913 100644 --- a/tests/e2e-chainsaw/v1beta2/setup/redis-cluster/ready-svc.yaml +++ b/tests/e2e-chainsaw/v1beta2/setup/redis-cluster/ready-svc.yaml @@ -230,3 +230,61 @@ spec: cluster: redis-cluster-v1beta2 redis-role: master redis_setup_type: cluster +--- +apiVersion: v1 +kind: Service +metadata: + name: redis-cluster-v1beta2-leader-metrics + labels: + app: redis-cluster-v1beta2-leader + redis_setup_type: cluster + role: leader + app.kubernetes.io/component: metrics + annotations: + prometheus.io/port: "9121" + prometheus.io/scrape: "true" + ownerReferences: + - apiVersion: redis.redis.opstreelabs.in/v1beta2 + controller: true + kind: RedisCluster + name: redis-cluster-v1beta2 +spec: + type: ClusterIP + ports: + - name: redis-exporter + port: 9121 + protocol: TCP + targetPort: 9121 + selector: + app: redis-cluster-v1beta2-leader + redis_setup_type: cluster + role: leader +--- +apiVersion: v1 +kind: Service +metadata: + name: redis-cluster-v1beta2-follower-metrics + labels: + app: redis-cluster-v1beta2-follower + redis_setup_type: cluster + role: follower + app.kubernetes.io/component: metrics + annotations: + prometheus.io/port: "9121" + prometheus.io/scrape: "true" + ownerReferences: + - apiVersion: redis.redis.opstreelabs.in/v1beta2 + controller: true + kind: RedisCluster + name: redis-cluster-v1beta2 +spec: + type: ClusterIP + ports: + - name: redis-exporter + port: 9121 + protocol: TCP + targetPort: 9121 + selector: + app: redis-cluster-v1beta2-follower + redis_setup_type: cluster + role: follower diff --git a/tests/e2e-chainsaw/v1beta2/setup/redis-ha/chainsaw-test.yaml b/tests/e2e-chainsaw/v1beta2/setup/redis-ha/chainsaw-test.yaml index beb8c7c934..df5d0c8214 100644 --- a/tests/e2e-chainsaw/v1beta2/setup/redis-ha/chainsaw-test.yaml +++ b/tests/e2e-chainsaw/v1beta2/setup/redis-ha/chainsaw-test.yaml @@ -46,11 +46,29 @@ spec: kind: Issuer secretName: redis-tls-cert EOF + # Wait for replication StatefulSet to be ready + - assert: + resource: + apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: redis-replication + status: + readyReplicas: 3 + # Wait for sentinel StatefulSet to be ready + - assert: + resource: + apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: redis-sentinel-sentinel + status: + readyReplicas: 1 - name: Test Master IP consistency try: - sleep: - duration: 60s + duration: 90s - script: timeout: 10s content: | @@ -64,7 +82,7 @@ spec: - name: Put data try: - script: - timeout: 30s + timeout: 90s content: > kubectl exec --namespace ${NAMESPACE} --container data-assert data-assert -- bash -c "cd /go/src/data-assert && go run main.go gen-redis-data \ @@ -86,7 +104,7 @@ spec: - name: Test Master IP consistency try: - sleep: - duration: 60s + duration: 90s - script: timeout: 10s content: | diff --git a/tests/e2e-chainsaw/v1beta2/setup/redis-replication/chainsaw-test.yaml b/tests/e2e-chainsaw/v1beta2/setup/redis-replication/chainsaw-test.yaml index 0801ec5e09..20bec1fc4b 100644 --- a/tests/e2e-chainsaw/v1beta2/setup/redis-replication/chainsaw-test.yaml +++ b/tests/e2e-chainsaw/v1beta2/setup/redis-replication/chainsaw-test.yaml @@ -56,7 +56,7 @@ spec: - name: Put data try: - script: - timeout: 30s + timeout: 90s content: > kubectl exec --namespace ${NAMESPACE} --container data-assert data-assert -- bash -c "cd /go/src/data-assert && go run main.go gen-redis-data \ diff --git a/tests/testdata/redis-cluster.yaml b/tests/testdata/redis-cluster.yaml index 448343a88a..6c3149831e 100644 --- a/tests/testdata/redis-cluster.yaml +++ b/tests/testdata/redis-cluster.yaml @@ -139,7 +139,7 @@ spec: secret: secretName: acl-secret TLS: - ca: ca.key + ca: ca.crt cert: tls.crt key: tls.key secret: diff --git a/tests/testdata/redis-replication.yaml b/tests/testdata/redis-replication.yaml index 8654056cd5..c646b25a0a 100644 --- a/tests/testdata/redis-replication.yaml +++ b/tests/testdata/redis-replication.yaml @@ -112,7 +112,7 @@ spec: secret: secretName: acl-secret TLS: - ca: ca.key + ca: ca.crt cert: tls.crt key: tls.key secret: diff --git a/tests/testdata/redis-sentinel.yaml b/tests/testdata/redis-sentinel.yaml index 3e22facac8..30d40d507a 100644 --- a/tests/testdata/redis-sentinel.yaml +++ b/tests/testdata/redis-sentinel.yaml @@ -91,7 +91,7 @@ spec: serviceAccountName: redis-sa terminationGracePeriodSeconds: 30 TLS: - ca: ca.key + ca: ca.crt cert: tls.crt key: tls.key secret: diff --git a/tests/testdata/redis-standalone.yaml b/tests/testdata/redis-standalone.yaml index c2019f8fa2..f732c2edd2 100644 --- a/tests/testdata/redis-standalone.yaml +++ b/tests/testdata/redis-standalone.yaml @@ -103,7 +103,7 @@ spec: secret: secretName: acl-secret TLS: - ca: ca.key + ca: ca.crt cert: tls.crt key: tls.key secret: