From 2c4d407c69b1ed9cadd96fe17c5c5c73176c04d3 Mon Sep 17 00:00:00 2001 From: biswas2200 Date: Fri, 13 Jun 2025 20:19:54 +0530 Subject: [PATCH 1/2] [FEAT] Add WarpStream Helm Chart - Cloud-Native Kafka Alternative Support IEEE --- charts/warpstream/Chart.yaml | 26 +++ charts/warpstream/templates/NOTES.txt | 32 ++++ charts/warpstream/templates/_helper.tpl | 88 +++++++++ charts/warpstream/templates/alerts.yaml | 40 +++++ charts/warpstream/templates/deployment.yaml | 114 ++++++++++++ charts/warpstream/templates/hpa.yaml | 32 ++++ .../templates/poddisruptionbudget.yaml | 18 ++ charts/warpstream/templates/rbac.yaml | 27 +++ charts/warpstream/templates/secret.yaml | 11 ++ charts/warpstream/templates/service.yaml | 24 +++ .../warpstream/templates/serviceMonitor.yaml | 20 +++ .../warpstream/templates/serviceaccount.yaml | 12 ++ charts/warpstream/values.schema.json | 170 ++++++++++++++++++ charts/warpstream/values.yaml | 164 +++++++++++++++++ 14 files changed, 778 insertions(+) create mode 100644 charts/warpstream/Chart.yaml create mode 100644 charts/warpstream/templates/NOTES.txt create mode 100644 charts/warpstream/templates/_helper.tpl create mode 100644 charts/warpstream/templates/alerts.yaml create mode 100644 charts/warpstream/templates/deployment.yaml create mode 100644 charts/warpstream/templates/hpa.yaml create mode 100644 charts/warpstream/templates/poddisruptionbudget.yaml create mode 100644 charts/warpstream/templates/rbac.yaml create mode 100644 charts/warpstream/templates/secret.yaml create mode 100644 charts/warpstream/templates/service.yaml create mode 100644 charts/warpstream/templates/serviceMonitor.yaml create mode 100644 charts/warpstream/templates/serviceaccount.yaml create mode 100644 charts/warpstream/values.schema.json create mode 100644 charts/warpstream/values.yaml diff --git a/charts/warpstream/Chart.yaml b/charts/warpstream/Chart.yaml new file mode 100644 index 00000000..74f7e7a1 --- /dev/null +++ b/charts/warpstream/Chart.yaml @@ -0,0 +1,26 @@ +apiVersion: v2 +name: warpstream +description: A Helm chart for WarpStream - a cloud-native, Kafka-compatible streaming platform +type: application +version: 0.0.1 +appVersion: "latest" +keywords: + - kafka + - streaming + - warpstream + - data + - messaging + - byoc +home: https://www.warpstream.com/ +sources: + - https://github.com/warpstreamlabs/charts + - https://github.com/warpstreamlabs/warpstream-docker-compose-example +maintainers: + - name: zopdev + url: https://zop.dev + - name: warpstream + url: https://www.warpstream.com +annotations: + # REQUIRED: zop.dev integration annotation + type: "application" + diff --git a/charts/warpstream/templates/NOTES.txt b/charts/warpstream/templates/NOTES.txt new file mode 100644 index 00000000..8c6de011 --- /dev/null +++ b/charts/warpstream/templates/NOTES.txt @@ -0,0 +1,32 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "warpstream.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo "Kafka endpoint: $NODE_IP:$NODE_PORT" +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "warpstream.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "warpstream.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo "Kafka endpoint: $SERVICE_IP:{{ .Values.service.port }}" +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "warpstream.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:9092 to use your WarpStream cluster" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 9092:$CONTAINER_PORT +{{- end }} + +2. Connect your Kafka clients to: + Bootstrap servers: {{ include "warpstream.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.service.port }} + +3. Monitor your WarpStream cluster: + Metrics endpoint: {{ include "warpstream.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.service.metricsPort }} + +4. Schema Registry endpoint: + {{ include "warpstream.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.service.schemaRegistryPort }} + \ No newline at end of file diff --git a/charts/warpstream/templates/_helper.tpl b/charts/warpstream/templates/_helper.tpl new file mode 100644 index 00000000..61175aa0 --- /dev/null +++ b/charts/warpstream/templates/_helper.tpl @@ -0,0 +1,88 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "warpstream.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "warpstream.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "warpstream.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "warpstream.labels" -}} +helm.sh/chart: {{ include "warpstream.chart" . }} +{{ include "warpstream.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "warpstream.selectorLabels" -}} +app.kubernetes.io/name: {{ include "warpstream.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "warpstream.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "warpstream.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +WarpStream agent roles as comma-separated string +*/}} +{{- define "warpstream.roles" -}} +{{- join "," .Values.warpstream.agent.roles }} +{{- end }} + +{{/* +WarpStream command args +*/}} +{{- define "warpstream.args" -}} +- "agent" +{{- if .Values.warpstream.storage.bucketURL }} +- "-bucketURL" +- {{ .Values.warpstream.storage.bucketURL | quote }} +{{- end }} +- "-defaultVirtualClusterID" +- {{ .Values.warpstream.agent.virtualClusterId | quote }} +- "-region" +- {{ .Values.warpstream.agent.region | quote }} +{{- if .Values.warpstream.agent.agentPoolId }} +- "-agentPoolID" +- {{ .Values.warpstream.agent.agentPoolId | quote }} +{{- end }} +- "-roles" +- {{ include "warpstream.roles" . | quote }} +{{- end }} diff --git a/charts/warpstream/templates/alerts.yaml b/charts/warpstream/templates/alerts.yaml new file mode 100644 index 00000000..6e43492b --- /dev/null +++ b/charts/warpstream/templates/alerts.yaml @@ -0,0 +1,40 @@ +{{- if .Values.monitoring.alerts.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "warpstream.fullname" . }}-alerts + labels: + {{- include "warpstream.labels" . | nindent 4 }} + grafana_alert: "1" +data: + warpstream-alerts.yaml: | + groups: + - name: warpstream + rules: + - alert: WarpStreamDown + expr: up{job="{{ include "warpstream.fullname" . }}"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "WarpStream agent is down" + description: "WarpStream agent {{ "{{ $labels.instance }}" }} has been down for more than 2 minutes." + + - alert: WarpStreamHighMemoryUsage + expr: (container_memory_usage_bytes{pod=~"{{ include "warpstream.fullname" . }}.*"} / container_spec_memory_limit_bytes) > 0.85 + for: 5m + labels: + severity: warning + annotations: + summary: "WarpStream agent high memory usage" + description: "WarpStream agent {{ "{{ $labels.pod }}" }} memory usage is above 85%." + + - alert: WarpStreamHighCPUUsage + expr: (rate(container_cpu_usage_seconds_total{pod=~"{{ include "warpstream.fullname" . }}.*"}[5m]) / container_spec_cpu_quota * container_spec_cpu_period) > 0.85 + for: 5m + labels: + severity: warning + annotations: + summary: "WarpStream agent high CPU usage" + description: "WarpStream agent {{ "{{ $labels.pod }}" }} CPU usage is above 85%." +{{- end }} diff --git a/charts/warpstream/templates/deployment.yaml b/charts/warpstream/templates/deployment.yaml new file mode 100644 index 00000000..433f589e --- /dev/null +++ b/charts/warpstream/templates/deployment.yaml @@ -0,0 +1,114 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "warpstream.fullname" . }} + labels: + {{- include "warpstream.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "warpstream.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "warpstream.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "warpstream.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- with .Values.topologySpreadConstraints }} + topologySpreadConstraints: + {{- range . }} + - {{- toYaml . | nindent 10 }} + labelSelector: + matchLabels: + {{- include "warpstream.selectorLabels" $ | nindent 14 }} + {{- end }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + {{- include "warpstream.args" . | nindent 12 }} + ports: + - name: kafka + containerPort: 9092 + protocol: TCP + - name: metrics + containerPort: 8080 + protocol: TCP + - name: schema-registry + containerPort: 9094 + protocol: TCP + env: + - name: WARPSTREAM_API_KEY + valueFrom: + secretKeyRef: + name: {{ include "warpstream.fullname" . }}-secret + key: api-key + - name: WARPSTREAM_LOG_LEVEL + value: {{ .Values.warpstream.agent.logLevel | quote }} + {{- if .Values.warpstream.agent.availabilityZone }} + - name: WARPSTREAM_AVAILABILITY_ZONE + value: {{ .Values.warpstream.agent.availabilityZone | quote }} + {{- end }} + {{- if .Values.warpstream.performance.gomaxprocs }} + - name: GOMAXPROCS + value: {{ .Values.warpstream.performance.gomaxprocs | quote }} + {{- end }} + {{- range .Values.warpstream.extraEnvVars }} + - name: {{ .name }} + value: {{ .value | quote }} + {{- end }} + livenessProbe: + httpGet: + path: /metrics + port: metrics + initialDelaySeconds: 30 + periodSeconds: 30 + readinessProbe: + httpGet: + path: /metrics + port: metrics + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- if .Values.persistence.enabled }} + volumeMounts: + - name: data + mountPath: /tmp/warpstream + {{- end }} + {{- if .Values.persistence.enabled }} + volumes: + - name: data + persistentVolumeClaim: + claimName: {{ include "warpstream.fullname" . }}-pvc + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + \ No newline at end of file diff --git a/charts/warpstream/templates/hpa.yaml b/charts/warpstream/templates/hpa.yaml new file mode 100644 index 00000000..6467323a --- /dev/null +++ b/charts/warpstream/templates/hpa.yaml @@ -0,0 +1,32 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "warpstream.fullname" . }} + labels: + {{- include "warpstream.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "warpstream.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/charts/warpstream/templates/poddisruptionbudget.yaml b/charts/warpstream/templates/poddisruptionbudget.yaml new file mode 100644 index 00000000..0664799a --- /dev/null +++ b/charts/warpstream/templates/poddisruptionbudget.yaml @@ -0,0 +1,18 @@ +{{- if .Values.podDisruptionBudget.enabled }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "warpstream.fullname" . }} + labels: + {{- include "warpstream.labels" . | nindent 4 }} +spec: + {{- if .Values.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + {{- include "warpstream.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/charts/warpstream/templates/rbac.yaml b/charts/warpstream/templates/rbac.yaml new file mode 100644 index 00000000..e7434dcb --- /dev/null +++ b/charts/warpstream/templates/rbac.yaml @@ -0,0 +1,27 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "warpstream.fullname" . }} + labels: + {{- include "warpstream.labels" . | nindent 4 }} +rules: +- apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "warpstream.fullname" . }} + labels: + {{- include "warpstream.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "warpstream.fullname" . }} +subjects: +- kind: ServiceAccount + name: {{ include "warpstream.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/charts/warpstream/templates/secret.yaml b/charts/warpstream/templates/secret.yaml new file mode 100644 index 00000000..537ee649 --- /dev/null +++ b/charts/warpstream/templates/secret.yaml @@ -0,0 +1,11 @@ +{{- if .Values.warpstream.agent.apiKey }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "warpstream.fullname" . }}-secret + labels: + {{- include "warpstream.labels" . | nindent 4 }} +type: Opaque +data: + api-key: {{ .Values.warpstream.agent.apiKey | b64enc }} +{{- end }} diff --git a/charts/warpstream/templates/service.yaml b/charts/warpstream/templates/service.yaml new file mode 100644 index 00000000..a91d2aea --- /dev/null +++ b/charts/warpstream/templates/service.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "warpstream.fullname" . }} + labels: + {{- include "warpstream.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: kafka + protocol: TCP + name: kafka + - port: {{ .Values.service.metricsPort }} + targetPort: metrics + protocol: TCP + name: metrics + - port: {{ .Values.service.schemaRegistryPort }} + targetPort: schema-registry + protocol: TCP + name: schema-registry + selector: + {{- include "warpstream.selectorLabels" . | nindent 4 }} + \ No newline at end of file diff --git a/charts/warpstream/templates/serviceMonitor.yaml b/charts/warpstream/templates/serviceMonitor.yaml new file mode 100644 index 00000000..e1d2b7f3 --- /dev/null +++ b/charts/warpstream/templates/serviceMonitor.yaml @@ -0,0 +1,20 @@ +{{- if .Values.monitoring.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "warpstream.fullname" . }} + {{- if .Values.monitoring.serviceMonitor.namespace }} + namespace: {{ .Values.monitoring.serviceMonitor.namespace }} + {{- end }} + labels: + {{- include "warpstream.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "warpstream.selectorLabels" . | nindent 6 }} + endpoints: + - port: {{ .Values.monitoring.serviceMonitor.port }} + interval: {{ .Values.monitoring.serviceMonitor.interval }} + scrapeTimeout: {{ .Values.monitoring.serviceMonitor.scrapeTimeout }} + path: {{ .Values.monitoring.serviceMonitor.path }} +{{- end }} diff --git a/charts/warpstream/templates/serviceaccount.yaml b/charts/warpstream/templates/serviceaccount.yaml new file mode 100644 index 00000000..666703df --- /dev/null +++ b/charts/warpstream/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "warpstream.serviceAccountName" . }} + labels: + {{- include "warpstream.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/warpstream/values.schema.json b/charts/warpstream/values.schema.json new file mode 100644 index 00000000..fa726385 --- /dev/null +++ b/charts/warpstream/values.schema.json @@ -0,0 +1,170 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "replicaCount": { + "type": "integer", + "minimum": 1, + "mutable": true, + "description": "Number of WarpStream agent replicas" + }, + "image": { + "type": "object", + "properties": { + "repository": { + "type": "string", + "mutable": false, + "description": "WarpStream agent Docker image repository" + }, + "tag": { + "type": "string", + "mutable": true, + "description": "WarpStream agent Docker image tag" + }, + "pullPolicy": { + "type": "string", + "enum": ["Always", "IfNotPresent", "Never"], + "mutable": true, + "description": "Image pull policy" + } + } + }, + "service": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["ClusterIP", "NodePort", "LoadBalancer"], + "mutable": true, + "description": "Kubernetes service type" + }, + "port": { + "type": "integer", + "minimum": 1, + "maximum": 65535, + "mutable": true, + "description": "Kafka protocol port" + }, + "metricsPort": { + "type": "integer", + "minimum": 1, + "maximum": 65535, + "mutable": true, + "description": "Metrics port" + } + } + }, + "resources": { + "type": "object", + "mutable": true, + "description": "Resource limits and requests" + }, + "autoscaling": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "mutable": true, + "description": "Enable horizontal pod autoscaling" + }, + "minReplicas": { + "type": "integer", + "minimum": 1, + "mutable": true + }, + "maxReplicas": { + "type": "integer", + "minimum": 1, + "mutable": true + } + } + }, + "warpstream": { + "type": "object", + "properties": { + "agent": { + "type": "object", + "properties": { + "apiKey": { + "type": "string", + "mutable": true, + "description": "WarpStream API key (required)" + }, + "virtualClusterId": { + "type": "string", + "mutable": true, + "description": "Virtual cluster ID (required)" + }, + "agentPoolId": { + "type": "string", + "mutable": true, + "description": "Agent Pool ID (optional)" + }, + "region": { + "type": "string", + "mutable": true, + "description": "Cluster region" + }, + "logLevel": { + "type": "string", + "enum": ["debug", "info", "warn", "error", "analytics"], + "mutable": true, + "description": "Log level" + }, + "roles": { + "type": "array", + "items": { + "type": "string", + "enum": ["agent", "proxy", "proxy-produce", "proxy-consume", "jobs"] + }, + "mutable": true, + "description": "Agent roles" + } + }, + "required": ["apiKey", "virtualClusterId", "region"] + }, + "storage": { + "type": "object", + "properties": { + "bucketURL": { + "type": "string", + "mutable": true, + "description": "S3-compatible storage bucket URL" + } + } + } + } + }, + "monitoring": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "mutable": true, + "description": "Enable monitoring" + }, + "serviceMonitor": { + "type": "object", + "mutable": true, + "description": "Prometheus ServiceMonitor configuration" + } + } + }, + "rbac": { + "type": "object", + "properties": { + "create": { + "type": "boolean", + "mutable": true, + "description": "Create RBAC resources for AZ detection" + } + } + }, + "ingress": { + "type": "object", + "mutable": true, + "description": "Ingress configuration" + } + }, + "required": ["warpstream"] +} diff --git a/charts/warpstream/values.yaml b/charts/warpstream/values.yaml new file mode 100644 index 00000000..8defd118 --- /dev/null +++ b/charts/warpstream/values.yaml @@ -0,0 +1,164 @@ +# charts/warpstream/values.yaml +# Default configuration for WarpStream +replicaCount: 1 + +image: + # Official WarpStream Docker image from Amazon ECR + repository: public.ecr.aws/warpstream-labs/warpstream_agent + pullPolicy: IfNotPresent + tag: "latest" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + name: "" + +podAnnotations: {} + +podSecurityContext: + fsGroup: 2000 + +securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + +service: + type: ClusterIP + # Kafka protocol port + port: 9092 + # Metrics port + metricsPort: 8080 + # Schema Registry port + schemaRegistryPort: 9094 + +resources: + limits: + cpu: 1000m + memory: 2Gi + requests: + cpu: 500m + memory: 1Gi + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 10 + targetCPUUtilizationPercentage: 80 + targetMemoryUtilizationPercentage: 80 + +nodeSelector: {} + +tolerations: [] + +affinity: + # Spread across availability zones + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: + - warpstream + topologyKey: topology.kubernetes.io/zone + +# Topology spread constraints for better distribution +topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app.kubernetes.io/name: warpstream + +# WarpStream specific configuration +warpstream: + # Agent configuration + agent: + # WarpStream API key (required) - will be stored in secret + apiKey: "" + # Virtual cluster ID (required) + virtualClusterId: "" + # Agent Pool ID (optional) + agentPoolId: "" + # Cluster region (required) + region: "us-east-1" + # Log level: debug, info, warn, error, analytics + logLevel: "info" + # Agent roles: agent, proxy, proxy-produce, proxy-consume, jobs + roles: + - "agent" + # Availability zone (auto-detected if not set) + availabilityZone: "" + + # Object storage configuration + storage: + # Bucket URL for S3-compatible storage (required for production) + bucketURL: "" + + # Performance tuning + performance: + # GOMAXPROCS setting (should match CPU limit) + gomaxprocs: "" + + # Additional environment variables + extraEnvVars: [] + # - name: CUSTOM_VAR + # value: "custom_value" + +# Kubernetes RBAC for availability zone detection +rbac: + # Specifies whether RBAC resources should be created + create: true + +# Pod Disruption Budget +podDisruptionBudget: + enabled: true + minAvailable: 1 + +# Monitoring configuration (following zopdev patterns) +monitoring: + enabled: true + serviceMonitor: + enabled: true + namespace: "" + interval: 30s + scrapeTimeout: 10s + path: /metrics + port: metrics + # Prometheus alerts + alerts: + enabled: true + +# Ingress configuration +ingress: + enabled: false + className: "" + annotations: {} + hosts: + - host: warpstream.local + paths: + - path: / + pathType: Prefix + tls: [] + +# Persistence for temporary data +persistence: + enabled: false + accessMode: ReadWriteOnce + size: 8Gi + storageClass: "" + \ No newline at end of file From 383c41c0dec1c664056240f417d0c6f0078dfd36 Mon Sep 17 00:00:00 2001 From: biswas2200 <122800409+biswas2200@users.noreply.github.com> Date: Wed, 3 Sep 2025 15:59:29 +0530 Subject: [PATCH 2/2] Reviewer Feedback Addressed --- charts/warpstream/Chart.yaml | 13 +- charts/warpstream/templates/_helper.tpl | 23 ++- charts/warpstream/templates/deployment.yaml | 4 +- charts/warpstream/templates/pvc.yaml | 17 ++ charts/warpstream/templates/secret.yaml | 7 +- charts/warpstream/values.schema.json | 169 ++++++-------------- charts/warpstream/values.yaml | 15 +- 7 files changed, 94 insertions(+), 154 deletions(-) create mode 100644 charts/warpstream/templates/pvc.yaml diff --git a/charts/warpstream/Chart.yaml b/charts/warpstream/Chart.yaml index 74f7e7a1..9ec6f99d 100644 --- a/charts/warpstream/Chart.yaml +++ b/charts/warpstream/Chart.yaml @@ -3,18 +3,8 @@ name: warpstream description: A Helm chart for WarpStream - a cloud-native, Kafka-compatible streaming platform type: application version: 0.0.1 -appVersion: "latest" -keywords: - - kafka - - streaming - - warpstream - - data - - messaging - - byoc +appVersion: "v694" home: https://www.warpstream.com/ -sources: - - https://github.com/warpstreamlabs/charts - - https://github.com/warpstreamlabs/warpstream-docker-compose-example maintainers: - name: zopdev url: https://zop.dev @@ -23,4 +13,3 @@ maintainers: annotations: # REQUIRED: zop.dev integration annotation type: "application" - diff --git a/charts/warpstream/templates/_helper.tpl b/charts/warpstream/templates/_helper.tpl index 61175aa0..0f8d6e75 100644 --- a/charts/warpstream/templates/_helper.tpl +++ b/charts/warpstream/templates/_helper.tpl @@ -60,29 +60,24 @@ Create the name of the service account to use {{- end }} {{/* -WarpStream agent roles as comma-separated string -*/}} -{{- define "warpstream.roles" -}} -{{- join "," .Values.warpstream.agent.roles }} -{{- end }} - -{{/* -WarpStream command args +WarpStream agent arguments */}} {{- define "warpstream.args" -}} - "agent" -{{- if .Values.warpstream.storage.bucketURL }} -- "-bucketURL" -- {{ .Values.warpstream.storage.bucketURL | quote }} -{{- end }} +{{- if .Values.warpstream.agent.virtualClusterId }} - "-defaultVirtualClusterID" - {{ .Values.warpstream.agent.virtualClusterId | quote }} +{{- end }} - "-region" - {{ .Values.warpstream.agent.region | quote }} +- "-roles" +- {{ join "," .Values.warpstream.agent.roles | quote }} {{- if .Values.warpstream.agent.agentPoolId }} - "-agentPoolID" - {{ .Values.warpstream.agent.agentPoolId | quote }} {{- end }} -- "-roles" -- {{ include "warpstream.roles" . | quote }} +{{- if .Values.warpstream.storage.bucketURL }} +- "-bucketURL" +- {{ .Values.warpstream.storage.bucketURL | quote }} +{{- end }} {{- end }} diff --git a/charts/warpstream/templates/deployment.yaml b/charts/warpstream/templates/deployment.yaml index 433f589e..71c427ed 100644 --- a/charts/warpstream/templates/deployment.yaml +++ b/charts/warpstream/templates/deployment.yaml @@ -30,7 +30,9 @@ spec: {{- with .Values.topologySpreadConstraints }} topologySpreadConstraints: {{- range . }} - - {{- toYaml . | nindent 10 }} + - maxSkew: {{ .maxSkew }} + topologyKey: {{ .topologyKey }} + whenUnsatisfiable: {{ .whenUnsatisfiable }} labelSelector: matchLabels: {{- include "warpstream.selectorLabels" $ | nindent 14 }} diff --git a/charts/warpstream/templates/pvc.yaml b/charts/warpstream/templates/pvc.yaml new file mode 100644 index 00000000..76255eb2 --- /dev/null +++ b/charts/warpstream/templates/pvc.yaml @@ -0,0 +1,17 @@ +{{- if .Values.persistence.enabled }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "warpstream.fullname" . }}-pvc + labels: + {{- include "warpstream.labels" . | nindent 4 }} +spec: + accessModes: + - {{ .Values.persistence.accessMode }} + resources: + requests: + storage: {{ .Values.persistence.size }} + {{- if .Values.persistence.storageClass }} + storageClassName: {{ .Values.persistence.storageClass }} + {{- end }} +{{- end }} diff --git a/charts/warpstream/templates/secret.yaml b/charts/warpstream/templates/secret.yaml index 537ee649..2c0a0662 100644 --- a/charts/warpstream/templates/secret.yaml +++ b/charts/warpstream/templates/secret.yaml @@ -1,4 +1,3 @@ -{{- if .Values.warpstream.agent.apiKey }} apiVersion: v1 kind: Secret metadata: @@ -7,5 +6,9 @@ metadata: {{- include "warpstream.labels" . | nindent 4 }} type: Opaque data: + {{- if .Values.warpstream.agent.apiKey }} api-key: {{ .Values.warpstream.agent.apiKey | b64enc }} -{{- end }} + {{- else }} + api-key: {{ "placeholder-key" | b64enc }} + {{- end }} + \ No newline at end of file diff --git a/charts/warpstream/values.schema.json b/charts/warpstream/values.schema.json index fa726385..fe0dc43d 100644 --- a/charts/warpstream/values.schema.json +++ b/charts/warpstream/values.schema.json @@ -2,82 +2,58 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { - "replicaCount": { - "type": "integer", - "minimum": 1, - "mutable": true, - "description": "Number of WarpStream agent replicas" - }, "image": { "type": "object", "properties": { - "repository": { - "type": "string", - "mutable": false, - "description": "WarpStream agent Docker image repository" - }, "tag": { "type": "string", "mutable": true, "description": "WarpStream agent Docker image tag" - }, - "pullPolicy": { - "type": "string", - "enum": ["Always", "IfNotPresent", "Never"], - "mutable": true, - "description": "Image pull policy" - } - } - }, - "service": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": ["ClusterIP", "NodePort", "LoadBalancer"], - "mutable": true, - "description": "Kubernetes service type" - }, - "port": { - "type": "integer", - "minimum": 1, - "maximum": 65535, - "mutable": true, - "description": "Kafka protocol port" - }, - "metricsPort": { - "type": "integer", - "minimum": 1, - "maximum": 65535, - "mutable": true, - "description": "Metrics port" } } }, - "resources": { - "type": "object", + "replicaCount": { + "type": "integer", + "minimum": 1, "mutable": true, - "description": "Resource limits and requests" + "description": "Number of WarpStream agent replicas" }, - "autoscaling": { + "resources": { "type": "object", "properties": { - "enabled": { - "type": "boolean", - "mutable": true, - "description": "Enable horizontal pod autoscaling" - }, - "minReplicas": { - "type": "integer", - "minimum": 1, - "mutable": true + "requests": { + "type": "object", + "properties": { + "cpu": { + "type": "string", + "default": "500m", + "mutable": true + }, + "memory": { + "type": "string", + "default": "1Gi", + "mutable": true + } + } }, - "maxReplicas": { - "type": "integer", - "minimum": 1, - "mutable": true + "limits": { + "type": "object", + "properties": { + "cpu": { + "type": "string", + "default": "1000m", + "mutable": true + }, + "memory": { + "type": "string", + "default": "2Gi", + "mutable": true + } + } } - } + }, + "mutable": true, + "description": "Resource limits and requests" }, "warpstream": { "type": "object", @@ -95,76 +71,29 @@ "mutable": true, "description": "Virtual cluster ID (required)" }, - "agentPoolId": { - "type": "string", - "mutable": true, - "description": "Agent Pool ID (optional)" - }, "region": { "type": "string", "mutable": true, "description": "Cluster region" - }, - "logLevel": { - "type": "string", - "enum": ["debug", "info", "warn", "error", "analytics"], - "mutable": true, - "description": "Log level" - }, - "roles": { - "type": "array", - "items": { - "type": "string", - "enum": ["agent", "proxy", "proxy-produce", "proxy-consume", "jobs"] - }, - "mutable": true, - "description": "Agent roles" - } - }, - "required": ["apiKey", "virtualClusterId", "region"] - }, - "storage": { - "type": "object", - "properties": { - "bucketURL": { - "type": "string", - "mutable": true, - "description": "S3-compatible storage bucket URL" } } } - } + }, + "mutable": true, + "description": "WarpStream specific configurations" }, - "monitoring": { - "type": "object", - "properties": { - "enabled": { - "type": "boolean", - "mutable": true, - "description": "Enable monitoring" + "services": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + } }, - "serviceMonitor": { - "type": "object", - "mutable": true, - "description": "Prometheus ServiceMonitor configuration" - } - } - }, - "rbac": { - "type": "object", - "properties": { - "create": { - "type": "boolean", - "mutable": true, - "description": "Create RBAC resources for AZ detection" - } + "required": ["name"] } - }, - "ingress": { - "type": "object", - "mutable": true, - "description": "Ingress configuration" } }, - "required": ["warpstream"] + "required": ["warpstream", "services"] } diff --git a/charts/warpstream/values.yaml b/charts/warpstream/values.yaml index 8defd118..80862d24 100644 --- a/charts/warpstream/values.yaml +++ b/charts/warpstream/values.yaml @@ -1,4 +1,3 @@ -# charts/warpstream/values.yaml # Default configuration for WarpStream replicaCount: 1 @@ -6,7 +5,7 @@ image: # Official WarpStream Docker image from Amazon ECR repository: public.ecr.aws/warpstream-labs/warpstream_agent pullPolicy: IfNotPresent - tag: "latest" + tag: "v694" imagePullSecrets: [] nameOverride: "" @@ -98,15 +97,20 @@ warpstream: region: "us-east-1" # Log level: debug, info, warn, error, analytics logLevel: "info" - # Agent roles: agent, proxy, proxy-produce, proxy-consume, jobs + # Agent roles: Valid roles are proxy, jobs, proxy-produce, proxy-consume, pipelines + # proxy: handles all Kafka requests (produce, consume, metadata) + # jobs: required for background tasks (compaction, cleanup) roles: - - "agent" + - "proxy" + - "jobs" # Availability zone (auto-detected if not set) availabilityZone: "" # Object storage configuration storage: # Bucket URL for S3-compatible storage (required for production) + # For development/testing, use: "mem://dev-bucket" + # For production, use: "s3://bucket-name?region=us-east-1" bucketURL: "" # Performance tuning @@ -161,4 +165,5 @@ persistence: accessMode: ReadWriteOnce size: 8Gi storageClass: "" - \ No newline at end of file + +services: [] \ No newline at end of file