From 83681022b7815053d780db67915cd859e1fc0d9c Mon Sep 17 00:00:00 2001 From: umignon Date: Mon, 14 Jul 2025 16:26:58 +0200 Subject: [PATCH] feat!: add clickhouse Bitnami --- .github/workflows/lint-test.yaml | 5 +- charts/sentry/Chart.lock | 11 ++-- charts/sentry/Chart.yaml | 8 +-- charts/sentry/ci/kind-values.yaml | 28 ++++++++++ charts/sentry/templates/_helper.tpl | 43 ++++++-------- .../templates/hooks/sentry-db-check.job.yaml | 15 +---- .../templates/hooks/sentry-db-init.job.yaml | 2 + .../templates/hooks/sentry-secret-create.yaml | 2 + .../templates/hooks/snuba-db-init.job.yaml | 2 + .../templates/hooks/snuba-migrate.job.yaml | 2 + .../sentry/templates/hooks/user-create.yaml | 2 + .../worker/deployment-sentry-worker.yaml | 5 ++ .../sentry/templates/snuba/_helper-snuba.tpl | 17 +++++- .../templates/snuba/secret-snuba-env.yaml | 3 - charts/sentry/values.yaml | 56 +------------------ 15 files changed, 86 insertions(+), 115 deletions(-) create mode 100644 charts/sentry/ci/kind-values.yaml diff --git a/.github/workflows/lint-test.yaml b/.github/workflows/lint-test.yaml index 869b26f9e..b428280cf 100644 --- a/.github/workflows/lint-test.yaml +++ b/.github/workflows/lint-test.yaml @@ -31,6 +31,7 @@ jobs: - name: Add Helm Repositories run: | helm repo add sentry-kubernetes https://sentry-kubernetes.github.io/charts + helm repo add bitnami https://charts.bitnami.com/bitnami helm repo update - name: Run chart-testing (list-changed) @@ -47,8 +48,8 @@ jobs: - name: Create kind cluster if: steps.list-changed.outputs.changed == 'true' - uses: helm/kind-action@v1.10.0 + uses: helm/kind-action@v1.12.0 - name: Run chart-testing (install) if: steps.list-changed.outputs.changed == 'true' - run: ct install --target-branch ${{ github.event.repository.default_branch }} --helm-extra-args "--timeout 1000s --set sentry.features.enableProfiling=true --set sentry.features.enableSessionReplay=true --set sentry.features.enableFeedback=true --set sentry.features.enableSpan=true" + run: ct install --debug --target-branch ${{ github.event.repository.default_branch }} --helm-extra-args "--timeout 1000s" diff --git a/charts/sentry/Chart.lock b/charts/sentry/Chart.lock index e062ea7f6..a6f821449 100644 --- a/charts/sentry/Chart.lock +++ b/charts/sentry/Chart.lock @@ -9,11 +9,8 @@ dependencies: repository: oci://registry-1.docker.io/bitnamicharts version: 29.3.14 - name: clickhouse - repository: https://sentry-kubernetes.github.io/charts - version: 4.0.1 -- name: zookeeper - repository: oci://registry-1.docker.io/bitnamicharts - version: 11.4.11 + repository: https://charts.bitnami.com/bitnami + version: 4.6.0 - name: rabbitmq repository: oci://registry-1.docker.io/bitnamicharts version: 11.16.2 @@ -23,5 +20,5 @@ dependencies: - name: nginx repository: oci://registry-1.docker.io/bitnamicharts version: 18.2.5 -digest: sha256:f57f1f1f8d455729a7e04eb41c658530d710c9febe34844e062f084c17c1a382 -generated: "2025-07-21T06:40:24.054695705Z" +digest: sha256:f2d82b26a693fed9ee3bd24fbcadafdacda7e525528bbbb347afceae607b2ecf +generated: "2025-08-05T08:10:43.800821278+02:00" diff --git a/charts/sentry/Chart.yaml b/charts/sentry/Chart.yaml index 03632ff64..4ae55c71a 100644 --- a/charts/sentry/Chart.yaml +++ b/charts/sentry/Chart.yaml @@ -19,13 +19,9 @@ dependencies: version: 29.3.14 condition: kafka.enabled - name: clickhouse - repository: https://sentry-kubernetes.github.io/charts - version: 4.0.1 + repository: https://charts.bitnami.com/bitnami + version: 4.6.0 condition: clickhouse.enabled - - name: zookeeper - repository: oci://registry-1.docker.io/bitnamicharts - version: 11.4.11 - condition: zookeeper.enabled - name: rabbitmq repository: oci://registry-1.docker.io/bitnamicharts version: 11.16.2 diff --git a/charts/sentry/ci/kind-values.yaml b/charts/sentry/ci/kind-values.yaml new file mode 100644 index 000000000..4cda9f6a6 --- /dev/null +++ b/charts/sentry/ci/kind-values.yaml @@ -0,0 +1,28 @@ +profiles: + - errors-only + +kafka: + enabled: true + kraft: + enabled: true + provisioning: + replicationFactor: 1 + controller: + replicaCount: 1 + broker: + replicaCount: 1 + +redis: + enabled: true + master.persistence.enabled: false + replica.replicaCount: 0 + +rabbitmq: + enabled: false + +clickhouse: + enabled: true + replicaCount: 1 + shards: 1 + keeper: + replicaCount: 1 diff --git a/charts/sentry/templates/_helper.tpl b/charts/sentry/templates/_helper.tpl index cc1ab85ce..f8bc04d0f 100644 --- a/charts/sentry/templates/_helper.tpl +++ b/charts/sentry/templates/_helper.tpl @@ -333,7 +333,7 @@ Set ClickHouse host */}} {{- define "sentry.clickhouse.host" -}} {{- if .Values.clickhouse.enabled -}} -{{- template "sentry.clickhouse.fullname" . -}} +{{- include "clickhouse.headlessServiceName" $.Subcharts.clickhouse -}} {{- else -}} {{ required "A valid .Values.externalClickhouse.host is required" .Values.externalClickhouse.host }} {{- end -}} @@ -344,7 +344,7 @@ Set ClickHouse port */}} {{- define "sentry.clickhouse.port" -}} {{- if .Values.clickhouse.enabled -}} -{{- default 9000 .Values.clickhouse.clickhouse.tcp_port }} +{{- default 9000 .Values.clickhouse.service.ports.tcp }} {{- else -}} {{ required "A valid .Values.externalClickhouse.tcpPort is required" .Values.externalClickhouse.tcpPort }} {{- end -}} @@ -355,7 +355,7 @@ Set ClickHouse HTTP port */}} {{- define "sentry.clickhouse.http_port" -}} {{- if .Values.clickhouse.enabled -}} -{{- default 8123 .Values.clickhouse.clickhouse.http_port }} +{{- default 8123 .Values.clickhouse.service.ports.http }} {{- else -}} {{ required "A valid .Values.externalClickhouse.httpPort is required" .Values.externalClickhouse.httpPort }} {{- end -}} @@ -377,11 +377,7 @@ Set ClickHouse User */}} {{- define "sentry.clickhouse.username" -}} {{- if .Values.clickhouse.enabled -}} - {{- if .Values.clickhouse.clickhouse.configmap.users.enabled -}} -{{ (index .Values.clickhouse.clickhouse.configmap.users.user 0).name }} - {{- else -}} -default - {{- end -}} +{{ default "default" .Values.clickhouse.auth.username }} {{- else -}} {{ required "A valid .Values.externalClickhouse.username is required" .Values.externalClickhouse.username }} {{- end -}} @@ -391,14 +387,9 @@ default Set ClickHouse Password */}} {{- define "sentry.clickhouse.password" -}} -{{- if .Values.clickhouse.enabled -}} - {{- if .Values.clickhouse.clickhouse.configmap.users.enabled -}} -{{ (index .Values.clickhouse.clickhouse.configmap.users.user 0).config.password }} - {{- else -}} - {{- end -}} -{{- else -}} +{{- if not .Values.clickhouse.enabled -}} {{ .Values.externalClickhouse.password }} -{{- end -}} +{{- end -}}F {{- end -}} {{/* @@ -406,23 +397,12 @@ Set ClickHouse cluster name */}} {{- define "sentry.clickhouse.cluster.name" -}} {{- if .Values.clickhouse.enabled -}} -{{ .Release.Name | printf "%s-clickhouse" }} +{{ include "clickhouse.headlessServiceName" $.Subcharts.clickhouse }} {{- else -}} {{ required "A valid .Values.externalClickhouse.clusterName is required" .Values.externalClickhouse.clusterName }} {{- end -}} {{- end -}} -{{/* -Set ClickHouse distributed cluster name -*/}} -{{- define "sentry.clickhouse.distributed.cluster.name" -}} -{{- if .Values.clickhouse.enabled -}} -{{ .Release.Name | printf "%s-clickhouse" }} -{{- else -}} -{{ default .Values.externalClickhouse.clusterName .Values.externalClickhouse.distributedClusterName }} -{{- end -}} -{{- end -}} - {{/* Set ClickHouse secure setting */}} @@ -667,6 +647,15 @@ Set external Clickhouse password from existingSecret name: {{ .Values.externalClickhouse.existingSecret }} key: {{ default "clickhouse-password" .Values.externalClickhouse.existingSecretKey }} {{- end }} +{{- if not .Values.externalClickhouse.existingSecret }} +- name: CLICKHOUSE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "clickhouse.secretName" $.Subcharts.clickhouse | quote }} + key: {{ include "clickhouse.secretKey" $.Subcharts.clickhouse | quote }} +{{- end }} +- name: CLICKHOUSE_HOST + value: {{ include "sentry.clickhouse.host" . | quote }} - name: CLICKHOUSE_MAX_CONNECTIONS value: {{ .Values.snuba.clickhouse.maxConnections | quote }} {{- if .Values.ipv6 }} diff --git a/charts/sentry/templates/hooks/sentry-db-check.job.yaml b/charts/sentry/templates/hooks/sentry-db-check.job.yaml index 082c02bde..cee9e4a6c 100644 --- a/charts/sentry/templates/hooks/sentry-db-check.job.yaml +++ b/charts/sentry/templates/hooks/sentry-db-check.job.yaml @@ -13,12 +13,14 @@ metadata: chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" release: "{{ .Release.Name }}" heritage: "{{ .Release.Service }}" + {{- if .Values.asHook }} annotations: # This is what defines this resource as a hook. Without this line, the # job is considered part of the release. "helm.sh/hook": "post-install,{{ if .Values.hooks.preUpgrade }}pre-upgrade{{ else }}post-upgrade{{ end }}" "helm.sh/hook-delete-policy": "{{ if .Values.hooks.removeOnSuccess }}hook-succeeded,{{ end }}before-hook-creation" "helm.sh/hook-weight": "-1" + {{- end }} spec: {{- if .Values.hooks.activeDeadlineSeconds }} activeDeadlineSeconds: {{ .Values.hooks.activeDeadlineSeconds }} @@ -87,10 +89,8 @@ spec: CLICKHOUSE_STATUS=0 while [ $CLICKHOUSE_STATUS -eq 0 ]; do CLICKHOUSE_STATUS=1 - CLICKHOUSE_REPLICAS={{ .Values.clickhouse.enabled | ternary .Values.clickhouse.clickhouse.replicas "1" }} - i=0; while [ $i -lt $CLICKHOUSE_REPLICAS ]; do {{- if .Values.clickhouse.enabled }} - CLICKHOUSE_HOST={{ $clickhouseHost }}-$i.{{ $clickhouseHost }}-headless + CLICKHOUSE_HOST={{ include "clickhouse.headlessServiceName" $.Subcharts.clickhouse }} {{- else }} CLICKHOUSE_HOST={{ .Values.externalClickhouse.host }} {{- end }} @@ -98,15 +98,6 @@ spec: CLICKHOUSE_STATUS=0 echo "$CLICKHOUSE_HOST is not available yet" fi - {{- if and .Values.clickhouse.enabled .Values.clickhouse.clickhouse.configmap.remote_servers.replica.backup.enabled }} - CLICKHOUSE_HOST={{ $clickhouseHost }}-replica-$i.{{ $clickhouseHost }}-replica-headless - if ! nc -z "$CLICKHOUSE_HOST" {{ $clickhousePort }}; then - CLICKHOUSE_STATUS=0 - echo "$CLICKHOUSE_HOST is not available yet" - fi - {{- end }} - i=$((i+1)) - done if [ "$CLICKHOUSE_STATUS" -eq 0 ]; then echo "Clickhouse not ready. Sleeping for 10s before trying again" sleep 10; diff --git a/charts/sentry/templates/hooks/sentry-db-init.job.yaml b/charts/sentry/templates/hooks/sentry-db-init.job.yaml index cdc185bd2..674e6eb9d 100644 --- a/charts/sentry/templates/hooks/sentry-db-init.job.yaml +++ b/charts/sentry/templates/hooks/sentry-db-init.job.yaml @@ -8,12 +8,14 @@ metadata: chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" release: "{{ .Release.Name }}" heritage: "{{ .Release.Service }}" + {{- if .Values.asHook }} annotations: # This is what defines this resource as a hook. Without this line, the # job is considered part of the release. "helm.sh/hook": "post-install,{{ if .Values.hooks.preUpgrade }}pre-upgrade{{ else }}post-upgrade{{ end }}" "helm.sh/hook-delete-policy": "{{ if .Values.hooks.removeOnSuccess }}hook-succeeded,{{ end }}before-hook-creation" "helm.sh/hook-weight": "6" + {{- end }} spec: {{- if .Values.hooks.activeDeadlineSeconds }} activeDeadlineSeconds: {{ .Values.hooks.activeDeadlineSeconds }} diff --git a/charts/sentry/templates/hooks/sentry-secret-create.yaml b/charts/sentry/templates/hooks/sentry-secret-create.yaml index 267503217..d2336ab4f 100644 --- a/charts/sentry/templates/hooks/sentry-secret-create.yaml +++ b/charts/sentry/templates/hooks/sentry-secret-create.yaml @@ -8,9 +8,11 @@ metadata: chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" release: "{{ .Release.Name }}" heritage: "{{ .Release.Service }}" + {{- if .Values.asHook }} annotations: "helm.sh/hook": "pre-install" "helm.sh/hook-weight": "3" + {{- end }} type: Opaque data: key: {{ randAlphaNum 50 | b64enc | quote }} diff --git a/charts/sentry/templates/hooks/snuba-db-init.job.yaml b/charts/sentry/templates/hooks/snuba-db-init.job.yaml index 5ae8c3059..555eab201 100644 --- a/charts/sentry/templates/hooks/snuba-db-init.job.yaml +++ b/charts/sentry/templates/hooks/snuba-db-init.job.yaml @@ -9,12 +9,14 @@ metadata: chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" release: "{{ .Release.Name }}" heritage: "{{ .Release.Service }}" + {{- if .Values.asHook }} annotations: # This is what defines this resource as a hook. Without this line, the # job is considered part of the release. "helm.sh/hook": "post-install,{{ if .Values.hooks.preUpgrade }}pre-upgrade{{ else }}post-upgrade{{ end }}" "helm.sh/hook-delete-policy": "{{ if .Values.hooks.removeOnSuccess }}hook-succeeded,{{ end }}before-hook-creation" "helm.sh/hook-weight": "3" + {{- end }} spec: {{- if .Values.hooks.activeDeadlineSeconds }} activeDeadlineSeconds: {{ .Values.hooks.activeDeadlineSeconds }} diff --git a/charts/sentry/templates/hooks/snuba-migrate.job.yaml b/charts/sentry/templates/hooks/snuba-migrate.job.yaml index 323432bdc..e95165648 100644 --- a/charts/sentry/templates/hooks/snuba-migrate.job.yaml +++ b/charts/sentry/templates/hooks/snuba-migrate.job.yaml @@ -9,12 +9,14 @@ metadata: chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" release: "{{ .Release.Name }}" heritage: "{{ .Release.Service }}" + {{- if .Values.asHook }} annotations: # This is what defines this resource as a hook. Without this line, the # job is considered part of the release. "helm.sh/hook": "post-install,{{ if .Values.hooks.preUpgrade }}pre-upgrade{{ else }}post-upgrade{{ end }}" "helm.sh/hook-delete-policy": "{{ if .Values.hooks.removeOnSuccess }}hook-succeeded,{{ end }}before-hook-creation" "helm.sh/hook-weight": "5" + {{- end }} spec: {{- if .Values.hooks.activeDeadlineSeconds }} activeDeadlineSeconds: {{ .Values.hooks.activeDeadlineSeconds }} diff --git a/charts/sentry/templates/hooks/user-create.yaml b/charts/sentry/templates/hooks/user-create.yaml index ad3bf7393..7f8634ad4 100644 --- a/charts/sentry/templates/hooks/user-create.yaml +++ b/charts/sentry/templates/hooks/user-create.yaml @@ -8,10 +8,12 @@ metadata: chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" release: "{{ .Release.Name }}" heritage: "{{ .Release.Service }}" + {{- if .Values.asHook }} annotations: "helm.sh/hook": "post-install,{{ if .Values.hooks.preUpgrade }}pre-upgrade{{ else }}post-upgrade{{ end }}" "helm.sh/hook-delete-policy": "{{ if .Values.hooks.removeOnSuccess }}hook-succeeded,{{ end }}before-hook-creation" "helm.sh/hook-weight": "9" + {{- end }} spec: {{- if .Values.hooks.activeDeadlineSeconds }} activeDeadlineSeconds: {{ .Values.hooks.activeDeadlineSeconds }} diff --git a/charts/sentry/templates/sentry/worker/deployment-sentry-worker.yaml b/charts/sentry/templates/sentry/worker/deployment-sentry-worker.yaml index fda9446e5..88d21989a 100644 --- a/charts/sentry/templates/sentry/worker/deployment-sentry-worker.yaml +++ b/charts/sentry/templates/sentry/worker/deployment-sentry-worker.yaml @@ -8,6 +8,11 @@ metadata: chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" release: "{{ .Release.Name }}" heritage: "{{ .Release.Service }}" + annotations: + meta.helm.sh/release-name: "{{ .Release.Name }}" + meta.helm.sh/release-namespace: "{{ .Release.Namespace }}" + "helm.sh/hook": "post-install,post-upgrade" + "helm.sh/hook-weight": "25" spec: revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} selector: diff --git a/charts/sentry/templates/snuba/_helper-snuba.tpl b/charts/sentry/templates/snuba/_helper-snuba.tpl index ada284223..dec786a28 100644 --- a/charts/sentry/templates/snuba/_helper-snuba.tpl +++ b/charts/sentry/templates/snuba/_helper-snuba.tpl @@ -31,6 +31,17 @@ settings.py: | KAFKA_TOPIC_MAP[topic.value] = f"{SENTRY_CHARTS_KAFKA_TOPIC_PREFIX}{topic.value}" {{- end }} + SENTRY_DISTRIBUTED_CLICKHOUSE_TABLES = True + + # Migration settings for distributed ClickHouse + MIGRATIONS_LOCK_TIMEOUT = int(env("MIGRATIONS_LOCK_TIMEOUT", "600")) + MIGRATIONS_BATCH_SIZE = int(env("MIGRATIONS_BATCH_SIZE", "1")) + + # ClickHouse distributed settings + CLICKHOUSE_MUTATIONS_SYNC = int(env("CLICKHOUSE_MUTATIONS_SYNC", "1")) + CLICKHOUSE_ALTER_SYNC = int(env("CLICKHOUSE_ALTER_SYNC", "1")) + CLICKHOUSE_REPLICATION_ALTER_PARTITIONS_SYNC = int(env("CLICKHOUSE_REPLICATION_ALTER_PARTITIONS_SYNC", "2")) + # Clickhouse Options CLUSTERS = [ { @@ -77,11 +88,11 @@ settings.py: | {{- if and .Values.externalClickhouse.singleNode (not .Values.clickhouse.enabled) }} "single_node": True, {{- else }} - "single_node": False, + "single_node": True, # IDK why this is needed, but it is required for the clickhouse client to work {{- end }} {{- if or .Values.clickhouse.enabled (not .Values.externalClickhouse.singleNode) }} - "cluster_name": {{ include "sentry.clickhouse.cluster.name" . | quote }}, - "distributed_cluster_name": {{ include "sentry.clickhouse.distributed.cluster.name" . | quote }}, + "cluster_name": {{ default "default" .Values.clickhouse.clusterName | quote }}, + "distributed_cluster_name": {{ default "default" .Values.clickhouse.clusterName | quote }}, {{- end }} }, ] diff --git a/charts/sentry/templates/snuba/secret-snuba-env.yaml b/charts/sentry/templates/snuba/secret-snuba-env.yaml index 38411057a..1e6148828 100644 --- a/charts/sentry/templates/snuba/secret-snuba-env.yaml +++ b/charts/sentry/templates/snuba/secret-snuba-env.yaml @@ -12,9 +12,6 @@ data: CLICKHOUSE_PORT: {{ include "sentry.clickhouse.port" . | b64enc | quote }} CLICKHOUSE_DATABASE: {{ include "sentry.clickhouse.database" . | b64enc | quote }} CLICKHOUSE_USER: {{ include "sentry.clickhouse.username" . | b64enc | quote }} -{{- if not .Values.externalClickhouse.existingSecret }} - CLICKHOUSE_PASSWORD: {{ include "sentry.clickhouse.password" . | b64enc | quote }} -{{- end }} {{- if .Values.externalClickhouse.secure }} CLICKHOUSE_SECURE: {{ include "sentry.clickhouse.secure" . | b64enc | quote }} {{- end }} diff --git a/charts/sentry/values.yaml b/charts/sentry/values.yaml index 9883d389e..05d6e01f1 100644 --- a/charts/sentry/values.yaml +++ b/charts/sentry/values.yaml @@ -35,7 +35,7 @@ user: # recommended to set false for updating the helm chart afterwards, # as you will have some downtime on each update if it's a hook # deploys relay & snuba consumers as post hooks -asHook: true +asHook: false images: sentry: @@ -2267,48 +2267,6 @@ config: clickhouse: enabled: true - nodeSelector: {} - # tolerations: [] - clickhouse: - replicas: "1" - configmap: - remote_servers: - internal_replication: true - replica: - backup: - enabled: false - zookeeper_servers: - enabled: true - config: - - index: "clickhouse" - hostTemplate: "{{ .Release.Name }}-zookeeper-clickhouse" - port: "2181" - users: - enabled: false - user: - # the first user will be used if enabled - - name: default - config: - password: "" - networks: - - ::/0 - profile: default - quota: default - - persistentVolumeClaim: - enabled: true - dataPersistentVolume: - enabled: true - accessModes: - - "ReadWriteOnce" - storage: "30Gi" - - ## Use this to enable an extra service account - # serviceAccount: - # annotations: {} - # enabled: false - # name: "sentry-clickhouse" - # automountServiceAccountToken: true ## This value is only used when clickhouse.enabled is set to false ## @@ -2335,18 +2293,6 @@ externalClickhouse: ## # distributedClusterName: distributed_test_shard_localhost -# Settings for Zookeeper. -# See https://github.com/bitnami/charts/tree/master/bitnami/zookeeper -zookeeper: - enabled: true - nameOverride: zookeeper-clickhouse - replicaCount: 1 - nodeSelector: {} - # tolerations: [] - ## When increasing the number of exceptions, you need to increase persistence.size - # persistence: - # size: 8Gi - # Settings for Kafka. # See https://github.com/bitnami/charts/tree/master/bitnami/kafka kafka: