From 59f8977e984ddc5c27eb89c86764cc4fd98947dd Mon Sep 17 00:00:00 2001 From: Ugo Date: Mon, 1 Sep 2025 07:17:47 +0000 Subject: [PATCH] feat(sentry): add db cleanup --- charts/sentry/README.md | 13 ++ charts/sentry/templates/_helper.tpl | 72 ++++++++++ .../cleanup/cronjob-sentry-db-cleanup.yaml | 134 ++++++++++++++++++ charts/sentry/values.yaml | 34 +++++ 4 files changed, 253 insertions(+) create mode 100644 charts/sentry/templates/sentry/cleanup/cronjob-sentry-db-cleanup.yaml diff --git a/charts/sentry/README.md b/charts/sentry/README.md index 0967cde69..316cdafd6 100644 --- a/charts/sentry/README.md +++ b/charts/sentry/README.md @@ -456,6 +456,19 @@ Note: this table is incomplete, so have a look at the values.yaml in case you mi | sentry.cleanup.sidecars | list | `[]` | | | sentry.cleanup.successfulJobsHistoryLimit | int | `5` | | | sentry.cleanup.volumes | list | `[]` | | +| sentry.cleanup.dbCleanup.enabled | bool | `false` | Enable database cleanup cronjob for nodestore_node table | +| sentry.cleanup.dbCleanup.schedule | string | `"0 1 * * *"` | Schedule for database cleanup cronjob | +| sentry.cleanup.dbCleanup.successfulJobsHistoryLimit | int | `5` | | +| sentry.cleanup.dbCleanup.failedJobsHistoryLimit | int | `5` | | +| sentry.cleanup.dbCleanup.activeDeadlineSeconds | int | `3600` | | +| sentry.cleanup.dbCleanup.concurrencyPolicy | string | `"Forbid"` | | +| sentry.cleanup.dbCleanup.image.repository | string | `"postgres"` | Docker image repository for database cleanup | +| sentry.cleanup.dbCleanup.image.tag | string | `"15-alpine"` | Docker image tag for database cleanup | +| sentry.cleanup.dbCleanup.image.pullPolicy | string | `"IfNotPresent"` | | +| sentry.cleanup.dbCleanup.resources | object | `{"limits":{"cpu":"500m","memory":"256Mi"},"requests":{"cpu":"100m","memory":"128Mi"}}` | | +| sentry.cleanup.dbCleanup.env | list | `[]` | | +| sentry.cleanup.dbCleanup.sidecars | list | `[]` | | +| sentry.cleanup.dbCleanup.volumes | list | `[]` | | | sentry.cron.affinity | object | `{}` | | | sentry.cron.enabled | bool | `true` | | | sentry.cron.env | list | `[]` | | diff --git a/charts/sentry/templates/_helper.tpl b/charts/sentry/templates/_helper.tpl index a98745774..2bd71d298 100644 --- a/charts/sentry/templates/_helper.tpl +++ b/charts/sentry/templates/_helper.tpl @@ -848,6 +848,78 @@ Set external Postgresql port from existingSecret {{- end }} {{- end }} +{{/* +PostgreSQL environment variables helper for database cleanup +*/}} +{{- define "sentry.postgresql.env" -}} +{{- if .Values.postgresql.enabled }} +- name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ default (include "sentry.postgresql.fullname" .) .Values.postgresql.auth.existingSecret }} + key: {{ default "postgres-password" .Values.postgresql.auth.secretKeys.adminPasswordKey }} +{{- else if .Values.externalPostgresql.password }} +- name: POSTGRES_PASSWORD + value: {{ .Values.externalPostgresql.password | quote }} +{{- else if .Values.externalPostgresql.existingSecret }} +- name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.externalPostgresql.existingSecret }} + key: {{ or .Values.externalPostgresql.existingSecretKeys.password .Values.externalPostgresql.existingSecretKey "postgresql-password" }} +{{- end }} +{{- if and .Values.externalPostgresql.existingSecret .Values.externalPostgresql.existingSecretKeys.username }} +- name: POSTGRES_USER + valueFrom: + secretKeyRef: + name: {{ .Values.externalPostgresql.existingSecret }} + key: {{ default .Values.externalPostgresql.existingSecretKeys.username }} +{{- else }} +- name: POSTGRES_USER + value: {{ include "sentry.postgresql.username" . | quote }} +{{- end }} +{{- if and .Values.externalPostgresql.existingSecret .Values.externalPostgresql.existingSecretKeys.database }} +- name: POSTGRES_NAME + valueFrom: + secretKeyRef: + name: {{ .Values.externalPostgresql.existingSecret }} + key: {{ default .Values.externalPostgresql.existingSecretKeys.database }} +{{- else }} +- name: POSTGRES_NAME + value: {{ include "sentry.postgresql.database" . | quote }} +{{- end }} +{{- if .Values.pgbouncer.enabled }} +- name: POSTGRES_HOST + value: {{ template "sentry.fullname" . }}-pgbouncer +{{- else }} +{{- if and .Values.externalPostgresql.existingSecret .Values.externalPostgresql.existingSecretKeys.host }} +- name: POSTGRES_HOST + valueFrom: + secretKeyRef: + name: {{ .Values.externalPostgresql.existingSecret }} + key: {{ default .Values.externalPostgresql.existingSecretKeys.host }} +{{- else }} +- name: POSTGRES_HOST + value: {{ include "sentry.postgresql.host" . | quote }} +{{- end }} +{{- end }} +{{- if .Values.pgbouncer.enabled }} +- name: POSTGRES_PORT + value: "5432" +{{- else }} +{{- if and .Values.externalPostgresql.existingSecret .Values.externalPostgresql.existingSecretKeys.port }} +- name: POSTGRES_PORT + valueFrom: + secretKeyRef: + name: {{ .Values.externalPostgresql.existingSecret }} + key: {{ default .Values.externalPostgresql.existingSecretKeys.port }} +{{- else }} +- name: POSTGRES_PORT + value: {{ include "sentry.postgresql.port" . | quote }} +{{- end }} +{{- end }} +{{- end -}} + {{/* Set S3 */}} diff --git a/charts/sentry/templates/sentry/cleanup/cronjob-sentry-db-cleanup.yaml b/charts/sentry/templates/sentry/cleanup/cronjob-sentry-db-cleanup.yaml new file mode 100644 index 000000000..4eb8d2072 --- /dev/null +++ b/charts/sentry/templates/sentry/cleanup/cronjob-sentry-db-cleanup.yaml @@ -0,0 +1,134 @@ +{{- if .Values.sentry.cleanup.dbCleanup.enabled }} +{{- $batchApiIsStable := eq (include "sentry.batch.isStable" .) "true" -}} +apiVersion: {{ include "sentry.batch.apiVersion" . }} +kind: CronJob +metadata: + name: {{ template "sentry.fullname" . }}-sentry-db-cleanup + labels: + app: {{ template "sentry.fullname" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +spec: + schedule: "{{ .Values.sentry.cleanup.dbCleanup.schedule }}" + successfulJobsHistoryLimit: {{ .Values.sentry.cleanup.dbCleanup.successfulJobsHistoryLimit }} + failedJobsHistoryLimit: {{ .Values.sentry.cleanup.dbCleanup.failedJobsHistoryLimit }} + concurrencyPolicy: "{{ .Values.sentry.cleanup.dbCleanup.concurrencyPolicy }}" + jobTemplate: + spec: + {{- if .Values.sentry.cleanup.dbCleanup.activeDeadlineSeconds }} + activeDeadlineSeconds: {{ .Values.sentry.cleanup.dbCleanup.activeDeadlineSeconds }} + {{- end}} + template: + metadata: + annotations: + {{- if .Values.sentry.cleanup.dbCleanup.annotations }} +{{ toYaml .Values.sentry.cleanup.dbCleanup.annotations | indent 12 }} + {{- end }} + labels: + app: {{ template "sentry.fullname" . }} + release: "{{ .Release.Name }}" + {{- if .Values.sentry.cleanup.dbCleanup.podLabels }} +{{ toYaml .Values.sentry.cleanup.dbCleanup.podLabels | indent 12 }} + {{- end }} + spec: + {{- if .Values.sentry.cleanup.dbCleanup.affinity }} + affinity: +{{ toYaml .Values.sentry.cleanup.dbCleanup.affinity | indent 12 }} + {{- end }} + {{- if .Values.sentry.cleanup.dbCleanup.nodeSelector }} + nodeSelector: +{{ toYaml .Values.sentry.cleanup.dbCleanup.nodeSelector | indent 12 }} + {{- else if .Values.global.nodeSelector }} + nodeSelector: +{{ toYaml .Values.global.nodeSelector | indent 12 }} + {{- end }} + {{- if .Values.sentry.cleanup.dbCleanup.tolerations }} + tolerations: +{{ toYaml .Values.sentry.cleanup.dbCleanup.tolerations | indent 12 }} + {{- else if .Values.global.tolerations }} + tolerations: +{{ toYaml .Values.global.tolerations | indent 12 }} + {{- end }} + {{- if .Values.dnsPolicy }} + dnsPolicy: {{ .Values.dnsPolicy | quote }} + {{- end }} + {{- if .Values.dnsConfig }} + dnsConfig: +{{ toYaml .Values.dnsConfig | indent 12 }} + {{- end }} + {{- if .Values.sentry.cleanup.dbCleanup.securityContext }} + securityContext: +{{ toYaml .Values.sentry.cleanup.dbCleanup.securityContext | indent 12 }} + {{- end }} + containers: + - name: {{ .Chart.Name }}-sentry-db-cleanup + image: "{{ .Values.sentry.cleanup.dbCleanup.image.repository }}:{{ .Values.sentry.cleanup.dbCleanup.image.tag }}" + imagePullPolicy: {{ .Values.sentry.cleanup.dbCleanup.image.pullPolicy }} + command: ["sh", "-c"] + args: + - | + export PGPASSWORD="${POSTGRES_PASSWORD}" + + echo "Starting database cleanup for nodestore_node table..." + echo "Retention days: {{ .Values.sentry.cleanup.days }}" + + # Delete old records from nodestore_node table + psql -h "${POSTGRES_HOST}" -p "${POSTGRES_PORT}" -U "${POSTGRES_USER}" -d "${POSTGRES_NAME}" -c " + DELETE FROM public.nodestore_node + WHERE \"timestamp\" < NOW() - INTERVAL '{{ .Values.sentry.cleanup.days }} days'; + " + + if [ $? -eq 0 ]; then + echo "Successfully deleted old records from nodestore_node table" + + # Run VACUUM FULL to reclaim space + echo "Running VACUUM FULL on nodestore_node table..." + psql -h "${POSTGRES_HOST}" -p "${POSTGRES_PORT}" -U "${POSTGRES_USER}" -d "${POSTGRES_NAME}" -c " + VACUUM FULL public.nodestore_node; + " + + if [ $? -eq 0 ]; then + echo "Successfully completed VACUUM FULL on nodestore_node table" + else + echo "Error: VACUUM FULL failed" + exit 1 + fi + else + echo "Error: Failed to delete old records" + exit 1 + fi + + echo "Database cleanup completed successfully" + env: +{{ include "sentry.postgresql.env" . | indent 12 }} +{{- if .Values.sentry.cleanup.dbCleanup.env }} +{{ toYaml .Values.sentry.cleanup.dbCleanup.env | indent 12 }} +{{- end }} + resources: +{{ toYaml .Values.sentry.cleanup.dbCleanup.resources | indent 14 }} +{{- if .Values.sentry.cleanup.dbCleanup.containerSecurityContext }} + securityContext: +{{ toYaml .Values.sentry.cleanup.dbCleanup.containerSecurityContext | indent 14 }} +{{- end }} +{{- if .Values.sentry.cleanup.dbCleanup.sidecars }} +{{ toYaml .Values.sentry.cleanup.dbCleanup.sidecars | indent 10 }} +{{- end }} +{{- if .Values.global.sidecars }} +{{ toYaml .Values.global.sidecars | indent 10 }} +{{- end }} + restartPolicy: Never + {{- if .Values.sentry.cleanup.dbCleanup.volumes }} + volumes: +{{ toYaml .Values.sentry.cleanup.dbCleanup.volumes | indent 10 }} + {{- else if .Values.global.volumes }} + volumes: +{{ toYaml .Values.global.volumes | indent 10 }} + {{- end }} + {{- if .Values.sentry.cleanup.dbCleanup.priorityClassName }} + priorityClassName: "{{ .Values.sentry.cleanup.dbCleanup.priorityClassName }}" + {{- end }} + {{- if .Values.serviceAccount.enabled }} + serviceAccountName: {{ .Values.serviceAccount.name }}-cleanup + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/sentry/values.yaml b/charts/sentry/values.yaml index ad97fdcff..4f99cff15 100644 --- a/charts/sentry/values.yaml +++ b/charts/sentry/values.yaml @@ -1190,6 +1190,40 @@ sentry: volumes: [] # volumeMounts: [] serviceAccount: {} + + # Database cleanup cronjob for nodestore_node table + # This runs direct SQL operations on the PostgreSQL database: + # DELETE FROM public.nodestore_node WHERE "timestamp" < NOW() - INTERVAL 'X days'; + # VACUUM FULL public.nodestore_node; + # Uses the same retention period as sentry.cleanup.days + dbCleanup: + enabled: false + schedule: "0 1 * * *" # Run at 1 AM daily, after main cleanup + successfulJobsHistoryLimit: 5 + failedJobsHistoryLimit: 5 + activeDeadlineSeconds: 3600 # 1 hour timeout + concurrencyPolicy: Forbid # Don't allow concurrent database operations + image: + repository: postgres + tag: "15-alpine" + pullPolicy: IfNotPresent + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 256Mi + # securityContext: {} + # containerSecurityContext: {} + # affinity: {} + # nodeSelector: {} + # tolerations: [] + # podLabels: {} + # annotations: {} + env: [] + sidecars: [] + volumes: [] # Sentry settings of connections to Kafka kafka: