diff --git a/charts/timescaledb-single/templates/_helpers.tpl b/charts/timescaledb-single/templates/_helpers.tpl index 43e56183..eddbd17a 100644 --- a/charts/timescaledb-single/templates/_helpers.tpl +++ b/charts/timescaledb-single/templates/_helpers.tpl @@ -66,6 +66,14 @@ ${HOME}/.pgbackrest_environment /etc/pgbackrest/bootstrap {{- end -}} +{{- define "pgbackrest_backup_pvc" -}} +{{ printf "%s-backup-volume" (include "clusterName" .) }} +{{- end -}} + +{{- define "pgbackrest_repo1_path" -}} +{{ printf "/%s/%s/" .Release.Namespace (include "clusterName" .) }} +{{- end -}} + {{- define "postgres.uid" -}} {{- default .Values.uid "1000" -}} {{- end -}} @@ -111,5 +119,5 @@ ${HOME}/.pgbackrest_environment {{- end -}} {{- define "secrets_pgbackrest" -}} -{{ printf "%s-pgbackrest" (include "clusterName" .) }} +{{ .Values.secrets.pgbackrestSecretName | default (printf "%s-pgbackrest" (include "clusterName" .)) }} {{- end -}} diff --git a/charts/timescaledb-single/templates/configmap-pgbackrest.yaml b/charts/timescaledb-single/templates/configmap-pgbackrest.yaml index ff151e6a..71f57dc9 100644 --- a/charts/timescaledb-single/templates/configmap-pgbackrest.yaml +++ b/charts/timescaledb-single/templates/configmap-pgbackrest.yaml @@ -11,7 +11,7 @@ metadata: chart: {{ template "timescaledb.chart" . }} release: {{ .Release.Name }} heritage: {{ .Release.Service }} -{{- $globalDefaults := dict "spool-path" (include "socket_directory" .) "compress-level" "3" "repo1-path" (printf "/%s/%s/" .Release.Namespace (include "clusterName" .)) }} +{{- $globalDefaults := dict "spool-path" (include "socket_directory" .) "compress-level" "3" }} {{- $globals := merge .Values.backup.pgBackRest $globalDefaults }} {{- $push := index .Values.backup "pgBackRest:archive-push" | default dict }} {{- $get := index .Values.backup "pgBackRest:archive-get" | default dict }} diff --git a/charts/timescaledb-single/templates/networkpolicy.yaml b/charts/timescaledb-single/templates/networkpolicy.yaml index 738b0bdf..95117420 100644 --- a/charts/timescaledb-single/templates/networkpolicy.yaml +++ b/charts/timescaledb-single/templates/networkpolicy.yaml @@ -28,7 +28,7 @@ spec: - port: 8081 protocol: TCP {{ if .Values.prometheus.enabled }} - # Prom server for scraping exporter + # Prometheus server for scraping exporter - from: - podSelector: matchLabels: diff --git a/charts/timescaledb-single/templates/pgbackrest.yaml b/charts/timescaledb-single/templates/pgbackrest.yaml index 5147cfdb..d2af646d 100644 --- a/charts/timescaledb-single/templates/pgbackrest.yaml +++ b/charts/timescaledb-single/templates/pgbackrest.yaml @@ -24,7 +24,7 @@ spec: ... {{- range .Values.backup.jobs }} --- -apiVersion: batch/v1beta1 +apiVersion: batch/v1 kind: CronJob metadata: name: {{ template "timescaledb.fullname" $ }}-{{ .name }} diff --git a/charts/timescaledb-single/templates/pvc-pgbackrest.yaml b/charts/timescaledb-single/templates/pvc-pgbackrest.yaml new file mode 100644 index 00000000..bc0ea794 --- /dev/null +++ b/charts/timescaledb-single/templates/pvc-pgbackrest.yaml @@ -0,0 +1,31 @@ +{{ if and .Values.backup.enabled .Values.persistentVolumes.backup.enabled }} +--- +# This PersistentVolumeClaim is only created if enabled. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ template "pgbackrest_backup_pvc" . }} + {{- if .Values.persistentVolumes.backup.annotations }} + annotations: +{{ toYaml .Values.persistentVolumes.backup.annotations | indent 4 }} + {{- end }} + labels: + app: {{ template "timescaledb.fullname" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + cluster-name: {{ template "clusterName" . }} + purpose: backup-storage +spec: + accessModes: +{{ toYaml .Values.persistentVolumes.backup.accessModes | indent 4 }} + resources: + requests: + storage: "{{ .Values.persistentVolumes.backup.size }}" +{{- if .Values.persistentVolumes.backup.storageClass }} +{{- if (eq "-" .Values.persistentVolumes.backup.storageClass) }} + storageClassName: "" +{{- else }} + storageClassName: "{{ .Values.persistentVolumes.backup.storageClass }}" +{{- end }} +{{- end }} +{{ end }} diff --git a/charts/timescaledb-single/templates/secret-pgbackrest.yaml b/charts/timescaledb-single/templates/secret-pgbackrest.yaml index 8b6fd035..fe360a56 100644 --- a/charts/timescaledb-single/templates/secret-pgbackrest.yaml +++ b/charts/timescaledb-single/templates/secret-pgbackrest.yaml @@ -1,6 +1,7 @@ +{{- if and (eq .Values.secrets.pgbackrestSecretName "") .Values.secrets.pgbackrest }} +--- # This file and its contents are licensed under the Apache License 2.0. # Please see the included NOTICE for copyright information and LICENSE for a copy of the license. -{{- if eq .Values.secrets.pgbackrestSecretName "" }} apiVersion: v1 kind: Secret metadata: diff --git a/charts/timescaledb-single/templates/statefulset-timescaledb.yaml b/charts/timescaledb-single/templates/statefulset-timescaledb.yaml index 9f32fcf3..848b3af6 100644 --- a/charts/timescaledb-single/templates/statefulset-timescaledb.yaml +++ b/charts/timescaledb-single/templates/statefulset-timescaledb.yaml @@ -50,6 +50,8 @@ spec: env: - name: TSTUNE_FILE value: {{ template "tstune_config" . }} + - name: POSTGRES_MAJOR_VERSION + value: {{ .Values.version | default 11 | quote }} - name: RESOURCES_WAL_VOLUME value: {{ if .Values.persistentVolumes.wal.enabled }}{{ .Values.persistentVolumes.wal.size }}{{ else }}"0"{{ end }} - name: RESOURCES_DATA_VOLUME @@ -93,7 +95,7 @@ spec: fi touch "${TSTUNE_FILE}" - timescaledb-tune -quiet -pg-version 11 -conf-path "${TSTUNE_FILE}" -cpus "${CPUS}" -memory "${MEMORY}MB" \ + timescaledb-tune -quiet -pg-version ${POSTGRES_MAJOR_VERSION} -conf-path "${TSTUNE_FILE}" -cpus "${CPUS}" -memory "${MEMORY}MB" \ {{ range $key, $value := .Values.timescaledbTune.args | default dict }}{{ printf "--%s %s " $key (quote $value)}}{{ end }} -yes # If there is a dedicated WAL Volume, we want to set max_wal_size to 60% of that volume @@ -195,79 +197,83 @@ spec: exec patroni /etc/timescaledb/patroni.yaml env: - # We use mixed case environment variables for Patroni User management, - # as the variable themselves are documented to be PATRONI__OPTIONS. - # Where possible, we want to have lowercase usernames in PostgreSQL as more complex postgres usernames - # requiring quoting to be done in certain contexts, which many tools do not do correctly, or even at all. - # https://patroni.readthedocs.io/en/latest/ENVIRONMENT.html#bootstrap-configuration - - name: PATRONI_admin_OPTIONS - value: createrole,createdb - - name: PATRONI_REPLICATION_USERNAME - value: standby - # To specify the PostgreSQL and Rest API connect addresses we need - # the PATRONI_KUBERNETES_POD_IP to be available as a bash variable, so we can compose an - # IP:PORT address later on - - name: PATRONI_KUBERNETES_POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: PATRONI_POSTGRESQL_CONNECT_ADDRESS - value: "$(PATRONI_KUBERNETES_POD_IP):5432" - - name: PATRONI_RESTAPI_CONNECT_ADDRESS - value: "$(PATRONI_KUBERNETES_POD_IP):8008" - - name: PATRONI_KUBERNETES_PORTS - {{- if .Values.pgBouncer.enabled }} - value: '[{"name": "postgresql", "port": 5432}, {"name": "pgbouncer", "port": 6432}]' - {{- else }} - value: '[{"name": "postgresql", "port": 5432}]' + # We use mixed case environment variables for Patroni User management, + # as the variable themselves are documented to be PATRONI__OPTIONS. + # Where possible, we want to have lowercase usernames in PostgreSQL as more complex postgres usernames + # requiring quoting to be done in certain contexts, which many tools do not do correctly, or even at all. + # https://patroni.readthedocs.io/en/latest/ENVIRONMENT.html#bootstrap-configuration + - name: PATRONI_admin_OPTIONS + value: createrole,createdb + - name: PATRONI_REPLICATION_USERNAME + value: standby + # To specify the PostgreSQL and Rest API connect addresses we need + # the PATRONI_KUBERNETES_POD_IP to be available as a bash variable, so we can compose an + # IP:PORT address later on + - name: PATRONI_KUBERNETES_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: PATRONI_POSTGRESQL_CONNECT_ADDRESS + value: "$(PATRONI_KUBERNETES_POD_IP):5432" + - name: PATRONI_RESTAPI_CONNECT_ADDRESS + value: "$(PATRONI_KUBERNETES_POD_IP):8008" + - name: PATRONI_KUBERNETES_PORTS + {{- if .Values.pgBouncer.enabled }} + value: '[{"name": "postgresql", "port": 5432}, {"name": "pgbouncer", "port": 6432}]' + {{- else }} + value: '[{"name": "postgresql", "port": 5432}]' + {{- end }} + - name: PATRONI_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: PATRONI_POSTGRESQL_DATA_DIR + value: {{ include "data_directory" . | quote }} + - name: PATRONI_KUBERNETES_NAMESPACE + value: {{ $.Release.Namespace }} + - name: PATRONI_KUBERNETES_LABELS + value: {{ printf "{app: %s, cluster-name: %s, release: %s}" (include "timescaledb.fullname" .) (include "clusterName" .) .Release.Name | quote }} + - name: PATRONI_SCOPE + value: {{ template "clusterName" . }} + - name: PGBACKREST_CONFIG + value: /etc/pgbackrest/pgbackrest.conf + {{- if and .Values.backup.enabled .Values.persistentVolumes.backup.enabled }} + - name: PGBACKREST_REPO1_PATH + value: {{ template "pgbackrest_repo1_path" . }} + {{- end }} + # PGDATA and PGHOST are not required to let Patroni/PostgreSQL run correctly, + # but for interactive sessions, callbacks and PostgreSQL tools they should be correct. + - name: PGDATA + value: "$(PATRONI_POSTGRESQL_DATA_DIR)" + - name: PGHOST + value: "{{ template "socket_directory" . }}" + - name: BOOTSTRAP_FROM_BACKUP + value: {{ .Values.bootstrapFromBackup.enabled | int | quote }} + {{- if .Values.env }}{{ .Values.env | default list | toYaml | nindent 10 }}{{- end }} + # pgBackRest is also called using the archive_command if the backup is enabled. + # this script will also need access to the environment variables specified for + # the backup. This can be removed once we do not directly invoke pgBackRest + # from inside the TimescaleDB container anymore + {{- if .Values.backup.env }}{{ .Values.backup.env | default list | toYaml | nindent 10 }}{{- end }} + {{- if .Values.version }} + - name: PATH + value: /usr/lib/postgresql/{{ .Values.version }}/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin {{- end }} - - name: PATRONI_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: PATRONI_POSTGRESQL_DATA_DIR - value: {{ include "data_directory" . | quote }} - - name: PATRONI_KUBERNETES_NAMESPACE - value: {{ $.Release.Namespace }} - - name: PATRONI_KUBERNETES_LABELS - value: {{ printf "{app: %s, cluster-name: %s, release: %s}" (include "timescaledb.fullname" .) (include "clusterName" .) .Release.Name | quote }} - - name: PATRONI_SCOPE - value: {{ template "clusterName" . }} - - name: PGBACKREST_CONFIG - value: /etc/pgbackrest/pgbackrest.conf - # PGDATA and PGHOST are not required to let Patroni/PostgreSQL run correctly, - # but for interactive sessions, callbacks and PostgreSQL tools they should be correct. - - name: PGDATA - value: "$(PATRONI_POSTGRESQL_DATA_DIR)" - - name: PGHOST - value: "{{ template "socket_directory" . }}" - - name: BOOTSTRAP_FROM_BACKUP - value: {{ .Values.bootstrapFromBackup.enabled | int | quote }} - {{- if .Values.env }}{{ .Values.env | default list | toYaml | nindent 8 }}{{- end }} - # pgBackRest is also called using the archive_command if the backup is enabled. - # this script will also need access to the environment variables specified for - # the backup. This can be removed once we do not directly invoke pgBackRest - # from inside the TimescaleDB container anymore - {{- if .Values.backup.env }}{{ .Values.backup.env | default list | toYaml | nindent 8 }}{{- end }} - {{- if .Values.version }} - - name: PATH - value: /usr/lib/postgresql/{{ .Values.version }}/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - {{- end }} envFrom: - - secretRef: - name: {{ template "secrets_credentials" . }} - optional: false - - secretRef: - name: {{ template "secrets_pgbackrest" . }} - optional: true - {{- if or .Values.envFrom .Values.backup.envFrom -}} - {{- .Values.backup.envFrom | default list | concat (.Values.envFrom | default list) | toYaml | nindent 8 -}} - {{- end }} + - secretRef: + name: {{ template "secrets_credentials" . }} + optional: false + - secretRef: + name: {{ template "secrets_pgbackrest" . }} + optional: true + {{- if or .Values.envFrom .Values.backup.envFrom -}} + {{- .Values.backup.envFrom | default list | concat (.Values.envFrom | default list) | toYaml | nindent 10 -}} + {{- end }} ports: - - containerPort: 8008 - name: patroni - - containerPort: 5432 - name: postgresql + - containerPort: 8008 + name: patroni + - containerPort: 5432 + name: postgresql {{- if .Values.readinessProbe.enabled }} readinessProbe: exec: @@ -290,6 +296,11 @@ spec: mountPath: {{ .Values.persistentVolumes.wal.mountPath | quote }} subPath: {{ .Values.persistentVolumes.wal.subPath | quote }} {{- end }} + {{- if and .Values.backup.enabled .Values.persistentVolumes.backup.enabled }} + - name: backup-volume + mountPath: {{ (include "pgbackrest_repo1_path" $) }} + subPath: {{ .Values.persistentVolumes.backup.subPath | quote }} + {{- end }} {{- if .Values.sharedMemory.useMount }} - name: shared-memory mountPath: /dev/shm @@ -382,27 +393,32 @@ spec: - containerPort: 8081 name: pgbackrest volumeMounts: - - name: socket-directory - mountPath: {{ template "socket_directory" . }} - readOnly: true - - name: storage-volume - mountPath: {{ .Values.persistentVolumes.data.mountPath | quote }} - subPath: {{ .Values.persistentVolumes.data.subPath | quote }} - {{- if .Values.persistentVolumes.wal.enabled }} - - name: wal-volume - mountPath: {{ .Values.persistentVolumes.wal.mountPath | quote }} - subPath: {{ .Values.persistentVolumes.wal.subPath | quote }} - {{- end }} + - name: socket-directory + mountPath: {{ template "socket_directory" . }} + readOnly: true + - name: storage-volume + mountPath: {{ .Values.persistentVolumes.data.mountPath | quote }} + subPath: {{ .Values.persistentVolumes.data.subPath | quote }} + {{- if .Values.persistentVolumes.wal.enabled }} + - name: wal-volume + mountPath: {{ .Values.persistentVolumes.wal.mountPath | quote }} + subPath: {{ .Values.persistentVolumes.wal.subPath | quote }} + {{- end }} + {{- if and .Values.backup.enabled .Values.persistentVolumes.backup.enabled }} + - name: backup-volume + mountPath: {{ (include "pgbackrest_repo1_path" $) }} + subPath: {{ .Values.persistentVolumes.backup.subPath | quote }} + {{- end }} {{- range $tablespaceName := ( .Values.persistentVolumes.tablespaces | default dict | keys ) }} - - name: {{ $tablespaceName }} - mountPath: {{ printf "%s/%s" (include "tablespaces_dir" $) $tablespaceName }} + - name: {{ $tablespaceName }} + mountPath: {{ printf "%s/%s" (include "tablespaces_dir" $) $tablespaceName }} {{- end }} - - mountPath: /etc/pgbackrest - name: pgbackrest - readOnly: true - - mountPath: {{ template "scripts_dir" . }} - name: timescaledb-scripts - readOnly: true + - name: pgbackrest + mountPath: /etc/pgbackrest + readOnly: true + - name: timescaledb-scripts + mountPath: {{ template "scripts_dir" . }} + readOnly: true env: - name: PGHOST value: {{ template "socket_directory" . }} @@ -410,17 +426,21 @@ spec: value: poddb - name: PGBACKREST_CONFIG value: /etc/pgbackrest/pgbackrest.conf + {{- if and .Values.backup.enabled .Values.persistentVolumes.backup.enabled }} + - name: PGBACKREST_REPO1_PATH + value: {{ template "pgbackrest_repo1_path" . }} + {{- end }} {{- if .Values.backup.env }}{{ .Values.backup.env | default list | toYaml | nindent 10 }}{{- end }} envFrom: - - secretRef: - name: {{ template "secrets_credentials" . }} - optional: false - - secretRef: - name: {{ template "secrets_pgbackrest" . }} - optional: false - {{- if or .Values.envFrom .Values.backup.envFrom -}} - {{- .Values.backup.envFrom | default list | concat (.Values.envFrom | default list) | toYaml | nindent 8 -}} - {{- end }} + - secretRef: + name: {{ template "secrets_credentials" . }} + optional: false + - secretRef: + name: {{ template "secrets_pgbackrest" . }} + optional: true + {{- if or .Values.envFrom .Values.backup.envFrom -}} + {{- .Values.backup.envFrom | default list | concat (.Values.envFrom | default list) | toYaml | nindent 10 -}} + {{- end }} {{ end }} {{- if .Values.prometheus.enabled }} @@ -510,6 +530,11 @@ spec: name: {{ template "timescaledb.fullname" . }}-pgbackrest defaultMode: 416 # 0640 permissions optional: true + {{- if and .Values.backup.enabled .Values.persistentVolumes.backup.enabled }} + - name: backup-volume + persistentVolumeClaim: + claimName: {{ template "pgbackrest_backup_pvc" . }} + {{- end }} - name: certificate secret: secretName: {{ template "secrets_certificate" . }} @@ -529,8 +554,8 @@ spec: {{- if .Values.persistentVolumes.data.enabled }} - metadata: name: storage-volume - annotations: {{- if .Values.persistentVolumes.data.annotations }} + annotations: {{ toYaml .Values.persistentVolumes.data.annotations | indent 10 }} {{- end }} labels: @@ -556,8 +581,8 @@ spec: {{- if .Values.persistentVolumes.wal.enabled }} - metadata: name: wal-volume - annotations: {{- if .Values.persistentVolumes.wal.annotations }} + annotations: {{ toYaml .Values.persistentVolumes.wal.annotations | indent 10 }} {{- end }} labels: @@ -583,8 +608,10 @@ spec: {{- range $tablespaceName, $volume := ($.Values.persistentVolumes.tablespaces | default dict ) }} - metadata: name: {{ $tablespaceName }} + {{- if $volume.annotations }} annotations: {{ $volume.annotations | default dict | toYaml | indent 10 }} + {{- end }} labels: app: {{ template "timescaledb.fullname" $ }} release: {{ $.Release.Name }} diff --git a/charts/timescaledb-single/values.schema.json b/charts/timescaledb-single/values.schema.json index be0ea471..62a346a6 100644 --- a/charts/timescaledb-single/values.schema.json +++ b/charts/timescaledb-single/values.schema.json @@ -543,6 +543,9 @@ }, "wal": { "$ref": "#/definitions/volume" + }, + "backup": { + "$ref": "#/definitions/volume" } }, "type": "object" diff --git a/charts/timescaledb-single/values.schema.yaml b/charts/timescaledb-single/values.schema.yaml index 4f7f47d4..34bfeca7 100644 --- a/charts/timescaledb-single/values.schema.yaml +++ b/charts/timescaledb-single/values.schema.yaml @@ -117,6 +117,8 @@ properties: type: object additionalProperties: "$ref": "#/definitions/volume" + backup: + "$ref": "#/definitions/volume" readinessProbe: type: object additionalProperties: false diff --git a/charts/timescaledb-single/values.yaml b/charts/timescaledb-single/values.yaml index a907b1bb..6439e6c1 100644 --- a/charts/timescaledb-single/values.yaml +++ b/charts/timescaledb-single/values.yaml @@ -370,9 +370,9 @@ persistentVolumes: enabled: true size: 2Gi ## database data Persistent Volume Storage Class - ## If defined, storageClassName: - ## If set to "-", storageClassName: "", which disables dynamic provisioning - ## If undefined (the default) or set to null, no storageClassName spec is + ## If defined, storageClass: + ## If set to "-", storageClass: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClass spec is ## set, choosing the default provisioner. (gp2 on AWS, standard on ## GKE, AWS & OpenStack) ## @@ -403,6 +403,19 @@ persistentVolumes: # example2: # size: 5Gi # storageClass: gp2 + # If you want to use [repo-type](https://pgbackrest.org/configuration.html#section-repository/option-repo-type) `posix` with pgbackrest + # enable this persistentVolume and apply a storageClass that supports `ReadWriteMany` (e.g. NFS see https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes) + # Also you need to adjust `Values.backup.pgbackrest` to your needs (i.e. at least set the repo-type). + # It is recommended to remove all S3 related environment variables, values from config maps and secrets. + # `mountPath` and `subPath` are not used for this volume. The value for `mountPath` is generated and provided as template (`pgbackrest_repo1_path`) + # used for volume mounts and the `PGBACKREST_REPO1_PATH` environment variable (except when bootstraping from backup) + backup: + enabled: false + size: 10Gi + # storageClass: "-" + annotations: {} + accessModes: + - ReadWriteMany # EXPERIMENTAL, please do *not* enable on production environments # if enabled, fullWalPrevention will switch the default transaction mode from read write diff --git a/charts/timescaledb-single/values/pgbackrest_backupvolume.example.yaml b/charts/timescaledb-single/values/pgbackrest_backupvolume.example.yaml new file mode 100644 index 00000000..9dcf4d3c --- /dev/null +++ b/charts/timescaledb-single/values/pgbackrest_backupvolume.example.yaml @@ -0,0 +1,35 @@ +# This file and its contents are licensed under the Apache License 2.0. +# Please see the included NOTICE for copyright information and LICENSE for a copy of the license. + +version: 14 + +image: + tag: pg14.1-ts2.5.1-oss-p2 + pullPolicy: IfNotPresent + +secrets: + pgbackrest: + # deleting s3 props from default values (see https://helm.sh/docs/chart_template_guide/values_files/#deleting-a-default-key) + PGBACKREST_REPO1_S3_REGION: null + PGBACKREST_REPO1_S3_KEY: null + PGBACKREST_REPO1_S3_KEY_SECRET: null + PGBACKREST_REPO1_S3_BUCKET: null + PGBACKREST_REPO1_S3_ENDPOINT: null + +backup: + enabled: true + pgBackRest: + # https://pgbackrest.org/configuration.html + # Although not impossible, care should be taken not to include secrets + # in these parameters. Use Kubernetes Secrets to specify S3 Keys, Secrets etc. + # deleting s3 props from default values (see https://helm.sh/docs/chart_template_guide/values_files/#deleting-a-default-key) + repo1-s3-region: null + repo1-s3-endpoint: null + # since the default repo-type is `posix` we could also delete this key - but being explicit about this also has advantages + repo1-type: posix + +persistentVolumes: + backup: + enabled: true + # creating that storageClass is not scope of this chart + storageClass: nfs