From a23a070c41cb4d2e9196e09e8b76ab42ce8b907a Mon Sep 17 00:00:00 2001 From: Mike Yoder Date: Sun, 13 Jul 2025 15:34:45 -0700 Subject: [PATCH] Modify the shield helm chart to work with cert-manager The certificates in the shield helm charts do not play nicely with argocd. Since argocd will essentially call "helm template" to acquire the chart's yaml, the "lookup" function will never work, and the certificate generation will create a different certificate each time. It's possible to tell ArgoCD to ignore this ever-changing certificate, but this also means that the certificate will never be rotated. The most graceful solution to this is to integrate with the widely-used cert-manager project. See https://github.com/cert-manager/cert-manager The idea is that you can create a certificate chain using cert-manager, and then refer to those certificate in the helm chart instead of asking the helm chart to generate them. For our ppurposes there are two interesting cert-manager resources. The "Issuer" is a resource that is used to sign a certificate. A "Certificate" is actually a request to create an x509 certificate. It emits a k8s Secret containing the actual TLS certificate details. See: * https://cert-manager.io/docs/concepts/issuer/ * https://cert-manager.io/docs/usage/certificate/ Imagine that the customer has the following setup: * A) a "self-signed" root Issuer * B) a "CA" Certificate representing the root (using A above). It will be long-lived, like all root CAs are. * C) a "root cert" Secret containing the actual x509 root CA cert (created by cert-manager as a result of B) * D) a "CA" Issuer that uses the Secret from C above At this point there is a long-lived root x509 certificate as well as an Issuer that will create certificates signed by that root. At this point, the shield helm chart can include a Certificate resource that refers to the Issuer from step D above. This will generate a Secret with the actual x509 cert. This secret can be used by the shield Deployment for the TLS webhook port. Additionally, the validating webhook can refer to the CA Certificate resource from step B above. It uses an annotation that tells cert-manger to edit the webhook and insert the appropriate caBundle information. --- charts/shield/templates/cluster/_tls.tpl | 44 +++- .../shield/templates/cluster/certificate.yaml | 39 ++++ ...ls-certificates-admissionregistration.yaml | 15 ++ ...rtificates-admissionregistration_test.yaml | 31 +++ .../tls-certificates-cert-manager_test.yaml | 192 ++++++++++++++++++ charts/shield/values.yaml | 9 + 6 files changed, 329 insertions(+), 1 deletion(-) create mode 100644 charts/shield/templates/cluster/certificate.yaml create mode 100644 charts/shield/tests/cluster/tls-certificates-cert-manager_test.yaml diff --git a/charts/shield/templates/cluster/_tls.tpl b/charts/shield/templates/cluster/_tls.tpl index 4b42ae8fd..453a9e437 100644 --- a/charts/shield/templates/cluster/_tls.tpl +++ b/charts/shield/templates/cluster/_tls.tpl @@ -9,7 +9,7 @@ {{- end }} {{- define "cluster.tls_certificates.secret_name" -}} - {{- if .Values.cluster.tls_certificates.create -}} + {{- if or .Values.cluster.tls_certificates.create .Values.cluster.tls_certificates.cert_manager.enabled -}} {{- include "cluster.fullname" . }}-tls-certificates {{- else if .Values.cluster.tls_certificates.secret_name -}} {{- .Values.cluster.tls_certificates.secret_name -}} @@ -18,6 +18,48 @@ {{- end -}} {{- end }} +{{- define "cluster.tls_certificates.check_conflicts" -}} + {{- if and .Values.cluster.tls_certificates.create .Values.cluster.tls_certificates.cert_manager.enabled -}} + {{- fail "Cannot specify both tls_certificates.create and tls_certificates.cert_manager.enabled" -}} + {{- end -}} + {{- if and (not (quote .Values.cluster.tls_certificates.secret_name | empty)) .Values.cluster.tls_certificates.cert_manager.enabled -}} + {{- fail "Cannot specify both tls_certificates.cert_manager.enabled and tls_certificates.secret_name" -}} + {{- end -}} +{{- end }} + +{{- define "cluster.tls_certificates.cert_manager_enabled" -}} + {{- if .Values.cluster | dig "tls_certificates" "cert_manager" "enabled" false -}} + {{- if .Values.cluster | dig "tls_certificates" "create" true -}} + {{- fail "Cannot specify both tls_certificates.create and tls_certificates.cert_manager.enabled" -}} + {{- end -}} + {{- if .Values.cluster | dig ".tls_certificates" "secret_name" "" -}} + {{- fail "Cannot specify both tls_certificates.secret_name and tls_certificates.cert_manager.enabled" -}} + {{- end -}} + true + {{- else -}} + false + {{- end -}} +{{- end }} + +{{- define "cluster.tls_certificates.cert_manager_certificate_name" -}} + shield-cluster-certificate +{{- end }} + +{{- define "cluster.tls_certificates.cert_manager_issuer_name" -}} + {{- if not .Values.cluster.tls_certificates.cert_manager.issuer_name -}} + {{- fail "cert_manager.issuer_name must be specified when cert_manager.enabled is true" -}} + {{- end -}} + {{- .Values.cluster.tls_certificates.cert_manager.issuer_name -}} +{{- end }} + +{{- define "cluster.tls_certificates.cert_manager_ca_cert_name" -}} + {{- if .Values.cluster.tls_certificates.cert_manager.ca_certificate_name -}} + {{- .Values.cluster.tls_certificates.cert_manager.ca_certificate_name -}} + {{- else -}} + {{- include "cluster.tls_certificates.cert_manager_certificate_name" . -}} + {{- end -}} +{{- end }} + {{- define "cluster.tls_certificates.mount_path" -}} /etc/sysdig/tls-certificates/ {{- end }} diff --git a/charts/shield/templates/cluster/certificate.yaml b/charts/shield/templates/cluster/certificate.yaml new file mode 100644 index 000000000..5d4d7af58 --- /dev/null +++ b/charts/shield/templates/cluster/certificate.yaml @@ -0,0 +1,39 @@ +{{- if .Values.cluster | dig "tls_certificates" "cert_manager" "enabled" false -}} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ include "cluster.tls_certificates.cert_manager_certificate_name" . }} + namespace: {{ .Release.Namespace }} +spec: + secretName: {{ include "cluster.tls_certificates.secret_name" . }} + + privateKey: + algorithm: RSA + encoding: PKCS1 + size: 2048 + + duration: 720h + renewBefore: 360h + + isCA: false + usages: + - server auth + - client auth + + subject: + organizations: + - "Sysdig, Inc." + + dnsNames: + {{- range $v := include "cluster.tls_certificates.dns_names" . | fromYamlArray }} + - {{ $v | quote }} + {{- end }} + + ipAddresses: + - 127.0.0.1 + + # Issuer references are always required. + issuerRef: + name: {{ include "cluster.tls_certificates.cert_manager_issuer_name" . }} + kind: Issuer +{{- end }} diff --git a/charts/shield/templates/cluster/tls-certificates-admissionregistration.yaml b/charts/shield/templates/cluster/tls-certificates-admissionregistration.yaml index 12331687b..4560159cc 100644 --- a/charts/shield/templates/cluster/tls-certificates-admissionregistration.yaml +++ b/charts/shield/templates/cluster/tls-certificates-admissionregistration.yaml @@ -1,5 +1,7 @@ {{- if (include "cluster.tls_certificates.required" .) }} +{{- include "cluster.tls_certificates.check_conflicts" . -}} {{- $cert := dict -}} +{{- if not (.Values.cluster | dig "tls_certificates" "cert_manager" "enabled" false) -}} {{- $existingTlsCertificatesSecret := lookup "v1" "Secret" .Release.Namespace (include "cluster.tls_certificates.secret_name" .) -}} {{- if $existingTlsCertificatesSecret -}} {{- $_ := set $cert "Cert" (index $existingTlsCertificatesSecret.data (include "cluster.tls_certificates.cert_file_name" .)) -}} @@ -32,12 +34,17 @@ data: {{ include "cluster.tls_certificates.private_key_file_name" . }}: {{ $cert.Key }} {{ include "cluster.tls_certificates.ca_cert_file_name" . }}: {{ $cert.CACert }} {{- end }} +{{- end }} {{- if and .Values.cluster.validatingwebhookconfiguration.create (include "cluster.audit_enabled" .) }} --- apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: name: {{ include "cluster.audit_webhook_name" . }} + {{- if .Values.cluster | dig "tls_certificates" "cert_manager" "enabled" false }} + annotations: + cert-manager.io/inject-ca-from: {{ (printf "%s/%s" .Release.Namespace (include "cluster.tls_certificates.cert_manager_ca_cert_name" .)) | quote }} + {{- end }} labels: {{- include "cluster.labels" . | nindent 4 }} webhooks: @@ -63,7 +70,9 @@ webhooks: name: {{ include "cluster.service_name" . }} path: /k8s-audit port: {{ include "cluster.audit_service_port" . }} + {{- if not (.Values.cluster | dig "tls_certificates" "cert_manager" "enabled" false) }} caBundle: {{ $cert.CACert }} + {{- end }} admissionReviewVersions: ["v1", "v1beta1"] sideEffects: None timeoutSeconds: {{ .Values.features.detections.kubernetes_audit.timeout }} @@ -75,6 +84,10 @@ apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: name: {{ include "cluster.admission_control_webhook_name" . }} + {{- if .Values.cluster | dig "tls_certificates" "cert_manager" "enabled" false }} + annotations: + cert-manager.io/inject-ca-from: {{ (printf "%s/%s" .Release.Namespace (include "cluster.tls_certificates.cert_manager_ca_cert_name" .)) | quote }} + {{- end }} labels: {{- include "cluster.labels" . | nindent 4 }} webhooks: @@ -112,7 +125,9 @@ webhooks: name: {{ include "cluster.service_name" . }} path: /validate port: {{ include "cluster.admission_control_service_port" . }} + {{- if not (.Values.cluster | dig "tls_certificates" "cert_manager" "enabled" false) }} caBundle: {{ $cert.CACert }} + {{- end }} admissionReviewVersions: ["v1", "v1beta1"] sideEffects: None timeoutSeconds: {{ .Values.features.admission_control.timeout }} diff --git a/charts/shield/tests/cluster/tls-certificates-admissionregistration_test.yaml b/charts/shield/tests/cluster/tls-certificates-admissionregistration_test.yaml index a249bfc4e..2e6463f13 100644 --- a/charts/shield/tests/cluster/tls-certificates-admissionregistration_test.yaml +++ b/charts/shield/tests/cluster/tls-certificates-admissionregistration_test.yaml @@ -635,3 +635,34 @@ tests: - pods scope: Namespaced documentIndex: 1 + + - it: Cert Manager Integration exclusive with tls create cert + set: + features: + detections: + kubernetes_audit: + enabled: true + cluster: + tls_certificates: + create: true + cert_manager: + enabled: true + asserts: + - failedTemplate: + errorPattern: "Cannot specify both" + + - it: Cert Manager Integration exclusive with tls secret name + set: + features: + detections: + kubernetes_audit: + enabled: true + cluster: + tls_certificates: + secret_name: foo + cert_manager: + enabled: true + asserts: + - failedTemplate: + errorPattern: "Cannot specify both" + diff --git a/charts/shield/tests/cluster/tls-certificates-cert-manager_test.yaml b/charts/shield/tests/cluster/tls-certificates-cert-manager_test.yaml new file mode 100644 index 000000000..7c507a79b --- /dev/null +++ b/charts/shield/tests/cluster/tls-certificates-cert-manager_test.yaml @@ -0,0 +1,192 @@ +suite: Cluster - TLS Certificates Cert Manager Integration +templates: + - templates/cluster/tls-certificates-admissionregistration.yaml + - templates/cluster/certificate.yaml +release: + name: shield-release + namespace: shield-namespace +values: + - ../values/base.yaml + +tests: + - it: Cert Manager Integration Basic Audit + set: + features: + detections: + kubernetes_audit: + enabled: true + cluster: + tls_certificates: + create: false + cert_manager: + enabled: true + issuer_name: shield-cluster-issuer + asserts: + - hasDocuments: + count: 1 + template: templates/cluster/certificate.yaml + - hasDocuments: + count: 1 + template: templates/cluster/tls-certificates-admissionregistration.yaml + - containsDocument: + kind: Certificate + apiVersion: cert-manager.io/v1 + name: shield-cluster-certificate + namespace: shield-namespace + template: templates/cluster/certificate.yaml + - containsDocument: + kind: ValidatingWebhookConfiguration + apiVersion: admissionregistration.k8s.io/v1 + name: shield-release-cluster-audit + template: templates/cluster/tls-certificates-admissionregistration.yaml + - contains: + path: spec.dnsNames + content: + localhost + template: templates/cluster/certificate.yaml + - contains: + path: spec.dnsNames + content: + shield-release-cluster + template: templates/cluster/certificate.yaml + - contains: + path: spec.dnsNames + content: + '*.shield-release-cluster' + template: templates/cluster/certificate.yaml + - contains: + path: spec.dnsNames + content: + '*.shield-release-cluster.shield-namespace.svc' + template: templates/cluster/certificate.yaml + - contains: + path: spec.dnsNames + content: + '*.shield-release-cluster-container-vm.shield-namespace.svc.cluster.local' + template: templates/cluster/certificate.yaml + - equal: + path: spec.secretName + value: shield-release-cluster-tls-certificates + template: templates/cluster/certificate.yaml + - equal: + path: spec.issuerRef.name + value: shield-cluster-issuer + template: templates/cluster/certificate.yaml + - equal: + path: metadata.annotations["cert-manager.io/inject-ca-from"] + value: shield-namespace/shield-cluster-certificate + template: templates/cluster/tls-certificates-admissionregistration.yaml + - notExists: + path: webhooks[0].clientConfig.caBundle + template: templates/cluster/tls-certificates-admissionregistration.yaml + + - it: Cert Manager Integration Basic Admission + set: + features: + admission_control: + enabled: true + cluster: + tls_certificates: + create: false + cert_manager: + enabled: true + issuer_name: shield-cluster-issuer + asserts: + - hasDocuments: + count: 1 + template: templates/cluster/certificate.yaml + - hasDocuments: + count: 1 + template: templates/cluster/tls-certificates-admissionregistration.yaml + - containsDocument: + kind: Certificate + apiVersion: cert-manager.io/v1 + name: shield-cluster-certificate + namespace: shield-namespace + template: templates/cluster/certificate.yaml + - containsDocument: + kind: ValidatingWebhookConfiguration + apiVersion: admissionregistration.k8s.io/v1 + name: shield-release-cluster-admission-control + template: templates/cluster/tls-certificates-admissionregistration.yaml + - equal: + path: metadata.annotations["cert-manager.io/inject-ca-from"] + value: shield-namespace/shield-cluster-certificate + template: templates/cluster/tls-certificates-admissionregistration.yaml + - notExists: + path: webhooks[0].clientConfig.caBundle + template: templates/cluster/tls-certificates-admissionregistration.yaml + + - it: Cert Manager Integration CA Cert + set: + features: + detections: + kubernetes_audit: + enabled: true + cluster: + tls_certificates: + create: false + cert_manager: + enabled: true + issuer_name: shield-cluster-issuer + ca_certificate_name: shield-cluster-ca-certificate + asserts: + - hasDocuments: + count: 1 + template: templates/cluster/certificate.yaml + - hasDocuments: + count: 1 + template: templates/cluster/tls-certificates-admissionregistration.yaml + - containsDocument: + kind: Certificate + apiVersion: cert-manager.io/v1 + name: shield-cluster-certificate + namespace: shield-namespace + template: templates/cluster/certificate.yaml + - containsDocument: + kind: ValidatingWebhookConfiguration + apiVersion: admissionregistration.k8s.io/v1 + name: shield-release-cluster-audit + template: templates/cluster/tls-certificates-admissionregistration.yaml + - contains: + path: spec.dnsNames + content: + localhost + template: templates/cluster/certificate.yaml + - contains: + path: spec.dnsNames + content: + shield-release-cluster + template: templates/cluster/certificate.yaml + - contains: + path: spec.dnsNames + content: + '*.shield-release-cluster' + template: templates/cluster/certificate.yaml + - contains: + path: spec.dnsNames + content: + '*.shield-release-cluster.shield-namespace.svc' + template: templates/cluster/certificate.yaml + - contains: + path: spec.dnsNames + content: + '*.shield-release-cluster-container-vm.shield-namespace.svc.cluster.local' + template: templates/cluster/certificate.yaml + - equal: + path: spec.secretName + value: shield-release-cluster-tls-certificates + template: templates/cluster/certificate.yaml + - equal: + path: spec.issuerRef.name + value: shield-cluster-issuer + template: templates/cluster/certificate.yaml + - equal: + path: metadata.annotations["cert-manager.io/inject-ca-from"] + value: shield-namespace/shield-cluster-ca-certificate + template: templates/cluster/tls-certificates-admissionregistration.yaml + - notExists: + path: webhooks[0].clientConfig.caBundle + template: templates/cluster/tls-certificates-admissionregistration.yaml + + diff --git a/charts/shield/values.yaml b/charts/shield/values.yaml index e70e82858..fad126853 100644 --- a/charts/shield/values.yaml +++ b/charts/shield/values.yaml @@ -427,6 +427,15 @@ cluster: create: true # The name of the secret that contains the TLS certificates secret_name: + # Ignore the above and use cert-manager to create the TLS certificates + cert_manager: + # Use cert-manager to create the TLS certificates + enabled: false + # The name of the cert-manager Issuer to use for the TLS certificates + issuer_name: + # If using a self-signed Issuer above, leave this empty. If using a CA Issuer, + # provide the name of the Certificate used to create the CA. + ca_certificate_name: resources: requests: # The CPU request for the cluster shield