From 7cf118983b3459a3c0dfc0e5998b940fd2836d7c Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Wed, 29 Jun 2022 17:17:35 +0300 Subject: [PATCH 1/5] Create a job to create the users --- .github/workflows/operatorBuildAndDeploy.yml | 7 ++- .../v1beta1/cassandradatacenter_types.go | 15 +++++ .../v1beta1/zz_generated.deepcopy.go | 27 ++++++++ ...dra.datastax.com_cassandradatacenters.yaml | 9 +++ config/manager/kustomization.yaml | 2 +- config/rbac/role.yaml | 12 ++++ .../cassandradatacenter_controller.go | 1 + pkg/reconciliation/reconcile_racks.go | 62 ++++++++++++++++++- ...ault-single-rack-single-node-dc-vault.yaml | 36 +++++++++++ 9 files changed, 164 insertions(+), 7 deletions(-) create mode 100644 tests/testdata/default-single-rack-single-node-dc-vault.yaml diff --git a/.github/workflows/operatorBuildAndDeploy.yml b/.github/workflows/operatorBuildAndDeploy.yml index ec3133598..ca13acc03 100644 --- a/.github/workflows/operatorBuildAndDeploy.yml +++ b/.github/workflows/operatorBuildAndDeploy.yml @@ -71,7 +71,7 @@ jobs: with: file: Dockerfile context: . - push: ${{ github.event_name != 'pull_request' }} + push: ${{ github.event_name != 'pull_request' && !env.ACT }} tags: k8ssandra/cass-operator:${{ steps.vars.outputs.sha_short }}, k8ssandra/cass-operator:latest, k8ssandra/cass-operator:${{ steps.vars.outputs.version }} platforms: linux/amd64 cache-from: type=local,src=/tmp/.buildx-cache @@ -81,7 +81,7 @@ jobs: uses: docker/build-push-action@v3 with: file: logger.Dockerfile - push: ${{ github.event_name != 'pull_request' }} + push: ${{ github.event_name != 'pull_request' && !env.ACT }} tags: k8ssandra/system-logger:${{ steps.vars.outputs.sha_short }}, k8ssandra/system-logger:latest platforms: linux/amd64 cache-from: type=local,src=/tmp/.buildx-cache @@ -106,11 +106,12 @@ jobs: build-args: | VERSION=${{ steps.vars.outputs.version }} context: . - push: ${{ !env.ACT }} + push: ${{ github.event_name != 'pull_request' && !env.ACT }} tags: k8ssandra/cass-operator-bundle:v${{ steps.vars.outputs.version }} platforms: linux/amd64 cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache - name: Build and update cass-operator-catalog run: | + # Load the images here? It wants the bundle.. make VERSION=${{ steps.vars.outputs.version }} CHANNEL=dev catalog-build catalog-push diff --git a/apis/cassandra/v1beta1/cassandradatacenter_types.go b/apis/cassandra/v1beta1/cassandradatacenter_types.go index 0e9f7dfd7..721f25d1d 100644 --- a/apis/cassandra/v1beta1/cassandradatacenter_types.go +++ b/apis/cassandra/v1beta1/cassandradatacenter_types.go @@ -77,6 +77,19 @@ type CassandraUser struct { Superuser bool `json:"superuser"` } +type UserInfo struct { + Annotations map[string]string `json:"annotations,omitempty"` + ServiceAccount string `json:"serviceAccountName,omitempty"` + // TODO Add CSI information here + /* + TODO Testing: + * Add back the Helm utilities + * Install Vault + Vault server there + * Install CSI driver + * Try to create clusters with both methods and see that it succeeds (and uses Vault to fetch the user rights) + */ +} + // CassandraDatacenterSpec defines the desired state of a CassandraDatacenter // +k8s:openapi-gen=true // +kubebuilder:pruning:PreserveUnknownFields @@ -184,6 +197,8 @@ type CassandraDatacenterSpec struct { // If it is omitted, we will generate a secret instead. SuperuserSecretName string `json:"superuserSecretName,omitempty"` + UserInfo *UserInfo `json:"userInfo,omitempty"` + // The k8s service account to use for the server pods ServiceAccount string `json:"serviceAccount,omitempty"` diff --git a/apis/cassandra/v1beta1/zz_generated.deepcopy.go b/apis/cassandra/v1beta1/zz_generated.deepcopy.go index 9cb1ed334..58c5977fc 100644 --- a/apis/cassandra/v1beta1/zz_generated.deepcopy.go +++ b/apis/cassandra/v1beta1/zz_generated.deepcopy.go @@ -285,6 +285,11 @@ func (in *CassandraDatacenterSpec) DeepCopyInto(out *CassandraDatacenterSpec) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.UserInfo != nil { + in, out := &in.UserInfo, &out.UserInfo + *out = new(UserInfo) + (*in).DeepCopyInto(*out) + } if in.NodeSelector != nil { in, out := &in.NodeSelector, &out.NodeSelector *out = make(map[string]string, len(*in)) @@ -667,3 +672,25 @@ func (in *StorageConfig) DeepCopy() *StorageConfig { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *UserInfo) DeepCopyInto(out *UserInfo) { + *out = *in + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UserInfo. +func (in *UserInfo) DeepCopy() *UserInfo { + if in == nil { + return nil + } + out := new(UserInfo) + in.DeepCopyInto(out) + return out +} diff --git a/config/crd/bases/cassandra.datastax.com_cassandradatacenters.yaml b/config/crd/bases/cassandra.datastax.com_cassandradatacenters.yaml index b42060c48..5f7e72b4f 100644 --- a/config/crd/bases/cassandra.datastax.com_cassandradatacenters.yaml +++ b/config/crd/bases/cassandra.datastax.com_cassandradatacenters.yaml @@ -8072,6 +8072,15 @@ spec: type: string type: object type: array + userInfo: + properties: + annotations: + additionalProperties: + type: string + type: object + serviceAccountName: + type: string + type: object users: description: Cassandra users to bootstrap items: diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 7621efdaa..ab5cc780a 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -14,4 +14,4 @@ kind: Kustomization images: - name: controller newName: k8ssandra/cass-operator - newTag: latest + newTag: v1.12.0-dev.28b8ea7-20220629 diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 8ee7de45f..4d7e1a528 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -5,6 +5,18 @@ metadata: creationTimestamp: null name: manager-role rules: +- apiGroups: + - batch + resources: + - jobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - "" resources: diff --git a/controllers/cassandra/cassandradatacenter_controller.go b/controllers/cassandra/cassandradatacenter_controller.go index 16d00c374..62e94b293 100644 --- a/controllers/cassandra/cassandradatacenter_controller.go +++ b/controllers/cassandra/cassandradatacenter_controller.go @@ -57,6 +57,7 @@ import ( // +kubebuilder:rbac:groups=core,namespace=cass-operator,resources=pods;endpoints;services;configmaps;secrets;persistentvolumeclaims;events,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=core,namespace=cass-operator,resources=namespaces,verbs=get // +kubebuilder:rbac:groups=core,resources=persistentvolumes;nodes,verbs=get;list;watch +// +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=policy,namespace=cass-operator,resources=poddisruptionbudgets,verbs=get;list;watch;create;update;patch;delete // CassandraDatacenterReconciler reconciles a cassandraDatacenter object diff --git a/pkg/reconciliation/reconcile_racks.go b/pkg/reconciliation/reconcile_racks.go index 38d6da6f1..bc1b7de3c 100644 --- a/pkg/reconciliation/reconcile_racks.go +++ b/pkg/reconciliation/reconcile_racks.go @@ -12,6 +12,7 @@ import ( "time" appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -105,6 +106,14 @@ func (rc *ReconciliationContext) CalculateRackInformation() error { func (rc *ReconciliationContext) CheckSuperuserSecretCreation() result.ReconcileResult { rc.ReqLogger.Info("reconcile_racks::CheckSuperuserSecretCreation") + if rc.Datacenter.Spec.UserInfo != nil { + return result.Continue() + } + + if rc.IsInitialized() { + return result.Continue() + } + _, err := rc.retrieveSuperuserSecretOrCreateDefault() if err != nil { rc.ReqLogger.Error(err, "error retrieving SuperuserSecret for CassandraDatacenter.") @@ -876,6 +885,10 @@ func (rc *ReconciliationContext) UpdateSecretWatches() error { func (rc *ReconciliationContext) CreateUsers() result.ReconcileResult { dc := rc.Datacenter + if rc.IsInitialized() { + return result.Continue() + } + if val, found := dc.Annotations[api.SkipUserCreationAnnotation]; found && val == "true" { rc.ReqLogger.Info(api.SkipUserCreationAnnotation + " is set, skipping CreateUser") return result.Continue() @@ -888,15 +901,58 @@ func (rc *ReconciliationContext) CreateUsers() result.ReconcileResult { rc.ReqLogger.Info("reconcile_racks::CreateUsers") + // TODO This should be cleaned up after a while (TTL) + when I delete CassandraDatacenter, so we need ownership also. And cass-operator labels + if dc.Spec.UserInfo != nil { + // Create the job + // TODO wait for it to complete before we continue.. + job := batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: dc.Namespace, + Name: fmt.Sprintf("usercreate-%s", dc.Name), + }, + Spec: batchv1.JobSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: dc.Spec.UserInfo.Annotations, + }, + Spec: corev1.PodSpec{ + // TODO Add volumes here if CSI was used + Containers: []corev1.Container{ + { + Name: "client", + Image: "burmanm/k8ssandra-client:latest", // k8ssandra/k8ssandra-client does not have this feature + ImagePullPolicy: corev1.PullIfNotPresent, + Args: []string{"users", "add", "--path", "/vault/secrets/database-config.txt", "--dc", dc.Name}, // TODO This path must be configurable + }, + }, + RestartPolicy: corev1.RestartPolicyNever, + }, + // Create the k8ssandra-client add users job here? + }, + }, + } + + // TODO Do I have this value somewhere..? + job.Spec.Template.Spec.ServiceAccountName = "cass-operator-controller-manager" + + if err := rc.Client.Create(rc.Ctx, &job); err != nil { + return result.Error(err) + } + return result.Continue() + } + err := rc.UpdateSecretWatches() if err != nil { rc.ReqLogger.Error(err, "Failed to update dynamic watches on secrets") } // make sure the default superuser secret exists - _, err = rc.retrieveSuperuserSecretOrCreateDefault() - if err != nil { - rc.ReqLogger.Error(err, "Failed to verify superuser secret status") + // TODO Nope.. + if dc.Spec.UserInfo == nil { + _, err = rc.retrieveSuperuserSecretOrCreateDefault() + if err != nil { + rc.ReqLogger.Error(err, "Failed to verify superuser secret status") + } } users := rc.GetUsers() diff --git a/tests/testdata/default-single-rack-single-node-dc-vault.yaml b/tests/testdata/default-single-rack-single-node-dc-vault.yaml new file mode 100644 index 000000000..d1330ba0f --- /dev/null +++ b/tests/testdata/default-single-rack-single-node-dc-vault.yaml @@ -0,0 +1,36 @@ +apiVersion: cassandra.datastax.com/v1beta1 +kind: CassandraDatacenter +metadata: + name: dc2 +spec: + clusterName: cluster2 + serverType: cassandra + serverVersion: "4.0.4" + userInfo: + annotations: + vault.hashicorp.com/agent-inject: 'true' + vault.hashicorp.com/role: 'internal-app-cass' + vault.hashicorp.com/agent-inject-secret-database-config.txt: 'internal/data/database/config' + vault.hashicorp.com/agent-inject-template-database-config.txt: | + {{- with secret "internal/data/database/config" -}} + {{- range $k, $v := .Data.data }} + {{ $k }}={{ $v}} + {{- end }} + {{- end -}} + managementApiAuth: + insecure: {} + size: 1 + storageConfig: + cassandraDataVolumeClaimSpec: + storageClassName: standard + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + racks: + - name: r1 + config: + jvm-server-options: + initial_heap_size: "512m" + max_heap_size: "512m" From b2ea8c162ba7742fdb2ee133e427395fd1880759 Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Thu, 30 Jun 2022 13:53:09 +0300 Subject: [PATCH 2/5] Change test annotations, add controller ref, labels and TTL to the job --- pkg/reconciliation/reconcile_racks.go | 14 +++++++- tests/external_secret/README.md | 33 +++++++++++++++++++ ...ault-single-rack-single-node-dc-vault.yaml | 3 +- 3 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 tests/external_secret/README.md diff --git a/pkg/reconciliation/reconcile_racks.go b/pkg/reconciliation/reconcile_racks.go index bc1b7de3c..f0cdd3630 100644 --- a/pkg/reconciliation/reconcile_racks.go +++ b/pkg/reconciliation/reconcile_racks.go @@ -901,9 +901,10 @@ func (rc *ReconciliationContext) CreateUsers() result.ReconcileResult { rc.ReqLogger.Info("reconcile_racks::CreateUsers") - // TODO This should be cleaned up after a while (TTL) + when I delete CassandraDatacenter, so we need ownership also. And cass-operator labels + // TODO This should be cleaned up after a while (TTL) if dc.Spec.UserInfo != nil { // Create the job + ttl := int32(86400) // TODO wait for it to complete before we continue.. job := batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ @@ -929,9 +930,20 @@ func (rc *ReconciliationContext) CreateUsers() result.ReconcileResult { }, // Create the k8ssandra-client add users job here? }, + TTLSecondsAfterFinished: &ttl, }, } + labels := dc.GetClusterLabels() + oplabels.AddOperatorLabels(labels, dc) + job.ObjectMeta.Labels = labels + + // Set CassandraDatacenter dc as the owner and controller + err := setControllerReference(dc, &job, rc.Scheme) + if err != nil { + return result.Error(err) + } + // TODO Do I have this value somewhere..? job.Spec.Template.Spec.ServiceAccountName = "cass-operator-controller-manager" diff --git a/tests/external_secret/README.md b/tests/external_secret/README.md new file mode 100644 index 000000000..7fe4c3232 --- /dev/null +++ b/tests/external_secret/README.md @@ -0,0 +1,33 @@ +## Install Vault + +helm install vault hashicorp/vault --set "server.dev.enabled=true" --namespace cass-operator +# --set "csi.enabled=true" + +## Go into Vault and exec certain commands.. + +kubectl exec -it vault-0 -- /bin/sh + +vault secrets enable -path=internal kv-v2 + +vault kv put internal/database/config username="db-readonly-username" password="db-secret-password" + +vault auth enable kubernetes + +vault write auth/kubernetes/config \ + kubernetes_host="https://$KUBERNETES_PORT_443_TCP_ADDR:443" + +vault policy write internal-app - < Date: Fri, 1 Jul 2022 16:53:22 +0300 Subject: [PATCH 3/5] Add CSI support --- .../v1beta1/cassandradatacenter_types.go | 12 ++-- .../v1beta1/zz_generated.deepcopy.go | 5 ++ ...dra.datastax.com_cassandradatacenters.yaml | 58 +++++++++++++++++-- pkg/reconciliation/reconcile_racks.go | 49 ++++++++++++++-- tests/external_secret/README.md | 45 +++++++++++++- tests/external_secret/secret_provider.yaml | 13 +++++ ...ault-single-rack-single-node-dc-vault.yaml | 6 ++ 7 files changed, 173 insertions(+), 15 deletions(-) create mode 100644 tests/external_secret/secret_provider.yaml diff --git a/apis/cassandra/v1beta1/cassandradatacenter_types.go b/apis/cassandra/v1beta1/cassandradatacenter_types.go index 721f25d1d..b7e78b2ab 100644 --- a/apis/cassandra/v1beta1/cassandradatacenter_types.go +++ b/apis/cassandra/v1beta1/cassandradatacenter_types.go @@ -78,8 +78,12 @@ type CassandraUser struct { } type UserInfo struct { - Annotations map[string]string `json:"annotations,omitempty"` - ServiceAccount string `json:"serviceAccountName,omitempty"` + Annotations map[string]string `json:"annotations,omitempty"` + // MountPath tells the script where to read the user information. Required if annotation injection is used, otherwise optional + MountPath string `json:"mountpath,omitempty"` + CSI *corev1.CSIVolumeSource `json:"csi,omitempty"` + SecretName string `json:"secretName,omitempty"` + // ServiceAccount string `json:"serviceAccountName,omitempty"` // TODO Add CSI information here /* TODO Testing: @@ -193,13 +197,13 @@ type CassandraDatacenterSpec struct { // podAntiAffinity and requiredDuringSchedulingIgnoredDuringExecution. AllowMultipleNodesPerWorker bool `json:"allowMultipleNodesPerWorker,omitempty"` - // This secret defines the username and password for the Cassandra server superuser. + // SuperuserSecretName is deprecated. Use UserInfo instead. This secret defines the username and password for the Cassandra server superuser. // If it is omitted, we will generate a secret instead. SuperuserSecretName string `json:"superuserSecretName,omitempty"` UserInfo *UserInfo `json:"userInfo,omitempty"` - // The k8s service account to use for the server pods + // ServiceAccount is the k8s service account to use for the server pods ServiceAccount string `json:"serviceAccount,omitempty"` // DEPRECATED. Use CassandraTask for rolling restarts. Whether to do a rolling restart at the next opportunity. The operator will set this back diff --git a/apis/cassandra/v1beta1/zz_generated.deepcopy.go b/apis/cassandra/v1beta1/zz_generated.deepcopy.go index 58c5977fc..cf00e11cb 100644 --- a/apis/cassandra/v1beta1/zz_generated.deepcopy.go +++ b/apis/cassandra/v1beta1/zz_generated.deepcopy.go @@ -683,6 +683,11 @@ func (in *UserInfo) DeepCopyInto(out *UserInfo) { (*out)[key] = val } } + if in.CSI != nil { + in, out := &in.CSI, &out.CSI + *out = new(v1.CSIVolumeSource) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UserInfo. diff --git a/config/crd/bases/cassandra.datastax.com_cassandradatacenters.yaml b/config/crd/bases/cassandra.datastax.com_cassandradatacenters.yaml index 5f7e72b4f..005eb3c41 100644 --- a/config/crd/bases/cassandra.datastax.com_cassandradatacenters.yaml +++ b/config/crd/bases/cassandra.datastax.com_cassandradatacenters.yaml @@ -7614,7 +7614,8 @@ spec: pattern: (6\.8\.\d+)|(3\.11\.\d+)|(4\.\d+\.\d+) type: string serviceAccount: - description: The k8s service account to use for the server pods + description: ServiceAccount is the k8s service account to use for + the server pods type: string size: description: Desired number of Cassandra server nodes @@ -7996,9 +7997,9 @@ spec: type: object type: object superuserSecretName: - description: This secret defines the username and password for the - Cassandra server superuser. If it is omitted, we will generate a - secret instead. + description: SuperuserSecretName is deprecated. Use UserInfo instead. + This secret defines the username and password for the Cassandra + server superuser. If it is omitted, we will generate a secret instead. type: string systemLoggerImage: description: Container image for the log tailing sidecar container. @@ -8078,7 +8079,54 @@ spec: additionalProperties: type: string type: object - serviceAccountName: + csi: + description: Represents a source location of a volume to mount, + managed by an external CSI driver + properties: + driver: + description: Driver is the name of the CSI driver that handles + this volume. Consult with your admin for the correct name + as registered in the cluster. + type: string + fsType: + description: Filesystem type to mount. Ex. "ext4", "xfs", + "ntfs". If not provided, the empty value is passed to the + associated CSI driver which will determine the default filesystem + to apply. + type: string + nodePublishSecretRef: + description: NodePublishSecretRef is a reference to the secret + object containing sensitive information to pass to the CSI + driver to complete the CSI NodePublishVolume and NodeUnpublishVolume + calls. This field is optional, and may be empty if no secret + is required. If the secret object contains more than one + secret, all secret references are passed. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + readOnly: + description: Specifies a read-only configuration for the volume. + Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: VolumeAttributes stores driver-specific properties + that are passed to the CSI driver. Consult your driver's + documentation for supported values. + type: object + required: + - driver + type: object + mountpath: + description: MountPath tells the script where to read the user + information. Required if annotation injection is used, otherwise + optional + type: string + secretName: type: string type: object users: diff --git a/pkg/reconciliation/reconcile_racks.go b/pkg/reconciliation/reconcile_racks.go index f0cdd3630..e27d81b0f 100644 --- a/pkg/reconciliation/reconcile_racks.go +++ b/pkg/reconciliation/reconcile_racks.go @@ -905,6 +905,16 @@ func (rc *ReconciliationContext) CreateUsers() result.ReconcileResult { if dc.Spec.UserInfo != nil { // Create the job ttl := int32(86400) + + // How to get this as input? + filePath := dc.Spec.UserInfo.MountPath + // filePath := "/vault/secrets/database-config.txt" + + // We want to mount it as a directory and read the files as usernames + if dc.Spec.UserInfo.CSI != nil && filePath != "" { + filePath = "/mnt/secrets/users" + } + // TODO wait for it to complete before we continue.. job := batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ @@ -913,9 +923,6 @@ func (rc *ReconciliationContext) CreateUsers() result.ReconcileResult { }, Spec: batchv1.JobSpec{ Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: dc.Spec.UserInfo.Annotations, - }, Spec: corev1.PodSpec{ // TODO Add volumes here if CSI was used Containers: []corev1.Container{ @@ -923,7 +930,7 @@ func (rc *ReconciliationContext) CreateUsers() result.ReconcileResult { Name: "client", Image: "burmanm/k8ssandra-client:latest", // k8ssandra/k8ssandra-client does not have this feature ImagePullPolicy: corev1.PullIfNotPresent, - Args: []string{"users", "add", "--path", "/vault/secrets/database-config.txt", "--dc", dc.Name}, // TODO This path must be configurable + Args: []string{"users", "add", "--path", filePath, "--dc", dc.Name}, // TODO This path must be configurable }, }, RestartPolicy: corev1.RestartPolicyNever, @@ -934,7 +941,39 @@ func (rc *ReconciliationContext) CreateUsers() result.ReconcileResult { }, } - labels := dc.GetClusterLabels() + // job.Spec.Template.Spec.Volumes + + if len(dc.Spec.UserInfo.Annotations) > 0 { + job.ObjectMeta.Annotations = dc.Spec.UserInfo.Annotations + } + + // TODO Add verification that we can't have dual injection (CSI + annotations) + + // TODO If Secret name is set, mount it just like the CSI + + if dc.Spec.UserInfo.SecretName != "" { + + } + + if dc.Spec.UserInfo.CSI != nil { + vol := corev1.Volume{ + Name: "user-source", + VolumeSource: corev1.VolumeSource{ + // TODO Add .. something? + CSI: dc.Spec.UserInfo.CSI, + }, + } + job.Spec.Template.Spec.Volumes = []corev1.Volume{vol} + job.Spec.Template.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{ + { + Name: "user-source", + ReadOnly: true, + MountPath: filePath, + }, + } + } + + labels := dc.GetDatacenterLabels() oplabels.AddOperatorLabels(labels, dc) job.ObjectMeta.Labels = labels diff --git a/tests/external_secret/README.md b/tests/external_secret/README.md index 7fe4c3232..acc04a682 100644 --- a/tests/external_secret/README.md +++ b/tests/external_secret/README.md @@ -1,15 +1,26 @@ +## Install Helm repositories + +``` +helm repo add secrets-store-csi-driver https://kubernetes-sigs.github.io/secrets-store-csi-driver/charts +helm repo add hashicorp https://helm.releases.hashicorp.com +helm repo update +``` + ## Install Vault +``` helm install vault hashicorp/vault --set "server.dev.enabled=true" --namespace cass-operator # --set "csi.enabled=true" +``` ## Go into Vault and exec certain commands.. +``` kubectl exec -it vault-0 -- /bin/sh vault secrets enable -path=internal kv-v2 -vault kv put internal/database/config username="db-readonly-username" password="db-secret-password" +vault kv put internal/database/config superuser="superpassword" vault auth enable kubernetes @@ -27,7 +38,39 @@ vault write auth/kubernetes/role/internal-app \ bound_service_account_namespaces=cass-operator \ policies=internal-app \ ttl=24h +``` + +## Install CSI driver: + +Not sure if syncSecret is needed, but Vault documentation wants it.. + +``` +helm install csi secrets-store-csi-driver/secrets-store-csi-driver \ + --set syncSecret.enabled=true --namespace cass-operator +``` + +Create the SecretProviderClass: + +```yaml +apiVersion: secrets-store.csi.x-k8s.io/v1 +kind: SecretProviderClass +metadata: + name: vault-database +spec: + provider: vault + parameters: + vaultAddress: "http://vault.default:8200" + roleName: "internal-app" + objects: | + - objectName: "superuser" + secretPath: "internal/database/config" + secretKey: "superuser" +``` + +The objectName becomes the username and the secretKey's data becomes the password. ## Now create the DC: +``` kubectl apply -f tests/testdata/default-single-rack-single-node-dc-vault.yaml +``` diff --git a/tests/external_secret/secret_provider.yaml b/tests/external_secret/secret_provider.yaml new file mode 100644 index 000000000..d96641caa --- /dev/null +++ b/tests/external_secret/secret_provider.yaml @@ -0,0 +1,13 @@ +apiVersion: secrets-store.csi.x-k8s.io/v1 +kind: SecretProviderClass +metadata: + name: vault-database +spec: + provider: vault + parameters: + vaultAddress: "http://vault.default:8200" + roleName: "internal-app" + objects: | + - objectName: "superuser" + secretPath: "internal/database/config" + secretKey: "superuser" diff --git a/tests/testdata/default-single-rack-single-node-dc-vault.yaml b/tests/testdata/default-single-rack-single-node-dc-vault.yaml index 281c34198..ff2559a86 100644 --- a/tests/testdata/default-single-rack-single-node-dc-vault.yaml +++ b/tests/testdata/default-single-rack-single-node-dc-vault.yaml @@ -18,6 +18,12 @@ spec: {{- end }} {{- end -}} vault.hashicorp.com/agent-pre-populate-only: 'true' + mountPath: /vault/secrets/database-config.txt + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: "vault-database" managementApiAuth: insecure: {} size: 1 From 992b0f8888cf0199e6c84cb112cf5e13ab1c2750 Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Fri, 1 Jul 2022 21:32:26 +0300 Subject: [PATCH 4/5] Mount a secret for user job --- pkg/reconciliation/reconcile_racks.go | 24 +++++++++++++++--------- tests/external_secret/README.md | 17 +++++++++-------- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/pkg/reconciliation/reconcile_racks.go b/pkg/reconciliation/reconcile_racks.go index e27d81b0f..1b4a997be 100644 --- a/pkg/reconciliation/reconcile_racks.go +++ b/pkg/reconciliation/reconcile_racks.go @@ -911,7 +911,7 @@ func (rc *ReconciliationContext) CreateUsers() result.ReconcileResult { // filePath := "/vault/secrets/database-config.txt" // We want to mount it as a directory and read the files as usernames - if dc.Spec.UserInfo.CSI != nil && filePath != "" { + if dc.Spec.UserInfo.CSI != nil || dc.Spec.UserInfo.SecretName != "" { filePath = "/mnt/secrets/users" } @@ -951,18 +951,24 @@ func (rc *ReconciliationContext) CreateUsers() result.ReconcileResult { // TODO If Secret name is set, mount it just like the CSI - if dc.Spec.UserInfo.SecretName != "" { - - } - - if dc.Spec.UserInfo.CSI != nil { + if dc.Spec.UserInfo.SecretName != "" || dc.Spec.UserInfo.CSI != nil { vol := corev1.Volume{ Name: "user-source", - VolumeSource: corev1.VolumeSource{ - // TODO Add .. something? + } + if dc.Spec.UserInfo.SecretName != "" { + vol.VolumeSource = corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: dc.Spec.UserInfo.SecretName, + }, + } + } + + if dc.Spec.UserInfo.CSI != nil { + vol.VolumeSource = corev1.VolumeSource{ CSI: dc.Spec.UserInfo.CSI, - }, + } } + job.Spec.Template.Spec.Volumes = []corev1.Volume{vol} job.Spec.Template.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{ { diff --git a/tests/external_secret/README.md b/tests/external_secret/README.md index acc04a682..cfd942e6a 100644 --- a/tests/external_secret/README.md +++ b/tests/external_secret/README.md @@ -20,7 +20,7 @@ kubectl exec -it vault-0 -- /bin/sh vault secrets enable -path=internal kv-v2 -vault kv put internal/database/config superuser="superpassword" +vault kv put internal/database/config username="superuser" password="superpassword" vault auth enable kubernetes @@ -42,11 +42,11 @@ vault write auth/kubernetes/role/internal-app \ ## Install CSI driver: -Not sure if syncSecret is needed, but Vault documentation wants it.. +Remember to enable CSI in the Install Vault step. ``` -helm install csi secrets-store-csi-driver/secrets-store-csi-driver \ - --set syncSecret.enabled=true --namespace cass-operator +helm install csi secrets-store-csi-driver/secrets-store-csi-driver --namespace cass-operator +# --set syncSecret.enabled=true ``` Create the SecretProviderClass: @@ -62,13 +62,14 @@ spec: vaultAddress: "http://vault.default:8200" roleName: "internal-app" objects: | - - objectName: "superuser" + - objectName: "username" secretPath: "internal/database/config" - secretKey: "superuser" + secretKey: "username" + - objectName: "password" + secretPath: "internal/database/config" + secretKey: "password" ``` -The objectName becomes the username and the secretKey's data becomes the password. - ## Now create the DC: ``` From 85e914e52df2b35057cad1001d863e17418aedbd Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Thu, 19 Jan 2023 17:15:04 +0200 Subject: [PATCH 5/5] Add a external tests test, fix some bugs, modify kubectl.Exec behavior, modify CRD a bit.. --- .../v1beta1/cassandradatacenter_types.go | 6 +- ...dra.datastax.com_cassandradatacenters.yaml | 26 +-- pkg/reconciliation/reconcile_racks.go | 36 ++-- tests/external_secret/README.md | 6 + .../external_secret/external_secrets_test.go | 154 ++++++++++++++++++ ...ault-single-rack-single-node-dc-vault.yaml | 17 +- tests/util/sh/lib.go | 11 -- 7 files changed, 210 insertions(+), 46 deletions(-) create mode 100644 tests/external_secret/external_secrets_test.go diff --git a/apis/cassandra/v1beta1/cassandradatacenter_types.go b/apis/cassandra/v1beta1/cassandradatacenter_types.go index b7e78b2ab..ae60b123e 100644 --- a/apis/cassandra/v1beta1/cassandradatacenter_types.go +++ b/apis/cassandra/v1beta1/cassandradatacenter_types.go @@ -80,10 +80,12 @@ type CassandraUser struct { type UserInfo struct { Annotations map[string]string `json:"annotations,omitempty"` // MountPath tells the script where to read the user information. Required if annotation injection is used, otherwise optional - MountPath string `json:"mountpath,omitempty"` + MountPath string `json:"mountPath,omitempty"` CSI *corev1.CSIVolumeSource `json:"csi,omitempty"` SecretName string `json:"secretName,omitempty"` - // ServiceAccount string `json:"serviceAccountName,omitempty"` + + // ServiceAccountName override job ServiceAccount, otherwise Spec.ServiceAccount (the one used to create the server pods) is used + ServiceAccountName string `json:"serviceAccountName,omitempty"` // TODO Add CSI information here /* TODO Testing: diff --git a/config/crd/bases/cassandra.datastax.com_cassandradatacenters.yaml b/config/crd/bases/cassandra.datastax.com_cassandradatacenters.yaml index 005eb3c41..4e6c12473 100644 --- a/config/crd/bases/cassandra.datastax.com_cassandradatacenters.yaml +++ b/config/crd/bases/cassandra.datastax.com_cassandradatacenters.yaml @@ -8084,18 +8084,18 @@ spec: managed by an external CSI driver properties: driver: - description: Driver is the name of the CSI driver that handles + description: driver is the name of the CSI driver that handles this volume. Consult with your admin for the correct name as registered in the cluster. type: string fsType: - description: Filesystem type to mount. Ex. "ext4", "xfs", - "ntfs". If not provided, the empty value is passed to the - associated CSI driver which will determine the default filesystem - to apply. + description: fsType to mount. Ex. "ext4", "xfs", "ntfs". If + not provided, the empty value is passed to the associated + CSI driver which will determine the default filesystem to + apply. type: string nodePublishSecretRef: - description: NodePublishSecretRef is a reference to the secret + description: nodePublishSecretRef is a reference to the secret object containing sensitive information to pass to the CSI driver to complete the CSI NodePublishVolume and NodeUnpublishVolume calls. This field is optional, and may be empty if no secret @@ -8107,27 +8107,33 @@ spec: TODO: Add other useful fields. apiVersion, kind, uid?' type: string type: object + x-kubernetes-map-type: atomic readOnly: - description: Specifies a read-only configuration for the volume. - Defaults to false (read/write). + description: readOnly specifies a read-only configuration + for the volume. Defaults to false (read/write). type: boolean volumeAttributes: additionalProperties: type: string - description: VolumeAttributes stores driver-specific properties + description: volumeAttributes stores driver-specific properties that are passed to the CSI driver. Consult your driver's documentation for supported values. type: object required: - driver type: object - mountpath: + mountPath: description: MountPath tells the script where to read the user information. Required if annotation injection is used, otherwise optional type: string secretName: type: string + serviceAccountName: + description: ServiceAccountName override job ServiceAccount, otherwise + Spec.ServiceAccount (the one used to create the server pods) + is used + type: string type: object users: description: Cassandra users to bootstrap diff --git a/pkg/reconciliation/reconcile_racks.go b/pkg/reconciliation/reconcile_racks.go index 1b4a997be..7716e7e51 100644 --- a/pkg/reconciliation/reconcile_racks.go +++ b/pkg/reconciliation/reconcile_racks.go @@ -901,14 +901,17 @@ func (rc *ReconciliationContext) CreateUsers() result.ReconcileResult { rc.ReqLogger.Info("reconcile_racks::CreateUsers") + // TODO We should check if we've already created this .. // TODO This should be cleaned up after a while (TTL) if dc.Spec.UserInfo != nil { // Create the job ttl := int32(86400) + // TODO Instead of this, I should probably have a "SecretRef" generic structure. That way we could use it for all Secrets (like mgmt-api auth), supporting + // both legacy as well as new structure + // How to get this as input? filePath := dc.Spec.UserInfo.MountPath - // filePath := "/vault/secrets/database-config.txt" // We want to mount it as a directory and read the files as usernames if dc.Spec.UserInfo.CSI != nil || dc.Spec.UserInfo.SecretName != "" { @@ -916,6 +919,7 @@ func (rc *ReconciliationContext) CreateUsers() result.ReconcileResult { } // TODO wait for it to complete before we continue.. + job := batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ Namespace: dc.Namespace, @@ -924,33 +928,26 @@ func (rc *ReconciliationContext) CreateUsers() result.ReconcileResult { Spec: batchv1.JobSpec{ Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ - // TODO Add volumes here if CSI was used Containers: []corev1.Container{ { Name: "client", - Image: "burmanm/k8ssandra-client:latest", // k8ssandra/k8ssandra-client does not have this feature + Image: "k8ssandra/k8ssandra-client:latest", ImagePullPolicy: corev1.PullIfNotPresent, - Args: []string{"users", "add", "--path", filePath, "--dc", dc.Name}, // TODO This path must be configurable + Args: []string{"users", "add", "--path", filePath, "--dc", dc.Name}, }, }, RestartPolicy: corev1.RestartPolicyNever, }, - // Create the k8ssandra-client add users job here? }, TTLSecondsAfterFinished: &ttl, }, } - // job.Spec.Template.Spec.Volumes - if len(dc.Spec.UserInfo.Annotations) > 0 { - job.ObjectMeta.Annotations = dc.Spec.UserInfo.Annotations + job.Spec.Template.ObjectMeta.Annotations = dc.Spec.UserInfo.Annotations } // TODO Add verification that we can't have dual injection (CSI + annotations) - - // TODO If Secret name is set, mount it just like the CSI - if dc.Spec.UserInfo.SecretName != "" || dc.Spec.UserInfo.CSI != nil { vol := corev1.Volume{ Name: "user-source", @@ -979,9 +976,8 @@ func (rc *ReconciliationContext) CreateUsers() result.ReconcileResult { } } - labels := dc.GetDatacenterLabels() - oplabels.AddOperatorLabels(labels, dc) - job.ObjectMeta.Labels = labels + // labels := dc.GetDatacenterLabels() + // oplabels.AddOperatorLabels(labels, dc) // Set CassandraDatacenter dc as the owner and controller err := setControllerReference(dc, &job, rc.Scheme) @@ -989,13 +985,19 @@ func (rc *ReconciliationContext) CreateUsers() result.ReconcileResult { return result.Error(err) } - // TODO Do I have this value somewhere..? - job.Spec.Template.Spec.ServiceAccountName = "cass-operator-controller-manager" + if dc.Spec.UserInfo.ServiceAccountName != "" { + job.Spec.Template.Spec.ServiceAccountName = dc.Spec.UserInfo.ServiceAccountName + } else { + job.Spec.Template.Spec.ServiceAccountName = dc.Spec.ServiceAccount + } if err := rc.Client.Create(rc.Ctx, &job); err != nil { + if errors.IsAlreadyExists(err) { + return result.Continue() + } return result.Error(err) } - return result.Continue() + return result.RequeueSoon(5) } err := rc.UpdateSecretWatches() diff --git a/tests/external_secret/README.md b/tests/external_secret/README.md index cfd942e6a..1a74ca96f 100644 --- a/tests/external_secret/README.md +++ b/tests/external_secret/README.md @@ -40,6 +40,12 @@ vault write auth/kubernetes/role/internal-app \ ttl=24h ``` +vault write auth/kubernetes/role/internal-app \ + bound_service_account_names=cass-operator-controller-manager \ + bound_service_account_namespaces=test-external-secrets \ + policies=internal-app \ + ttl=24h + ## Install CSI driver: Remember to enable CSI in the Install Vault step. diff --git a/tests/external_secret/external_secrets_test.go b/tests/external_secret/external_secrets_test.go new file mode 100644 index 000000000..25b8877be --- /dev/null +++ b/tests/external_secret/external_secrets_test.go @@ -0,0 +1,154 @@ +// Copyright DataStax, Inc. +// Please see the included license file for details. + +package external_secrets + +import ( + "fmt" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/k8ssandra/cass-operator/tests/kustomize" + ginkgo_util "github.com/k8ssandra/cass-operator/tests/util/ginkgo" + "github.com/k8ssandra/cass-operator/tests/util/kubectl" + shutil "github.com/k8ssandra/cass-operator/tests/util/sh" +) + +var ( + testName = "External secrets test with Vault" + namespace = "test-external-secrets" + dcName = "dc2" + dcYaml = "../testdata/default-single-rack-single-node-dc-vault.yaml" + ns = ginkgo_util.NewWrapper(testName, namespace) +) + +func TestLifecycle(t *testing.T) { + AfterSuite(func() { + logPath := fmt.Sprintf("%s/aftersuite", ns.LogDir) + err := kubectl.DumpAllLogs(logPath).ExecV() + if err != nil { + t.Logf("Failed to dump all the logs: %v", err) + } + + fmt.Printf("\n\tPost-run logs dumped at: %s\n\n", logPath) + ns.Terminate() + err = kustomize.Undeploy(namespace) + if err != nil { + t.Logf("Failed to undeploy cass-operator: %v", err) + } + }) + + RegisterFailHandler(Fail) + RunSpecs(t, testName) +} + +var _ = Describe(testName, func() { + Context("when in a new cluster with Vault installed", func() { + Specify("Vault is installed with cass-operator", func() { + + By("creating a namespace for the cass-operator") + err := kubectl.CreateNamespace(namespace).ExecV() + Expect(err).ToNot(HaveOccurred()) + + By("deploy cass-operator with kustomize") + err = kustomize.Deploy(namespace) + Expect(err).ToNot(HaveOccurred()) + ns.WaitForOperatorReady() + + By("deploying Vault") + err = shutil.RunV("helm", "repo", "add", "hashicorp", "https://helm.releases.hashicorp.com") + Expect(err).ShouldNot(HaveOccurred()) + + err = shutil.RunV("helm", "repo", "update") + Expect(err).ShouldNot(HaveOccurred()) + + err = shutil.RunV("helm", "install", "-n", namespace, "vault", "hashicorp/vault", "--set", "server.dev.enabled=true") + Expect(err).ShouldNot(HaveOccurred()) + + By("Waiting for all components to be ready") + readyGetter := kubectl.Get("pods"). + WithFlag("selector", "component=server"). + FormatOutput("jsonpath={.items[0].status.conditions[?(@.type=='Ready')].status}") + err = kubectl.WaitForOutputContains(readyGetter, "True", 1800) + Expect(err).ShouldNot(HaveOccurred()) + + podName := "vault-0" + + // Write settings to the Vault pod + k := kubectl.ExecOnPod(podName, "--", "sh", "-c", "vault secrets enable -path=internal kv-v2") + err = ns.ExecV(k) + Expect(err).ShouldNot(HaveOccurred()) + + username := "superuser" + password := "superpassword" + + k = kubectl.ExecOnPod(podName, "--", "sh", "-c", fmt.Sprintf("vault kv put internal/database/config username=%s password=%s", username, password)) + err = ns.ExecV(k) + Expect(err).ShouldNot(HaveOccurred()) + + k = kubectl.ExecOnPod(podName, "--", "sh", "-c", "vault auth enable kubernetes") + err = ns.ExecV(k) + Expect(err).ShouldNot(HaveOccurred()) + + k = kubectl.ExecOnPod(podName, "--", "sh", "-c", `vault write auth/kubernetes/config kubernetes_host="https://$KUBERNETES_PORT_443_TCP_ADDR:443"`) + err = ns.ExecV(k) + Expect(err).ShouldNot(HaveOccurred()) + + k = kubectl.ExecOnPod(podName, "--", "sh", "-c", `vault policy write internal-app - <