From 48059f49f74b83df60b935b41a1f504139efb5e9 Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Fri, 21 Apr 2023 14:12:50 +0300 Subject: [PATCH 1/3] Add new config item to set SecretProvider implementation --- apis/config/v1beta1/operatorconfig_types.go | 16 ++++++ apis/config/v1beta1/zz_generated.deepcopy.go | 52 ++++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/apis/config/v1beta1/operatorconfig_types.go b/apis/config/v1beta1/operatorconfig_types.go index 7dc087d25..b758b9788 100644 --- a/apis/config/v1beta1/operatorconfig_types.go +++ b/apis/config/v1beta1/operatorconfig_types.go @@ -38,6 +38,22 @@ type OperatorConfig struct { // ImageConfigFile indicates the path where to load the imageConfig from ImageConfigFile string `json:"imageConfigFile,omitempty"` + + // SecretProvider sets where the Secrets are loaded from. If unset, loads them from Kubernetes + SecretProvider SecretProviderConfigSpec `json:"secretProvider,omitempty"` +} + +type SecretProviderConfigSpec struct { + Source SecretProviderSource `json:",inline,omitempty"` +} + +type SecretProviderSource struct { + Filesystem *SecretProviderFilesystemSource `json:"filesystem,omitempty"` +} + +type SecretProviderFilesystemSource struct { + Path string `json:"path"` + NamespacedPaths bool `json:"namespaced,omitempty"` } func init() { diff --git a/apis/config/v1beta1/zz_generated.deepcopy.go b/apis/config/v1beta1/zz_generated.deepcopy.go index 79386c320..f512afae9 100644 --- a/apis/config/v1beta1/zz_generated.deepcopy.go +++ b/apis/config/v1beta1/zz_generated.deepcopy.go @@ -136,6 +136,7 @@ func (in *OperatorConfig) DeepCopyInto(out *OperatorConfig) { *out = *in out.TypeMeta = in.TypeMeta in.ControllerManagerConfigurationSpec.DeepCopyInto(&out.ControllerManagerConfigurationSpec) + in.SecretProvider.DeepCopyInto(&out.SecretProvider) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OperatorConfig. @@ -155,3 +156,54 @@ func (in *OperatorConfig) DeepCopyObject() runtime.Object { } return nil } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecretProviderConfigSpec) DeepCopyInto(out *SecretProviderConfigSpec) { + *out = *in + in.Source.DeepCopyInto(&out.Source) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretProviderConfigSpec. +func (in *SecretProviderConfigSpec) DeepCopy() *SecretProviderConfigSpec { + if in == nil { + return nil + } + out := new(SecretProviderConfigSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecretProviderFilesystemSource) DeepCopyInto(out *SecretProviderFilesystemSource) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretProviderFilesystemSource. +func (in *SecretProviderFilesystemSource) DeepCopy() *SecretProviderFilesystemSource { + if in == nil { + return nil + } + out := new(SecretProviderFilesystemSource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecretProviderSource) DeepCopyInto(out *SecretProviderSource) { + *out = *in + if in.Filesystem != nil { + in, out := &in.Filesystem, &out.Filesystem + *out = new(SecretProviderFilesystemSource) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretProviderSource. +func (in *SecretProviderSource) DeepCopy() *SecretProviderSource { + if in == nil { + return nil + } + out := new(SecretProviderSource) + in.DeepCopyInto(out) + return out +} From 374dbaa49b717a7a508713f2b0e8411189acd707 Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Fri, 21 Apr 2023 16:31:20 +0300 Subject: [PATCH 2/3] Add SecretProvider interface and an implementation that reads Secrets from the filesystem --- pkg/secrets/provider.go | 37 +++++++++++++ pkg/secrets/provider_file.go | 51 ++++++++++++++++++ pkg/secrets/provider_file_test.go | 72 +++++++++++++++++++++++++ pkg/secrets/provider_kubernetes.go | 22 ++++++++ pkg/secrets/provider_kubernetes_test.go | 3 ++ 5 files changed, 185 insertions(+) create mode 100644 pkg/secrets/provider.go create mode 100644 pkg/secrets/provider_file.go create mode 100644 pkg/secrets/provider_file_test.go create mode 100644 pkg/secrets/provider_kubernetes.go create mode 100644 pkg/secrets/provider_kubernetes_test.go diff --git a/pkg/secrets/provider.go b/pkg/secrets/provider.go new file mode 100644 index 000000000..34ef46eab --- /dev/null +++ b/pkg/secrets/provider.go @@ -0,0 +1,37 @@ +package secrets + +import ( + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/scheme" +) + +type SecretProvider interface { + RetrieveSecret(types.NamespacedName) (*corev1.Secret, error) + StoreOrUpdateSecret(*corev1.Secret) error +} + +// Common methods that could be used by multiple implementations + +func decodeSecret(data []byte) (*corev1.Secret, error) { + decoder := serializer.NewCodecFactory(scheme.Scheme).UniversalDecoder() + object := &corev1.Secret{} + + err := runtime.DecodeInto(decoder, data, object) + if err != nil { + return nil, err + } + return object, nil +} + +func encodeSecret(secret *corev1.Secret) ([]byte, error) { + encoder := serializer.NewCodecFactory(scheme.Scheme).LegacyCodec(corev1.SchemeGroupVersion) + b, err := runtime.Encode(encoder, secret) + if err != nil { + return nil, err + } + return b, nil + +} diff --git a/pkg/secrets/provider_file.go b/pkg/secrets/provider_file.go new file mode 100644 index 000000000..3047395c4 --- /dev/null +++ b/pkg/secrets/provider_file.go @@ -0,0 +1,51 @@ +package secrets + +import ( + "os" + "path/filepath" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" +) + +type FileProvider struct { + path string + namespacePrefixed bool +} + +func NewFileProvider(directoryPath string, namespacePrefix bool) *FileProvider { + return &FileProvider{ + path: directoryPath, + namespacePrefixed: namespacePrefix, + } +} + +// SecretProvider interface + +func (f *FileProvider) RetrieveSecret(name types.NamespacedName) (*corev1.Secret, error) { + filename := f.secretPath(name) + return readSecret(filename) +} + +func (f *FileProvider) StoreOrUpdateSecret(secret *corev1.Secret) error { + // This provider does not store updates to the disk + return nil +} + +// Rest of the implementation + +func (f *FileProvider) secretPath(name types.NamespacedName) string { + if f.namespacePrefixed { + return filepath.Join(f.path, name.Namespace, name.Name) + } + return filepath.Join(f.path, name.Name) +} + +func readSecret(path string) (*corev1.Secret, error) { + b, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + return decodeSecret(b) +} diff --git a/pkg/secrets/provider_file_test.go b/pkg/secrets/provider_file_test.go new file mode 100644 index 000000000..f8122f216 --- /dev/null +++ b/pkg/secrets/provider_file_test.go @@ -0,0 +1,72 @@ +package secrets + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +var _ SecretProvider = &FileProvider{} + +func TestSecretPathsNoPrefix(t *testing.T) { + assert := assert.New(t) + f := &FileProvider{ + path: "/etc/secrets", + namespacePrefixed: false, + } + + sName := types.NamespacedName{Name: "cert-client", Namespace: "cluster1"} + path := f.secretPath(sName) + assert.Equal("/etc/secrets/cert-client", path) +} + +func TestSecretPathsWithPrefix(t *testing.T) { + assert := assert.New(t) + f := &FileProvider{ + path: "/etc/secrets", + namespacePrefixed: true, + } + + sName := types.NamespacedName{Name: "cert-client", Namespace: "cluster1"} + path := f.secretPath(sName) + assert.Equal("/etc/secrets/cluster1/cert-client", path) +} + +func TestSecretReading(t *testing.T) { + require := require.New(t) + s := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-secret", + Namespace: "test-ns", + }, + StringData: map[string]string{ + "a": "b", + "c": "d", + }, + } + + data, err := encodeSecret(s) + require.NoError(err) + + tempDir, err := os.MkdirTemp("", "") + require.NoError(err) + defer os.RemoveAll(tempDir) + + err = os.WriteFile(filepath.Join(tempDir, "test-secret"), data, 0755) + require.NoError(err) + + f := NewFileProvider(tempDir, false) + + s2, err := f.RetrieveSecret(types.NamespacedName{Name: s.Name, Namespace: s.Namespace}) + require.NoError(err) + + require.Equal(s.Name, s2.Name) + require.Equal(s.Namespace, s2.Namespace) + require.Equal(s.StringData, s2.StringData) +} diff --git a/pkg/secrets/provider_kubernetes.go b/pkg/secrets/provider_kubernetes.go new file mode 100644 index 000000000..fb81a0fe4 --- /dev/null +++ b/pkg/secrets/provider_kubernetes.go @@ -0,0 +1,22 @@ +package secrets + +import ( + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type KubernetesProvider struct { + client.Client +} + +// SecretProvider interfaces + +func (k *KubernetesProvider) RetrieveSecret(name types.NamespacedName) (*corev1.Secret, error) { + return nil, nil +} + +func (k *KubernetesProvider) StoreOrUpdateSecret(secret *corev1.Secret) error { + + return nil +} diff --git a/pkg/secrets/provider_kubernetes_test.go b/pkg/secrets/provider_kubernetes_test.go new file mode 100644 index 000000000..4c6f76bfa --- /dev/null +++ b/pkg/secrets/provider_kubernetes_test.go @@ -0,0 +1,3 @@ +package secrets + +var _ SecretProvider = &KubernetesProvider{} From 6901976a02c76c9f1bd8f1278cfa3111433e044b Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Tue, 25 Apr 2023 09:31:54 +0300 Subject: [PATCH 3/3] Add Kubernetes secretProvider impl --- pkg/secrets/provider.go | 6 +- pkg/secrets/provider_file.go | 8 +- pkg/secrets/provider_file_test.go | 3 +- pkg/secrets/provider_kubernetes.go | 25 +++++- pkg/secrets/provider_kubernetes_test.go | 106 ++++++++++++++++++++++++ 5 files changed, 138 insertions(+), 10 deletions(-) diff --git a/pkg/secrets/provider.go b/pkg/secrets/provider.go index 34ef46eab..b415f678a 100644 --- a/pkg/secrets/provider.go +++ b/pkg/secrets/provider.go @@ -1,6 +1,8 @@ package secrets import ( + "context" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" @@ -9,8 +11,8 @@ import ( ) type SecretProvider interface { - RetrieveSecret(types.NamespacedName) (*corev1.Secret, error) - StoreOrUpdateSecret(*corev1.Secret) error + RetrieveSecret(context.Context, types.NamespacedName) (*corev1.Secret, error) + StoreOrUpdateSecret(context.Context, *corev1.Secret) error } // Common methods that could be used by multiple implementations diff --git a/pkg/secrets/provider_file.go b/pkg/secrets/provider_file.go index 3047395c4..69bc08578 100644 --- a/pkg/secrets/provider_file.go +++ b/pkg/secrets/provider_file.go @@ -1,6 +1,7 @@ package secrets import ( + "context" "os" "path/filepath" @@ -22,13 +23,14 @@ func NewFileProvider(directoryPath string, namespacePrefix bool) *FileProvider { // SecretProvider interface -func (f *FileProvider) RetrieveSecret(name types.NamespacedName) (*corev1.Secret, error) { +func (f *FileProvider) RetrieveSecret(ctx context.Context, name types.NamespacedName) (*corev1.Secret, error) { filename := f.secretPath(name) return readSecret(filename) } -func (f *FileProvider) StoreOrUpdateSecret(secret *corev1.Secret) error { - // This provider does not store updates to the disk +func (f *FileProvider) StoreOrUpdateSecret(ctx context.Context, secret *corev1.Secret) error { + // This provider does not store updates to the disk. This is intended for use-cases where the secrets are externally + // applied to operator's volume return nil } diff --git a/pkg/secrets/provider_file_test.go b/pkg/secrets/provider_file_test.go index f8122f216..0ae46e55d 100644 --- a/pkg/secrets/provider_file_test.go +++ b/pkg/secrets/provider_file_test.go @@ -1,6 +1,7 @@ package secrets import ( + "context" "os" "path/filepath" "testing" @@ -63,7 +64,7 @@ func TestSecretReading(t *testing.T) { f := NewFileProvider(tempDir, false) - s2, err := f.RetrieveSecret(types.NamespacedName{Name: s.Name, Namespace: s.Namespace}) + s2, err := f.RetrieveSecret(context.TODO(), types.NamespacedName{Name: s.Name, Namespace: s.Namespace}) require.NoError(err) require.Equal(s.Name, s2.Name) diff --git a/pkg/secrets/provider_kubernetes.go b/pkg/secrets/provider_kubernetes.go index fb81a0fe4..70674a77c 100644 --- a/pkg/secrets/provider_kubernetes.go +++ b/pkg/secrets/provider_kubernetes.go @@ -1,7 +1,10 @@ package secrets import ( + "context" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -10,13 +13,27 @@ type KubernetesProvider struct { client.Client } +func NewKubernetesProvider(cli client.Client) *KubernetesProvider { + return &KubernetesProvider{ + Client: cli, + } +} + // SecretProvider interfaces -func (k *KubernetesProvider) RetrieveSecret(name types.NamespacedName) (*corev1.Secret, error) { - return nil, nil +func (k *KubernetesProvider) RetrieveSecret(ctx context.Context, name types.NamespacedName) (*corev1.Secret, error) { + secret := &corev1.Secret{} + if err := k.Client.Get(ctx, name, secret); err != nil { + return nil, err + } + return secret, nil } -func (k *KubernetesProvider) StoreOrUpdateSecret(secret *corev1.Secret) error { - +func (k *KubernetesProvider) StoreOrUpdateSecret(ctx context.Context, secret *corev1.Secret) error { + if err := k.Client.Create(ctx, secret); err != nil { + if errors.IsAlreadyExists(err) { + return k.Client.Update(ctx, secret) + } + } return nil } diff --git a/pkg/secrets/provider_kubernetes_test.go b/pkg/secrets/provider_kubernetes_test.go index 4c6f76bfa..1ef24d7cc 100644 --- a/pkg/secrets/provider_kubernetes_test.go +++ b/pkg/secrets/provider_kubernetes_test.go @@ -1,3 +1,109 @@ package secrets +import ( + "context" + "fmt" + "testing" + + "github.com/k8ssandra/cass-operator/pkg/mocks" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + var _ SecretProvider = &KubernetesProvider{} + +func TestStoreAndReadSecret(t *testing.T) { + require := require.New(t) + mockClient := &mocks.Client{} + + var storedSecret *corev1.Secret + var errToRespond error + + k := NewKubernetesProvider(mockClient) + + mockClient.On("Get", + mock.MatchedBy( + func(ctx context.Context) bool { + return ctx != nil + }), + mock.MatchedBy( + func(key client.ObjectKey) bool { + return key != client.ObjectKey{} + }), + mock.MatchedBy( + func(obj runtime.Object) bool { + return obj != nil + })). + Return(nil). + Run(func(args mock.Arguments) { + arg := args.Get(2).(*corev1.Secret) + arg.StringData = storedSecret.StringData + // return &corev1.Endpoints{} + }).Times(2) + + mockClient.On("Create", + mock.MatchedBy( + func(ctx context.Context) bool { + return ctx != nil + }), + mock.MatchedBy( + func(s *corev1.Secret) bool { + return true + })). + Return(nil). + Run(func(args mock.Arguments) { + arg := args.Get(1).(*corev1.Secret) + storedSecret = arg + }) + + mockClient.On("Create", + mock.MatchedBy( + func(ctx context.Context) bool { + return ctx != nil + }), + mock.MatchedBy( + func(s *corev1.Secret) bool { + fmt.Printf("We're here?\n") + if storedSecret != nil { + errToRespond = errors.NewAlreadyExists(schema.ParseGroupResource("Secret"), storedSecret.Name) + return false + } + return true + })). + Return(errToRespond) + + s := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ns", + Namespace: "testnamespace", + }, + StringData: map[string]string{ + "a": "b", + }, + } + + err := k.StoreOrUpdateSecret(context.TODO(), s) + require.NoError(err) + + s2, err := k.RetrieveSecret(context.TODO(), types.NamespacedName{Name: s.Name, Namespace: s.Namespace}) + require.NoError(err) + + require.Equal(s.StringData, s2.StringData) + + s2.StringData["c"] = "e" + + err = k.StoreOrUpdateSecret(context.TODO(), s2) + require.NoError(err) + + s3, err := k.RetrieveSecret(context.TODO(), types.NamespacedName{Name: s.Name, Namespace: s.Namespace}) + require.NoError(err) + + require.Equal("e", s3.StringData["c"]) +}