Secret-aware pod scheduling (#125)

## Description

Fixes #57

This requires pod volume definitions to switch from using the `csi` volume type (which are fully contained within the Kubelet's Pod lifecycle) to the `ephemeral` volume type (which is reified into a full `PersistentVolumeClaim` by the scheduler). For example:

```yaml
---
apiVersion: v1
kind: Pod
metadata:
  name: example-secret-consumer
spec:
  volumes:
    - name: tls
      csi:
        driver: secrets.stackable.tech
        volumeAttributes:
          secrets.stackable.tech/class: tls
          secrets.stackable.tech/scope: node,pod,service=secret-consumer-nginx
  containers:
    - name: ubuntu
      image: ubuntu
      stdin: true
      tty: true
      volumeMounts:
        - name: tls
          mountPath: /tls
```

should be changed into:

```yaml
---
apiVersion: v1
kind: Pod
metadata:
  name: example-secret-consumer
spec:
  volumes:
    - name: tls
      ephemeral:
        volumeClaimTemplate:
          metadata:
            annotations:
              secrets.stackable.tech/class: secret
              secrets.stackable.tech/scope: node,pod,service=secret-consumer-nginx
          spec:
            storageClassName: secrets.stackable.tech
            accessModes:
              - ReadWriteOnce
            resources:
              requests:
                storage: "1"
  containers:
    - name: ubuntu
      image: ubuntu
      stdin: true
      tty: true
      volumeMounts:
        - name: tls
          mountPath: /tls
```

`csi` volumes still work for now, but won't be able to take advantage of secret-aware scheduling. We will probably want to remove it after all operators and documentation have been migrated.

Author: nightkr

CHANGELOG.md

@@ -4,6 +4,19 @@ All notable changes to this project will be documented in this file.
 
 ## [Unreleased]
 
+### Added
+
+- Pods that consume Node-scoped `k8sSearch` secrets will now only be scheduled to Nodes that have the secret provisioned ([#125]).
+  - This is only supported for pods that use the new-style `ephemeral` volume definitions rather than `csi`.
+
+### Changed
+
+- Pods that consume secrets should now use the `ephemeral` volume type rather than `csi` ([#125]).
+  - `csi` volumes will keep working for now, but should be considered deprecated, and will not be compatible
+    with all new features.
+
+[#125]: https://github.com/stackabletech/secret-operator/pull/125
+
 ## [0.3.0] - 2022-05-05
 
 ### Added

Cargo.lock

@@ -5,8 +5,26 @@ custom_build(
     # ignore=['result*', 'Cargo.nix', 'target', *.yaml],
     outputs_image_ref_to='result/ref',
 )
-k8s_yaml('provisioner.yaml')
-k8s_yaml('examples/simple-consumer-nginx.yaml')
+
+# Load the latest CRDs from Nix
 watch_file('result')
 if os.path.exists('result'):
    k8s_yaml('result/crds.yaml')
+
+# Exclude stale CRDs from Helm chart, and apply the rest
+helm_crds, helm_non_crds = filter_yaml(
+   helm(
+      'deploy/helm/secret-operator',
+      name='secret-operator',
+      set=[
+         'image.repository=docker.stackable.tech/teozkr/secret-provisioner',
+      ],
+   ),
+   api_version = "^apiextensions\\.k8s\\.io/.*$",
+   kind = "^CustomResourceDefinition$",
+)
+k8s_yaml(helm_non_crds)
+
+# Load examples
+k8s_yaml('examples/simple-consumer-nginx.yaml')
+k8s_yaml('examples/simple-consumer-shell.yaml')

Tiltfile

@@ -9,3 +9,4 @@ spec:
   fsGroupPolicy: File
   volumeLifecycleModes:
     - Ephemeral
+    - Persistent

deploy/helm/secret-operator/templates/csidriver.yaml

@@ -36,12 +36,26 @@ spec:
           env:
             - name: CSI_ENDPOINT
               value: /csi/csi.sock
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: spec.nodeName
           volumeMounts:
             - name: csi
               mountPath: /csi
             - name: mountpoint
               mountPath: /var/lib/kubelet/pods
               mountPropagation: Bidirectional
+        - name: external-provisioner
+          image: k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0
+          args:
+            - --csi-address=/csi/csi.sock
+            - --feature-gates=Topology=true
+            - --extra-create-metadata
+          volumeMounts:
+            - name: csi
+              mountPath: /csi
         - name: node-driver-registrar
           image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.4.0
           args:

deploy/helm/secret-operator/templates/daemonset.yaml

@@ -8,16 +8,40 @@ rules:
       - ""
     resources:
       - secrets
+      - events
     verbs:
+      - get
       - list
+      - watch
       - create
+  - apiGroups:
+      - ""
+    resources:
+      - persistentvolumes
+    verbs:
       - get
+      - list
+      - watch
+      - create
+      - delete
   - apiGroups:
       - ""
     resources:
       - nodes
+      - persistentvolumeclaims
     verbs:
       - get
+      - list
+      - watch
+  - apiGroups:
+      - storage.k8s.io
+    resources:
+      - csinodes
+      - storageclasses
+    verbs:
+      - get
+      - list
+      - watch
   - apiGroups:
       - ""
     resources:

deploy/helm/secret-operator/templates/roles.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: secrets.stackable.tech
+provisioner: secrets.stackable.tech

deploy/helm/secret-operator/templates/storageclass.yaml

@@ -9,3 +9,4 @@ spec:
   fsGroupPolicy: File
   volumeLifecycleModes:
     - Ephemeral
+    - Persistent

deploy/manifests/csidriver.yaml

@@ -31,12 +31,26 @@ spec:
           env:
             - name: CSI_ENDPOINT
               value: /csi/csi.sock
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: spec.nodeName
           volumeMounts:
             - name: csi
               mountPath: /csi
             - name: mountpoint
               mountPath: /var/lib/kubelet/pods
               mountPropagation: Bidirectional
+        - name: external-provisioner
+          image: k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0
+          args:
+            - --csi-address=/csi/csi.sock
+            - --feature-gates=Topology=true
+            - --extra-create-metadata
+          volumeMounts:
+            - name: csi
+              mountPath: /csi
         - name: node-driver-registrar
           image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.4.0
           args:

deploy/manifests/daemonset.yaml

@@ -8,16 +8,40 @@ rules:
       - ""
     resources:
       - secrets
+      - events
     verbs:
+      - get
       - list
+      - watch
       - create
+  - apiGroups:
+      - ""
+    resources:
+      - persistentvolumes
+    verbs:
       - get
+      - list
+      - watch
+      - create
+      - delete
   - apiGroups:
       - ""
     resources:
       - nodes
+      - persistentvolumeclaims
     verbs:
       - get
+      - list
+      - watch
+  - apiGroups:
+      - storage.k8s.io
+    resources:
+      - csinodes
+      - storageclasses
+    verbs:
+      - get
+      - list
+      - watch
   - apiGroups:
       - ""
     resources: