sgx: set epc limits via NRI annotations

Signed-off-by: Mikko Ylinen <mikko.ylinen@intel.com>

Author: mythi

deployments/operator/crd/bases/deviceplugin.intel.com_sgxdeviceplugins.yaml

@@ -72,6 +72,11 @@ spec:
                 description: NodeSelector provides a simple way to constrain device
                   plugin pods to nodes with particular labels.
                 type: object
+              nriImage:
+                description: 'NRIImage is a container image with SGX Node Resource
+                  Interface (NRI) plugin executable. Set this value if SGX EPC cgroups
+                  limits enforcement is wanted. TODO: is this a good name?'
+                type: string
               provisionLimit:
                 description: ProvisionLimit is a number of containers that can share
                   the same SGX provision device.

deployments/operator/samples/deviceplugin_v1_sgxdeviceplugin.yaml

@@ -4,6 +4,7 @@ metadata:
   name: sgxdeviceplugin-sample
 spec:
   image: intel/intel-sgx-plugin:0.29.0
+  nriImage: ghcr.io/containers/nri-plugins/nri-sgx-epc:v0.3.2
   enclaveLimit: 110
   provisionLimit: 110
   logLevel: 4

deployments/sgx_plugin/overlays/epc-cgroups/kustomization.yaml

@@ -0,0 +1,7 @@
+resources:
+  - ../../base
+
+patches:
+- path: nri_plugin_patch.yaml
+  target:
+    name: intel-sgx-plugin

deployments/sgx_plugin/overlays/epc-cgroups/nri_plugin_patch.yaml

@@ -0,0 +1,21 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: intel-sgx-plugin
+spec:
+  template:
+    spec:
+      containers:
+      - name: nri-sgx-epc
+        image: ghcr.io/containers/nri-plugins/nri-sgx-epc:unstable
+        securityContext:
+          readOnlyRootFilesystem: true
+          allowPrivilegeEscalation: false
+        imagePullPolicy: IfNotPresent
+        volumeMounts:
+        - name: nrisockets
+          mountPath: /var/run/nri
+      volumes:
+      - name: nrisockets
+        hostPath:
+          path: /var/run/nri

pkg/apis/deviceplugin/v1/sgxdeviceplugin_types.go

@@ -35,6 +35,11 @@ type SgxDevicePluginSpec struct {
 	// Recommendation is to leave this unset and prefer the SGX NodeFeatureRule instead.
 	InitImage string `json:"initImage,omitempty"`
 
+	// NRIImage is a container image with SGX Node Resource Interface (NRI) plugin executable. Set
+	// this value if SGX EPC cgroups limits enforcement is wanted.
+	// TODO: is this a good name?
+	NRIImage string `json:"nriImage,omitempty"`
+
 	// EnclaveLimit is a number of containers that can share the same SGX enclave device.
 	// +kubebuilder:validation:Minimum=1
 	EnclaveLimit int `json:"enclaveLimit,omitempty"`

pkg/controllers/sgx/controller.go

@@ -112,6 +112,27 @@ func setInitContainer(spec *v1.PodSpec, imageName string) {
 	addVolumeIfMissing(spec, "nfd-features", "/etc/kubernetes/node-feature-discovery/source.d/", v1.HostPathDirectoryOrCreate)
 }
 
+func setNRIContainer(spec *v1.PodSpec, imageName string) {
+	yes := true
+	no := false
+	spec.Containers = append(spec.Containers, v1.Container{
+		Name:            "nri-sgx-epc",
+		Image:           imageName,
+		ImagePullPolicy: "IfNotPresent",
+		SecurityContext: &v1.SecurityContext{
+			ReadOnlyRootFilesystem:   &yes,
+			AllowPrivilegeEscalation: &no,
+		},
+		VolumeMounts: []v1.VolumeMount{
+			{
+				Name:      "nrisockets",
+				MountPath: "/var/run/nri",
+			},
+		},
+	})
+	addVolumeIfMissing(spec, "nrisockets", "/var/run/nri", v1.HostPathDirectoryOrCreate)
+}
+
 func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet {
 	devicePlugin := rawObj.(*devicepluginv1.SgxDevicePlugin)
 
@@ -131,6 +152,10 @@ func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet {
 	if devicePlugin.Spec.InitImage != "" {
 		setInitContainer(&daemonSet.Spec.Template.Spec, devicePlugin.Spec.InitImage)
 	}
+	// add the optional NRI plugin container
+	if devicePlugin.Spec.NRIImage != "" {
+		setNRIContainer(&daemonSet.Spec.Template.Spec, devicePlugin.Spec.NRIImage)
+	}
 
 	return daemonSet
 }
@@ -166,6 +191,26 @@ func (c *controller) UpdateDaemonSet(rawObj client.Object, ds *apps.DaemonSet) (
 		updated = true
 	}
 
+	// remove NRI plugin
+	if len(ds.Spec.Template.Spec.Containers) > 1 && dp.Spec.NRIImage == "" {
+		ds.Spec.Template.Spec.Containers = []v1.Container{ds.Spec.Template.Spec.Containers[0]}
+		ds.Spec.Template.Spec.Volumes = removeVolume(ds.Spec.Template.Spec.Volumes, "nrisockets")
+		updated = true
+	}
+
+	// update NRI plugin image
+	if len(ds.Spec.Template.Spec.Containers) > 1 && ds.Spec.Template.Spec.Containers[1].Image != dp.Spec.NRIImage {
+		ds.Spec.Template.Spec.Containers[1].Image = dp.Spec.NRIImage
+		updated = true
+	}
+
+	// add NRI plugin image
+	if len(ds.Spec.Template.Spec.Containers) == 1 && dp.Spec.NRIImage != "" {
+		setNRIContainer(&ds.Spec.Template.Spec, dp.Spec.NRIImage)
+
+		updated = true
+	}
+
 	if len(dp.Spec.NodeSelector) > 0 {
 		if !reflect.DeepEqual(ds.Spec.Template.Spec.NodeSelector, dp.Spec.NodeSelector) {
 			ds.Spec.Template.Spec.NodeSelector = dp.Spec.NodeSelector

pkg/webhooks/sgx/sgx.go

@@ -35,6 +35,7 @@ var ErrObjectType = errors.New("invalid runtime object type")
 type Mutator struct{}
 
 const (
+	epcLimitKey              = "epc-limit.nri.io/container"
 	namespace                = "sgx.intel.com"
 	encl                     = namespace + "/enclave"
 	epc                      = namespace + "/epc"
@@ -148,6 +149,8 @@ func (s *Mutator) Default(ctx context.Context, obj runtime.Object) error {
 			continue
 		}
 
+		pod.Annotations[fmt.Sprintf("%s.%s", epcLimitKey, container.Name)] = fmt.Sprintf("%d", epcSize)
+
 		totalEpc += epcSize
 
 		// Quote Generation Modes:

test/e2e/sgxadmissionwebhook/sgxaadmissionwebhook.go

@@ -69,6 +69,7 @@ func describe() {
 
 		ginkgo.By("checking the pod total EPC size annotation is correctly set")
 		gomega.Expect(pod.Annotations["sgx.intel.com/epc"]).To(gomega.Equal("1Mi"))
+		gomega.Expect(pod.Annotations["epc-limit.nri.io/container.test"]).To(gomega.Equal("1048576"))
 	})
 	ginkgo.It("mutates created pods when the container contains the quote generation libraries", func(ctx context.Context) {
 		ginkgo.By("submitting the pod")
@@ -79,6 +80,7 @@ func describe() {
 
 		ginkgo.By("checking the pod total EPC size annotation is correctly set")
 		gomega.Expect(pod.Annotations["sgx.intel.com/epc"]).To(gomega.Equal("1Mi"))
+		gomega.Expect(pod.Annotations["epc-limit.nri.io/container.test"]).To(gomega.Equal("1048576"))
 	})
 	ginkgo.It("mutates created pods when the container uses aesmd from a side-car container to generate quotes", func(ctx context.Context) {
 		ginkgo.By("submitting the pod")
@@ -93,6 +95,8 @@ func describe() {
 		gomega.Expect(pod.Spec.Containers[0].Env[0].Value).To(gomega.Equal("1"))
 		ginkgo.By("checking the pod total EPC size annotation is correctly set")
 		gomega.Expect(pod.Annotations["sgx.intel.com/epc"]).To(gomega.Equal("2Mi"))
+		gomega.Expect(pod.Annotations["epc-limit.nri.io/container.test"]).To(gomega.Equal("1048576"))
+		gomega.Expect(pod.Annotations["epc-limit.nri.io/container.aesmd"]).To(gomega.Equal("1048576"))
 	})
 	ginkgo.It("mutates created pods where one container uses host/daemonset aesmd to generate quotes", func(ctx context.Context) {
 		ginkgo.By("submitting the pod")
@@ -106,6 +110,7 @@ func describe() {
 		gomega.Expect(pod.Spec.Containers[0].Env[0].Value).To(gomega.Equal("1"))
 		ginkgo.By("checking the pod total EPC size annotation is correctly set")
 		gomega.Expect(pod.Annotations["sgx.intel.com/epc"]).To(gomega.Equal("1Mi"))
+		gomega.Expect(pod.Annotations["epc-limit.nri.io/container.test"]).To(gomega.Equal("1048576"))
 	})
 	ginkgo.It("mutates created pods where three containers use host/daemonset aesmd to generate quotes", func(ctx context.Context) {
 		ginkgo.By("submitting the pod")
@@ -125,6 +130,9 @@ func describe() {
 		gomega.Expect(pod.Spec.Containers[2].Env[0].Value).To(gomega.Equal("1"))
 		ginkgo.By("checking the pod total EPC size annotation is correctly set")
 		gomega.Expect(pod.Annotations["sgx.intel.com/epc"]).To(gomega.Equal("3Mi"))
+		gomega.Expect(pod.Annotations["epc-limit.nri.io/container.test1"]).To(gomega.Equal("1048576"))
+		gomega.Expect(pod.Annotations["epc-limit.nri.io/container.test2"]).To(gomega.Equal("1048576"))
+		gomega.Expect(pod.Annotations["epc-limit.nri.io/container.test3"]).To(gomega.Equal("1048576"))
 	})
 	ginkgo.It("checks that Volumes and VolumeMounts are created only once", func(ctx context.Context) {
 		ginkgo.By("submitting the pod")

test/envtest/sgxdeviceplugin_controller_test.go

@@ -38,6 +38,7 @@ var _ = Describe("SgxDevicePlugin Controller", func() {
 			spec := devicepluginv1.SgxDevicePluginSpec{
 				Image:        "sgx-testimage",
 				InitImage:    "sgx-testinitimage",
+				NRIImage:     "sgx-testnriimage",
 				NodeSelector: map[string]string{"sgx-nodeselector": "true"},
 			}
 
@@ -76,13 +77,15 @@ var _ = Describe("SgxDevicePlugin Controller", func() {
 			By("updating SgxDevicePlugin successfully")
 			updatedImage := "updated-sgx-testimage"
 			updatedInitImage := "updated-sgx-testinitimage"
+			updatedNRIImage := "updated-sgx-testnriimage"
 			updatedLogLevel := 2
 			updatedEnclaveLimit := 2
 			updatedProvisionLimit := 2
 			updatedNodeSelector := map[string]string{"updated-sgx-nodeselector": "true"}
 
 			fetched.Spec.Image = updatedImage
 			fetched.Spec.InitImage = updatedInitImage
+			fetched.Spec.NRIImage = updatedNRIImage
 			fetched.Spec.LogLevel = updatedLogLevel
 			fetched.Spec.EnclaveLimit = updatedEnclaveLimit
 			fetched.Spec.ProvisionLimit = updatedProvisionLimit
@@ -112,13 +115,17 @@ var _ = Describe("SgxDevicePlugin Controller", func() {
 			Expect(ds.Spec.Template.Spec.Containers[0].Args).Should(ConsistOf(expectArgs))
 			Expect(ds.Spec.Template.Spec.Containers[0].Image).Should(Equal(updatedImage))
 			Expect(ds.Spec.Template.Spec.InitContainers).To(HaveLen(1))
+			Expect(ds.Spec.Template.Spec.Containers).To(HaveLen(2))
+			Expect(ds.Spec.Template.Spec.Containers[1].Image).Should(Equal(updatedNRIImage))
 			Expect(ds.Spec.Template.Spec.InitContainers[0].Image).To(Equal(updatedInitImage))
 			Expect(ds.Spec.Template.Spec.NodeSelector).Should(Equal(updatedNodeSelector))
 
 			By("updating SgxDevicePlugin with different values successfully")
 			updatedInitImage = ""
+			updatedNRIImage = ""
 			updatedNodeSelector = map[string]string{}
 			fetched.Spec.InitImage = updatedInitImage
+			fetched.Spec.NRIImage = updatedNRIImage
 			fetched.Spec.NodeSelector = updatedNodeSelector
 
 			Expect(k8sClient.Update(context.Background(), fetched)).Should(Succeed())
@@ -128,6 +135,7 @@ var _ = Describe("SgxDevicePlugin Controller", func() {
 			err = k8sClient.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: expectedDsName}, ds)
 			Expect(err).To(BeNil())
 			Expect(ds.Spec.Template.Spec.InitContainers).To(HaveLen(0))
+			Expect(ds.Spec.Template.Spec.Containers).To(HaveLen(1))
 			Expect(ds.Spec.Template.Spec.NodeSelector).Should(And(HaveLen(1), HaveKeyWithValue("kubernetes.io/arch", "amd64")))
 
 			By("deleting SgxDevicePlugin successfully")