sgx: set epc limits via NRI annotations

Signed-off-by: Mikko Ylinen <mikko.ylinen@intel.com>

Author: mythi

deployments/sgx_plugin/base/intel-sgx-plugin.yaml

@@ -15,6 +15,15 @@ spec:
     spec:
       automountServiceAccountToken: false
       containers:
+      - name: nri-sgx-epc
+        image: ghcr.io/containers/nri-plugins/nri-sgx-epc:unstable
+        securityContext:
+          readOnlyRootFilesystem: true
+          allowPrivilegeEscalation: false
+        imagePullPolicy: IfNotPresent
+        volumeMounts:
+        - name: nrisockets
+          mountPath: /var/run/nri
       - name: intel-sgx-plugin
         image: intel/intel-sgx-plugin:devel
         securityContext:
@@ -36,6 +45,9 @@ spec:
       - name: kubeletsockets
         hostPath:
           path: /var/lib/kubelet/device-plugins
+      - name: nrisockets
+        hostPath:
+          path: /var/run/nri
       - name: sgx-enclave
         hostPath:
           path: /dev/sgx_enclave

pkg/controllers/sgx/controller.go

@@ -131,8 +131,8 @@ func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet {
 
 	daemonSet.ObjectMeta.Namespace = c.ns
 
-	daemonSet.Spec.Template.Spec.Containers[0].Args = getPodArgs(devicePlugin)
-	daemonSet.Spec.Template.Spec.Containers[0].Image = devicePlugin.Spec.Image
+	daemonSet.Spec.Template.Spec.Containers[1].Args = getPodArgs(devicePlugin)
+	daemonSet.Spec.Template.Spec.Containers[1].Image = devicePlugin.Spec.Image
 
 	// add the optional init container
 	if devicePlugin.Spec.InitImage != "" {
@@ -157,8 +157,8 @@ func removeVolume(volumes []v1.Volume, name string) []v1.Volume {
 func (c *controller) UpdateDaemonSet(rawObj client.Object, ds *apps.DaemonSet) (updated bool) {
 	dp := rawObj.(*devicepluginv1.SgxDevicePlugin)
 
-	if ds.Spec.Template.Spec.Containers[0].Image != dp.Spec.Image {
-		ds.Spec.Template.Spec.Containers[0].Image = dp.Spec.Image
+	if ds.Spec.Template.Spec.Containers[1].Image != dp.Spec.Image {
+		ds.Spec.Template.Spec.Containers[1].Image = dp.Spec.Image
 		updated = true
 	}
 
@@ -184,8 +184,8 @@ func (c *controller) UpdateDaemonSet(rawObj client.Object, ds *apps.DaemonSet) (
 	}
 
 	newargs := getPodArgs(dp)
-	if strings.Join(ds.Spec.Template.Spec.Containers[0].Args, " ") != strings.Join(newargs, " ") {
-		ds.Spec.Template.Spec.Containers[0].Args = newargs
+	if strings.Join(ds.Spec.Template.Spec.Containers[1].Args, " ") != strings.Join(newargs, " ") {
+		ds.Spec.Template.Spec.Containers[1].Args = newargs
 		updated = true
 	}
 

pkg/controllers/sgx/controller_test.go

@@ -65,6 +65,21 @@ func (c *controller) newDaemonSetExpected(rawObj client.Object) *apps.DaemonSet
 				Spec: v1.PodSpec{
 					AutomountServiceAccountToken: &no,
 					Containers: []v1.Container{
+						{
+							Name:            "nri-sgx-epc",
+							Image:           "ghcr.io/containers/nri-plugins/nri-sgx-epc:unstable",
+							ImagePullPolicy: "IfNotPresent",
+							SecurityContext: &v1.SecurityContext{
+								ReadOnlyRootFilesystem:   &yes,
+								AllowPrivilegeEscalation: &no,
+							},
+							VolumeMounts: []v1.VolumeMount{
+								{
+									Name:      "nrisockets",
+									MountPath: "/var/run/nri",
+								},
+							},
+						},
 						{
 							Name:            appLabel,
 							Args:            getPodArgs(devicePlugin),
@@ -105,6 +120,14 @@ func (c *controller) newDaemonSetExpected(rawObj client.Object) *apps.DaemonSet
 								},
 							},
 						},
+						{
+							Name: "nrisockets",
+							VolumeSource: v1.VolumeSource{
+								HostPath: &v1.HostPathVolumeSource{
+									Path: "/var/run/nri",
+								},
+							},
+						},
 						{
 							Name: "sgx-enclave",
 							VolumeSource: v1.VolumeSource{

pkg/webhooks/sgx/sgx.go

@@ -35,6 +35,7 @@ var ErrObjectType = errors.New("invalid runtime object type")
 type Mutator struct{}
 
 const (
+	epcLimitKey              = "epc-limit.nri.io"
 	namespace                = "sgx.intel.com"
 	encl                     = namespace + "/enclave"
 	epc                      = namespace + "/epc"
@@ -148,6 +149,9 @@ func (s *Mutator) Default(ctx context.Context, obj runtime.Object) error {
 			continue
 		}
 
+		epcLimitAnnotation := fmt.Sprintf("%s/container.%s", epcLimitKey, container.Name)
+		pod.Annotations[epcLimitAnnotation] = fmt.Sprintf("%d", epcSize)
+
 		totalEpc += epcSize
 
 		// Quote Generation Modes:

test/e2e/sgxadmissionwebhook/sgxaadmissionwebhook.go

@@ -69,6 +69,7 @@ func describe() {
 
 		ginkgo.By("checking the pod total EPC size annotation is correctly set")
 		gomega.Expect(pod.Annotations["sgx.intel.com/epc"]).To(gomega.Equal("1Mi"))
+		gomega.Expect(pod.Annotations["epc-limit.nri.io/container.test"]).To(gomega.Equal("1Mi"))
 	})
 	ginkgo.It("mutates created pods when the container contains the quote generation libraries", func(ctx context.Context) {
 		ginkgo.By("submitting the pod")
@@ -79,6 +80,7 @@ func describe() {
 
 		ginkgo.By("checking the pod total EPC size annotation is correctly set")
 		gomega.Expect(pod.Annotations["sgx.intel.com/epc"]).To(gomega.Equal("1Mi"))
+		gomega.Expect(pod.Annotations["epc-limit.nri.io/container.test"]).To(gomega.Equal("1Mi"))
 	})
 	ginkgo.It("mutates created pods when the container uses aesmd from a side-car container to generate quotes", func(ctx context.Context) {
 		ginkgo.By("submitting the pod")
@@ -93,6 +95,8 @@ func describe() {
 		gomega.Expect(pod.Spec.Containers[0].Env[0].Value).To(gomega.Equal("1"))
 		ginkgo.By("checking the pod total EPC size annotation is correctly set")
 		gomega.Expect(pod.Annotations["sgx.intel.com/epc"]).To(gomega.Equal("2Mi"))
+		gomega.Expect(pod.Annotations["epc-limit.nri.io/container.test"]).To(gomega.Equal("1Mi"))
+		gomega.Expect(pod.Annotations["epc-limit.nri.io/container.aesmd"]).To(gomega.Equal("1Mi"))
 	})
 	ginkgo.It("mutates created pods where one container uses host/daemonset aesmd to generate quotes", func(ctx context.Context) {
 		ginkgo.By("submitting the pod")
@@ -106,6 +110,7 @@ func describe() {
 		gomega.Expect(pod.Spec.Containers[0].Env[0].Value).To(gomega.Equal("1"))
 		ginkgo.By("checking the pod total EPC size annotation is correctly set")
 		gomega.Expect(pod.Annotations["sgx.intel.com/epc"]).To(gomega.Equal("1Mi"))
+		gomega.Expect(pod.Annotations["epc-limit.nri.io/container.test"]).To(gomega.Equal("1Mi"))
 	})
 	ginkgo.It("mutates created pods where three containers use host/daemonset aesmd to generate quotes", func(ctx context.Context) {
 		ginkgo.By("submitting the pod")
@@ -125,6 +130,9 @@ func describe() {
 		gomega.Expect(pod.Spec.Containers[2].Env[0].Value).To(gomega.Equal("1"))
 		ginkgo.By("checking the pod total EPC size annotation is correctly set")
 		gomega.Expect(pod.Annotations["sgx.intel.com/epc"]).To(gomega.Equal("3Mi"))
+		gomega.Expect(pod.Annotations["epc-limit.nri.io/container.test1"]).To(gomega.Equal("1Mi"))
+		gomega.Expect(pod.Annotations["epc-limit.nri.io/container.test2"]).To(gomega.Equal("1Mi"))
+		gomega.Expect(pod.Annotations["epc-limit.nri.io/container.test3"]).To(gomega.Equal("1Mi"))
 	})
 	ginkgo.It("checks that Volumes and VolumeMounts are created only once", func(ctx context.Context) {
 		ginkgo.By("submitting the pod")