WIP: e2e: add initial SGX EPC cgroups tests

Signed-off-by: Mikko Ylinen <mikko.ylinen@intel.com>

Author: mythi

.github/workflows/lib-e2e.yaml

@@ -44,6 +44,7 @@ jobs:
           - name: e2e-sgx
             runner: sgx
             images: intel-sgx-plugin intel-sgx-initcontainer intel-sgx-admissionwebhook sgx-sdk-demo intel-deviceplugin-operator stress-ng-gramine
+            targetjob: e2e-sgx SKIP=App:sgx-epc-cgroup
 
     name: ${{ matrix.name }}
     runs-on: ${{ matrix.runner }}

scripts/set-version.sh

@@ -15,8 +15,8 @@ if [ $# != 1 ] || [ "$1" = "?" ] || [ "$1" = "--help" ]; then
     exit 1
 fi
 
-files=$(git grep -l '^TAG?*=\|intel/accel-config-demo:\|intel/crypto-perf:\|intel/opae-nlb-demo:\|intel/openssl-qat-engine:\|intel/dlb-libdlb-demo:\|intel/sgx-sdk-demo:\|intel/intel-[^ ]*:\|version=\|appVersion:\|tag:' Makefile deployments demo/*accel-config*.yaml demo/*fpga*.yaml demo/*openssl*.yaml demo/dlb-libdlb*.yaml pkg/controllers/*/*_test.go build/docker/*.Dockerfile test/e2e/*/*.go)
+files=$(git grep -l '^TAG?*=\|intel/accel-config-demo:\|intel/crypto-perf:\|intel/opae-nlb-demo:\|intel/openssl-qat-engine:\|intel/dlb-libdlb-demo:\|intel/stress-ng-gramine:\|intel/sgx-sdk-demo:\|intel/intel-[^ ]*:\|version=\|appVersion:\|tag:' Makefile deployments demo/*accel-config*.yaml demo/*fpga*.yaml demo/*openssl*.yaml demo/dlb-libdlb*.yaml pkg/controllers/*/*_test.go build/docker/*.Dockerfile test/e2e/*/*.go)
 
 for file in $files; do
-    sed -i -e "s;\(^TAG?*=\|intel/accel-config-demo:\|intel/crypto-perf:\|intel/opae-nlb-demo:\|intel/openssl-qat-engine:\|intel/dlb-libdlb-demo:\|intel/sgx-sdk-demo:\|intel/intel-[^ ]*:\|version=\|appVersion: [^ ]\|tag: [^ ]\)[^ \"]*;\1$1;g" "$file";
+    sed -i -e "s;\(^TAG?*=\|intel/accel-config-demo:\|intel/crypto-perf:\|intel/opae-nlb-demo:\|intel/openssl-qat-engine:\|intel/dlb-libdlb-demo:\|intel/stress-ng-gramine:\|intel/sgx-sdk-demo:\|intel/intel-[^ ]*:\|version=\|appVersion: [^ ]\|tag: [^ ]\)[^ \"]*;\1$1;g" "$file";
 done

test/e2e/sgx/sgx.go

@@ -16,6 +16,7 @@ package sgx
 
 import (
 	"context"
+	"fmt"
 	"path/filepath"
 	"time"
 
@@ -28,6 +29,7 @@ import (
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/kubernetes/test/e2e/framework"
 	e2edebug "k8s.io/kubernetes/test/e2e/framework/debug"
+	e2ejob "k8s.io/kubernetes/test/e2e/framework/job"
 	e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl"
 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
 	admissionapi "k8s.io/pod-security-admission/api"
@@ -37,7 +39,8 @@ const (
 	ns                   = "inteldeviceplugins-system"
 	timeout              = time.Second * 120
 	kustomizationWebhook = "deployments/sgx_admissionwebhook/overlays/default-with-certmanager/kustomization.yaml"
-	kustomizationPlugin  = "deployments/sgx_plugin/base/kustomization.yaml"
+	kustomizationPlugin  = "deployments/sgx_plugin/overlays/epc-cgroups/kustomization.yaml"
+	stressNGImage        = "intel/stress-ng-gramine:devel"
 )
 
 func init() {
@@ -80,6 +83,9 @@ func describe() {
 	})
 
 	ginkgo.Context("When SGX resources are available", func() {
+		var nodeWithEPC string
+		var epcCapacity int64
+
 		ginkgo.BeforeEach(func(ctx context.Context) {
 			ginkgo.By("checking if the resource is allocatable")
 			if err := utils.WaitForNodesWithResource(ctx, f.ClientSet, "sgx.intel.com/epc", 150*time.Second); err != nil {
@@ -91,6 +97,20 @@ func describe() {
 			if err := utils.WaitForNodesWithResource(ctx, f.ClientSet, "sgx.intel.com/provision", 30*time.Second); err != nil {
 				framework.Failf("unable to wait for nodes to have positive allocatable provision resource: %v", err)
 			}
+
+			nodelist, err := f.ClientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
+			if err != nil {
+				framework.Failf("failed to list Nodes: %v", err)
+			}
+
+			// we have at least one node with sgx.intel.com/epc capacity
+			for _, item := range nodelist.Items {
+				if q, ok := item.Status.Allocatable["sgx.intel.com/epc"]; ok && q.Value() > 0 {
+					epcCapacity = q.Value()
+					nodeWithEPC = item.Name
+					break
+				}
+			}
 		})
 
 		ginkgo.It("deploys a sgx-sdk-demo pod requesting SGX enclave resources [App:sgx-sdk-demo]", func(ctx context.Context) {
@@ -120,6 +140,95 @@ func describe() {
 			gomega.Expect(err).To(gomega.BeNil(), utils.GetPodLogs(ctx, f, pod.ObjectMeta.Name, "testcontainer"))
 		})
 
+		ginkgo.It("deploys simultaneous SGX EPC stressor jobs with equal EPC limits but no memory limits [App:sgx-epc-cgroup]", func(ctx context.Context) {
+			parallelism := int32(20)
+			completions := int32(20)
+			quantity := resource.NewQuantity(epcCapacity/int64(parallelism), resource.BinarySI)
+
+			testArgs := []string{
+				"stress-ng-edmm",
+				"--vm",
+				"1",
+				"--vm-bytes",
+				fmt.Sprintf("%db", epcCapacity/int64(parallelism)),
+				"--page-in",
+				"-t",
+				"30",
+			}
+			job := e2ejob.NewTestJobOnNode("success", "sgx-epc-stressjob", v1.RestartPolicyNever, parallelism, completions, nil, 1, nodeWithEPC)
+
+			job.Spec.Template.Spec.Containers[0].Image = stressNGImage
+			job.Spec.Template.Spec.Containers[0].Args = testArgs
+			job.Spec.Template.Spec.Containers[0].Resources = v1.ResourceRequirements{
+				Requests: v1.ResourceList{"sgx.intel.com/epc": *quantity},
+				Limits:   v1.ResourceList{"sgx.intel.com/epc": *quantity},
+			}
+
+			job, err := e2ejob.CreateJob(ctx, f.ClientSet, f.Namespace.Name, job)
+			framework.ExpectNoError(err, "failed to create job in namespace: %s", f.Namespace.Name)
+
+			err = e2ejob.WaitForJobComplete(ctx, f.ClientSet, f.Namespace.Name, job.Name, completions)
+			framework.ExpectNoError(err, "failed to ensure job completion in namespace: %s", f.Namespace.Name)
+		})
+
+		ginkgo.It("deploys one SGX EPC stressor job with a known enclave size and no memory limits [App:sgx-epc-cgroup]", func(ctx context.Context) {
+			quantity := resource.NewQuantity(epcCapacity, resource.BinarySI)
+
+			testArgs := []string{
+				"stress-ng",
+				"--vm",
+				"1",
+				"--vm-bytes",
+				"128m",
+				"--page-in",
+				"-t",
+				"30",
+			}
+			job := e2ejob.NewTestJobOnNode("success", "sgx-epc-stressjob", v1.RestartPolicyNever, 1, 1, nil, 1, nodeWithEPC)
+
+			job.Spec.Template.Spec.Containers[0].Image = stressNGImage
+			job.Spec.Template.Spec.Containers[0].Args = testArgs
+			job.Spec.Template.Spec.Containers[0].Resources = v1.ResourceRequirements{
+				Requests: v1.ResourceList{"sgx.intel.com/epc": *quantity},
+				Limits:   v1.ResourceList{"sgx.intel.com/epc": *quantity},
+			}
+
+			job, err := e2ejob.CreateJob(ctx, f.ClientSet, f.Namespace.Name, job)
+			framework.ExpectNoError(err, "failed to create job in namespace: %s", f.Namespace.Name)
+
+			err = e2ejob.WaitForJobComplete(ctx, f.ClientSet, f.Namespace.Name, job.Name, 1)
+			framework.ExpectNoError(err, "failed to ensure job completion in namespace: %s", f.Namespace.Name)
+		})
+
+		ginkgo.It("deploys one SGX EPC stressor job with dynamic EPC allocation and memory limit set to kill once enough EPC pages are reclaimed [App:sgx-epc-cgroup]", func(ctx context.Context) {
+			quantity := resource.NewQuantity(epcCapacity/10, resource.BinarySI)
+
+			testArgs := []string{
+				"stress-ng-edmm",
+				"--bigheap",
+				"1",
+				"--bigheap-growth",
+				"1M",
+				"--page-in",
+				"-t",
+				"60",
+			}
+			job := e2ejob.NewTestJobOnNode("success", "sgx-epc-stressjob", v1.RestartPolicyNever, 1, 1, nil, 1, nodeWithEPC)
+
+			job.Spec.Template.Spec.Containers[0].Image = stressNGImage
+			job.Spec.Template.Spec.Containers[0].Args = testArgs
+			job.Spec.Template.Spec.Containers[0].Resources = v1.ResourceRequirements{
+				Requests: v1.ResourceList{"sgx.intel.com/epc": *quantity},
+				Limits: v1.ResourceList{"sgx.intel.com/epc": *quantity,
+					v1.ResourceMemory: *quantity},
+			}
+
+			job, err := e2ejob.CreateJob(ctx, f.ClientSet, f.Namespace.Name, job)
+			framework.ExpectNoError(err, "failed to create job in namespace: %s", f.Namespace.Name)
+			err = e2ejob.WaitForJobFailed(f.ClientSet, f.Namespace.Name, job.Name)
+			framework.ExpectNoError(err, "failed to ensure job completion in namespace: %s", f.Namespace.Name)
+		})
+
 		ginkgo.When("there is no app to run [App:noapp]", func() {
 			ginkgo.It("does nothing", func() {})
 		})