qat,e2e: add heartbeat and auto-reset validations

Signed-off-by: Hyeongju Johannes Lee <hyeongju.lee@intel.com>

Author: hj-johannes-lee

test/e2e/qat/qatplugin_dpdk.go

@@ -29,15 +29,19 @@ import (
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/kubernetes/test/e2e/framework"
 	e2edebug "k8s.io/kubernetes/test/e2e/framework/debug"
+	e2ejob "k8s.io/kubernetes/test/e2e/framework/job"
 	e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl"
 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
+	imageutils "k8s.io/kubernetes/test/utils/image"
 	admissionapi "k8s.io/pod-security-admission/api"
 )
 
 const (
 	qatPluginKustomizationYaml = "deployments/qat_plugin/overlays/e2e/kustomization.yaml"
 	cryptoTestYaml             = "deployments/qat_dpdk_app/crypto-perf/crypto-perf-dpdk-pod-requesting-qat-cy.yaml"
 	compressTestYaml           = "deployments/qat_dpdk_app/compress-perf/compress-perf-dpdk-pod-requesting-qat-dc.yaml"
+	cy                         = "qat.intel.com/cy"
+	dc                         = "qat.intel.com/dc"
 )
 
 const (
@@ -77,6 +81,7 @@ func describeQatDpdkPlugin() {
 	var dpPodName string
 
 	var resourceName v1.ResourceName
+	var nodeName string
 
 	ginkgo.JustBeforeEach(func(ctx context.Context) {
 		ginkgo.By("deploying QAT plugin in DPDK mode")
@@ -97,6 +102,18 @@ func describeQatDpdkPlugin() {
 			framework.Failf("container filesystem info checks failed: %v", err)
 		}
 
+		// when running [Functionality] e2e tests
+		if resourceName == "" {
+			if err := utils.WaitForNodesWithResource(ctx, f.ClientSet, cy, 100*time.Second, utils.WaitForPositiveResource); err != nil {
+				resourceName = dc
+				nodeName, _ = utils.FindNodeAndResourceCapacity(f, ctx, dc)
+				return
+			}
+			resourceName = cy
+			nodeName, _ = utils.FindNodeAndResourceCapacity(f, ctx, cy)
+			return
+		}
+
 		ginkgo.By("checking if the resource is allocatable")
 		if err := utils.WaitForNodesWithResource(ctx, f.ClientSet, resourceName, 30*time.Second, utils.WaitForPositiveResource); err != nil {
 			framework.Failf("unable to wait for nodes to have positive allocatable resource: %v", err)
@@ -165,6 +182,52 @@ func describeQatDpdkPlugin() {
 			ginkgo.It("does nothing", func() {})
 		})
 	})
+
+	ginkgo.Context("When QAT's autoreset is off [Functionality:heartbeat]", func() {
+		ginkgo.BeforeEach(func() {
+			ginkgo.By("creating a configMap before plugin gets deployed")
+			e2ekubectl.RunKubectlOrDie(f.Namespace.Name, "create", "configmap", "--from-literal", "qat.conf=AutoresetEnabled=off", "qat-config")
+		})
+
+		ginkgo.It("checks if the heartbeat is read correctly", func(ctx context.Context) {
+			injectError(ctx, f, nodeName)
+
+			ginkgo.By("seeing if there is zero resource")
+			if err := utils.WaitForNodesWithResource(ctx, f.ClientSet, resourceName, 100*time.Second, utils.WaitForZeroResource); err != nil {
+				framework.Failf("unable to wait for nodes to have no resource: %v", err)
+			}
+		})
+	})
+
+	ginkgo.Context("When QAT's autoreset is on [Functionality:auto-reset]", func() {
+		ginkgo.BeforeEach(func() {
+			ginkgo.By("creating a configMap before plugin gets deployed")
+			e2ekubectl.RunKubectlOrDie(f.Namespace.Name, "create", "configmap", "--from-literal", "qat.conf=AutoresetEnabled=on", "qat-config")
+		})
+
+		ginkgo.It("checks if an injected error gets solved", func(ctx context.Context) {
+			injectError(ctx, f, nodeName)
+
+			ginkgo.By("seeing if there is zero resource")
+			if err := utils.WaitForNodesWithResource(ctx, f.ClientSet, resourceName, 100*time.Second, utils.WaitForZeroResource); err != nil {
+				// It should not fail since there can be a case auto-reset function worked before plugin notices.
+				// It is still useful to wait for some time to ensure that injected error affects other specs.
+				framework.Logf("unable to wait for nodes to have no resource: %v", err)
+			}
+
+			ginkgo.By("seeing if there is positive allocatable resource")
+			if err := utils.WaitForNodesWithResource(ctx, f.ClientSet, resourceName, 300*time.Second, utils.WaitForPositiveResource); err != nil {
+				framework.Failf("unable to wait for nodes to have positive allocatable resource: %v", err)
+			}
+
+			ginkgo.By("checking if openssl pod runs successfully")
+			if resourceName == cy {
+				runCpaSampleCode(ctx, f, symmetric, resourceName)
+			} else {
+				runCpaSampleCode(ctx, f, compression, resourceName)
+			}
+		})
+	})
 }
 
 func runCpaSampleCode(ctx context.Context, f *framework.Framework, runTests int, resourceName v1.ResourceName) {
@@ -199,3 +262,36 @@ func runCpaSampleCode(ctx context.Context, f *framework.Framework, runTests int,
 	err = e2epod.WaitForPodSuccessInNamespaceTimeout(ctx, f.ClientSet, pod.ObjectMeta.Name, f.Namespace.Name, 300*time.Second)
 	gomega.Expect(err).To(gomega.BeNil(), utils.GetPodLogs(ctx, f, pod.ObjectMeta.Name, pod.Spec.Containers[0].Name))
 }
+
+func injectError(ctx context.Context, f *framework.Framework, nodeName string) {
+	job := e2ejob.NewTestJobOnNode("success", "qat-inject-error", v1.RestartPolicyNever, 1, 1, nil, 0, nodeName)
+
+	job.Spec.Template.Spec.Containers[0].Image = imageutils.GetE2EImage(imageutils.BusyBox)
+	job.Spec.Template.Spec.Containers[0].Command = []string{
+		"/bin/sh",
+		"-c",
+		"find /sys/kernel/debug/qat_*/heartbeat/ -name inject_error -exec sh -c 'echo 1 > {}' \\;",
+	}
+	job.Spec.Template.Spec.Containers[0].VolumeMounts = []v1.VolumeMount{{
+		Name:      "debugfs",
+		MountPath: "/sys/kernel/debug/",
+	}}
+	job.Spec.Template.Spec.Volumes = []v1.Volume{{
+		Name: "debugfs",
+		VolumeSource: v1.VolumeSource{
+			HostPath: &v1.HostPathVolumeSource{
+				Path: "/sys/kernel/debug/",
+			},
+		},
+	}}
+	yes := true
+	job.Spec.Template.Spec.Containers[0].SecurityContext = &v1.SecurityContext{
+		Privileged: &yes,
+	}
+
+	job, err := e2ejob.CreateJob(ctx, f.ClientSet, f.Namespace.Name, job)
+	framework.ExpectNoError(err, "failed to create job in namespace: %s", f.Namespace.Name)
+
+	err = e2ejob.WaitForJobComplete(ctx, f.ClientSet, f.Namespace.Name, job.Name, nil, 1)
+	framework.ExpectNoError(err, "failed to ensure job completion in namespace: %s", f.Namespace.Name)
+}

test/e2e/utils/utils.go

@@ -365,3 +365,19 @@ func Kubectl(ns string, cmd string, opt string, file string) {
 	msg := e2ekubectl.RunKubectlOrDie(ns, cmd, opt, path)
 	framework.Logf("%s", msg)
 }
+
+func FindNodeAndResourceCapacity(f *framework.Framework, ctx context.Context, resourceName string) (string, int64) {
+	nodelist, err := f.ClientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
+	if err != nil {
+		framework.Failf("failed to list Nodes: %v", err)
+	}
+
+	// we have at least one node with resource capacity
+	for _, item := range nodelist.Items {
+		if q, ok := item.Status.Allocatable[v1.ResourceName(resourceName)]; ok && q.Value() > 0 {
+			return item.Name, q.Value()
+		}
+	}
+
+	return "", 0
+}