qat,e2e: add heartbeat and auto-reset validations

Signed-off-by: Hyeongju Johannes Lee <hyeongju.lee@intel.com>

Author: hj-johannes-lee

test/e2e/qat/qatplugin_dpdk.go

@@ -31,6 +31,7 @@ import (
 	e2edebug "k8s.io/kubernetes/test/e2e/framework/debug"
 	e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl"
 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
+	imageutils "k8s.io/kubernetes/test/utils/image"
 	admissionapi "k8s.io/pod-security-admission/api"
 )
 
@@ -165,6 +166,34 @@ func describeQatDpdkPlugin() {
 			ginkgo.It("does nothing", func() {})
 		})
 	})
+
+	ginkgo.Context("When QAT's autoreset is on [Functionality:auto-reset]", func() {
+		ginkgo.BeforeEach(func() {
+			ginkgo.By("creating a configMap before plugin gets deployed")
+			e2ekubectl.RunKubectlOrDie(f.Namespace.Name, "create", "configmap", "--from-literal", "qat.conf=AutoresetEnabled=on", "qat-config")
+		})
+
+		ginkgo.It("checks if an injected error gets solved", func(ctx context.Context) {
+			injectError(ctx, f)
+			if err := utils.WaitForNodesWithResource(ctx, f.ClientSet, resourceName, 300*time.Second, utils.WaitForPositiveResource); err != nil {
+				framework.Failf("unable to wait for nodes to have positive allocatable resource: %v", err)
+			}
+		})
+	})
+
+	ginkgo.Context("When QAT's autoreset is off [Functionality:auto-reset]", func() {
+		ginkgo.BeforeEach(func() {
+			ginkgo.By("creating a configMap before plugin gets deployed")
+			e2ekubectl.RunKubectlOrDie(f.Namespace.Name, "create", "configmap", "--from-literal", "qat.conf=AutoresetEnabled=off", "qat-config")
+		})
+
+		ginkgo.It("checks if the heartbeat is read correctly", func(ctx context.Context) {
+			injectError(ctx, f)
+			if err := utils.WaitForNodesWithResource(ctx, f.ClientSet, resourceName, 100*time.Second, utils.WaitForZeroResource); err != nil {
+				framework.Failf("unable to wait for nodes to have zero resource: %v", err)
+			}
+		})
+	})
 }
 
 func runCpaSampleCode(ctx context.Context, f *framework.Framework, runTests int, resourceName v1.ResourceName) {
@@ -199,3 +228,45 @@ func runCpaSampleCode(ctx context.Context, f *framework.Framework, runTests int,
 	err = e2epod.WaitForPodSuccessInNamespaceTimeout(ctx, f.ClientSet, pod.ObjectMeta.Name, f.Namespace.Name, 300*time.Second)
 	gomega.Expect(err).To(gomega.BeNil(), utils.GetPodLogs(ctx, f, pod.ObjectMeta.Name, pod.Spec.Containers[0].Name))
 }
+
+func injectError(ctx context.Context, f *framework.Framework) {
+	yes := true
+	podSpec := &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{Name: "qat-inject-error"},
+		Spec: v1.PodSpec{
+			Containers: []v1.Container{
+				{
+					Name:            "qat-inject-error",
+					Image:           imageutils.GetE2EImage(imageutils.BusyBox),
+					ImagePullPolicy: "IfNotPresent",
+					Command:         []string{"find", "/sys/kernel/debug/qat_*/heartbeat/", "-name", "inject_error", "-exec", "sh", "-c", "'echo 1 > {}'", "\\;"},
+					SecurityContext: &v1.SecurityContext{
+						Privileged: &yes,
+					},
+					VolumeMounts: []v1.VolumeMount{
+						{
+							Name:      "debugfs",
+							MountPath: "/sys/kernel/debug/",
+						},
+					},
+				},
+			},
+			Volumes: []v1.Volume{
+				{
+					Name: "debugfs",
+					VolumeSource: v1.VolumeSource{
+						HostPath: &v1.HostPathVolumeSource{
+							Path: "/sys/kernel/debug/",
+						},
+					},
+				},
+			},
+			RestartPolicy: v1.RestartPolicyNever,
+		},
+	}
+	pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(ctx, podSpec, metav1.CreateOptions{})
+	framework.ExpectNoError(err, "pod Create API error")
+
+	err = e2epod.WaitForPodSuccessInNamespaceTimeout(ctx, f.ClientSet, pod.ObjectMeta.Name, f.Namespace.Name, 300*time.Second)
+	gomega.Expect(err).To(gomega.BeNil(), utils.GetPodLogs(ctx, f, pod.ObjectMeta.Name, pod.Spec.Containers[0].Name))
+}