qat,e2e: add heartbeat and auto-reset validation

Signed-off-by: Hyeongju Johannes Lee <hyeongju.lee@intel.com>

Author: hj-johannes-lee

test/e2e/qat/qatplugin_dpdk.go

@@ -16,6 +16,8 @@ package qat
 
 import (
 	"context"
+	"fmt"
+	"os"
 	"path/filepath"
 	"strconv"
 	"time"
@@ -38,6 +40,8 @@ const (
 	qatPluginKustomizationYaml = "deployments/qat_plugin/overlays/e2e/kustomization.yaml"
 	cryptoTestYaml             = "deployments/qat_dpdk_app/crypto-perf/crypto-perf-dpdk-pod-requesting-qat-cy.yaml"
 	compressTestYaml           = "deployments/qat_dpdk_app/compress-perf/compress-perf-dpdk-pod-requesting-qat-dc.yaml"
+	autoResetPattern           = "/sys/bus/pci/drivers/*/*/qat/auto_reset"
+	injectErrorPattern         = "/sys/kernel/debug/qat_*/heartbeat/inject_error"
 )
 
 const (
@@ -55,6 +59,41 @@ func init() {
 	ginkgo.Describe("QAT plugin in DPDK mode [Device:qat] [Mode:dpdk]", describeQatDpdkPlugin)
 }
 
+func setQatAutoReset(cfgVal string) error {
+	matches, err := filepath.Glob(autoResetPattern)
+	if err != nil {
+		return err
+	}
+
+	for _, filePath := range matches {
+		err = os.WriteFile(filePath, []byte(cfgVal), 0600)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func injectError() error {
+	matches, err := filepath.Glob(injectErrorPattern)
+	if err != nil {
+		return err
+	}
+
+	for _, filePath := range matches {
+		file, err := os.OpenFile(filePath, os.O_WRONLY|os.O_TRUNC, 0600)
+		if err != nil {
+			return err
+		}
+		defer file.Close()
+
+		os.Stdout = file
+		fmt.Println("1")
+	}
+
+	return nil
+}
+
 func describeQatDpdkPlugin() {
 	f := framework.NewDefaultFramework("qatplugindpdk")
 	f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged
@@ -101,6 +140,28 @@ func describeQatDpdkPlugin() {
 		if err := utils.WaitForNodesWithResource(ctx, f.ClientSet, resourceName, 30*time.Second, utils.WaitForPositiveResource); err != nil {
 			framework.Failf("unable to wait for nodes to have positive allocatable resource: %v", err)
 		}
+
+		ginkgo.By("checking if injected errors are recognized")
+		if err := setQatAutoReset("off"); err != nil {
+			framework.Logf("unable to set auto reset in this system: %v", err)
+			return
+		}
+		if err := injectError(); err != nil {
+			framework.Logf("unable to test error injection in this system: %v", err)
+			return
+		}
+		if err := utils.WaitForNodesWithResource(ctx, f.ClientSet, resourceName, 100*time.Second, utils.WaitForZeroResource); err != nil {
+			framework.Failf("unable to wait for nodes to have zero resource: %v", err)
+		}
+
+		ginkgo.By("checking if auto_reset works to solve the errors")
+		if err := setQatAutoReset("on"); err != nil {
+			framework.Logf("unable to set auto reset in this system: %v", err)
+			return
+		}
+		if err := utils.WaitForNodesWithResource(ctx, f.ClientSet, resourceName, 100*time.Second, utils.WaitForPositiveResource); err != nil {
+			framework.Failf("unable to wait for nodes to have positive allocatable resource: %v", err)
+		}
 	})
 
 	ginkgo.AfterEach(func(ctx context.Context) {