add sweeper resources scan (#2490)

* add sweeper resources scan

* update file common function

* add log

* update log print

* update file generate path

* update sweeper resource scan dir

* generate csv file add Classification

* delete cloud_common file

* optimization add scanning resources

* add scanning some resources

* fix DescribeTargetGroups request Filters param

* optimize Filters logic

* update add scanning resources logic

* csv file add CreationDuration

* added handling of non-keep resources

* add e2e scanning resources

* update csv file title

* update NonKeepResourceScanHeader

* add err log print

Author: bruceybian

tencentcloud/common/file_common.go

@@ -0,0 +1,90 @@
+package common
+
+import (
+	"encoding/csv"
+	"log"
+	"os"
+	"path/filepath"
+	"time"
+)
+
+const (
+	SweeperResourceScanDir        = "../../../tmp/resource_scan/"
+	SweeperNonKeepResourceScanDir = "../../../tmp/non_keep_resource_scan/"
+)
+
+var ResourceScanHeader = []string{"资源类型", "资源名称", "实例ID", "实例名称", "分类", "创建时长(天)"}
+var NonKeepResourceScanHeader = []string{"ResourceType", "ResourceName", "InstanceId", "InstanceName"}
+
+// WriteCsvFileData write data to csv file
+func WriteCsvFileData(dirPath string, header []string, data [][]string) error {
+	log.Printf("[INFO] write csv file data[%v] to path[%v] start", len(data), dirPath)
+
+	count := 0
+	defer func() {
+		log.Printf("[INFO] write csv file data to path[%v] success count[%v]", dirPath, count)
+	}()
+
+	if len(data) == 0 {
+		return nil
+	}
+
+	err := os.MkdirAll(dirPath, 0755)
+	if err != nil {
+		log.Printf("[CRITAL] create directory %s error: %v", dirPath, err.Error())
+		return err
+	}
+
+	currentDate := time.Now().Format("20060102")
+	filePath := filepath.Join(dirPath, currentDate+".csv")
+
+	_, err = os.Stat(filePath)
+	if os.IsNotExist(err) {
+		err = GenerateCsvFile(filePath, header)
+		if err != nil {
+			log.Printf("[CRITAL] generate csv file error: %v", err.Error())
+			return err
+		}
+	}
+
+	file, err := os.OpenFile(filePath, os.O_APPEND|os.O_WRONLY, 0644)
+	if err != nil {
+		log.Printf("[CRITAL] open csv file error: %v", err.Error())
+		return err
+	}
+	defer file.Close()
+
+	writer := csv.NewWriter(file)
+
+	for _, row := range data {
+		err = writer.Write(row)
+		if err != nil {
+			log.Printf("[CRITAL] write data[%v] to csv file error: %v", row, err.Error())
+			return err
+		}
+		count++
+	}
+	writer.Flush()
+
+	return nil
+}
+
+// GenerateCsvFile generate when csv file does not exist
+func GenerateCsvFile(filePath string, header []string) error {
+	file, err := os.Create(filePath)
+	if err != nil {
+		log.Printf("[CRITAL] create csv file error: %v", err.Error())
+		return err
+	}
+	defer file.Close()
+
+	writer := csv.NewWriter(file)
+	err = writer.Write(header)
+	if err != nil {
+		log.Printf("[CRITAL] write header to csv file error: %v", err.Error())
+		return err
+	}
+	writer.Flush()
+
+	return nil
+}

tencentcloud/common/resource_scan.go

@@ -0,0 +1,148 @@
+package common
+
+import (
+	"fmt"
+	"log"
+	"regexp"
+	"strconv"
+	"time"
+)
+
+const (
+	KeepResource    = "keep"
+	NonKeepResource = "non-keep"
+)
+
+// TimeFormats add all possible time formats
+var TimeFormats = []string{
+	time.RFC3339, //ISO8601 UTC time
+	"2006-01-02 15:04:05",
+	// add other time formats here
+}
+
+type ResourceInstance struct {
+	Id          string
+	Name        string
+	CreatTime   string
+	DefaultKeep bool
+}
+
+func ProcessScanCloudResources(resources, nonKeepResources []*ResourceInstance, resourceType, resourceName string) {
+	ProcessResources(resources, resourceType, resourceName)
+
+	ProcessNonKeepResources(nonKeepResources, resourceType, resourceName)
+}
+
+// ProcessResources Process all scanned cloud resources
+func ProcessResources(resources []*ResourceInstance, resourceType, resourceName string) {
+	data := make([][]string, len(resources))
+	for i, r := range resources {
+		isResourceKeep := CheckResourceNameKeep(r.Name)
+		// some resources default to keep
+		if r.DefaultKeep {
+			isResourceKeep = KeepResource
+		}
+
+		creationDuration, err := DaysSinceCreation(r.CreatTime)
+		if err != nil {
+			log.Printf("[CRITAL] compute resource creation duration error: %v", err.Error())
+		}
+
+		data[i] = []string{
+			resourceType,
+			resourceName,
+			r.Id,
+			r.Name,
+			isResourceKeep,
+			creationDuration,
+		}
+	}
+	err := WriteCsvFileData(SweeperResourceScanDir, ResourceScanHeader, data)
+	if err != nil {
+		log.Printf("[CRITAL] write csv file data error: %v", err.Error())
+	}
+}
+
+// ProcessNonKeepResources Processing scanned non-keep cloud resources
+func ProcessNonKeepResources(nonKeepResources []*ResourceInstance, resourceType, resourceName string) {
+	data := make([][]string, len(nonKeepResources))
+	for i, r := range nonKeepResources {
+		data[i] = []string{
+			resourceType,
+			resourceName,
+			r.Id,
+			r.Name,
+		}
+	}
+	err := WriteCsvFileData(SweeperNonKeepResourceScanDir, NonKeepResourceScanHeader, data)
+	if err != nil {
+		log.Printf("[CRITAL] write csv file data error: %v", err.Error())
+	}
+}
+
+// CheckResourceNameKeep check whether to keep resource name
+func CheckResourceNameKeep(name string) string {
+	flag := CheckResourcePersist(name, "")
+	if flag {
+		return KeepResource
+	}
+	return NonKeepResource
+}
+
+// CheckResourcePersist check whether to persist resource
+func CheckResourcePersist(name, createTime string) bool {
+	if name == "" && createTime == "" {
+		return false
+	}
+	parsedTime, _ := ParsedTime(createTime)
+
+	createdWithin30Minutes := false
+	if parsedTime != nil {
+		createdWithin30Minutes = parsedTime.Add(time.Minute * 30).After(time.Now())
+	}
+
+	flag := regexp.MustCompile("^(keep|Default)").MatchString(name)
+	return flag || createdWithin30Minutes
+}
+
+// DaysSinceCreation compute resource creation duration
+func DaysSinceCreation(createTime string) (string, error) {
+	parsedTime, err := ParsedTime(createTime)
+	if err != nil {
+		return "", err
+	}
+
+	duration := time.Since(*parsedTime)
+	days := duration.Hours() / 24
+
+	return fmt.Sprintf("%.2f", days), nil
+}
+
+// ParsedTime parse time
+func ParsedTime(createTime string) (*time.Time, error) {
+	if createTime == "" {
+		return nil, nil
+	}
+
+	var parsedTime time.Time
+	var err error
+
+	timestamp, err := strconv.ParseInt(createTime, 10, 64)
+	if err == nil {
+		parsedTime = time.Unix(timestamp, 0)
+	} else {
+		// try parsing input strings using different time formats
+		for _, format := range TimeFormats {
+			parsedTime, err = time.Parse(format, createTime)
+			if err == nil {
+				break
+			}
+		}
+	}
+
+	if err != nil {
+		log.Printf("[CRITAL] unable to parse create time[%s]", createTime)
+		return nil, fmt.Errorf("unable to parse create time: %v", err.Error())
+	}
+	return &parsedTime, nil
+}

tencentcloud/services/as/resource_tc_as_attachment_test.go

@@ -35,6 +35,23 @@ func testSweepAsAttachment(r string) error {
 		return fmt.Errorf("list scaling group error: %s", err.Error())
 	}
 
+	// add scanning resources
+	var resources, nonKeepResources []*tccommon.ResourceInstance
+	for _, v := range scalingGroups {
+		if !tccommon.CheckResourcePersist(*v.AutoScalingGroupName, *v.CreatedTime) {
+			nonKeepResources = append(nonKeepResources, &tccommon.ResourceInstance{
+				Id:   *v.AutoScalingGroupId,
+				Name: *v.AutoScalingGroupName,
+			})
+		}
+		resources = append(resources, &tccommon.ResourceInstance{
+			Id:        *v.AutoScalingGroupId,
+			Name:      *v.AutoScalingGroupName,
+			CreatTime: *v.CreatedTime,
+		})
+	}
+	tccommon.ProcessScanCloudResources(resources, nonKeepResources, "as", "attachment")
+
 	for _, v := range scalingGroups {
 		scalingGroupId := *v.AutoScalingGroupId
 		scalingGroupName := *v.AutoScalingGroupName

tencentcloud/services/cam/resource_tc_cam_group_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"log"
+	"strconv"
 	"testing"
 
 	tcacctest "github.com/tencentcloudstack/terraform-provider-tencentcloud/tencentcloud/acctest"
@@ -33,6 +34,24 @@ func init() {
 			if err != nil {
 				return err
 			}
+
+			// add scanning resources
+			var resources, nonKeepResources []*tccommon.ResourceInstance
+			for _, v := range groups {
+				if !tccommon.CheckResourcePersist(*v.GroupName, *v.CreateTime) {
+					nonKeepResources = append(nonKeepResources, &tccommon.ResourceInstance{
+						Id:   strconv.FormatUint(*v.GroupId, 10),
+						Name: *v.GroupName,
+					})
+				}
+				resources = append(resources, &tccommon.ResourceInstance{
+					Id:        strconv.FormatUint(*v.GroupId, 10),
+					Name:      *v.GroupName,
+					CreatTime: *v.CreateTime,
+				})
+			}
+			tccommon.ProcessScanCloudResources(resources, nonKeepResources, "cam", "group")
+
 			for _, v := range groups {
 				name := *v.GroupName
 

tencentcloud/services/cbs/resource_tc_cbs_storage_test.go

@@ -27,11 +27,27 @@ func init() {
 			service := localcbs.NewCbsService(client)
 
 			disks, err := service.DescribeDisksByFilter(ctx, nil)
-
 			if err != nil {
 				return err
 			}
 
+			// add scanning resources
+			var resources, nonKeepResources []*tccommon.ResourceInstance
+			for _, v := range disks {
+				if !tccommon.CheckResourcePersist(*v.DiskName, *v.CreateTime) {
+					nonKeepResources = append(nonKeepResources, &tccommon.ResourceInstance{
+						Id:   *v.DiskId,
+						Name: *v.DiskName,
+					})
+				}
+				resources = append(resources, &tccommon.ResourceInstance{
+					Id:        *v.DiskId,
+					Name:      *v.DiskName,
+					CreatTime: *v.CreateTime,
+				})
+			}
+			tccommon.ProcessScanCloudResources(resources, nonKeepResources, "cbs", "storage")
+
 			for i := range disks {
 				disk := disks[i]
 				id := *disk.DiskId

tencentcloud/services/ccn/resource_tc_ccn_test.go

@@ -41,6 +41,23 @@ func testSweepCcnInstance(region string) error {
 		return fmt.Errorf("get instance list error: %s", err.Error())
 	}
 
+	// add scanning resources
+	var resources, nonKeepResources []*tccommon.ResourceInstance
+	for _, v := range instances {
+		if !tccommon.CheckResourcePersist(v.Name(), v.CreateTime()) {
+			nonKeepResources = append(nonKeepResources, &tccommon.ResourceInstance{
+				Id:   v.CcnId(),
+				Name: v.Name(),
+			})
+		}
+		resources = append(resources, &tccommon.ResourceInstance{
+			Id:        v.CcnId(),
+			Name:      v.Name(),
+			CreatTime: v.CreateTime(),
+		})
+	}
+	tccommon.ProcessScanCloudResources(resources, nonKeepResources, "ccn", "")
+
 	for _, v := range instances {
 		instanceId := v.CcnId()
 		instanceName := v.Name()

tencentcloud/services/cdb/resource_tc_mysql_instance_test.go

@@ -47,12 +47,30 @@ func testSweepMySQLInstance(region string) error {
 	if err != nil {
 		return err
 	}
+	items := response.Response.Items
 
-	if len(response.Response.Items) == 0 {
+	if len(items) == 0 {
 		return nil
 	}
 
-	for _, v := range response.Response.Items {
+	// add scanning resources
+	var resources, nonKeepResources []*tccommon.ResourceInstance
+	for _, v := range items {
+		if !tccommon.CheckResourcePersist(*v.InstanceId, *v.CreateTime) {
+			nonKeepResources = append(nonKeepResources, &tccommon.ResourceInstance{
+				Id:   *v.InstanceId,
+				Name: *v.InstanceName,
+			})
+		}
+		resources = append(resources, &tccommon.ResourceInstance{
+			Id:        *v.InstanceId,
+			Name:      *v.InstanceName,
+			CreatTime: *v.CreateTime,
+		})
+	}
+	tccommon.ProcessScanCloudResources(resources, nonKeepResources, "msql", "instance")
+
+	for _, v := range items {
 		id := *v.InstanceId
 		name := *v.InstanceName
 		if tcacctest.IsResourcePersist(name, nil) {