From 1cd175839cc208fd3e5da8756ae5e5e874fdfc55 Mon Sep 17 00:00:00 2001 From: hatfieldbrian Date: Sat, 10 Dec 2022 23:07:58 -0800 Subject: [PATCH 1/2] log download object routine Signed-off-by: hatfieldbrian --- ...edvolumereplicationgrouplist_controller.go | 1 + controllers/s3utils.go | 26 ++++++++++++++++--- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/controllers/protectedvolumereplicationgrouplist_controller.go b/controllers/protectedvolumereplicationgrouplist_controller.go index e76c124..78793ee 100644 --- a/controllers/protectedvolumereplicationgrouplist_controller.go +++ b/controllers/protectedvolumereplicationgrouplist_controller.go @@ -148,6 +148,7 @@ func (s *ProtectedVolumeReplicationGroupListInstance) getVrgContentsFromS3(prefi // download VRGs prefixInS3 := fmt.Sprintf("%s/%s/", namespace, vrgName) + s.log.Info("downloadVRGs", "namespace", namespace, "vrg", vrgName, "prefix", prefixInS3) vrgs, err := DownloadVRGs(objectStore, prefixInS3) if err != nil { return vrgsAll, fmt.Errorf("error during DownloadVRGs on '%s': %w", prefixInS3, err) diff --git a/controllers/s3utils.go b/controllers/s3utils.go index 9222456..9ce51d7 100644 --- a/controllers/s3utils.go +++ b/controllers/s3utils.go @@ -34,6 +34,7 @@ import ( ramen "github.com/ramendr/ramen/api/v1alpha1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -172,6 +173,7 @@ func (s3ObjectStoreGetter) ObjectStore(ctx context.Context, s3Bucket: s3StoreProfile.S3Bucket, callerTag: callerTag, name: s3ProfileName, + log: log, } return s3Conn, s3StoreProfile, nil @@ -210,6 +212,7 @@ type s3ObjectStore struct { s3Bucket string callerTag string name string + log logr.Logger } // CreateBucket creates the given bucket; does not return an error if the bucket @@ -462,8 +465,13 @@ func DownloadTypedObjects(s ObjectStorer, keyPrefix string, objectsPointer inter objects := reflect.MakeSlice(reflect.SliceOf(objectType), len(keys), len(keys)) + log := ctrl.Log + log.Info("s3 download typed objects", "keys", keys, "objectsValue", objectsValue, "objectType", objectType) for i := range keys { objectReceiver := objects.Index(i).Addr().Interface() + + log.Info("s3 download typed object", "i", i, "key", keys[i], "objectReceiver", objectReceiver) + if err := s.DownloadObject(keys[i], objectReceiver); err != nil { return fmt.Errorf("unable to DownloadObject of key %s, %w", keys[i], err) @@ -544,11 +552,23 @@ func (s *s3ObjectStore) DownloadObject(key string, } gzReader, err := gzip.NewReader(bytes.NewReader(writerAt.Bytes())) - if err != nil && !errorswrapper.Is(err, io.EOF) { - return fmt.Errorf("failed to unzip data of %s:%s, %w", - bucket, key, err) + if err != nil { + if !errorswrapper.Is(err, io.EOF) { + return fmt.Errorf("failed to unzip data of %s:%s, %w", + bucket, key, err) + } + + s.log.Info("s3 download object end-of-file", "error", err) } + s.log.Info("s3 download object decode", + "bucket", bucket, + "key", key, + "downloadContent", downloadContent, + "writerAt", writerAt, + "gzReader", gzReader, + ) + if err := json.NewDecoder(gzReader).Decode(downloadContent); err != nil { return fmt.Errorf("failed to decode json decoder of %s:%s, %w", bucket, key, err) From 35be3c53d9d3d84b852828253d3f96211d977fac Mon Sep 17 00:00:00 2001 From: hatfieldbrian Date: Sun, 12 Mar 2023 07:00:25 -0700 Subject: [PATCH 2/2] s3: log byte count downloaded Signed-off-by: hatfieldbrian --- controllers/s3utils.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/controllers/s3utils.go b/controllers/s3utils.go index 9ce51d7..1c0416e 100644 --- a/controllers/s3utils.go +++ b/controllers/s3utils.go @@ -467,6 +467,7 @@ func DownloadTypedObjects(s ObjectStorer, keyPrefix string, objectsPointer inter log := ctrl.Log log.Info("s3 download typed objects", "keys", keys, "objectsValue", objectsValue, "objectType", objectType) + for i := range keys { objectReceiver := objects.Index(i).Addr().Interface() @@ -543,19 +544,20 @@ func (s *s3ObjectStore) DownloadObject(key string, ctx, cancel := context.WithDeadline(context.TODO(), time.Now().Add(s3Timeout)) defer cancel() - if _, err := s.downloader.DownloadWithContext(ctx, writerAt, &s3.GetObjectInput{ + byteCount, err := s.downloader.DownloadWithContext(ctx, writerAt, &s3.GetObjectInput{ Bucket: &bucket, Key: &key, - }); err != nil { - return fmt.Errorf("failed to download data of %s:%s, %w", - bucket, key, err) + }) + if err != nil { + return fmt.Errorf("failed to download data of %s:%s byte count: %d, %w", + bucket, key, byteCount, err) } gzReader, err := gzip.NewReader(bytes.NewReader(writerAt.Bytes())) if err != nil { if !errorswrapper.Is(err, io.EOF) { - return fmt.Errorf("failed to unzip data of %s:%s, %w", - bucket, key, err) + return fmt.Errorf("failed to unzip data of %s:%s byte count: %d, %w", + bucket, key, byteCount, err) } s.log.Info("s3 download object end-of-file", "error", err) @@ -564,6 +566,7 @@ func (s *s3ObjectStore) DownloadObject(key string, s.log.Info("s3 download object decode", "bucket", bucket, "key", key, + "byte count", byteCount, "downloadContent", downloadContent, "writerAt", writerAt, "gzReader", gzReader,