From ad66fcd9a92200d35a430dd510b26cfc798bd387 Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Wed, 30 Mar 2022 22:38:52 +0000 Subject: [PATCH 1/3] Simplify bucket emptying mechanism --- task/aws/resources/resource_bucket.go | 47 +++------------------------ task/aws/task.go | 2 +- task/az/task.go | 2 +- task/gcp/resources/resource_bucket.go | 22 +++---------- task/gcp/task.go | 2 +- 5 files changed, 12 insertions(+), 63 deletions(-) diff --git a/task/aws/resources/resource_bucket.go b/task/aws/resources/resource_bucket.go index 76e56722..f99add55 100644 --- a/task/aws/resources/resource_bucket.go +++ b/task/aws/resources/resource_bucket.go @@ -78,54 +78,17 @@ func (b *Bucket) Update(ctx context.Context) error { } func (b *Bucket) Delete(ctx context.Context) error { - listInput := s3.ListObjectsV2Input{ + input := s3.DeleteBucketInput{ Bucket: aws.String(b.Identifier), } - - for paginator := s3.NewListObjectsV2Paginator(b.Client.Services.S3, &listInput); paginator.HasMorePages(); { - page, err := paginator.NextPage(ctx) - - if err != nil { - var e smithy.APIError - if errors.As(err, &e) && e.ErrorCode() == "NoSuchBucket" { - b.Resource = nil - return nil - } - return err - } - - if len(page.Contents) == 0 { - break - } - - var objects []types.ObjectIdentifier - for _, object := range page.Contents { - objects = append(objects, types.ObjectIdentifier{ - Key: object.Key, - }) - } - - input := s3.DeleteObjectsInput{ - Bucket: aws.String(b.Identifier), - Delete: &types.Delete{ - Objects: objects, - }, - } - - if _, err = b.Client.Services.S3.DeleteObjects(ctx, &input); err != nil { + + if _, err := b.Client.Services.S3.DeleteBucket(ctx, &input); err != nil { + var e smithy.APIError + if errors.As(err, &e) && e.ErrorCode() != "NoSuchBucket" { return err } } - deleteInput := s3.DeleteBucketInput{ - Bucket: aws.String(b.Identifier), - } - - _, err := b.Client.Services.S3.DeleteBucket(ctx, &deleteInput) - if err != nil { - return err - } - b.Resource = nil return nil } diff --git a/task/aws/task.go b/task/aws/task.go index 3fd4c56a..33ad3e6c 100644 --- a/task/aws/task.go +++ b/task/aws/task.go @@ -196,7 +196,7 @@ func (t *Task) Read(ctx context.Context) error { func (t *Task) Delete(ctx context.Context) error { logrus.Debug("Downloading Directory...") - if t.Read(ctx) == nil { + if t.Resources.Bucket.Read(ctx) == nil { if t.Attributes.Environment.DirectoryOut != "" { if err := t.Pull(ctx, t.Attributes.Environment.Directory, t.Attributes.Environment.DirectoryOut); err != nil && err != common.NotFoundError { return err diff --git a/task/az/task.go b/task/az/task.go index ff5baa5a..ea1db832 100644 --- a/task/az/task.go +++ b/task/az/task.go @@ -185,7 +185,7 @@ func (t *Task) Read(ctx context.Context) error { func (t *Task) Delete(ctx context.Context) error { logrus.Debug("Downloading Directory...") - if t.Read(ctx) == nil { + if t.Resources.BlobContainer.Read(ctx) == nil { if t.Attributes.Environment.DirectoryOut != "" { if err := t.Pull(ctx, t.Attributes.Environment.Directory, t.Attributes.Environment.DirectoryOut); err != nil && err != common.NotFoundError { return err diff --git a/task/gcp/resources/resource_bucket.go b/task/gcp/resources/resource_bucket.go index b7c23b79..8d9d017b 100644 --- a/task/gcp/resources/resource_bucket.go +++ b/task/gcp/resources/resource_bucket.go @@ -61,25 +61,11 @@ func (b *Bucket) Update(ctx context.Context) error { } func (b *Bucket) Delete(ctx context.Context) error { - if b.Read(ctx) == common.NotFoundError { - return nil - } - - deletePage := func(objects *storage.Objects) error { - for _, object := range objects.Items { - if err := b.Client.Services.Storage.Objects.Delete(b.Identifier, object.Name).Do(); err != nil { - return err - } - } - return nil - } - - if err := b.Client.Services.Storage.Objects.List(b.Identifier).Pages(ctx, deletePage); err != nil { - return err - } - if err := b.Client.Services.Storage.Buckets.Delete(b.Identifier).Do(); err != nil { - return err + var e *googleapi.Error + if errors.As(err, &e) && e.Code != 404 { + return err + } } b.Resource = nil diff --git a/task/gcp/task.go b/task/gcp/task.go index 86020fed..5860e4d5 100644 --- a/task/gcp/task.go +++ b/task/gcp/task.go @@ -264,7 +264,7 @@ func (t *Task) Read(ctx context.Context) error { func (t *Task) Delete(ctx context.Context) error { logrus.Debug("Downloading Directory...") - if t.Read(ctx) == nil { + if t.Resources.Bucket.Read(ctx) == nil { if t.Attributes.Environment.DirectoryOut != "" { if err := t.Pull(ctx, t.Attributes.Environment.Directory, t.Attributes.Environment.DirectoryOut); err != nil && err != common.NotFoundError { return err From 1d0bc33260576f74d2800d0c7ccbf75e109a7981 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Wed, 30 Mar 2022 22:44:12 +0000 Subject: [PATCH 2/3] Restyled by gofmt --- task/aws/resources/resource_bucket.go | 2 +- task/common/machine/storage.go | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/task/aws/resources/resource_bucket.go b/task/aws/resources/resource_bucket.go index f99add55..12025fda 100644 --- a/task/aws/resources/resource_bucket.go +++ b/task/aws/resources/resource_bucket.go @@ -81,7 +81,7 @@ func (b *Bucket) Delete(ctx context.Context) error { input := s3.DeleteBucketInput{ Bucket: aws.String(b.Identifier), } - + if _, err := b.Client.Services.S3.DeleteBucket(ctx, &input); err != nil { var e smithy.APIError if errors.As(err, &e) && e.ErrorCode() != "NoSuchBucket" { diff --git a/task/common/machine/storage.go b/task/common/machine/storage.go index d36f7c2b..93ca697a 100644 --- a/task/common/machine/storage.go +++ b/task/common/machine/storage.go @@ -139,6 +139,11 @@ func Delete(ctx context.Context, destination string) error { return err } + ctx, fi := filter.AddConfig(ctx) + if err := fi.AddRule("+ **"); err != nil { + return err + } + actions := []func(context.Context) error{ func(ctx context.Context) error { return operations.Delete(ctx, destinationFileSystem) From fd21d82f8c71aceeb0310a87b1495ba50bf37759 Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Thu, 31 Mar 2022 20:49:58 +0000 Subject: [PATCH 3/3] Add debug logs --- task/common/machine/storage.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/task/common/machine/storage.go b/task/common/machine/storage.go index 213dccee..a52939bc 100644 --- a/task/common/machine/storage.go +++ b/task/common/machine/storage.go @@ -148,6 +148,17 @@ func Delete(ctx context.Context, destination string) error { return err } + operations.SyncPrintf = func(format string, a ...interface{}) { + logrus.Infof(format, a...) + } + fs.LogPrint = func(level fs.LogLevel, text string) { + logrus.Info(text) + } + + ctx, ci := fs.AddConfig(ctx) + ci.LogLevel = fs.LogLevelDebug + ci.StatsLogLevel = fs.LogLevelDebug + actions := []func(context.Context) error{ func(ctx context.Context) error { return operations.Delete(ctx, destinationFileSystem)