Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion internal/controller/store_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ func (r *StoreReconciler) SetupWithManager(mgr ctrl.Manager, logger *zap.Sugared
}
return ctrl.NewControllerManagedBy(mgr).
For(&v1.Store{}).
// We get triggerd by every update on the created resources, this leeads to high reconciles at the start.
// We get triggered by every update on the created resources, this leads to high reconciles at the start.
Owns(&corev1.Secret{}).
Owns(&corev1.Service{}).
Owns(&networkingv1.Ingress{}).
Expand Down
34 changes: 24 additions & 10 deletions internal/snapshot/s3.go
Original file line number Diff line number Diff line change
Expand Up @@ -162,12 +162,19 @@ func (s *SnapshotService) createAssetBackup(
defer wg.Done()
logger := logger.With(zap.String("bucket", "private"))
logger.Info("Starting S3 bucket read")

// make sure to create the directory for the private bucket backup, even if the bucket is empty
routeFilePath := filepath.Join(snapshotCtx.TempArchiveDir, "private")
if err := os.MkdirAll(routeFilePath, 0755); err != nil {
logger.Errorw("failed to create directory for bucket backup", zap.String("path", routeFilePath), zap.Error(err))
errChan <- fmt.Errorf("failed to create directory for private bucket backup: %w", err)
return
}

downloader := util.NewS3Downloader(minioClient, cfg.S3.PrivateBucket)
err := downloader.DownloadBucket(ctx, parallelDownloads,
processDownloadObject(
filepath.Join(snapshotCtx.TempArchiveDir, "private"),
logger,
))
err := downloader.DownloadBucket(ctx,
parallelDownloads,
processDownloadObject(routeFilePath, logger))
if err != nil {
logger.Errorw("bucket backup failed", zap.Error(err))
errChan <- fmt.Errorf("private bucket backup: %w", err)
Expand All @@ -183,12 +190,19 @@ func (s *SnapshotService) createAssetBackup(

logger := logger.With(zap.String("bucket", "public"))
logger.Info("Starting S3 bucket read")

// make sure to create the directory for the public bucket backup, even if the bucket is empty
rootDirPath := filepath.Join(snapshotCtx.TempArchiveDir, "public")
if err := os.MkdirAll(rootDirPath, 0755); err != nil {
logger.Errorw("failed to create directory for bucket backup", zap.String("path", rootDirPath), zap.Error(err))
errChan <- fmt.Errorf("failed to create directory for public bucket backup: %w", err)
return
}

downloader := util.NewS3Downloader(minioClient, cfg.S3.PublicBucket)
err := downloader.DownloadBucket(ctx, parallelDownloads,
processDownloadObject(
filepath.Join(snapshotCtx.TempArchiveDir, "public"),
logger,
))
err := downloader.DownloadBucket(ctx,
parallelDownloads,
processDownloadObject(rootDirPath, logger))
if err != nil {
logger.Errorw("bucket backup failed", zap.Error(err))
errChan <- fmt.Errorf("public bucket backup: %w", err)
Expand Down
14 changes: 14 additions & 0 deletions internal/util/s3.go
Original file line number Diff line number Diff line change
Expand Up @@ -251,6 +251,16 @@ func uploadBucket(ctx context.Context, s3Client *minio.Client, bucketName, sourc
}

logger.Info("Scanning source directory for files")
if _, err := os.Stat(sourcePath); err != nil {
if os.IsNotExist(err) {
logger.Warnw("Source path does not exist", zap.String("source_path", sourcePath))
logger.Info("No files to upload, operation complete")
return nil
}
logger.Errorw("Failed to stat source path", zap.String("source_path", sourcePath), zap.Error(err))
return fmt.Errorf("failed to stat source path: %w", err)
}

var files []string
err := filepath.WalkDir(sourcePath, func(path string, d fs.DirEntry, err error) error {
if err != nil {
Expand Down Expand Up @@ -435,6 +445,10 @@ func downloadBucket(ctx context.Context, s3Client *minio.Client, bucketName stri
errCh := make(chan error, workers)
sem := make(chan struct{}, workers)

if err := os.MkdirAll(destinationPath, 0755); err != nil {
return fmt.Errorf("error creating destination directory: %w", err)
}

for object := range objects {
if object.Err != nil {
return fmt.Errorf("error listing objects: %w", object.Err)
Expand Down
Loading