S3 Purge Local Backup Feature + Fallbacks when local files are purged

apps/api/internal/backup/backup.go

@@ -3,6 +3,7 @@ package backup
 import (
 	"database/sql"
 	"encoding/json"
+	"fmt"
 	"io"
 	"net/http"
 	"os"
@@ -194,6 +195,12 @@ func (h *BackupHandler) DownloadBackup(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	backupID := vars["id"]
 
+	userID, err := common.GetUserIDFromContext(r.Context())
+	if err != nil {
+		response.SendError(w, http.StatusBadRequest, err.Error())
+		return
+	}
+
 	backup, err := h.backupService.GetBackup(backupID)
 	if err != nil {
 		if err == sql.ErrNoRows {
@@ -204,7 +211,23 @@ func (h *BackupHandler) DownloadBackup(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	file, err := os.Open(backup.Path)
+	// Ensure backup file is available (local or download from S3)
+	filePath, isTemp, err := h.backupService.ensureBackupFileAvailable(backup, userID)
+	if err != nil {
+		response.SendError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+
+	// Clean up temp file after download if needed
+	if isTemp {
+		defer func() {
+			if err := os.Remove(filePath); err != nil {
+				fmt.Printf("Warning: Failed to remove temp file %s: %v\n", filePath, err)
+			}
+		}()
+	}
+
+	file, err := os.Open(filePath)
 	if err != nil {
 		response.SendError(w, http.StatusInternalServerError, "Failed to open backup file")
 		return

apps/api/internal/backup/backup_diff.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"strings"
 
+	"github.com/dendianugerah/velld/internal/common"
 	"github.com/dendianugerah/velld/internal/common/response"
 	"github.com/gorilla/mux"
 )
@@ -33,6 +34,12 @@ func (h *BackupHandler) CompareBackups(w http.ResponseWriter, r *http.Request) {
 	sourceID := vars["sourceId"]
 	targetID := vars["targetId"]
 
+	userID, err := common.GetUserIDFromContext(r.Context())
+	if err != nil {
+		response.SendError(w, http.StatusBadRequest, err.Error())
+		return
+	}
+
 	sourceBackup, err := h.backupService.GetBackup(sourceID)
 	if err != nil {
 		response.SendError(w, http.StatusNotFound, "Source backup not found")
@@ -45,13 +52,40 @@ func (h *BackupHandler) CompareBackups(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	sourceContent, err := readBackupFile(sourceBackup.Path)
+	// Ensure both backup files are available (local or download from S3)
+	sourceFilePath, sourceIsTemp, err := h.backupService.ensureBackupFileAvailable(sourceBackup, userID)
+	if err != nil {
+		response.SendError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to access source backup: %v", err))
+		return
+	}
+
+	targetFilePath, targetIsTemp, err := h.backupService.ensureBackupFileAvailable(targetBackup, userID)
+	if err != nil {
+		response.SendError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to access target backup: %v", err))
+		return
+	}
+
+	// Clean up temp files after comparison
+	defer func() {
+		if sourceIsTemp {
+			if err := os.Remove(sourceFilePath); err != nil {
+				fmt.Printf("Warning: Failed to remove temp source file %s: %v\n", sourceFilePath, err)
+			}
+		}
+		if targetIsTemp {
+			if err := os.Remove(targetFilePath); err != nil {
+				fmt.Printf("Warning: Failed to remove temp target file %s: %v\n", targetFilePath, err)
+			}
+		}
+	}()
+
+	sourceContent, err := readBackupFile(sourceFilePath)
 	if err != nil {
 		response.SendError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to read source backup: %v", err))
 		return
 	}
 
-	targetContent, err := readBackupFile(targetBackup.Path)
+	targetContent, err := readBackupFile(targetFilePath)
 	if err != nil {
 		response.SendError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to read target backup: %v", err))
 		return

apps/api/internal/backup/backup_restore.go

@@ -30,15 +30,26 @@ func (s *BackupService) RestoreBackup(backupID string, connectionID string) erro
 		return fmt.Errorf("failed to get backup: %v", err)
 	}
 
-	if _, err := os.Stat(backup.Path); os.IsNotExist(err) {
-		return fmt.Errorf("backup file not found: %s", backup.Path)
-	}
-
 	conn, err := s.connStorage.GetConnection(connectionID)
 	if err != nil {
 		return fmt.Errorf("failed to get connection: %v", err)
 	}
 
+	// Ensure backup file is available (local or download from S3)
+	filePath, isTemp, err := s.ensureBackupFileAvailable(backup, conn.UserID)
+	if err != nil {
+		return err
+	}
+
+	// Clean up temp file after restore if needed
+	if isTemp {
+		defer func() {
+			if err := os.Remove(filePath); err != nil {
+				fmt.Printf("Warning: Failed to remove temp file %s: %v\n", filePath, err)
+			}
+		}()
+	}
+
 	if err := s.verifyRestoreTools(conn.Type); err != nil {
 		return err
 	}
@@ -56,11 +67,11 @@ func (s *BackupService) RestoreBackup(backupID string, connectionID string) erro
 	var cmd *exec.Cmd
 	switch conn.Type {
 	case "postgresql":
-		cmd = s.createPsqlRestoreCmd(conn, backup.Path)
+		cmd = s.createPsqlRestoreCmd(conn, filePath)
 	case "mysql", "mariadb":
-		cmd = s.createMySQLRestoreCmd(conn, backup.Path)
+		cmd = s.createMySQLRestoreCmd(conn, filePath)
 	case "mongodb":
-		cmd = s.createMongoRestoreCmd(conn, backup.Path)
+		cmd = s.createMongoRestoreCmd(conn, filePath)
 	default:
 		return fmt.Errorf("unsupported database type for restore: %s", conn.Type)
 	}

apps/api/internal/backup/backup_service.go

@@ -189,7 +189,7 @@ func (s *BackupService) createMultiDatabaseBackup(conn *connection.StoredConnect
 		now := time.Now()
 		backup.CompletedTime = &now
 
-		if err := s.uploadToS3IfEnabled(backup, conn.UserID); err != nil {
+		if err := s.uploadToS3IfEnabled(backup, conn.UserID, conn.Name); err != nil {
 			fmt.Printf("Warning: Failed to upload backup '%s' to S3: %v\n", dbName, err)
 		}
 
@@ -297,7 +297,7 @@ func (s *BackupService) createSingleDatabaseBackup(conn *connection.StoredConnec
 	now := time.Now()
 	backup.CompletedTime = &now
 
-	if err := s.uploadToS3IfEnabled(backup, conn.UserID); err != nil {
+	if err := s.uploadToS3IfEnabled(backup, conn.UserID, conn.Name); err != nil {
 		fmt.Printf("Warning: Failed to upload backup to S3: %v\n", err)
 	}
 
@@ -330,7 +330,7 @@ func (s *BackupService) GetBackupStats(userID uuid.UUID) (*BackupStats, error) {
 	return s.backupRepo.GetBackupStats(userID)
 }
 
-func (s *BackupService) uploadToS3IfEnabled(backup *Backup, userID uuid.UUID) error {
+func (s *BackupService) uploadToS3IfEnabled(backup *Backup, userID uuid.UUID, connectionName string) error {
 	userSettings, err := s.settingsService.GetUserSettingsInternal(userID)
 	if err != nil {
 		return fmt.Errorf("failed to get user settings: %w", err)
@@ -385,13 +385,115 @@ func (s *BackupService) uploadToS3IfEnabled(backup *Backup, userID uuid.UUID) er
 	}
 
 	ctx := context.Background()
-	objectKey, err := s3Storage.UploadFile(ctx, backup.Path)
+	// Use sanitized connection name as subfolder
+	sanitizedConnectionName := common.SanitizeConnectionName(connectionName)
+	objectKey, err := s3Storage.UploadFileWithPath(ctx, backup.Path, sanitizedConnectionName)
 	if err != nil {
 		return fmt.Errorf("failed to upload backup to S3: %w", err)
 	}
 
 	backup.S3ObjectKey = &objectKey
 
 	fmt.Printf("Successfully uploaded backup %s to S3: %s\n", backup.ID, objectKey)
+
+	// Purge local backup file if enabled
+	if userSettings.S3PurgeLocal {
+		if err := os.Remove(backup.Path); err != nil {
+			fmt.Printf("Warning: Failed to purge local backup file %s: %v\n", backup.Path, err)
+		} else {
+			fmt.Printf("Successfully purged local backup file: %s\n", backup.Path)
+		}
+	}
+
 	return nil
 }
+
+// ensureBackupFileAvailable checks if backup file exists locally, if not downloads from S3
+// Returns the path to use and a boolean indicating if it's a temporary file that should be cleaned up
+func (s *BackupService) ensureBackupFileAvailable(backup *Backup, userID uuid.UUID) (string, bool, error) {
+	// Check if local file exists
+	if _, err := os.Stat(backup.Path); err == nil {
+		// Local file exists, use it
+		return backup.Path, false, nil
+	}
+
+	// Local file doesn't exist, check if we have S3 object key
+	if backup.S3ObjectKey == nil || *backup.S3ObjectKey == "" {
+		return "", false, fmt.Errorf("backup file not found locally and no S3 object key available")
+	}
+
+	// Get user settings to configure S3 client
+	userSettings, err := s.settingsService.GetUserSettingsInternal(userID)
+	if err != nil {
+		return "", false, fmt.Errorf("failed to get user settings: %w", err)
+	}
+
+	if !userSettings.S3Enabled {
+		return "", false, fmt.Errorf("backup file not found locally and S3 is not enabled")
+	}
+
+	// Validate S3 configuration
+	if userSettings.S3Endpoint == nil || *userSettings.S3Endpoint == "" {
+		return "", false, fmt.Errorf("S3 endpoint not configured")
+	}
+	if userSettings.S3Bucket == nil || *userSettings.S3Bucket == "" {
+		return "", false, fmt.Errorf("S3 bucket not configured")
+	}
+	if userSettings.S3AccessKey == nil || *userSettings.S3AccessKey == "" {
+		return "", false, fmt.Errorf("S3 access key not configured")
+	}
+	if userSettings.S3SecretKey == nil || *userSettings.S3SecretKey == "" {
+		return "", false, fmt.Errorf("S3 secret key not configured")
+	}
+
+	// Decrypt S3 secret key
+	secretKey, err := s.cryptoService.Decrypt(*userSettings.S3SecretKey)
+	if err != nil {
+		return "", false, fmt.Errorf("failed to decrypt S3 secret key: %w", err)
+	}
+
+	// Configure S3 client
+	region := "us-east-1"
+	if userSettings.S3Region != nil && *userSettings.S3Region != "" {
+		region = *userSettings.S3Region
+	}
+
+	pathPrefix := ""
+	if userSettings.S3PathPrefix != nil {
+		pathPrefix = *userSettings.S3PathPrefix
+	}
+
+	s3Config := S3Config{
+		Endpoint:   *userSettings.S3Endpoint,
+		Region:     region,
+		Bucket:     *userSettings.S3Bucket,
+		AccessKey:  *userSettings.S3AccessKey,
+		SecretKey:  secretKey,
+		UseSSL:     userSettings.S3UseSSL,
+		PathPrefix: pathPrefix,
+	}
+
+	s3Storage, err := NewS3Storage(s3Config)
+	if err != nil {
+		return "", false, fmt.Errorf("failed to create S3 storage client: %w", err)
+	}
+
+	// Create temp file path
+	tempDir := filepath.Join(os.TempDir(), "velld-s3-downloads")
+	if err := os.MkdirAll(tempDir, 0755); err != nil {
+		return "", false, fmt.Errorf("failed to create temp directory: %w", err)
+	}
+
+	tempFilePath := filepath.Join(tempDir, filepath.Base(backup.Path))
+
+	// Download from S3
+	ctx := context.Background()
+	if err := s3Storage.DownloadFile(ctx, *backup.S3ObjectKey, tempFilePath); err != nil {
+		return "", false, fmt.Errorf("failed to download backup from S3: %w", err)
+	}
+
+	fmt.Printf("Successfully downloaded backup %s from S3 to temp location: %s\n", backup.ID, tempFilePath)
+
+	// Return temp file path and indicate it should be cleaned up
+	return tempFilePath, true, nil
+}

apps/api/internal/backup/s3_storage.go

@@ -85,6 +85,32 @@ func (s *S3Storage) UploadFile(ctx context.Context, localPath string) (string, e
 	return objectKey, nil
 }
 
+// UploadFileWithPath uploads a file to S3 with a custom subfolder path
+func (s *S3Storage) UploadFileWithPath(ctx context.Context, localPath string, subfolder string) (string, error) {
+	file, err := os.Open(localPath)
+	if err != nil {
+		return "", fmt.Errorf("failed to open file: %w", err)
+	}
+	defer file.Close()
+
+	fileInfo, err := file.Stat()
+	if err != nil {
+		return "", fmt.Errorf("failed to stat file: %w", err)
+	}
+
+	fileName := filepath.Base(localPath)
+	objectKey := s.getObjectKeyWithPath(fileName, subfolder)
+
+	_, err = s.client.PutObject(ctx, s.bucket, objectKey, file, fileInfo.Size(), minio.PutObjectOptions{
+		ContentType: "application/octet-stream",
+	})
+	if err != nil {
+		return "", fmt.Errorf("failed to upload to S3: %w", err)
+	}
+
+	return objectKey, nil
+}
+
 func (s *S3Storage) DownloadFile(ctx context.Context, objectKey, localPath string) error {
 	object, err := s.client.GetObject(ctx, s.bucket, objectKey, minio.GetObjectOptions{})
 	if err != nil {
@@ -161,3 +187,20 @@ func (s *S3Storage) getObjectKey(fileName string) string {
 	fileName = strings.TrimPrefix(fileName, "/")
 	return fmt.Sprintf("%s/%s", prefix, fileName)
 }
+
+func (s *S3Storage) getObjectKeyWithPath(fileName string, subfolder string) string {
+	// Build path: prefix/subfolder/fileName
+	var parts []string
+
+	if s.prefix != "" {
+		parts = append(parts, strings.TrimSuffix(s.prefix, "/"))
+	}
+
+	if subfolder != "" {
+		parts = append(parts, strings.Trim(subfolder, "/"))
+	}
+
+	parts = append(parts, strings.TrimPrefix(fileName, "/"))
+
+	return strings.Join(parts, "/")
+}

apps/api/internal/database/migrations/20251113153839_add_s3_purge_local.sql

@@ -0,0 +1,15 @@
+-- +goose Up
+-- +goose StatementBegin
+SELECT 'Adding s3_purge_local setting to user_settings';
+
+ALTER TABLE user_settings ADD COLUMN s3_purge_local INTEGER DEFAULT 0;
+
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+SELECT 'Removing s3_purge_local setting from user_settings';
+
+ALTER TABLE user_settings DROP COLUMN s3_purge_local;
+
+-- +goose StatementEnd

apps/api/internal/settings/model.go

@@ -27,6 +27,7 @@ type UserSettings struct {
 	S3SecretKey  *string   `json:"s3_secret_key,omitempty"`
 	S3UseSSL     bool      `json:"s3_use_ssl"`
 	S3PathPrefix *string   `json:"s3_path_prefix,omitempty"`
+	S3PurgeLocal bool      `json:"s3_purge_local"`
 	CreatedAt    time.Time `json:"created_at"`
 	UpdatedAt    time.Time `json:"updated_at"`
 	EnvConfigured map[string]bool `json:"env_configured,omitempty"`
@@ -51,4 +52,5 @@ type UpdateSettingsRequest struct {
 	S3SecretKey  *string `json:"s3_secret_key,omitempty"`
 	S3UseSSL     *bool   `json:"s3_use_ssl,omitempty"`
 	S3PathPrefix *string `json:"s3_path_prefix,omitempty"`
+	S3PurgeLocal *bool   `json:"s3_purge_local,omitempty"`
 }