diff --git a/apps/api/internal/backup/backup.go b/apps/api/internal/backup/backup.go index f0a639b..1f418b2 100644 --- a/apps/api/internal/backup/backup.go +++ b/apps/api/internal/backup/backup.go @@ -3,6 +3,7 @@ package backup import ( "database/sql" "encoding/json" + "fmt" "io" "net/http" "os" @@ -194,6 +195,12 @@ func (h *BackupHandler) DownloadBackup(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) backupID := vars["id"] + userID, err := common.GetUserIDFromContext(r.Context()) + if err != nil { + response.SendError(w, http.StatusBadRequest, err.Error()) + return + } + backup, err := h.backupService.GetBackup(backupID) if err != nil { if err == sql.ErrNoRows { @@ -204,7 +211,23 @@ func (h *BackupHandler) DownloadBackup(w http.ResponseWriter, r *http.Request) { return } - file, err := os.Open(backup.Path) + // Ensure backup file is available (local or download from S3) + filePath, isTemp, err := h.backupService.ensureBackupFileAvailable(backup, userID) + if err != nil { + response.SendError(w, http.StatusInternalServerError, err.Error()) + return + } + + // Clean up temp file after download if needed + if isTemp { + defer func() { + if err := os.Remove(filePath); err != nil { + fmt.Printf("Warning: Failed to remove temp file %s: %v\n", filePath, err) + } + }() + } + + file, err := os.Open(filePath) if err != nil { response.SendError(w, http.StatusInternalServerError, "Failed to open backup file") return diff --git a/apps/api/internal/backup/backup_diff.go b/apps/api/internal/backup/backup_diff.go index 26491d2..3d3ea22 100644 --- a/apps/api/internal/backup/backup_diff.go +++ b/apps/api/internal/backup/backup_diff.go @@ -7,6 +7,7 @@ import ( "os" "strings" + "github.com/dendianugerah/velld/internal/common" "github.com/dendianugerah/velld/internal/common/response" "github.com/gorilla/mux" ) @@ -33,6 +34,12 @@ func (h *BackupHandler) CompareBackups(w http.ResponseWriter, r *http.Request) { sourceID := vars["sourceId"] targetID := vars["targetId"] + userID, err := common.GetUserIDFromContext(r.Context()) + if err != nil { + response.SendError(w, http.StatusBadRequest, err.Error()) + return + } + sourceBackup, err := h.backupService.GetBackup(sourceID) if err != nil { response.SendError(w, http.StatusNotFound, "Source backup not found") @@ -45,13 +52,40 @@ func (h *BackupHandler) CompareBackups(w http.ResponseWriter, r *http.Request) { return } - sourceContent, err := readBackupFile(sourceBackup.Path) + // Ensure both backup files are available (local or download from S3) + sourceFilePath, sourceIsTemp, err := h.backupService.ensureBackupFileAvailable(sourceBackup, userID) + if err != nil { + response.SendError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to access source backup: %v", err)) + return + } + + targetFilePath, targetIsTemp, err := h.backupService.ensureBackupFileAvailable(targetBackup, userID) + if err != nil { + response.SendError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to access target backup: %v", err)) + return + } + + // Clean up temp files after comparison + defer func() { + if sourceIsTemp { + if err := os.Remove(sourceFilePath); err != nil { + fmt.Printf("Warning: Failed to remove temp source file %s: %v\n", sourceFilePath, err) + } + } + if targetIsTemp { + if err := os.Remove(targetFilePath); err != nil { + fmt.Printf("Warning: Failed to remove temp target file %s: %v\n", targetFilePath, err) + } + } + }() + + sourceContent, err := readBackupFile(sourceFilePath) if err != nil { response.SendError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to read source backup: %v", err)) return } - targetContent, err := readBackupFile(targetBackup.Path) + targetContent, err := readBackupFile(targetFilePath) if err != nil { response.SendError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to read target backup: %v", err)) return diff --git a/apps/api/internal/backup/backup_restore.go b/apps/api/internal/backup/backup_restore.go index 847ef25..38fcc34 100644 --- a/apps/api/internal/backup/backup_restore.go +++ b/apps/api/internal/backup/backup_restore.go @@ -30,15 +30,26 @@ func (s *BackupService) RestoreBackup(backupID string, connectionID string) erro return fmt.Errorf("failed to get backup: %v", err) } - if _, err := os.Stat(backup.Path); os.IsNotExist(err) { - return fmt.Errorf("backup file not found: %s", backup.Path) - } - conn, err := s.connStorage.GetConnection(connectionID) if err != nil { return fmt.Errorf("failed to get connection: %v", err) } + // Ensure backup file is available (local or download from S3) + filePath, isTemp, err := s.ensureBackupFileAvailable(backup, conn.UserID) + if err != nil { + return err + } + + // Clean up temp file after restore if needed + if isTemp { + defer func() { + if err := os.Remove(filePath); err != nil { + fmt.Printf("Warning: Failed to remove temp file %s: %v\n", filePath, err) + } + }() + } + if err := s.verifyRestoreTools(conn.Type); err != nil { return err } @@ -56,11 +67,11 @@ func (s *BackupService) RestoreBackup(backupID string, connectionID string) erro var cmd *exec.Cmd switch conn.Type { case "postgresql": - cmd = s.createPsqlRestoreCmd(conn, backup.Path) + cmd = s.createPsqlRestoreCmd(conn, filePath) case "mysql", "mariadb": - cmd = s.createMySQLRestoreCmd(conn, backup.Path) + cmd = s.createMySQLRestoreCmd(conn, filePath) case "mongodb": - cmd = s.createMongoRestoreCmd(conn, backup.Path) + cmd = s.createMongoRestoreCmd(conn, filePath) default: return fmt.Errorf("unsupported database type for restore: %s", conn.Type) } diff --git a/apps/api/internal/backup/backup_service.go b/apps/api/internal/backup/backup_service.go index 863656c..90f772f 100644 --- a/apps/api/internal/backup/backup_service.go +++ b/apps/api/internal/backup/backup_service.go @@ -189,7 +189,7 @@ func (s *BackupService) createMultiDatabaseBackup(conn *connection.StoredConnect now := time.Now() backup.CompletedTime = &now - if err := s.uploadToS3IfEnabled(backup, conn.UserID); err != nil { + if err := s.uploadToS3IfEnabled(backup, conn.UserID, conn.Name); err != nil { fmt.Printf("Warning: Failed to upload backup '%s' to S3: %v\n", dbName, err) } @@ -297,7 +297,7 @@ func (s *BackupService) createSingleDatabaseBackup(conn *connection.StoredConnec now := time.Now() backup.CompletedTime = &now - if err := s.uploadToS3IfEnabled(backup, conn.UserID); err != nil { + if err := s.uploadToS3IfEnabled(backup, conn.UserID, conn.Name); err != nil { fmt.Printf("Warning: Failed to upload backup to S3: %v\n", err) } @@ -330,7 +330,7 @@ func (s *BackupService) GetBackupStats(userID uuid.UUID) (*BackupStats, error) { return s.backupRepo.GetBackupStats(userID) } -func (s *BackupService) uploadToS3IfEnabled(backup *Backup, userID uuid.UUID) error { +func (s *BackupService) uploadToS3IfEnabled(backup *Backup, userID uuid.UUID, connectionName string) error { userSettings, err := s.settingsService.GetUserSettingsInternal(userID) if err != nil { return fmt.Errorf("failed to get user settings: %w", err) @@ -385,7 +385,9 @@ func (s *BackupService) uploadToS3IfEnabled(backup *Backup, userID uuid.UUID) er } ctx := context.Background() - objectKey, err := s3Storage.UploadFile(ctx, backup.Path) + // Use sanitized connection name as subfolder + sanitizedConnectionName := common.SanitizeConnectionName(connectionName) + objectKey, err := s3Storage.UploadFileWithPath(ctx, backup.Path, sanitizedConnectionName) if err != nil { return fmt.Errorf("failed to upload backup to S3: %w", err) } @@ -393,5 +395,105 @@ func (s *BackupService) uploadToS3IfEnabled(backup *Backup, userID uuid.UUID) er backup.S3ObjectKey = &objectKey fmt.Printf("Successfully uploaded backup %s to S3: %s\n", backup.ID, objectKey) + + // Purge local backup file if enabled + if userSettings.S3PurgeLocal { + if err := os.Remove(backup.Path); err != nil { + fmt.Printf("Warning: Failed to purge local backup file %s: %v\n", backup.Path, err) + } else { + fmt.Printf("Successfully purged local backup file: %s\n", backup.Path) + } + } + return nil } + +// ensureBackupFileAvailable checks if backup file exists locally, if not downloads from S3 +// Returns the path to use and a boolean indicating if it's a temporary file that should be cleaned up +func (s *BackupService) ensureBackupFileAvailable(backup *Backup, userID uuid.UUID) (string, bool, error) { + // Check if local file exists + if _, err := os.Stat(backup.Path); err == nil { + // Local file exists, use it + return backup.Path, false, nil + } + + // Local file doesn't exist, check if we have S3 object key + if backup.S3ObjectKey == nil || *backup.S3ObjectKey == "" { + return "", false, fmt.Errorf("backup file not found locally and no S3 object key available") + } + + // Get user settings to configure S3 client + userSettings, err := s.settingsService.GetUserSettingsInternal(userID) + if err != nil { + return "", false, fmt.Errorf("failed to get user settings: %w", err) + } + + if !userSettings.S3Enabled { + return "", false, fmt.Errorf("backup file not found locally and S3 is not enabled") + } + + // Validate S3 configuration + if userSettings.S3Endpoint == nil || *userSettings.S3Endpoint == "" { + return "", false, fmt.Errorf("S3 endpoint not configured") + } + if userSettings.S3Bucket == nil || *userSettings.S3Bucket == "" { + return "", false, fmt.Errorf("S3 bucket not configured") + } + if userSettings.S3AccessKey == nil || *userSettings.S3AccessKey == "" { + return "", false, fmt.Errorf("S3 access key not configured") + } + if userSettings.S3SecretKey == nil || *userSettings.S3SecretKey == "" { + return "", false, fmt.Errorf("S3 secret key not configured") + } + + // Decrypt S3 secret key + secretKey, err := s.cryptoService.Decrypt(*userSettings.S3SecretKey) + if err != nil { + return "", false, fmt.Errorf("failed to decrypt S3 secret key: %w", err) + } + + // Configure S3 client + region := "us-east-1" + if userSettings.S3Region != nil && *userSettings.S3Region != "" { + region = *userSettings.S3Region + } + + pathPrefix := "" + if userSettings.S3PathPrefix != nil { + pathPrefix = *userSettings.S3PathPrefix + } + + s3Config := S3Config{ + Endpoint: *userSettings.S3Endpoint, + Region: region, + Bucket: *userSettings.S3Bucket, + AccessKey: *userSettings.S3AccessKey, + SecretKey: secretKey, + UseSSL: userSettings.S3UseSSL, + PathPrefix: pathPrefix, + } + + s3Storage, err := NewS3Storage(s3Config) + if err != nil { + return "", false, fmt.Errorf("failed to create S3 storage client: %w", err) + } + + // Create temp file path + tempDir := filepath.Join(os.TempDir(), "velld-s3-downloads") + if err := os.MkdirAll(tempDir, 0755); err != nil { + return "", false, fmt.Errorf("failed to create temp directory: %w", err) + } + + tempFilePath := filepath.Join(tempDir, filepath.Base(backup.Path)) + + // Download from S3 + ctx := context.Background() + if err := s3Storage.DownloadFile(ctx, *backup.S3ObjectKey, tempFilePath); err != nil { + return "", false, fmt.Errorf("failed to download backup from S3: %w", err) + } + + fmt.Printf("Successfully downloaded backup %s from S3 to temp location: %s\n", backup.ID, tempFilePath) + + // Return temp file path and indicate it should be cleaned up + return tempFilePath, true, nil +} diff --git a/apps/api/internal/backup/s3_storage.go b/apps/api/internal/backup/s3_storage.go index 34c4b8f..6518813 100644 --- a/apps/api/internal/backup/s3_storage.go +++ b/apps/api/internal/backup/s3_storage.go @@ -85,6 +85,32 @@ func (s *S3Storage) UploadFile(ctx context.Context, localPath string) (string, e return objectKey, nil } +// UploadFileWithPath uploads a file to S3 with a custom subfolder path +func (s *S3Storage) UploadFileWithPath(ctx context.Context, localPath string, subfolder string) (string, error) { + file, err := os.Open(localPath) + if err != nil { + return "", fmt.Errorf("failed to open file: %w", err) + } + defer file.Close() + + fileInfo, err := file.Stat() + if err != nil { + return "", fmt.Errorf("failed to stat file: %w", err) + } + + fileName := filepath.Base(localPath) + objectKey := s.getObjectKeyWithPath(fileName, subfolder) + + _, err = s.client.PutObject(ctx, s.bucket, objectKey, file, fileInfo.Size(), minio.PutObjectOptions{ + ContentType: "application/octet-stream", + }) + if err != nil { + return "", fmt.Errorf("failed to upload to S3: %w", err) + } + + return objectKey, nil +} + func (s *S3Storage) DownloadFile(ctx context.Context, objectKey, localPath string) error { object, err := s.client.GetObject(ctx, s.bucket, objectKey, minio.GetObjectOptions{}) if err != nil { @@ -161,3 +187,20 @@ func (s *S3Storage) getObjectKey(fileName string) string { fileName = strings.TrimPrefix(fileName, "/") return fmt.Sprintf("%s/%s", prefix, fileName) } + +func (s *S3Storage) getObjectKeyWithPath(fileName string, subfolder string) string { + // Build path: prefix/subfolder/fileName + var parts []string + + if s.prefix != "" { + parts = append(parts, strings.TrimSuffix(s.prefix, "/")) + } + + if subfolder != "" { + parts = append(parts, strings.Trim(subfolder, "/")) + } + + parts = append(parts, strings.TrimPrefix(fileName, "/")) + + return strings.Join(parts, "/") +} diff --git a/apps/api/internal/database/migrations/20251113153839_add_s3_purge_local.sql b/apps/api/internal/database/migrations/20251113153839_add_s3_purge_local.sql new file mode 100644 index 0000000..993dfb9 --- /dev/null +++ b/apps/api/internal/database/migrations/20251113153839_add_s3_purge_local.sql @@ -0,0 +1,15 @@ +-- +goose Up +-- +goose StatementBegin +SELECT 'Adding s3_purge_local setting to user_settings'; + +ALTER TABLE user_settings ADD COLUMN s3_purge_local INTEGER DEFAULT 0; + +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +SELECT 'Removing s3_purge_local setting from user_settings'; + +ALTER TABLE user_settings DROP COLUMN s3_purge_local; + +-- +goose StatementEnd diff --git a/apps/api/internal/settings/model.go b/apps/api/internal/settings/model.go index 46cc529..e5e3020 100644 --- a/apps/api/internal/settings/model.go +++ b/apps/api/internal/settings/model.go @@ -27,6 +27,7 @@ type UserSettings struct { S3SecretKey *string `json:"s3_secret_key,omitempty"` S3UseSSL bool `json:"s3_use_ssl"` S3PathPrefix *string `json:"s3_path_prefix,omitempty"` + S3PurgeLocal bool `json:"s3_purge_local"` CreatedAt time.Time `json:"created_at"` UpdatedAt time.Time `json:"updated_at"` EnvConfigured map[string]bool `json:"env_configured,omitempty"` @@ -51,4 +52,5 @@ type UpdateSettingsRequest struct { S3SecretKey *string `json:"s3_secret_key,omitempty"` S3UseSSL *bool `json:"s3_use_ssl,omitempty"` S3PathPrefix *string `json:"s3_path_prefix,omitempty"` + S3PurgeLocal *bool `json:"s3_purge_local,omitempty"` } diff --git a/apps/api/internal/settings/settings_repository.go b/apps/api/internal/settings/settings_repository.go index 9141b8a..59b41f5 100644 --- a/apps/api/internal/settings/settings_repository.go +++ b/apps/api/internal/settings/settings_repository.go @@ -24,7 +24,7 @@ func (r *SettingsRepository) GetUserSettings(userID uuid.UUID) (*UserSettings, e SELECT id, user_id, notify_dashboard, notify_email, notify_webhook, webhook_url, email, smtp_host, smtp_port, smtp_username, smtp_password, s3_enabled, s3_endpoint, s3_region, s3_bucket, - s3_access_key, s3_secret_key, s3_use_ssl, s3_path_prefix, + s3_access_key, s3_secret_key, s3_use_ssl, s3_path_prefix, s3_purge_local, created_at, updated_at FROM user_settings WHERE user_id = $1`, userID).Scan( @@ -34,6 +34,7 @@ func (r *SettingsRepository) GetUserSettings(userID uuid.UUID) (*UserSettings, e &settings.SMTPUsername, &settings.SMTPPassword, &settings.S3Enabled, &settings.S3Endpoint, &settings.S3Region, &settings.S3Bucket, &settings.S3AccessKey, &settings.S3SecretKey, &settings.S3UseSSL, &settings.S3PathPrefix, + &settings.S3PurgeLocal, &createdAtStr, &updatedAtStr) if err == sql.ErrNoRows { @@ -74,15 +75,16 @@ func (r *SettingsRepository) CreateUserSettings(settings *UserSettings) error { id, user_id, notify_dashboard, notify_email, notify_webhook, webhook_url, email, smtp_host, smtp_port, smtp_username, smtp_password, s3_enabled, s3_endpoint, s3_region, s3_bucket, - s3_access_key, s3_secret_key, s3_use_ssl, s3_path_prefix, + s3_access_key, s3_secret_key, s3_use_ssl, s3_path_prefix, s3_purge_local, created_at, updated_at - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21)`, + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22)`, settings.ID, settings.UserID, settings.NotifyDashboard, settings.NotifyEmail, settings.NotifyWebhook, settings.WebhookURL, settings.Email, settings.SMTPHost, settings.SMTPPort, settings.SMTPUsername, settings.SMTPPassword, settings.S3Enabled, settings.S3Endpoint, settings.S3Region, settings.S3Bucket, settings.S3AccessKey, settings.S3SecretKey, settings.S3UseSSL, settings.S3PathPrefix, + settings.S3PurgeLocal, settings.CreatedAt, settings.UpdatedAt) return err } @@ -96,13 +98,14 @@ func (r *SettingsRepository) UpdateUserSettings(settings *UserSettings) error { smtp_username = $8, smtp_password = $9, s3_enabled = $10, s3_endpoint = $11, s3_region = $12, s3_bucket = $13, s3_access_key = $14, s3_secret_key = $15, s3_use_ssl = $16, - s3_path_prefix = $17, updated_at = $18 - WHERE user_id = $19`, + s3_path_prefix = $17, s3_purge_local = $18, updated_at = $19 + WHERE user_id = $20`, settings.NotifyDashboard, settings.NotifyEmail, settings.NotifyWebhook, settings.WebhookURL, settings.Email, settings.SMTPHost, settings.SMTPPort, settings.SMTPUsername, settings.SMTPPassword, settings.S3Enabled, settings.S3Endpoint, settings.S3Region, settings.S3Bucket, settings.S3AccessKey, settings.S3SecretKey, settings.S3UseSSL, settings.S3PathPrefix, + settings.S3PurgeLocal, settings.UpdatedAt, settings.UserID) return err } diff --git a/apps/api/internal/settings/settings_service.go b/apps/api/internal/settings/settings_service.go index 95459e2..2a93364 100644 --- a/apps/api/internal/settings/settings_service.go +++ b/apps/api/internal/settings/settings_service.go @@ -152,6 +152,9 @@ func (s *SettingsService) UpdateUserSettings(userID uuid.UUID, req *UpdateSettin if req.S3PathPrefix != nil { settings.S3PathPrefix = req.S3PathPrefix } + if req.S3PurgeLocal != nil { + settings.S3PurgeLocal = *req.S3PurgeLocal + } if err := s.repo.UpdateUserSettings(settings); err != nil { return nil, err diff --git a/apps/web/components/views/dashboard/activity-list.tsx b/apps/web/components/views/dashboard/activity-list.tsx index b21d0c4..e642433 100644 --- a/apps/web/components/views/dashboard/activity-list.tsx +++ b/apps/web/components/views/dashboard/activity-list.tsx @@ -6,7 +6,7 @@ import { HistoryListSkeleton } from "@/components/ui/skeleton/history-list"; import { ConnectionListSkeleton } from "@/components/ui/skeleton/connection-list"; import { EmptyState } from "@/components/ui/empty-state"; -import { Database, Clock, HardDrive, Calendar, Activity, Timer } from "lucide-react"; +import { Database, Clock, HardDrive, Calendar, Activity, Timer, Cloud } from "lucide-react"; import { formatDistanceToNow, parseISO } from "date-fns"; import { formatSize, getScheduleFrequency } from "@/lib/helper"; @@ -52,6 +52,12 @@ export function ActivityList() { {typeLabels[item.database_type as DatabaseType]} + {item.s3_object_key && ( + + + S3 + + )} {item.status} + {item.s3_object_key && ( + + + S3 + + )}

{formatDistanceToNow(parseISO(item.created_at), { addSuffix: true })} @@ -208,6 +214,12 @@ export function HistoryList() { {item.database_type} + {item.s3_object_key && ( + + + S3 + + )}

{formatDistanceToNow(parseISO(item.created_at), { addSuffix: true })} | {formatSize(item.size)} diff --git a/apps/web/components/views/settings/s3-storage-settings.tsx b/apps/web/components/views/settings/s3-storage-settings.tsx index 724b6be..e568f99 100644 --- a/apps/web/components/views/settings/s3-storage-settings.tsx +++ b/apps/web/components/views/settings/s3-storage-settings.tsx @@ -38,6 +38,7 @@ export function S3StorageSettings() { s3_secret_key: "", s3_use_ssl: true, s3_path_prefix: "", + s3_purge_local: false, }); // Sync form data when settings load @@ -52,6 +53,7 @@ export function S3StorageSettings() { s3_secret_key: "", // Don't show the secret key for security s3_use_ssl: settings.s3_use_ssl ?? true, s3_path_prefix: settings.s3_path_prefix || "", + s3_purge_local: settings.s3_purge_local || false, }); } }, [settings]); @@ -358,6 +360,23 @@ export function S3StorageSettings() { /> + {/* Purge Local Backup */} +

+
+ +

+ Automatically delete local backup files after successful S3 upload to save disk space +

+
+ setFormData({ ...formData, s3_purge_local: checked })} + /> +
+ {/* Test Connection Result */} {testResult && (
; }