S3 lifecycle management features

apps/api/cmd/api-server/main.go

@@ -64,7 +64,6 @@ func main() {
 	connRepo := connection.NewConnectionRepository(db, cryptoService)
 	connService := connection.NewConnectionService(connRepo, connManager)
 
-	connHandler := connection.NewConnectionHandler(connService)
 	authHandler := auth.NewAuthHandler(authService)
 
 	authMiddleware := middleware.NewAuthMiddleware(secrets.JWTSecret)
@@ -85,15 +84,6 @@ func main() {
 	protected.Use(authMiddleware.RequireAuth)
 	protected.HandleFunc("/auth/profile", authHandler.GetProfile).Methods("GET", "OPTIONS")
 
-	protected.HandleFunc("/connections/test", connHandler.TestConnection).Methods("POST", "OPTIONS")
-	protected.HandleFunc("/connections/{id}/discover", connHandler.DiscoverDatabases).Methods("GET", "OPTIONS")
-	protected.HandleFunc("/connections/{id}/databases", connHandler.UpdateSelectedDatabases).Methods("PUT", "OPTIONS")
-	protected.HandleFunc("/connections/{id}", connHandler.GetConnection).Methods("GET", "OPTIONS")
-	protected.HandleFunc("/connections/{id}", connHandler.DeleteConnection).Methods("DELETE", "OPTIONS")
-	protected.HandleFunc("/connections", connHandler.SaveConnection).Methods("POST", "OPTIONS")
-	protected.HandleFunc("/connections", connHandler.ListConnections).Methods("GET", "OPTIONS")
-	protected.HandleFunc("/connections", connHandler.UpdateConnection).Methods("PUT", "OPTIONS")
-
 	backupRepo := backup.NewBackupRepository(db)
 	settingsRepo := settings.NewSettingsRepository(db)
 	notificationRepo := notification.NewNotificationRepository(db)
@@ -108,6 +98,19 @@ func main() {
 		cryptoService,
 	)
 
+	// Create connHandler after backupService is available
+	connHandler := connection.NewConnectionHandler(connService, backupService)
+
+	protected.HandleFunc("/connections/test", connHandler.TestConnection).Methods("POST", "OPTIONS")
+	protected.HandleFunc("/connections/{id}/discover", connHandler.DiscoverDatabases).Methods("GET", "OPTIONS")
+	protected.HandleFunc("/connections/{id}/databases", connHandler.UpdateSelectedDatabases).Methods("PUT", "OPTIONS")
+	protected.HandleFunc("/connections/{id}/settings", connHandler.UpdateConnectionSettings).Methods("POST", "OPTIONS")
+	protected.HandleFunc("/connections/{id}", connHandler.GetConnection).Methods("GET", "OPTIONS")
+	protected.HandleFunc("/connections/{id}", connHandler.DeleteConnection).Methods("DELETE", "OPTIONS")
+	protected.HandleFunc("/connections", connHandler.SaveConnection).Methods("POST", "OPTIONS")
+	protected.HandleFunc("/connections", connHandler.ListConnections).Methods("GET", "OPTIONS")
+	protected.HandleFunc("/connections", connHandler.UpdateConnection).Methods("PUT", "OPTIONS")
+
 	backupHandler := backup.NewBackupHandler(backupService)
 
 	protected.HandleFunc("/backups/stats", backupHandler.GetBackupStats).Methods("GET", "OPTIONS")

apps/api/internal/backup/backup_repository.go

@@ -229,7 +229,7 @@ func (r *BackupRepository) UpdateBackupStatus(id string, status string) error {
 
 func (r *BackupRepository) GetBackupsOlderThan(connectionID string, cutoffTime time.Time) ([]*Backup, error) {
 	rows, err := r.db.Query(`
-		SELECT id, path, created_at 
+		SELECT id, path, s3_object_key, created_at 
 		FROM backups 
 		WHERE connection_id = $1 
 		AND created_at < $2 
@@ -244,7 +244,7 @@ func (r *BackupRepository) GetBackupsOlderThan(connectionID string, cutoffTime t
 	for rows.Next() {
 		backup := &Backup{}
 		var createdAtStr string
-		err := rows.Scan(&backup.ID, &backup.Path, &createdAtStr)
+		err := rows.Scan(&backup.ID, &backup.Path, &backup.S3ObjectKey, &createdAtStr)
 		if err != nil {
 			return nil, err
 		}
@@ -472,3 +472,70 @@ func (r *BackupRepository) GetBackupStats(userID uuid.UUID) (*BackupStats, error
 
 	return stats, nil
 }
+
+func (r *BackupRepository) GetBackupsByConnectionID(connectionID string) ([]*Backup, error) {
+	rows, err := r.db.Query(`
+		SELECT id, connection_id, schedule_id, status, path, s3_object_key, size,
+		       started_time, completed_time, created_at, updated_at
+		FROM backups
+		WHERE connection_id = $1
+		ORDER BY created_at DESC`,
+		connectionID)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var backups []*Backup
+	for rows.Next() {
+		backup := &Backup{}
+		var startedTimeStr, createdAtStr, updatedAtStr string
+		var completedTimeStr sql.NullString
+
+		err := rows.Scan(
+			&backup.ID, &backup.ConnectionID, &backup.ScheduleID, &backup.Status,
+			&backup.Path, &backup.S3ObjectKey, &backup.Size,
+			&startedTimeStr, &completedTimeStr, &createdAtStr, &updatedAtStr,
+		)
+		if err != nil {
+			return nil, err
+		}
+
+		backup.StartedTime, err = common.ParseTime(startedTimeStr)
+		if err != nil {
+			return nil, err
+		}
+
+		if completedTimeStr.Valid {
+			completedTime, err := common.ParseTime(completedTimeStr.String)
+			if err != nil {
+				return nil, err
+			}
+			backup.CompletedTime = &completedTime
+		}
+
+		backup.CreatedAt, err = common.ParseTime(createdAtStr)
+		if err != nil {
+			return nil, err
+		}
+
+		backup.UpdatedAt, err = common.ParseTime(updatedAtStr)
+		if err != nil {
+			return nil, err
+		}
+
+		backups = append(backups, backup)
+	}
+
+	return backups, rows.Err()
+}
+
+
+func (r *BackupRepository) UpdateBackupS3ObjectKey(backupID string, s3ObjectKey string) error {
+	_, err := r.db.Exec(`
+		UPDATE backups 
+		SET s3_object_key = $1, updated_at = datetime('now') 
+		WHERE id = $2`,
+		s3ObjectKey, backupID)
+	return err
+}

apps/api/internal/backup/backup_scheduler.go

@@ -1,6 +1,7 @@
 package backup
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
 	"os"
@@ -125,13 +126,105 @@ func (s *BackupService) cleanupOldBackups(connectionID string, retentionDays int
 	cutoffTime := time.Now().AddDate(0, 0, -retentionDays)
 	oldBackups, err := s.backupRepo.GetBackupsOlderThan(connectionID, cutoffTime)
 	if err != nil {
+		fmt.Printf("Error fetching old backups for cleanup: %v\n", err)
 		return
 	}
 
+	if len(oldBackups) == 0 {
+		return
+	}
+
+	// Get connection to retrieve user settings for S3
+	conn, err := s.connStorage.GetConnection(connectionID)
+	if err != nil {
+		fmt.Printf("Error getting connection for cleanup: %v\n", err)
+		return
+	}
+
+	// Get user settings to check if S3 is enabled
+	userSettings, err := s.settingsService.GetUserSettingsInternal(conn.UserID)
+	if err != nil {
+		fmt.Printf("Warning: Failed to get user settings for cleanup: %v\n", err)
+		// Continue with local cleanup even if we can't get S3 settings
+	}
+
+	// Initialize S3 client if S3 is enabled and configured
+	var s3Storage *S3Storage
+	if userSettings != nil && userSettings.S3Enabled && 
+	   userSettings.S3Endpoint != nil && *userSettings.S3Endpoint != "" &&
+	   userSettings.S3Bucket != nil && *userSettings.S3Bucket != "" &&
+	   userSettings.S3AccessKey != nil && *userSettings.S3AccessKey != "" &&
+	   userSettings.S3SecretKey != nil && *userSettings.S3SecretKey != "" {
+
+		secretKey, err := s.cryptoService.Decrypt(*userSettings.S3SecretKey)
+		if err != nil {
+			fmt.Printf("Warning: Failed to decrypt S3 secret key for cleanup: %v\n", err)
+		} else {
+			region := "us-east-1"
+			if userSettings.S3Region != nil && *userSettings.S3Region != "" {
+				region = *userSettings.S3Region
+			}
+
+			pathPrefix := ""
+			if userSettings.S3PathPrefix != nil {
+				pathPrefix = *userSettings.S3PathPrefix
+			}
+
+			s3Config := S3Config{
+				Endpoint:   *userSettings.S3Endpoint,
+				Region:     region,
+				Bucket:     *userSettings.S3Bucket,
+				AccessKey:  *userSettings.S3AccessKey,
+				SecretKey:  secretKey,
+				UseSSL:     userSettings.S3UseSSL,
+				PathPrefix: pathPrefix,
+			}
+
+			s3Storage, err = NewS3Storage(s3Config)
+			if err != nil {
+				fmt.Printf("Warning: Failed to create S3 storage client for cleanup: %v\n", err)
+				s3Storage = nil
+			}
+		}
+	}
+
+	// Clean up old backups
+	ctx := context.Background()
 	for _, backup := range oldBackups {
-		os.Remove(backup.Path)
-		s.backupRepo.DeleteBackup(backup.ID.String())
+		backupID := backup.ID.String()
+
+		// Delete from S3 if object key exists, S3 is configured, and connection has S3 cleanup enabled
+		if backup.S3ObjectKey != nil && *backup.S3ObjectKey != "" && s3Storage != nil && conn.S3CleanupOnRetention {
+			if err := s3Storage.DeleteFile(ctx, *backup.S3ObjectKey); err != nil {
+				fmt.Printf("Warning: Failed to delete S3 object %s for backup %s: %v\n", 
+					*backup.S3ObjectKey, backupID, err)
+			} else {
+				fmt.Printf("Deleted S3 object %s for backup %s (retention cleanup)\n", 
+					*backup.S3ObjectKey, backupID)
+			}
+		}
+
+		// Delete local file if it exists
+		if _, err := os.Stat(backup.Path); err == nil {
+			if err := os.Remove(backup.Path); err != nil {
+				fmt.Printf("Warning: Failed to delete local file %s for backup %s: %v\n", 
+					backup.Path, backupID, err)
+			} else {
+				fmt.Printf("Deleted local file %s for backup %s (retention cleanup)\n", 
+					backup.Path, backupID)
+			}
+		}
+
+		// Delete backup record from database
+		if err := s.backupRepo.DeleteBackup(backupID); err != nil {
+			fmt.Printf("Error deleting backup record %s: %v\n", backupID, err)
+		} else {
+			fmt.Printf("Deleted backup record %s (retention cleanup)\n", backupID)
+		}
 	}
+
+	fmt.Printf("Retention cleanup completed: processed %d old backups for connection %s\n", 
+		len(oldBackups), connectionID)
 }
 
 func (s *BackupService) DisableBackupSchedule(connectionID string) error {