From a3af5e5b162039a1ae2a40173c3ca770d72aca3c Mon Sep 17 00:00:00 2001 From: Remco Beckers Date: Tue, 3 Feb 2026 13:44:45 +0100 Subject: [PATCH 1/7] STAC-23457 Restore stackpacks as well --- cmd/settings/restore.go | 6 ++- cmd/stackgraph/restore.go | 6 ++- internal/foundation/config/config.go | 18 +++++---- .../config/testdata/validConfigMapConfig.yaml | 3 ++ .../scripts/restore-settings-backup.sh | 38 +++++++++++++++++++ .../scripts/restore-stackgraph-backup.sh | 23 +++++++++++ 6 files changed, 84 insertions(+), 10 deletions(-) diff --git a/cmd/settings/restore.go b/cmd/settings/restore.go index 905f488..41e24ad 100644 --- a/cmd/settings/restore.go +++ b/cmd/settings/restore.go @@ -28,13 +28,14 @@ var ( useLatest bool background bool skipConfirmation bool + skipStackpacks bool ) func restoreCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command { cmd := &cobra.Command{ Use: "restore", Short: "Restore Settings from a backup archive", - Long: `Restore Settings data from a backup archive stored in S3. Can use --latest or --archive to specify which backup to restore.`, + Long: `Restore Settings data from a backup archive stored in S3. Automatically also restores Stackpacks backup that was made at the same time, it can be skipped with --skip-stackpacks. Can use --latest or --archive to specify which backup to restore.`, Run: func(_ *cobra.Command, _ []string) { cmdutils.Run(globalFlags, runRestore, cmdutils.StorageIsNotRequired) }, @@ -45,6 +46,7 @@ func restoreCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command { cmd.Flags().BoolVar(&background, "background", false, "Run restore job in background without waiting for completion") cmd.Flags().BoolVarP(&skipConfirmation, "yes", "y", false, "Skip confirmation prompt") cmd.Flags().BoolVar(&fromPVC, "from-old-pvc", false, "Restore backup from legacy PVC instead of S3") + cmd.Flags().BoolVar(&skipStackpacks, "skip-stackpacks", false, "Skip restoring stackpacks backup") cmd.MarkFlagsMutuallyExclusive("archive", "latest") cmd.MarkFlagsOneRequired("archive", "latest") @@ -192,12 +194,14 @@ func buildEnvVar(extraEnvVar []corev1.EnvVar, config *config.Config) []corev1.En commonVar := []corev1.EnvVar{ {Name: "BACKUP_CONFIGURATION_BUCKET_NAME", Value: config.Settings.Bucket}, {Name: "BACKUP_CONFIGURATION_S3_PREFIX", Value: config.Settings.S3Prefix}, + {Name: "BACKUP_CONFIGURATION_STACKPACKS_S3_PREFIX", Value: config.Settings.StackpacksS3Prefix}, {Name: "MINIO_ENDPOINT", Value: fmt.Sprintf("%s:%d", storageService.Name, storageService.Port)}, {Name: "STACKSTATE_BASE_URL", Value: config.Settings.Restore.BaseURL}, {Name: "RECEIVER_BASE_URL", Value: config.Settings.Restore.ReceiverBaseURL}, {Name: "PLATFORM_VERSION", Value: config.Settings.Restore.PlatformVersion}, {Name: "ZOOKEEPER_QUORUM", Value: config.Settings.Restore.ZookeeperQuorum}, {Name: "BACKUP_CONFIGURATION_UPLOAD_REMOTE", Value: strconv.FormatBool(config.GlobalBackupEnabled())}, + {Name: "SKIP_STACKPACKS", Value: strconv.FormatBool(skipStackpacks)}, } if fromPVC { // Force PVC mode in the shell script, suppress local bucket diff --git a/cmd/stackgraph/restore.go b/cmd/stackgraph/restore.go index a56fc82..19cd50b 100644 --- a/cmd/stackgraph/restore.go +++ b/cmd/stackgraph/restore.go @@ -35,13 +35,14 @@ var ( useLatest bool background bool skipConfirmation bool + skipStackpacks bool ) func restoreCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command { cmd := &cobra.Command{ Use: "restore", Short: "Restore Stackgraph from a backup archive", - Long: `Restore Stackgraph data from a backup archive stored in S3. Can use --latest or --archive to specify which backup to restore.`, + Long: `Restore Stackgraph data from a backup archive stored in S3. Automatically also restores Stackpacks backup that was made at the same time, it can be skipped with --skip-stackpacks. Can use --latest or --archive to specify which backup to restore.`, Run: func(_ *cobra.Command, _ []string) { cmdutils.Run(globalFlags, runRestore, cmdutils.StorageIsRequired) }, @@ -51,6 +52,7 @@ func restoreCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command { cmd.Flags().BoolVar(&useLatest, "latest", false, "Restore from the most recent backup") cmd.Flags().BoolVar(&background, "background", false, "Run restore job in background without waiting for completion") cmd.Flags().BoolVarP(&skipConfirmation, "yes", "y", false, "Skip confirmation prompt") + cmd.Flags().BoolVar(&skipStackpacks, "skip-stackpacks", false, "Skip restoring stackpacks backup") cmd.MarkFlagsMutuallyExclusive("archive", "latest") cmd.MarkFlagsOneRequired("archive", "latest") @@ -268,9 +270,11 @@ func buildRestoreEnvVars(backupFile string, config *config.Config) []corev1.EnvV {Name: "FORCE_DELETE", Value: purgeStackgraphDataFlag}, {Name: "BACKUP_STACKGRAPH_BUCKET_NAME", Value: config.Stackgraph.Bucket}, {Name: "BACKUP_STACKGRAPH_S3_PREFIX", Value: config.Stackgraph.S3Prefix}, + {Name: "BACKUP_STACKGRAPH_STACKPACKS_S3_PREFIX", Value: config.Stackgraph.StackpacksS3Prefix}, {Name: "BACKUP_STACKGRAPH_MULTIPART_ARCHIVE", Value: strconv.FormatBool(config.Stackgraph.MultipartArchive)}, {Name: "MINIO_ENDPOINT", Value: fmt.Sprintf("%s:%d", storageService.Name, storageService.Port)}, {Name: "ZOOKEEPER_QUORUM", Value: config.Stackgraph.Restore.ZookeeperQuorum}, + {Name: "SKIP_STACKPACKS", Value: strconv.FormatBool(skipStackpacks)}, } } diff --git a/internal/foundation/config/config.go b/internal/foundation/config/config.go index e944301..2f7c054 100644 --- a/internal/foundation/config/config.go +++ b/internal/foundation/config/config.go @@ -135,10 +135,11 @@ type StorageConfig struct { // StackgraphConfig holds Stackgraph backup-specific configuration type StackgraphConfig struct { - Bucket string `yaml:"bucket" validate:"required"` - S3Prefix string `yaml:"s3Prefix"` - MultipartArchive bool `yaml:"multipartArchive" validate:"boolean"` - Restore StackgraphRestoreConfig `yaml:"restore" validate:"required"` + Bucket string `yaml:"bucket" validate:"required"` + S3Prefix string `yaml:"s3Prefix"` + StackpacksS3Prefix string `yaml:"stackpacksS3Prefix"` + MultipartArchive bool `yaml:"multipartArchive" validate:"boolean"` + Restore StackgraphRestoreConfig `yaml:"restore" validate:"required"` } type VictoriaMetricsConfig struct { @@ -169,10 +170,11 @@ type StackgraphRestoreConfig struct { } type SettingsConfig struct { - Bucket string `yaml:"bucket" validate:"required"` - S3Prefix string `yaml:"s3Prefix"` - LocalBucket string `yaml:"localBucket"` - Restore SettingsRestoreConfig `yaml:"restore" validate:"required"` + Bucket string `yaml:"bucket" validate:"required"` + S3Prefix string `yaml:"s3Prefix"` + StackpacksS3Prefix string `yaml:"stackpacksS3Prefix"` + LocalBucket string `yaml:"localBucket"` + Restore SettingsRestoreConfig `yaml:"restore" validate:"required"` } type SettingsRestoreConfig struct { diff --git a/internal/foundation/config/testdata/validConfigMapConfig.yaml b/internal/foundation/config/testdata/validConfigMapConfig.yaml index 76aca9a..e36900e 100644 --- a/internal/foundation/config/testdata/validConfigMapConfig.yaml +++ b/internal/foundation/config/testdata/validConfigMapConfig.yaml @@ -76,6 +76,8 @@ stackgraph: bucket: sts-stackgraph-backup # S3 prefix path for backups s3Prefix: "" + # S3 prefix path for stackpacks backups + stackpacksS3Prefix: "stackpacks/" # Archive split to multiple parts multipartArchive: true # Restore configuration @@ -139,6 +141,7 @@ victoriaMetrics: settings: bucket: sts-settings-backup s3Prefix: "" + stackpacksS3Prefix: "stackpacks/" restore: scaleDownLabelSelector: "observability.suse.com/scalable-during-settings-restore=true" loggingConfigConfigMap: suse-observability-logging diff --git a/internal/scripts/scripts/restore-settings-backup.sh b/internal/scripts/scripts/restore-settings-backup.sh index 68b95c4..2309054 100644 --- a/internal/scripts/scripts/restore-settings-backup.sh +++ b/internal/scripts/scripts/restore-settings-backup.sh @@ -55,4 +55,42 @@ fi echo "=== Restoring settings backup from \"${BACKUP_FILE}\"..." /opt/docker/bin/settings-backup -Dlogback.configurationFile=/opt/docker/etc_log/logback.xml -restore "${RESTORE_FILE}" +echo "=== Settings restore complete" + +# === StackPacks Restore === +if [ "${SKIP_STACKPACKS:-false}" == "true" ]; then + echo "=== Skipping StackPacks restore (--skip-stackpacks flag set)" +else + export STACKPACKS_BACKUP_DIR="${BACKUP_DIR}/stackpacks" + + # Construct stackpacks backup filename from the original backup file + STACKPACKS_FILE="${BACKUP_FILE}.stackpacks.zip" + STACKPACKS_RESTORE_FILE="${STACKPACKS_BACKUP_DIR}/${STACKPACKS_FILE}" + + echo "=== Checking for StackPacks backup \"${STACKPACKS_FILE}\"..." + + # Check local PVC first, then try S3 if not found and remote is enabled + if [ ! -f "${STACKPACKS_RESTORE_FILE}" ] && [ "$BACKUP_CONFIGURATION_UPLOAD_REMOTE" == "true" ]; then + # Ensure AWS credentials are set for S3 access + export AWS_ACCESS_KEY_ID + AWS_ACCESS_KEY_ID="$(cat /aws-keys/accesskey)" + export AWS_SECRET_ACCESS_KEY + AWS_SECRET_ACCESS_KEY="$(cat /aws-keys/secretkey)" + + # Check if file exists in S3 + if sts-toolbox aws s3 ls --endpoint "http://${MINIO_ENDPOINT}" --region minio --bucket "${BACKUP_CONFIGURATION_BUCKET_NAME}" --prefix "${BACKUP_CONFIGURATION_STACKPACKS_S3_PREFIX}${STACKPACKS_FILE}" 2>/dev/null | grep -q "${STACKPACKS_FILE}"; then + echo "=== Downloading StackPacks backup from S3..." + sts-toolbox aws s3 cp --endpoint "http://${MINIO_ENDPOINT}" --region minio "s3://${BACKUP_CONFIGURATION_BUCKET_NAME}/${BACKUP_CONFIGURATION_STACKPACKS_S3_PREFIX}${STACKPACKS_FILE}" "${TMP_DIR}/${STACKPACKS_FILE}" + STACKPACKS_RESTORE_FILE="${TMP_DIR}/${STACKPACKS_FILE}" + fi + fi + + if [ -f "${STACKPACKS_RESTORE_FILE}" ]; then + echo "=== Restoring StackPacks from \"${STACKPACKS_FILE}\"..." + /opt/docker/bin/stack-packs-backup -Dlogback.configurationFile=/opt/docker/etc_log/logback.xml -restore "${STACKPACKS_RESTORE_FILE}" + echo "=== StackPacks restore complete" + else + echo "=== WARNING: StackPacks backup \"${STACKPACKS_FILE}\" not found, skipping StackPacks restore" + fi +fi echo "===" diff --git a/internal/scripts/scripts/restore-stackgraph-backup.sh b/internal/scripts/scripts/restore-stackgraph-backup.sh index 2e89c30..0f3a9c4 100644 --- a/internal/scripts/scripts/restore-stackgraph-backup.sh +++ b/internal/scripts/scripts/restore-stackgraph-backup.sh @@ -30,4 +30,27 @@ fi echo "=== Importing StackGraph data from \"${BACKUP_FILE}\"..." /opt/docker/bin/stackstate-server -Dlogback.configurationFile=/opt/docker/etc_log/logback.xml -import "${TMP_DIR}/${BACKUP_FILE}" "${FORCE_DELETE}" +echo "=== StackGraph restore complete" + +# === StackPacks Restore === +if [ "${SKIP_STACKPACKS:-false}" == "true" ]; then + echo "=== Skipping StackPacks restore (--skip-stackpacks flag set)" +else + # Construct stackpacks backup filename from the original backup file + STACKPACKS_FILE="${BACKUP_FILE}.stackpacks.zip" + + echo "=== Checking for StackPacks backup \"${STACKPACKS_FILE}\" in bucket \"${BACKUP_STACKGRAPH_BUCKET_NAME}\"..." + + # Check if stackpacks backup exists in S3 + if sts-toolbox aws s3 ls --endpoint "http://${MINIO_ENDPOINT}" --region minio --bucket "${BACKUP_STACKGRAPH_BUCKET_NAME}" --prefix "${BACKUP_STACKGRAPH_STACKPACKS_S3_PREFIX}${STACKPACKS_FILE}" 2>/dev/null | grep -q "${STACKPACKS_FILE}"; then + echo "=== Downloading StackPacks backup..." + sts-toolbox aws s3 cp --endpoint "http://${MINIO_ENDPOINT}" --region minio "s3://${BACKUP_STACKGRAPH_BUCKET_NAME}/${BACKUP_STACKGRAPH_STACKPACKS_S3_PREFIX}${STACKPACKS_FILE}" "${TMP_DIR}/${STACKPACKS_FILE}" + + echo "=== Restoring StackPacks from \"${STACKPACKS_FILE}\"..." + /opt/docker/bin/stack-packs-backup -Dlogback.configurationFile=/opt/docker/etc_log/logback.xml -restore "${TMP_DIR}/${STACKPACKS_FILE}" + echo "=== StackPacks restore complete" + else + echo "=== WARNING: StackPacks backup \"${STACKPACKS_FILE}\" not found in S3, skipping StackPacks restore" + fi +fi echo "===" From 13531bd0f83fb20cf57c51ccafc984935e06f390 Mon Sep 17 00:00:00 2001 From: Remco Beckers Date: Fri, 6 Feb 2026 14:39:05 +0100 Subject: [PATCH 2/7] STAC-23457 Rename filterBackupObjects --- cmd/settings/list.go | 3 ++- cmd/stackgraph/list.go | 2 +- cmd/stackgraph/restore.go | 2 +- internal/clients/s3/filter.go | 4 ++-- internal/clients/s3/filter_test.go | 14 +++++++------- 5 files changed, 13 insertions(+), 12 deletions(-) diff --git a/cmd/settings/list.go b/cmd/settings/list.go index 6072e3a..388d91c 100644 --- a/cmd/settings/list.go +++ b/cmd/settings/list.go @@ -29,6 +29,7 @@ const ( isMultiPartArchive = false expectedListJobPodCount = 1 expectedListJobContainerCount = 1 + backupFileNameRegex = `^sts-backup-.*\.sty$` ) // Shared flag for --from-old-pvc, used by both list and restore commands @@ -182,7 +183,7 @@ func getBackupListFromS3(appCtx *app.Context) ([]BackupFileInfo, error) { } // Filter objects based on whether the archive is split or not - filteredObjects := s3client.FilterBackupObjects(result.Contents, isMultiPartArchive) + filteredObjects := s3client.FilterMultipartBackupObjects(result.Contents, isMultiPartArchive) var backups []BackupFileInfo for _, obj := range filteredObjects { diff --git a/cmd/stackgraph/list.go b/cmd/stackgraph/list.go index 1d169d9..5b242a5 100644 --- a/cmd/stackgraph/list.go +++ b/cmd/stackgraph/list.go @@ -63,7 +63,7 @@ func runList(appCtx *app.Context) error { } // Filter objects based on whether the archive is split or not - filteredObjects := s3client.FilterBackupObjects(result.Contents, multipartArchive) + filteredObjects := s3client.FilterMultipartBackupObjects(result.Contents, multipartArchive) // Sort by LastModified time (most recent first) sort.Slice(filteredObjects, func(i, j int) bool { diff --git a/cmd/stackgraph/restore.go b/cmd/stackgraph/restore.go index 19cd50b..1bb8688 100644 --- a/cmd/stackgraph/restore.go +++ b/cmd/stackgraph/restore.go @@ -180,7 +180,7 @@ func getLatestBackup(k8sClient *k8s.Client, namespace string, config *config.Con } // Filter objects based on whether the archive is split or not - filteredObjects := s3client.FilterBackupObjects(result.Contents, multipartArchive) + filteredObjects := s3client.FilterMultipartBackupObjects(result.Contents, multipartArchive) if len(filteredObjects) == 0 { return "", fmt.Errorf("no backups found in bucket %s", bucket) diff --git a/internal/clients/s3/filter.go b/internal/clients/s3/filter.go index 5787352..403aecb 100644 --- a/internal/clients/s3/filter.go +++ b/internal/clients/s3/filter.go @@ -19,10 +19,10 @@ type Object struct { Size int64 } -// FilterBackupObjects filters S3 objects based on whether the archive is split or not +// FilterMultipartBackupObjects filters S3 objects based on whether the archive is split or not // If it is not multipartArchive, it filters out multipart archives (files ending with .digits) // Otherwise, it groups multipart archives by base name and sums their sizes -func FilterBackupObjects(objects []s3types.Object, multipartArchive bool) []Object { +func FilterMultipartBackupObjects(objects []s3types.Object, multipartArchive bool) []Object { if !multipartArchive { return filterNonMultipart(objects) } diff --git a/internal/clients/s3/filter_test.go b/internal/clients/s3/filter_test.go index 2cf9b86..ef88d88 100644 --- a/internal/clients/s3/filter_test.go +++ b/internal/clients/s3/filter_test.go @@ -79,7 +79,7 @@ func TestFilterBackupObjects_SingleFileMode(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result := FilterBackupObjects(tt.objects, false) + result := FilterMultipartBackupObjects(tt.objects, false) assert.Equal(t, tt.expectedCount, len(result)) @@ -158,7 +158,7 @@ func TestFilterBackupObjects_MultipartMode(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result := FilterBackupObjects(tt.objects, tt.multipartArchive) + result := FilterMultipartBackupObjects(tt.objects, tt.multipartArchive) assert.Equal(t, tt.expectedCount, len(result)) @@ -196,14 +196,14 @@ func TestFilterBackupObjects_ObjectMetadata(t *testing.T) { } // Test single file mode - result := FilterBackupObjects(objects, false) + result := FilterMultipartBackupObjects(objects, false) assert.Equal(t, 1, len(result)) assert.Equal(t, "backup-2024-01-01.tar.gz", result[0].Key) assert.Equal(t, int64(1234567890), result[0].Size) assert.Equal(t, now.Unix(), result[0].LastModified.Unix()) // Test multipart mode - should group parts and sum sizes - result = FilterBackupObjects(objects, true) + result = FilterMultipartBackupObjects(objects, true) assert.Equal(t, 2, len(result)) // tar.gz file + grouped multipart // Find the multipart archive result @@ -275,7 +275,7 @@ func TestFilterBackupObjects_EdgeCases(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result := FilterBackupObjects(tt.objects, tt.multipartArchive) + result := FilterMultipartBackupObjects(tt.objects, tt.multipartArchive) assert.Equal(t, tt.expectedCount, len(result)) }) } @@ -331,7 +331,7 @@ func TestFilterBackupObjects_RealWorldScenarios(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result := FilterBackupObjects(tt.objects, tt.multipartArchive) + result := FilterMultipartBackupObjects(tt.objects, tt.multipartArchive) assert.Equal(t, tt.expectedCount, len(result), "Scenario: %s", tt.scenario) }) } @@ -348,7 +348,7 @@ func TestFilterBackupObjects_SizeSummation(t *testing.T) { {Key: aws.String("sts-backup-20251029-0924.graph.01"), Size: aws.Int64(6567239)}, } - result := FilterBackupObjects(objects, true) + result := FilterMultipartBackupObjects(objects, true) // Should have 3 grouped archives assert.Equal(t, 3, len(result)) From bcb426ab3941674ab2493d08beb1777347a48cd1 Mon Sep 17 00:00:00 2001 From: Remco Beckers Date: Fri, 6 Feb 2026 15:18:55 +0100 Subject: [PATCH 3/7] STAC-23457 Apply more filtering to settings/stackgraph backups To hide the settings backups and to make the backup name copy/pasteable --- cmd/settings/list.go | 23 ++++ cmd/stackgraph/list.go | 11 ++ internal/clients/s3/filter.go | 45 ++++++++ internal/clients/s3/filter_test.go | 170 +++++++++++++++++++++++++++++ 4 files changed, 249 insertions(+) diff --git a/cmd/settings/list.go b/cmd/settings/list.go index 388d91c..fc99fc1 100644 --- a/cmd/settings/list.go +++ b/cmd/settings/list.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "regexp" "slices" "sort" "strconv" @@ -185,6 +186,13 @@ func getBackupListFromS3(appCtx *app.Context) ([]BackupFileInfo, error) { // Filter objects based on whether the archive is split or not filteredObjects := s3client.FilterMultipartBackupObjects(result.Contents, isMultiPartArchive) + // Filter to only include direct children of the prefix that match the backup filename pattern, + // and strip the prefix from the key + filteredObjects, err = s3client.FilterByPrefixAndRegex(filteredObjects, prefix, backupFileNameRegex) + if err != nil { + return nil, fmt.Errorf("failed to filter objects: %w", err) + } + var backups []BackupFileInfo for _, obj := range filteredObjects { row := BackupFileInfo{ @@ -299,6 +307,9 @@ func getBackupListFromPVC(appCtx *app.Context) ([]BackupFileInfo, error) { return nil, fmt.Errorf("failed to parse list job output: %w", err) } + // Filter by backup filename pattern + files = filterBackupsByRegex(files, backupFileNameRegex) + return files, nil } @@ -377,3 +388,15 @@ func ParseListJobOutput(input string) ([]BackupFileInfo, error) { return files, nil } + +// filterBackupsByRegex filters BackupFileInfo by matching filename against a regex pattern +func filterBackupsByRegex(backups []BackupFileInfo, pattern string) []BackupFileInfo { + re := regexp.MustCompile(pattern) + var filtered []BackupFileInfo + for _, b := range backups { + if re.MatchString(b.Filename) { + filtered = append(filtered, b) + } + } + return filtered +} diff --git a/cmd/stackgraph/list.go b/cmd/stackgraph/list.go index 5b242a5..c9b86b7 100644 --- a/cmd/stackgraph/list.go +++ b/cmd/stackgraph/list.go @@ -17,6 +17,10 @@ import ( "github.com/stackvista/stackstate-backup-cli/internal/orchestration/portforward" ) +const ( + backupFileNameRegex = `^sts-backup-.*\.graph$` +) + func listCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command { return &cobra.Command{ Use: "list", @@ -65,6 +69,13 @@ func runList(appCtx *app.Context) error { // Filter objects based on whether the archive is split or not filteredObjects := s3client.FilterMultipartBackupObjects(result.Contents, multipartArchive) + // Filter to only include direct children of the prefix that match the backup filename pattern, + // and strip the prefix from the key + filteredObjects, err = s3client.FilterByPrefixAndRegex(filteredObjects, prefix, backupFileNameRegex) + if err != nil { + return fmt.Errorf("failed to filter objects: %w", err) + } + // Sort by LastModified time (most recent first) sort.Slice(filteredObjects, func(i, j int) bool { return filteredObjects[i].LastModified.After(filteredObjects[j].LastModified) diff --git a/internal/clients/s3/filter.go b/internal/clients/s3/filter.go index 403aecb..f44ceb7 100644 --- a/internal/clients/s3/filter.go +++ b/internal/clients/s3/filter.go @@ -1,6 +1,8 @@ package s3 import ( + "fmt" + "regexp" "strings" "time" @@ -141,6 +143,49 @@ func getBaseName(key string) (string, bool) { return key, false } +// FilterByPrefixAndRegex filters objects to only include direct children of the given prefix +// that match the specified regex pattern. It excludes objects in nested subdirectories and +// strips the prefix from the key, returning just the filename portion. +// +// For example, with prefix "backups/" and pattern `^sts-backup-.*\.graph$`: +// - "backups/sts-backup-20240101.graph" -> included, Key becomes "sts-backup-20240101.graph" +// - "backups/other-file.txt" -> excluded (doesn't match pattern) +// - "backups/subdir/sts-backup-20240101.graph" -> excluded (nested) +func FilterByPrefixAndRegex(objects []Object, prefix string, pattern string) ([]Object, error) { + re, err := regexp.Compile(pattern) + if err != nil { + return nil, fmt.Errorf("invalid regex pattern: %w", err) + } + + var filtered []Object + for _, obj := range objects { + // Strip the prefix from the key + relativePath := strings.TrimPrefix(obj.Key, prefix) + + // Skip if the relative path contains a slash (indicating nested directory) + if strings.Contains(relativePath, "/") { + continue + } + + // Skip empty relative paths (the prefix itself) + if relativePath == "" { + continue + } + + // Check if the filename matches the regex pattern + if !re.MatchString(relativePath) { + continue + } + + filtered = append(filtered, Object{ + Key: relativePath, + LastModified: obj.LastModified, + Size: obj.Size, + }) + } + return filtered, nil +} + func FilterByCommonPrefix(objects []s3types.CommonPrefix) []Object { var filteredObjects []Object diff --git a/internal/clients/s3/filter_test.go b/internal/clients/s3/filter_test.go index ef88d88..1e64a86 100644 --- a/internal/clients/s3/filter_test.go +++ b/internal/clients/s3/filter_test.go @@ -364,3 +364,173 @@ func TestFilterBackupObjects_SizeSummation(t *testing.T) { assert.Equal(t, int64(109206155), sizeMap["sts-backup-20251029-0300.graph"]) // 104857600 + 4348555 assert.Equal(t, int64(111424839), sizeMap["sts-backup-20251029-0924.graph"]) // 104857600 + 6567239 } + +// TestFilterByPrefixAndRegex tests the combined filtering by prefix and regex pattern +func TestFilterByPrefixAndRegex(t *testing.T) { + now := time.Now() + + tests := []struct { + name string + objects []Object + prefix string + pattern string + expectedKeys []string + expectError bool + }{ + { + name: "filters stackgraph backups with prefix and .graph extension", + objects: []Object{ + {Key: "backups/sts-backup-20240101.graph", Size: 1000, LastModified: now}, + {Key: "backups/sts-backup-20240102.graph", Size: 2000, LastModified: now}, + {Key: "backups/other-file.txt", Size: 500, LastModified: now}, + {Key: "backups/sts-backup-20240103.tar.gz", Size: 3000, LastModified: now}, + }, + prefix: "backups/", + pattern: `^sts-backup-.*\.graph$`, + expectedKeys: []string{"sts-backup-20240101.graph", "sts-backup-20240102.graph"}, + expectError: false, + }, + { + name: "filters settings backups with .sty extension", + objects: []Object{ + {Key: "settings/sts-backup-20240101.sty", Size: 1000, LastModified: now}, + {Key: "settings/sts-backup-20240102.sty", Size: 2000, LastModified: now}, + {Key: "settings/other-file.txt", Size: 500, LastModified: now}, + {Key: "settings/sts-backup-20240103.graph", Size: 3000, LastModified: now}, + }, + prefix: "settings/", + pattern: `^sts-backup-.*\.sty$`, + expectedKeys: []string{"sts-backup-20240101.sty", "sts-backup-20240102.sty"}, + expectError: false, + }, + { + name: "excludes nested files even if they match pattern", + objects: []Object{ + {Key: "backups/sts-backup-20240101.graph", Size: 1000, LastModified: now}, + {Key: "backups/old/sts-backup-20240102.graph", Size: 2000, LastModified: now}, + {Key: "backups/archive/2023/sts-backup-20230101.graph", Size: 3000, LastModified: now}, + }, + prefix: "backups/", + pattern: `^sts-backup-.*\.graph$`, + expectedKeys: []string{"sts-backup-20240101.graph"}, + expectError: false, + }, + { + name: "works with empty prefix", + objects: []Object{ + {Key: "sts-backup-20240101.graph", Size: 1000, LastModified: now}, + {Key: "sts-backup-20240102.graph", Size: 2000, LastModified: now}, + {Key: "subdir/sts-backup-20240103.graph", Size: 3000, LastModified: now}, + {Key: "other-file.txt", Size: 500, LastModified: now}, + }, + prefix: "", + pattern: `^sts-backup-.*\.graph$`, + expectedKeys: []string{"sts-backup-20240101.graph", "sts-backup-20240102.graph"}, + expectError: false, + }, + { + name: "returns empty slice when no matches", + objects: []Object{ + {Key: "backups/other-file.txt", Size: 500, LastModified: now}, + {Key: "backups/another-file.log", Size: 100, LastModified: now}, + }, + prefix: "backups/", + pattern: `^sts-backup-.*\.graph$`, + expectedKeys: []string{}, + expectError: false, + }, + { + name: "handles empty object list", + objects: []Object{}, + prefix: "backups/", + pattern: `^sts-backup-.*\.graph$`, + expectedKeys: []string{}, + expectError: false, + }, + { + name: "returns error for invalid regex", + objects: []Object{ + {Key: "backups/sts-backup-20240101.graph", Size: 1000, LastModified: now}, + }, + prefix: "backups/", + pattern: `[invalid`, + expectedKeys: nil, + expectError: true, + }, + { + name: "excludes the prefix directory itself", + objects: []Object{ + {Key: "backups/", Size: 0, LastModified: now}, + {Key: "backups/sts-backup-20240101.graph", Size: 1000, LastModified: now}, + }, + prefix: "backups/", + pattern: `^sts-backup-.*\.graph$`, + expectedKeys: []string{"sts-backup-20240101.graph"}, + expectError: false, + }, + { + name: "returns empty when all files are nested", + objects: []Object{ + {Key: "backups/old/sts-backup-20240101.graph", Size: 1000, LastModified: now}, + {Key: "backups/archive/sts-backup-20240102.graph", Size: 2000, LastModified: now}, + }, + prefix: "backups/", + pattern: `^sts-backup-.*\.graph$`, + expectedKeys: []string{}, + expectError: false, + }, + { + name: "filters with complex regex pattern", + objects: []Object{ + {Key: "backups/sts-backup-20240101-1200.graph", Size: 1000, LastModified: now}, + {Key: "backups/sts-backup-20240102-1300.graph", Size: 2000, LastModified: now}, + {Key: "backups/sts-backup-invalid.graph", Size: 500, LastModified: now}, + {Key: "backups/sts-backup-20240103.graph", Size: 3000, LastModified: now}, + }, + prefix: "backups/", + pattern: `^sts-backup-\d{8}-\d{4}\.graph$`, + expectedKeys: []string{"sts-backup-20240101-1200.graph", "sts-backup-20240102-1300.graph"}, + expectError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := FilterByPrefixAndRegex(tt.objects, tt.prefix, tt.pattern) + + if tt.expectError { + assert.Error(t, err) + assert.Nil(t, result) + return + } + + assert.NoError(t, err) + + resultKeys := make([]string, len(result)) + for i, obj := range result { + resultKeys[i] = obj.Key + } + + assert.Equal(t, tt.expectedKeys, resultKeys) + }) + } +} + +// TestFilterByPrefixAndRegex_PreservesMetadata tests that object metadata is preserved after filtering +func TestFilterByPrefixAndRegex_PreservesMetadata(t *testing.T) { + now := time.Now() + + objects := []Object{ + {Key: "backups/sts-backup-20240101.graph", Size: 1234567890, LastModified: now}, + {Key: "backups/other-file.txt", Size: 500, LastModified: now.Add(-24 * time.Hour)}, + {Key: "backups/nested/sts-backup-20240102.graph", Size: 999, LastModified: now}, + } + + result, err := FilterByPrefixAndRegex(objects, "backups/", `^sts-backup-.*\.graph$`) + + assert.NoError(t, err) + assert.Equal(t, 1, len(result)) + assert.Equal(t, "sts-backup-20240101.graph", result[0].Key) + assert.Equal(t, int64(1234567890), result[0].Size) + assert.Equal(t, now.Unix(), result[0].LastModified.Unix()) +} From b1b53859fa3fa022cc87a4e3eceaea9ce704a292 Mon Sep 17 00:00:00 2001 From: Remco Beckers Date: Fri, 6 Feb 2026 17:04:59 +0100 Subject: [PATCH 4/7] STAC-23457 Configuration for stackpacks restore --- cmd/settings/restore.go | 36 ++++++++++++++++++++++--- cmd/stackgraph/restore.go | 40 +++++++++++++++++++++++++--- internal/foundation/config/config.go | 30 ++++++++++++--------- 3 files changed, 86 insertions(+), 20 deletions(-) diff --git a/cmd/settings/restore.go b/cmd/settings/restore.go index 41e24ad..a27a4cb 100644 --- a/cmd/settings/restore.go +++ b/cmd/settings/restore.go @@ -215,17 +215,26 @@ func buildEnvVar(extraEnvVar []corev1.EnvVar, config *config.Config) []corev1.En // buildVolumeMounts constructs volume mounts for the restore job container func buildVolumeMounts(config *config.Config) []corev1.VolumeMount { - mounts := []corev1.VolumeMount{ + volumeMounts := []corev1.VolumeMount{ {Name: "backup-log", MountPath: "/opt/docker/etc_log"}, + {Name: "config-volume", MountPath: "/opt/docker/etc/application_stackstate.conf", SubPath: "application_stackstate.conf"}, {Name: "backup-restore-scripts", MountPath: "/backup-restore-scripts"}, {Name: "minio-keys", MountPath: "/aws-keys"}, {Name: "tmp-data", MountPath: "/tmp-data"}, } // Mount PVC in legacy mode or when --from-old-pvc is set if config.IsLegacyMode() || fromPVC { - mounts = append(mounts, corev1.VolumeMount{Name: "settings-backup-data", MountPath: "/settings-backup-data"}) + volumeMounts = append(volumeMounts, corev1.VolumeMount{Name: "settings-backup-data", MountPath: "/settings-backup-data"}) } - return mounts + + if config.Settings.Restore.StackpacksPVCName != "" { + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: "stackpacks-local", + MountPath: "/var/stackpacks_local", + }) + } + + return volumeMounts } // buildVolumes constructs volumes for the restore job pod @@ -241,6 +250,16 @@ func buildVolumes(config *config.Config, defaultMode int32) []corev1.Volume { }, }, }, + { + Name: "config-volume", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: config.Settings.Restore.StsBackupConfigConfigMapName, + }, + }, + }, + }, { Name: "backup-restore-scripts", VolumeSource: corev1.VolumeSource{ @@ -278,6 +297,17 @@ func buildVolumes(config *config.Config, defaultMode int32) []corev1.Volume { }, }) } + if config.Settings.Restore.StackpacksPVCName != "" { + volumes = append(volumes, corev1.Volume{ + Name: "stackpacks-local", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: config.Settings.Restore.StackpacksPVCName, + }, + }, + }) + } + return volumes } diff --git a/cmd/stackgraph/restore.go b/cmd/stackgraph/restore.go index 1bb8688..cd0569f 100644 --- a/cmd/stackgraph/restore.go +++ b/cmd/stackgraph/restore.go @@ -279,13 +279,23 @@ func buildRestoreEnvVars(backupFile string, config *config.Config) []corev1.EnvV } // buildRestoreVolumeMounts constructs volume mounts for the restore job container -func buildRestoreVolumeMounts() []corev1.VolumeMount { - return []corev1.VolumeMount{ +func buildRestoreVolumeMounts(config *config.Config) []corev1.VolumeMount { + volumeMounts := []corev1.VolumeMount{ {Name: "backup-log", MountPath: "/opt/docker/etc_log"}, + {Name: "config-volume", MountPath: "/opt/docker/etc/application_stackstate.conf", SubPath: "application_stackstate.conf"}, {Name: "backup-restore-scripts", MountPath: "/backup-restore-scripts"}, {Name: "minio-keys", MountPath: "/aws-keys"}, {Name: "tmp-data", MountPath: "/tmp-data"}, } + + if config.Settings.Restore.StackpacksPVCName != "" { + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: "stackpacks-local", + MountPath: "/var/stackpacks_local", + }) + } + + return volumeMounts } // buildRestoreInitContainers constructs init containers for the restore job @@ -308,7 +318,7 @@ func buildRestoreInitContainers(config *config.Config) []corev1.Container { // buildRestoreVolumes constructs volumes for the restore job pod func buildRestoreVolumes(jobName string, config *config.Config, defaultMode int32) []corev1.Volume { - return []corev1.Volume{ + volumes := []corev1.Volume{ { Name: "backup-log", VolumeSource: corev1.VolumeSource{ @@ -319,6 +329,16 @@ func buildRestoreVolumes(jobName string, config *config.Config, defaultMode int3 }, }, }, + { + Name: "config-volume", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: config.Stackgraph.Restore.StsBackupConfigConfigMapName, + }, + }, + }, + }, { Name: "backup-restore-scripts", VolumeSource: corev1.VolumeSource{ @@ -347,6 +367,18 @@ func buildRestoreVolumes(jobName string, config *config.Config, defaultMode int3 }, }, } + if config.Settings.Restore.StackpacksPVCName != "" { + volumes = append(volumes, corev1.Volume{ + Name: "stackpacks-local", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: config.Settings.Restore.StackpacksPVCName, + }, + }, + }) + } + + return volumes } // buildRestoreContainers constructs containers for the restore job @@ -360,7 +392,7 @@ func buildRestoreContainers(backupFile string, config *config.Config) []corev1.C Command: []string{"/backup-restore-scripts/restore-stackgraph-backup.sh"}, Env: buildRestoreEnvVars(backupFile, config), Resources: k8s.ConvertResources(config.Stackgraph.Restore.Job.Resources), - VolumeMounts: buildRestoreVolumeMounts(), + VolumeMounts: buildRestoreVolumeMounts(config), }, } } diff --git a/internal/foundation/config/config.go b/internal/foundation/config/config.go index 2f7c054..1c0c0d9 100644 --- a/internal/foundation/config/config.go +++ b/internal/foundation/config/config.go @@ -162,11 +162,13 @@ type S3Location struct { // StackgraphRestoreConfig holds Stackgraph restore-specific configuration type StackgraphRestoreConfig struct { - ScaleDownLabelSelector string `yaml:"scaleDownLabelSelector" validate:"required"` - LoggingConfigConfigMapName string `yaml:"loggingConfigConfigMap" validate:"required"` - ZookeeperQuorum string `yaml:"zookeeperQuorum" validate:"required"` - Job JobConfig `yaml:"job" validate:"required"` - PVC PVCConfig `yaml:"pvc" validate:"required"` + ScaleDownLabelSelector string `yaml:"scaleDownLabelSelector" validate:"required"` + LoggingConfigConfigMapName string `yaml:"loggingConfigConfigMap" validate:"required"` + StsBackupConfigConfigMapName string `yaml:"stsBackupConfigConfigMap" validate:"required"` + ZookeeperQuorum string `yaml:"zookeeperQuorum" validate:"required"` + Job JobConfig `yaml:"job" validate:"required"` + PVC PVCConfig `yaml:"pvc" validate:"required"` + StackpacksPVCName string `yaml:"stackpacksPvc"` } type SettingsConfig struct { @@ -178,14 +180,16 @@ type SettingsConfig struct { } type SettingsRestoreConfig struct { - ScaleDownLabelSelector string `yaml:"scaleDownLabelSelector" validate:"required"` - LoggingConfigConfigMapName string `yaml:"loggingConfigConfigMap" validate:"required"` - BaseURL string `yaml:"baseUrl" validate:"required"` - ReceiverBaseURL string `yaml:"receiverBaseUrl" validate:"required"` - PlatformVersion string `yaml:"platformVersion" validate:"required"` - ZookeeperQuorum string `yaml:"zookeeperQuorum" validate:"required"` - Job JobConfig `yaml:"job" validate:"required"` - PVC string `yaml:"pvc"` // Required only in legacy mode + ScaleDownLabelSelector string `yaml:"scaleDownLabelSelector" validate:"required"` + LoggingConfigConfigMapName string `yaml:"loggingConfigConfigMap" validate:"required"` + StsBackupConfigConfigMapName string `yaml:"stsBackupConfigConfigMap" validate:"required"` + BaseURL string `yaml:"baseUrl" validate:"required"` + ReceiverBaseURL string `yaml:"receiverBaseUrl" validate:"required"` + PlatformVersion string `yaml:"platformVersion" validate:"required"` + ZookeeperQuorum string `yaml:"zookeeperQuorum" validate:"required"` + Job JobConfig `yaml:"job" validate:"required"` + PVC string `yaml:"pvc"` // Required only in legacy mode + StackpacksPVCName string `yaml:"stackpacksPvc"` } // ClickhouseConfig holds Clickhouse-specific configuration From 75df5339bd64352f37ffc3b88fee05cc22ac410f Mon Sep 17 00:00:00 2001 From: Remco Beckers Date: Mon, 30 Mar 2026 14:31:46 +0200 Subject: [PATCH 5/7] STAC-23457 Update for storing backups on s3proxy --- cmd/elasticsearch/list_test.go | 4 ++ cmd/settings/list.go | 2 +- cmd/settings/restore.go | 21 +--------- cmd/stackgraph/restore.go | 25 ++--------- internal/foundation/config/config.go | 2 - internal/foundation/config/config_test.go | 42 ++++++++++--------- .../config/testdata/validConfigMapConfig.yaml | 2 + .../config/testdata/validConfigMapOnly.yaml | 2 + .../testdata/validStorageConfigMapConfig.yaml | 2 + .../testdata/validStorageConfigMapOnly.yaml | 2 + .../scripts/restore-settings-backup.sh | 30 ++++--------- 11 files changed, 51 insertions(+), 83 deletions(-) diff --git a/cmd/elasticsearch/list_test.go b/cmd/elasticsearch/list_test.go index e66cfb4..36e9a82 100644 --- a/cmd/elasticsearch/list_test.go +++ b/cmd/elasticsearch/list_test.go @@ -68,6 +68,7 @@ stackgraph: restore: scaleDownLabelSelector: "app=stackgraph" loggingConfigConfigMap: logging-config + stsBackupConfigConfigMap: backup-config zookeeperQuorum: "zookeeper:2181" job: image: backup:latest @@ -107,6 +108,7 @@ settings: restore: scaleDownLabelSelector: "app=settings" loggingConfigConfigMap: logging-config + stsBackupConfigConfigMap: backup-config baseUrl: "http://server:7070" receiverBaseUrl: "http://receiver:7077" platformVersion: "5.2.0" @@ -152,6 +154,7 @@ stackgraph: restore: scaleDownLabelSelector: "app=stackgraph" loggingConfigConfigMap: logging-config + stsBackupConfigConfigMap: backup-config zookeeperQuorum: "zookeeper:2181" job: image: backup:latest @@ -192,6 +195,7 @@ settings: restore: scaleDownLabelSelector: "app=settings" loggingConfigConfigMap: logging-config + stsBackupConfigConfigMap: backup-config baseUrl: "http://server:7070" receiverBaseUrl: "http://receiver:7077" platformVersion: "5.2.0" diff --git a/cmd/settings/list.go b/cmd/settings/list.go index fc99fc1..6682b24 100644 --- a/cmd/settings/list.go +++ b/cmd/settings/list.go @@ -238,7 +238,7 @@ func getBackupListFromLocalBucket(appCtx *app.Context) ([]BackupFileInfo, error) return nil, fmt.Errorf("failed to list objects in local bucket: %w", err) } - filteredObjects := s3client.FilterBackupObjects(result.Contents, isMultiPartArchive) + filteredObjects := s3client.FilterMultipartBackupObjects(result.Contents, isMultiPartArchive) var backups []BackupFileInfo for _, obj := range filteredObjects { diff --git a/cmd/settings/restore.go b/cmd/settings/restore.go index a27a4cb..88100a4 100644 --- a/cmd/settings/restore.go +++ b/cmd/settings/restore.go @@ -35,7 +35,8 @@ func restoreCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command { cmd := &cobra.Command{ Use: "restore", Short: "Restore Settings from a backup archive", - Long: `Restore Settings data from a backup archive stored in S3. Automatically also restores Stackpacks backup that was made at the same time, it can be skipped with --skip-stackpacks. Can use --latest or --archive to specify which backup to restore.`, + Long: `Restore Settings data from a backup archive stored in S3. Automatically also restores Stackpacks backup that was made at the same time, +it can be skipped with --skip-stackpacks. Can use --latest or --archive to specify which backup to restore.`, Run: func(_ *cobra.Command, _ []string) { cmdutils.Run(globalFlags, runRestore, cmdutils.StorageIsNotRequired) }, @@ -227,13 +228,6 @@ func buildVolumeMounts(config *config.Config) []corev1.VolumeMount { volumeMounts = append(volumeMounts, corev1.VolumeMount{Name: "settings-backup-data", MountPath: "/settings-backup-data"}) } - if config.Settings.Restore.StackpacksPVCName != "" { - volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: "stackpacks-local", - MountPath: "/var/stackpacks_local", - }) - } - return volumeMounts } @@ -297,17 +291,6 @@ func buildVolumes(config *config.Config, defaultMode int32) []corev1.Volume { }, }) } - if config.Settings.Restore.StackpacksPVCName != "" { - volumes = append(volumes, corev1.Volume{ - Name: "stackpacks-local", - VolumeSource: corev1.VolumeSource{ - PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: config.Settings.Restore.StackpacksPVCName, - }, - }, - }) - } - return volumes } diff --git a/cmd/stackgraph/restore.go b/cmd/stackgraph/restore.go index cd0569f..8e4fad8 100644 --- a/cmd/stackgraph/restore.go +++ b/cmd/stackgraph/restore.go @@ -42,7 +42,8 @@ func restoreCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command { cmd := &cobra.Command{ Use: "restore", Short: "Restore Stackgraph from a backup archive", - Long: `Restore Stackgraph data from a backup archive stored in S3. Automatically also restores Stackpacks backup that was made at the same time, it can be skipped with --skip-stackpacks. Can use --latest or --archive to specify which backup to restore.`, + Long: `Restore Stackgraph data from a backup archive stored in S3. Automatically also restores Stackpacks backup that was made at the same time, +it can be skipped with --skip-stackpacks. Can use --latest or --archive to specify which backup to restore.`, Run: func(_ *cobra.Command, _ []string) { cmdutils.Run(globalFlags, runRestore, cmdutils.StorageIsRequired) }, @@ -279,7 +280,7 @@ func buildRestoreEnvVars(backupFile string, config *config.Config) []corev1.EnvV } // buildRestoreVolumeMounts constructs volume mounts for the restore job container -func buildRestoreVolumeMounts(config *config.Config) []corev1.VolumeMount { +func buildRestoreVolumeMounts() []corev1.VolumeMount { volumeMounts := []corev1.VolumeMount{ {Name: "backup-log", MountPath: "/opt/docker/etc_log"}, {Name: "config-volume", MountPath: "/opt/docker/etc/application_stackstate.conf", SubPath: "application_stackstate.conf"}, @@ -288,13 +289,6 @@ func buildRestoreVolumeMounts(config *config.Config) []corev1.VolumeMount { {Name: "tmp-data", MountPath: "/tmp-data"}, } - if config.Settings.Restore.StackpacksPVCName != "" { - volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: "stackpacks-local", - MountPath: "/var/stackpacks_local", - }) - } - return volumeMounts } @@ -367,17 +361,6 @@ func buildRestoreVolumes(jobName string, config *config.Config, defaultMode int3 }, }, } - if config.Settings.Restore.StackpacksPVCName != "" { - volumes = append(volumes, corev1.Volume{ - Name: "stackpacks-local", - VolumeSource: corev1.VolumeSource{ - PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: config.Settings.Restore.StackpacksPVCName, - }, - }, - }) - } - return volumes } @@ -392,7 +375,7 @@ func buildRestoreContainers(backupFile string, config *config.Config) []corev1.C Command: []string{"/backup-restore-scripts/restore-stackgraph-backup.sh"}, Env: buildRestoreEnvVars(backupFile, config), Resources: k8s.ConvertResources(config.Stackgraph.Restore.Job.Resources), - VolumeMounts: buildRestoreVolumeMounts(config), + VolumeMounts: buildRestoreVolumeMounts(), }, } } diff --git a/internal/foundation/config/config.go b/internal/foundation/config/config.go index 1c0c0d9..20381e4 100644 --- a/internal/foundation/config/config.go +++ b/internal/foundation/config/config.go @@ -168,7 +168,6 @@ type StackgraphRestoreConfig struct { ZookeeperQuorum string `yaml:"zookeeperQuorum" validate:"required"` Job JobConfig `yaml:"job" validate:"required"` PVC PVCConfig `yaml:"pvc" validate:"required"` - StackpacksPVCName string `yaml:"stackpacksPvc"` } type SettingsConfig struct { @@ -189,7 +188,6 @@ type SettingsRestoreConfig struct { ZookeeperQuorum string `yaml:"zookeeperQuorum" validate:"required"` Job JobConfig `yaml:"job" validate:"required"` PVC string `yaml:"pvc"` // Required only in legacy mode - StackpacksPVCName string `yaml:"stackpacksPvc"` } // ClickhouseConfig holds Clickhouse-specific configuration diff --git a/internal/foundation/config/config_test.go b/internal/foundation/config/config_test.go index 2719e87..2b36732 100644 --- a/internal/foundation/config/config_test.go +++ b/internal/foundation/config/config_test.go @@ -625,9 +625,10 @@ func TestConfig_StructValidation(t *testing.T) { S3Prefix: "", MultipartArchive: true, Restore: StackgraphRestoreConfig{ - ScaleDownLabelSelector: "app=stackgraph", - LoggingConfigConfigMapName: "logging-config", - ZookeeperQuorum: "zookeeper:2181", + ScaleDownLabelSelector: "app=stackgraph", + LoggingConfigConfigMapName: "logging-config", + StsBackupConfigConfigMapName: "backup-config", + ZookeeperQuorum: "zookeeper:2181", Job: JobConfig{ Image: "backup:latest", WaitImage: "wait:latest", @@ -682,13 +683,14 @@ func TestConfig_StructValidation(t *testing.T) { Bucket: "settings-backup", S3Prefix: "", Restore: SettingsRestoreConfig{ - ScaleDownLabelSelector: "app=settings", - LoggingConfigConfigMapName: "logging-config", - BaseURL: "http://server:7070", - ReceiverBaseURL: "http://receiver:7077", - PlatformVersion: "5.2.0", - ZookeeperQuorum: "zookeeper:2181", - PVC: "suse-observability-settings-backup-data", + ScaleDownLabelSelector: "app=settings", + LoggingConfigConfigMapName: "logging-config", + StsBackupConfigConfigMapName: "backup-config", + BaseURL: "http://server:7070", + ReceiverBaseURL: "http://receiver:7077", + PlatformVersion: "5.2.0", + ZookeeperQuorum: "zookeeper:2181", + PVC: "suse-observability-settings-backup-data", Job: JobConfig{ Image: "settings-backup:latest", WaitImage: "wait:latest", @@ -772,9 +774,10 @@ func TestConfig_StructValidation(t *testing.T) { S3Prefix: "", MultipartArchive: true, Restore: StackgraphRestoreConfig{ - ScaleDownLabelSelector: "app=stackgraph", - LoggingConfigConfigMapName: "logging-config", - ZookeeperQuorum: "zookeeper:2181", + ScaleDownLabelSelector: "app=stackgraph", + LoggingConfigConfigMapName: "logging-config", + StsBackupConfigConfigMapName: "backup-config", + ZookeeperQuorum: "zookeeper:2181", Job: JobConfig{ Image: "backup:latest", WaitImage: "wait:latest", @@ -829,12 +832,13 @@ func TestConfig_StructValidation(t *testing.T) { Bucket: "settings-backup", S3Prefix: "", Restore: SettingsRestoreConfig{ - ScaleDownLabelSelector: "app=settings", - LoggingConfigConfigMapName: "logging-config", - BaseURL: "http://server:7070", - ReceiverBaseURL: "http://receiver:7077", - PlatformVersion: "5.2.0", - ZookeeperQuorum: "zookeeper:2181", + ScaleDownLabelSelector: "app=settings", + LoggingConfigConfigMapName: "logging-config", + StsBackupConfigConfigMapName: "backup-config", + BaseURL: "http://server:7070", + ReceiverBaseURL: "http://receiver:7077", + PlatformVersion: "5.2.0", + ZookeeperQuorum: "zookeeper:2181", Job: JobConfig{ Image: "settings-backup:latest", WaitImage: "wait:latest", diff --git a/internal/foundation/config/testdata/validConfigMapConfig.yaml b/internal/foundation/config/testdata/validConfigMapConfig.yaml index e36900e..8477294 100644 --- a/internal/foundation/config/testdata/validConfigMapConfig.yaml +++ b/internal/foundation/config/testdata/validConfigMapConfig.yaml @@ -86,6 +86,7 @@ stackgraph: scaleDownLabelSelector: "observability.suse.com/scalable-during-stackgraph-restore=true" # ConfigMap containing logging configuration loggingConfigConfigMap: suse-observability-logging + stsBackupConfigConfigMap: suse-observability-backup-config # Zookeeper quorum connection string zookeeperQuorum: "suse-observability-zookeeper:2181" # Job configuration @@ -145,6 +146,7 @@ settings: restore: scaleDownLabelSelector: "observability.suse.com/scalable-during-settings-restore=true" loggingConfigConfigMap: suse-observability-logging + stsBackupConfigConfigMap: suse-observability-backup-config baseUrl: "http://suse-observability-server:7070" receiverBaseUrl: "http://suse-observability-receiver:7077" platformVersion: "5.2.0" diff --git a/internal/foundation/config/testdata/validConfigMapOnly.yaml b/internal/foundation/config/testdata/validConfigMapOnly.yaml index dbf8ea1..40bb36a 100644 --- a/internal/foundation/config/testdata/validConfigMapOnly.yaml +++ b/internal/foundation/config/testdata/validConfigMapOnly.yaml @@ -83,6 +83,7 @@ stackgraph: restore: scaleDownLabelSelector: "observability.suse.com/scalable-during-stackgraph-restore=true" loggingConfigConfigMap: suse-observability-logging + stsBackupConfigConfigMap: suse-observability-backup-config zookeeperQuorum: "suse-observability-zookeeper:2181" job: labels: @@ -132,6 +133,7 @@ settings: restore: scaleDownLabelSelector: "observability.suse.com/scalable-during-settings-restore=true" loggingConfigConfigMap: suse-observability-logging + stsBackupConfigConfigMap: suse-observability-backup-config baseUrl: "http://suse-observability-server:7070" receiverBaseUrl: "http://suse-observability-receiver:7077" platformVersion: "5.2.0" diff --git a/internal/foundation/config/testdata/validStorageConfigMapConfig.yaml b/internal/foundation/config/testdata/validStorageConfigMapConfig.yaml index 5b928f8..e962362 100644 --- a/internal/foundation/config/testdata/validStorageConfigMapConfig.yaml +++ b/internal/foundation/config/testdata/validStorageConfigMapConfig.yaml @@ -87,6 +87,7 @@ stackgraph: scaleDownLabelSelector: "observability.suse.com/scalable-during-stackgraph-restore=true" # ConfigMap containing logging configuration loggingConfigConfigMap: suse-observability-logging + stsBackupConfigConfigMap: suse-observability-backup-config # Zookeeper quorum connection string zookeeperQuorum: "suse-observability-zookeeper:2181" # Job configuration @@ -147,6 +148,7 @@ settings: restore: scaleDownLabelSelector: "observability.suse.com/scalable-during-settings-restore=true" loggingConfigConfigMap: suse-observability-logging + stsBackupConfigConfigMap: suse-observability-backup-config baseUrl: "http://suse-observability-server:7070" receiverBaseUrl: "http://suse-observability-receiver:7077" platformVersion: "5.2.0" diff --git a/internal/foundation/config/testdata/validStorageConfigMapOnly.yaml b/internal/foundation/config/testdata/validStorageConfigMapOnly.yaml index 9498e59..891d7af 100644 --- a/internal/foundation/config/testdata/validStorageConfigMapOnly.yaml +++ b/internal/foundation/config/testdata/validStorageConfigMapOnly.yaml @@ -86,6 +86,7 @@ stackgraph: restore: scaleDownLabelSelector: "observability.suse.com/scalable-during-stackgraph-restore=true" loggingConfigConfigMap: suse-observability-logging + stsBackupConfigConfigMap: suse-observability-backup-config zookeeperQuorum: "suse-observability-zookeeper:2181" job: labels: @@ -137,6 +138,7 @@ settings: restore: scaleDownLabelSelector: "observability.suse.com/scalable-during-settings-restore=true" loggingConfigConfigMap: suse-observability-logging + stsBackupConfigConfigMap: suse-observability-backup-config baseUrl: "http://suse-observability-server:7070" receiverBaseUrl: "http://suse-observability-receiver:7077" platformVersion: "5.2.0" diff --git a/internal/scripts/scripts/restore-settings-backup.sh b/internal/scripts/scripts/restore-settings-backup.sh index 2309054..6e85322 100644 --- a/internal/scripts/scripts/restore-settings-backup.sh +++ b/internal/scripts/scripts/restore-settings-backup.sh @@ -58,39 +58,27 @@ echo "=== Restoring settings backup from \"${BACKUP_FILE}\"..." echo "=== Settings restore complete" # === StackPacks Restore === +# StackPacks backups are always stored in S3, next to the settings backup. if [ "${SKIP_STACKPACKS:-false}" == "true" ]; then echo "=== Skipping StackPacks restore (--skip-stackpacks flag set)" else - export STACKPACKS_BACKUP_DIR="${BACKUP_DIR}/stackpacks" - # Construct stackpacks backup filename from the original backup file STACKPACKS_FILE="${BACKUP_FILE}.stackpacks.zip" - STACKPACKS_RESTORE_FILE="${STACKPACKS_BACKUP_DIR}/${STACKPACKS_FILE}" - echo "=== Checking for StackPacks backup \"${STACKPACKS_FILE}\"..." + echo "=== Checking for StackPacks backup \"${STACKPACKS_FILE}\" in bucket \"${BACKUP_CONFIGURATION_BUCKET_NAME}\"..." - # Check local PVC first, then try S3 if not found and remote is enabled - if [ ! -f "${STACKPACKS_RESTORE_FILE}" ] && [ "$BACKUP_CONFIGURATION_UPLOAD_REMOTE" == "true" ]; then - # Ensure AWS credentials are set for S3 access - export AWS_ACCESS_KEY_ID - AWS_ACCESS_KEY_ID="$(cat /aws-keys/accesskey)" - export AWS_SECRET_ACCESS_KEY - AWS_SECRET_ACCESS_KEY="$(cat /aws-keys/secretkey)" + setup_aws_credentials - # Check if file exists in S3 - if sts-toolbox aws s3 ls --endpoint "http://${MINIO_ENDPOINT}" --region minio --bucket "${BACKUP_CONFIGURATION_BUCKET_NAME}" --prefix "${BACKUP_CONFIGURATION_STACKPACKS_S3_PREFIX}${STACKPACKS_FILE}" 2>/dev/null | grep -q "${STACKPACKS_FILE}"; then - echo "=== Downloading StackPacks backup from S3..." - sts-toolbox aws s3 cp --endpoint "http://${MINIO_ENDPOINT}" --region minio "s3://${BACKUP_CONFIGURATION_BUCKET_NAME}/${BACKUP_CONFIGURATION_STACKPACKS_S3_PREFIX}${STACKPACKS_FILE}" "${TMP_DIR}/${STACKPACKS_FILE}" - STACKPACKS_RESTORE_FILE="${TMP_DIR}/${STACKPACKS_FILE}" - fi - fi + # Check if stackpacks backup exists in S3 + if sts-toolbox aws s3 ls --endpoint "http://${MINIO_ENDPOINT}" --region minio --bucket "${BACKUP_CONFIGURATION_BUCKET_NAME}" --prefix "${BACKUP_CONFIGURATION_STACKPACKS_S3_PREFIX}${STACKPACKS_FILE}" 2>/dev/null | grep -q "${STACKPACKS_FILE}"; then + echo "=== Downloading StackPacks backup from S3..." + sts-toolbox aws s3 cp --endpoint "http://${MINIO_ENDPOINT}" --region minio "s3://${BACKUP_CONFIGURATION_BUCKET_NAME}/${BACKUP_CONFIGURATION_STACKPACKS_S3_PREFIX}${STACKPACKS_FILE}" "${TMP_DIR}/${STACKPACKS_FILE}" - if [ -f "${STACKPACKS_RESTORE_FILE}" ]; then echo "=== Restoring StackPacks from \"${STACKPACKS_FILE}\"..." - /opt/docker/bin/stack-packs-backup -Dlogback.configurationFile=/opt/docker/etc_log/logback.xml -restore "${STACKPACKS_RESTORE_FILE}" + /opt/docker/bin/stack-packs-backup -Dlogback.configurationFile=/opt/docker/etc_log/logback.xml -restore "${TMP_DIR}/${STACKPACKS_FILE}" echo "=== StackPacks restore complete" else - echo "=== WARNING: StackPacks backup \"${STACKPACKS_FILE}\" not found, skipping StackPacks restore" + echo "=== WARNING: StackPacks backup \"${STACKPACKS_FILE}\" not found in S3, skipping StackPacks restore" fi fi echo "===" From 5ee7e3aa9a412c144c5f8fcf88fc723d7a7c698d Mon Sep 17 00:00:00 2001 From: Remco Beckers Date: Wed, 1 Apr 2026 08:57:14 +0200 Subject: [PATCH 6/7] Fix linter warning --- internal/clients/s3/filter_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/clients/s3/filter_test.go b/internal/clients/s3/filter_test.go index 1e64a86..b7d5d7d 100644 --- a/internal/clients/s3/filter_test.go +++ b/internal/clients/s3/filter_test.go @@ -366,7 +366,7 @@ func TestFilterBackupObjects_SizeSummation(t *testing.T) { } // TestFilterByPrefixAndRegex tests the combined filtering by prefix and regex pattern -func TestFilterByPrefixAndRegex(t *testing.T) { +func TestFilterByPrefixAndRegex(t *testing.T) { //nolint:funlen // Table-driven test now := time.Now() tests := []struct { From 233aa73d33e656a752b1faea192c69ab477f2c9a Mon Sep 17 00:00:00 2001 From: Remco Beckers Date: Thu, 2 Apr 2026 15:13:17 +0200 Subject: [PATCH 7/7] STAC-23457 Filter out stackpacks from backup listing --- cmd/settings/list.go | 5 +++++ internal/clients/s3/filter_test.go | 13 +++++++++++++ 2 files changed, 18 insertions(+) diff --git a/cmd/settings/list.go b/cmd/settings/list.go index 6682b24..da8c341 100644 --- a/cmd/settings/list.go +++ b/cmd/settings/list.go @@ -240,6 +240,11 @@ func getBackupListFromLocalBucket(appCtx *app.Context) ([]BackupFileInfo, error) filteredObjects := s3client.FilterMultipartBackupObjects(result.Contents, isMultiPartArchive) + filteredObjects, err = s3client.FilterByPrefixAndRegex(filteredObjects, "", backupFileNameRegex) + if err != nil { + return nil, fmt.Errorf("failed to filter objects: %w", err) + } + var backups []BackupFileInfo for _, obj := range filteredObjects { backups = append(backups, BackupFileInfo{ diff --git a/internal/clients/s3/filter_test.go b/internal/clients/s3/filter_test.go index b7d5d7d..7880c3a 100644 --- a/internal/clients/s3/filter_test.go +++ b/internal/clients/s3/filter_test.go @@ -479,6 +479,19 @@ func TestFilterByPrefixAndRegex(t *testing.T) { //nolint:funlen // Table-driven expectedKeys: []string{}, expectError: false, }, + { + name: "excludes stackpacks backups when listing settings local bucket (empty prefix)", + objects: []Object{ + {Key: "sts-backup-20240101.sty", Size: 1000, LastModified: now}, + {Key: "sts-backup-20240101.sty.stackpacks.zip", Size: 500, LastModified: now}, + {Key: "sts-backup-20240102.sty", Size: 2000, LastModified: now}, + {Key: "sts-backup-20240102.sty.stackpacks.zip", Size: 300, LastModified: now}, + }, + prefix: "", + pattern: `^sts-backup-.*\.sty$`, + expectedKeys: []string{"sts-backup-20240101.sty", "sts-backup-20240102.sty"}, + expectError: false, + }, { name: "filters with complex regex pattern", objects: []Object{