diff --git a/COMMANDS.md b/COMMANDS.md index b6797ac..a0cb552 100644 --- a/COMMANDS.md +++ b/COMMANDS.md @@ -270,8 +270,8 @@ Display information about backups. By default, only active backups or backups with deletion status "In progress" from gpbackup_history.db are displayed. -To additional display deleted backups, use the --deleted option. -To additional display failed backups, use the --failed option. +To display deleted backups, use the --deleted option. +To display failed backups, use the --failed option. To display all backups, use --deleted and --failed options together. To display backups of a specific type, use the --type option. @@ -286,7 +286,21 @@ To display backups that exclude the specified table, use the --table and --exclu The formatting rules for . match those of the --exclude-table option in gpbackup. To display backups that exclude the specified schema, use the --schema and --exclude options. -The formatting rules for match those of the --exclude-schema option in gpbackup. +The formatting rules for match those of the --exclude-schema option in gpbackup. + +To display details about object filtering, use the --detail option. +The details are presented as follows, depending on the active filtering type: + * include-table / exclude-table: a comma-separated list of fully-qualified table names in the format .
; + * include-schema / exclude-schema: a comma-separated list of schema names; + * if no object filtering was used, the value is empty. + +To display a backup chain for a specific backup, use the --timestamp option. +In this mode, the backup with the specified timestamp and all of its dependent backups will be displayed. +The deleted and failed backups are always included in this mode. +The information about object filtering details is always included in this mode. +When --timestamp is set, the following options cannot be used: --type, --table, --schema, --exclude, --failed, --deleted, --detail. + +To display the "object filtering details" column for all backups without using --timestamp, use the --detail option. The gpbackup_history.db file location can be set using the --history-db option. Can be specified only once. The full path to the file is required. @@ -296,13 +310,15 @@ Usage: gpbackman backup-info [flags] Flags: - --deleted show deleted backups - --exclude show backups that exclude the specific table (format .
) or schema - --failed show failed backups - -h, --help help for backup-info - --schema string show backups that include the specified schema - --table string show backups that include the specified table (format .
) - --type string backup type filter (full, incremental, data-only, metadata-only) + --deleted show deleted backups + --detail show object filtering details + --exclude show backups that exclude the specific table (format .
) or schema + --failed show failed backups + -h, --help help for backup-info + --schema string show backups that include the specified schema + --table string show backups that include the specified table (format .
) + --timestamp string show backup info and its dependent backups for the specified timestamp + --type string backup type filter (full, incremental, data-only, metadata-only) Global Flags: --history-db string full path to the gpbackup_history.db file @@ -338,6 +354,12 @@ The following information is provided about each backup: - `""` - if backup is active; - date in format `Mon Jan 02 2006 15:04:05` - if backup is deleted and deletion timestamp is set. +If the `--timestamp` option is specified, the following additional information is provided: +* `OBJECT FILTERING DETAILS` - details about object filtering: + - if `include-table` or `exclude-table` filtering was used, a comma-separated list of fully-qualified table names in the format `.
`; + - if `include-schema` or `exclude-schema` filtering was used, a comma-separated list of schema names; + - if no object filtering was used, the value is empty. + If gpbackup is launched without specifying `--metadata-only` flag, but there were no tables that contain data for backup, then gpbackup will only perform a `metadata-only` backup. The logs will contain messages like `No tables in backup set contain data. Performing metadata-only backup instead.` As a result, gpBackMan will display such backups as `metadata-only`. ## Examples @@ -371,7 +393,7 @@ Display info for active full backups from `gpbackup_history.db`: ./gpbackman backup-info \ --type full -TIMESTAMP | DATE | STATUS | DATABASE | TYPE | OBJECT FILTERING | PLUGIN | DURATION | DATE DELETED + TIMESTAMP | DATE | STATUS | DATABASE | TYPE | OBJECT FILTERING | PLUGIN | DURATION | DATE DELETED ----------------+--------------------------+---------+----------+------+------------------+--------------------+----------+-------------- 20230809232817 | Wed Aug 09 2023 23:28:17 | Success | demo | full | | | 04:00:03 | 20230725101115 | Tue Jul 25 2023 10:11:15 | Success | demo | full | | gpbackup_s3_plugin | 00:00:20 | @@ -386,7 +408,7 @@ Find all backups, including deleted ones, containing the `test1` schema. --deleted \ --schema test1 -TIMESTAMP | DATE | STATUS | DATABASE | TYPE | OBJECT FILTERING | PLUGIN | DURATION | DATE DELETED + TIMESTAMP | DATE | STATUS | DATABASE | TYPE | OBJECT FILTERING | PLUGIN | DURATION | DATE DELETED ----------------+--------------------------+---------+----------+-------------+------------------+--------------------+----------+-------------------------- 20230525101152 | Thu May 25 2023 10:11:52 | Success | demo | incremental | include-schema | gpbackup_s3_plugin | 00:30:00 | Sun Jun 25 2023 10:11:52 20230524101152 | Wed May 24 2023 10:11:52 | Success | demo | incremental | include-schema | gpbackup_s3_plugin | 00:30:00 | @@ -400,7 +422,7 @@ Display info for all backups, including deleted and failed ones, from `gpbackup_ --failed \ --history-db /data/master/gpseg-1/gpbackup_history.db -TIMESTAMP | DATE | STATUS | DATABASE | TYPE | OBJECT FILTERING | PLUGIN | DURATION | DATE DELETED + TIMESTAMP | DATE | STATUS | DATABASE | TYPE | OBJECT FILTERING | PLUGIN | DURATION | DATE DELETED ----------------+--------------------------+---------+----------+---------------+------------------+--------------------+----------+----------------------------- 20230809232817 | Wed Aug 09 2023 23:28:17 | Success | demo | full | | | 04:00:03 | 20230806230400 | Sun Aug 06 2023 23:04:00 | Failure | demo | full | | gpbackup_s3_plugin | 00:00:38 | @@ -425,6 +447,39 @@ TIMESTAMP | DATE | STATUS | DATABASE | TYPE | 20230523101115 | Tue May 23 2023 10:11:15 | Success | demo | full | include-schema | gpbackup_s3_plugin | 01:01:00 | ``` +Display full backup with object filtering details: +```bash +./gpbackman backup-info \ + --type full \ + --detail + + TIMESTAMP | DATE | STATUS | DATABASE | TYPE | OBJECT FILTERING | PLUGIN | DURATION | DATE DELETED | OBJECT FILTERING DETAILS +----------------+--------------------------+---------+----------+------+------------------+--------------------+----------+--------------+-------------------------- + 20250915221743 | Mon Sep 15 2025 22:17:43 | Success | demo | full | | | 00:00:01 | | + 20250915221643 | Mon Sep 15 2025 22:16:43 | Success | demo | full | exclude-schema | gpbackup_s3_plugin | 00:00:01 | | sch1 + 20250915221631 | Mon Sep 15 2025 22:16:31 | Success | demo | full | include-table | gpbackup_s3_plugin | 00:00:01 | | sch2.tbl_c, sch2.tbl_d + 20250915221616 | Mon Sep 15 2025 22:16:16 | Success | demo | full | | gpbackup_s3_plugin | 00:00:05 | | + 20250915221553 | Mon Sep 15 2025 22:15:53 | Success | demo | full | exclude-table | | 00:00:02 | | sch1.tbl_b + 20250915221542 | Mon Sep 15 2025 22:15:42 | Success | demo | full | include-table | | 00:00:01 | | sch1.tbl_a + 20250915221531 | Mon Sep 15 2025 22:15:31 | Success | demo | full | | | 00:00:01 | | + +``` + +Display info for the backup chain for a specific backup. In this example, the backup with timestamp `20250913210921` is a full backup, and all its dependent incremental backups are displayed as well: +```bash +./gpbackman backup-info \ + --timestamp 20250913210921 + + TIMESTAMP | DATE | STATUS | DATABASE | TYPE | OBJECT FILTERING | PLUGIN | DURATION | DATE DELETED | OBJECT FILTERING DETAILS +----------------+--------------------------+---------+----------+-------------+------------------+--------------------+----------+--------------------------+-------------------------- + 20250915201446 | Mon Sep 15 2025 20:14:46 | Success | demo | incremental | include-table | gpbackup_s3_plugin | 00:00:02 | | sch2.tbl_c + 20250915201439 | Mon Sep 15 2025 20:14:39 | Success | demo | incremental | include-table | gpbackup_s3_plugin | 00:00:01 | | sch2.tbl_c + 20250915201307 | Mon Sep 15 2025 20:13:07 | Success | demo | incremental | include-table | gpbackup_s3_plugin | 00:00:02 | Mon Sep 15 2025 20:17:56 | sch2.tbl_c + 20250915200929 | Mon Sep 15 2025 20:09:29 | Success | demo | incremental | include-table | gpbackup_s3_plugin | 00:00:01 | | sch2.tbl_c + 20250913210957 | Sat Sep 13 2025 21:09:57 | Success | demo | incremental | include-table | gpbackup_s3_plugin | 00:00:01 | | sch2.tbl_c + 20250913210921 | Sat Sep 13 2025 21:09:21 | Success | demo | full | include-table | gpbackup_s3_plugin | 00:00:02 | | sch2.tbl_c +``` + ## Using container ```bash diff --git a/cmd/backup_info.go b/cmd/backup_info.go index f28df60..7e7ab27 100644 --- a/cmd/backup_info.go +++ b/cmd/backup_info.go @@ -20,8 +20,22 @@ var ( backupInfoTableNameFilter string backupInfoSchemaNameFilter string backupInfoExcludeFilter bool + backupInfoTimestamp string + backupInfoShowDetails bool ) +// Options for the backup-info command. +type BackupInfoOptions struct { + ShowDeleted bool + ShowFailed bool + BackupTypeFilter string + TableNameFilter string + SchemaNameFilter string + ExcludeFilter bool + Timestamp string + ShowDetails bool +} + var backupInfoCmd = &cobra.Command{ Use: "backup-info", Short: "Display information about backups", @@ -29,8 +43,8 @@ var backupInfoCmd = &cobra.Command{ By default, only active backups or backups with deletion status "In progress" from gpbackup_history.db are displayed. -To additional display deleted backups, use the --deleted option. -To additional display failed backups, use the --failed option. +To display deleted backups, use the --deleted option. +To display failed backups, use the --failed option. To display all backups, use --deleted and --failed options together. To display backups of a specific type, use the --type option. @@ -45,7 +59,21 @@ To display backups that exclude the specified table, use the --table and --exclu The formatting rules for .
match those of the --exclude-table option in gpbackup. To display backups that exclude the specified schema, use the --schema and --exclude options. -The formatting rules for match those of the --exclude-schema option in gpbackup. +The formatting rules for match those of the --exclude-schema option in gpbackup. + +To display details about object filtering, use the --detail option. +The details are presented as follows, depending on the active filtering type: + * include-table / exclude-table: a comma-separated list of fully-qualified table names in the format .
; + * include-schema / exclude-schema: a comma-separated list of schema names; + * if no object filtering was used, the value is empty. + +To display a backup chain for a specific backup, use the --timestamp option. +In this mode, the backup with the specified timestamp and all of its dependent backups will be displayed. +The deleted and failed backups are always included in this mode. +The information about object filtering details is always included in this mode. +When --timestamp is set, the following options cannot be used: --type, --table, --schema, --exclude, --failed, --deleted, --detail. + +To display the "object filtering details" column for all backups without using --timestamp, use the --detail option. The gpbackup_history.db file location can be set using the --history-db option. Can be specified only once. The full path to the file is required. @@ -60,6 +88,12 @@ If the --history-db option is not specified, the history database will be search func init() { rootCmd.AddCommand(backupInfoCmd) + backupInfoCmd.Flags().StringVar( + &backupInfoTimestamp, + timestampFlagName, + "", + "show backup info and its dependent backups for the specified timestamp", + ) backupInfoCmd.Flags().BoolVar( &backupInfoShowDeleted, deletedFlagName, @@ -96,11 +130,31 @@ func init() { false, "show backups that exclude the specific table (format .
) or schema", ) + backupInfoCmd.Flags().BoolVar( + &backupInfoShowDetails, + detailFlagName, + false, + "show object filtering details", + ) } // These flag checks are applied only for backup-info commands. func doBackupInfoFlagValidation(flags *pflag.FlagSet) { var err error + if flags.Changed(timestampFlagName) { + err = gpbckpconfig.CheckTimestamp(backupInfoTimestamp) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableValidateFlag(backupInfoTimestamp, timestampFlagName, err)) + execOSExit(exitErrorCode) + } + // --timestamp is not compatible with --type, --table, --schema, --exclude, --failed, --deleted, --detail + err = checkCompatibleFlags(flags, timestampFlagName, + typeFlagName, tableFlagName, schemaFlagName, excludeFlagName, failedFlagName, deletedFlagName, detailFlagName) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableCompatibleFlags(err, timestampFlagName, typeFlagName, tableFlagName, schemaFlagName, excludeFlagName, failedFlagName, deletedFlagName, detailFlagName)) + execOSExit(exitErrorCode) + } + } // If type is specified and have correct values. if flags.Changed(typeFlagName) { err = checkBackupType(backupInfoBackupTypeFilter) @@ -140,7 +194,18 @@ func doBackupInfo() { func backupInfo() error { t := table.NewWriter() - initTable(t) + opts := BackupInfoOptions{ + ShowDeleted: backupInfoShowDeleted, + ShowFailed: backupInfoShowFailed, + BackupTypeFilter: backupInfoBackupTypeFilter, + TableNameFilter: backupInfoTableNameFilter, + SchemaNameFilter: backupInfoSchemaNameFilter, + ExcludeFilter: backupInfoExcludeFilter, + Timestamp: backupInfoTimestamp, + ShowDetails: backupInfoShowDetails, + } + includeDetails := opts.Timestamp != "" || opts.ShowDetails + initTable(t, includeDetails) hDB, err := gpbckpconfig.OpenHistoryDB(getHistoryDBPath(rootHistoryDB)) if err != nil { gplog.Error("%s", textmsg.ErrorTextUnableActionHistoryDB("open", err)) @@ -152,15 +217,7 @@ func backupInfo() error { gplog.Error("%s", textmsg.ErrorTextUnableActionHistoryDB("close", closeErr)) } }() - err = backupInfoDB( - backupInfoShowDeleted, - backupInfoShowFailed, - backupInfoExcludeFilter, - backupInfoBackupTypeFilter, - backupInfoTableNameFilter, - backupInfoSchemaNameFilter, - hDB, - t) + err = backupInfoDB(opts, hDB, t) if err != nil { return err } @@ -168,28 +225,57 @@ func backupInfo() error { return nil } -func backupInfoDB(showDeleted, showFailed, backupExcludeFilter bool, backupTypeFilter, backupTableFilter, backupSchemaFilter string, hDB *sql.DB, t table.Writer) error { - backupList, err := gpbckpconfig.GetBackupNamesDB(showDeleted, showFailed, hDB) +func backupInfoDB(opts BackupInfoOptions, hDB *sql.DB, t table.Writer) error { + // List all according to showDeleted/showFailed + if opts.Timestamp == "" { + backupList, err := gpbckpconfig.GetBackupNamesDB(opts.ShowDeleted, opts.ShowFailed, hDB) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableReadHistoryDB(err)) + return err + } + for _, backupName := range backupList { + backupData, err := gpbckpconfig.GetBackupDataDB(backupName, hDB) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableGetBackupInfo(backupName, err)) + return err + } + // In legacy mode (no timestamp specified), include details if the --detail flag is set + includeObjectFilteringDetails := opts.ShowDetails + addBackupToTable(opts.BackupTypeFilter, opts.TableNameFilter, opts.SchemaNameFilter, opts.ExcludeFilter, includeObjectFilteringDetails, backupData, t) + } + return nil + } + // Timestamp mode: show base backup and only its dependent backups + // Verify base backup exists + baseBackupData, err := gpbckpconfig.GetBackupDataDB(opts.Timestamp, hDB) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableGetBackupInfo(opts.Timestamp, err)) + return err + } + // In timestamp mode, include the extra details column to match the header + includeObjectFilteringDetails := true + addBackupToTable("", "", "", false, includeObjectFilteringDetails, baseBackupData, t) + backupDependenciesList, err := gpbckpconfig.GetBackupDependencies(opts.Timestamp, hDB) if err != nil { gplog.Error("%s", textmsg.ErrorTextUnableReadHistoryDB(err)) return err } - for _, backupName := range backupList { - backupData, err := gpbckpconfig.GetBackupDataDB(backupName, hDB) + for _, depTimestamp := range backupDependenciesList { + backupData, err := gpbckpconfig.GetBackupDataDB(depTimestamp, hDB) if err != nil { - gplog.Error("%s", textmsg.ErrorTextUnableGetBackupInfo(backupName, err)) + gplog.Error("%s", textmsg.ErrorTextUnableGetBackupInfo(depTimestamp, err)) return err } - addBackupToTable(backupTypeFilter, backupTableFilter, backupSchemaFilter, backupExcludeFilter, backupData, t) + addBackupToTable("", "", "", false, includeObjectFilteringDetails, backupData, t) } return nil } -func initTable(t table.Writer) { +func initTable(t table.Writer, includeDetails bool) { t.SetOutputMirror(os.Stdout) t.SetStyle(table.StyleDefault) t.Style().Options.DrawBorder = false - t.AppendHeader(table.Row{ + header := table.Row{ "timestamp", "date", "status", @@ -199,7 +285,11 @@ func initTable(t table.Writer) { "plugin", "duration", "date deleted", - }) + } + if includeDetails { + header = append(header, "object filtering details") + } + t.AppendHeader(header) t.SortBy([]table.SortBy{{Name: "timestamp", Mode: table.Dsc}}) } @@ -208,7 +298,7 @@ func initTable(t table.Writer) { // If errors occur, they are logged, but they are not returned. // The main idea is to show the maximum available information and display all errors that occur. // But do not fall when errors occur. So, display anyway. -func addBackupToTable(backupTypeFilter, backupTableFilter, backupSchemaFilter string, backupExcludeFilter bool, backupData gpbckpconfig.BackupConfig, t table.Writer) { +func addBackupToTable(backupTypeFilter, backupTableFilter, backupSchemaFilter string, backupExcludeFilter, includeDetails bool, backupData gpbckpconfig.BackupConfig, t table.Writer) { var matchToObjectFilter bool backupDate, err := backupData.GetBackupDate() if err != nil { @@ -232,7 +322,7 @@ func addBackupToTable(backupTypeFilter, backupTableFilter, backupSchemaFilter st } matchToObjectFilter = backupData.CheckObjectFilteringExists(backupTableFilter, backupSchemaFilter, backupFilter, backupExcludeFilter) if (backupTypeFilter == "" || backupTypeFilter == backupType) && matchToObjectFilter { - t.AppendRow([]interface{}{ + row := []interface{}{ backupData.Timestamp, backupDate, backupData.Status, @@ -242,6 +332,10 @@ func addBackupToTable(backupTypeFilter, backupTableFilter, backupSchemaFilter st backupData.Plugin, formatBackupDuration(backupDuration), backupDateDeleted, - }) + } + if includeDetails { + row = append(row, backupData.GetObjectFilteringDetails()) + } + t.AppendRow(row) } } diff --git a/cmd/constants.go b/cmd/constants.go index 5af66bc..7a91714 100644 --- a/cmd/constants.go +++ b/cmd/constants.go @@ -40,6 +40,7 @@ const ( backupDirFlagName = "backup-dir" parallelProcessesFlagName = "parallel-processes" ignoreErrorsFlagName = "ignore-errors" + detailFlagName = "detail" exitErrorCode = 1 diff --git a/e2e_tests/README.md b/e2e_tests/README.md index b2c5dc6..fc08569 100644 --- a/e2e_tests/README.md +++ b/e2e_tests/README.md @@ -2,7 +2,7 @@ The following architecture is used to run the tests: -* Separate containers for MinIO and nginx. Official images [minio/minio](https://hub.docker.com/r/minio/minio), [minio/mc](https://hub.docker.com/r/minio/mc) and [nginx](https://hub.docker.com/_/nginx) are used. It's necessary for S3 compatible storage for WAL archiving and backups. +* Separate containers for MinIO and nginx. Official images [minio/minio](https://hub.docker.com/r/minio/minio), [minio/mc](https://hub.docker.com/r/minio/mc) and [nginx](https://hub.docker.com/_/nginx) are used. It's necessary for S3 compatible storage for backups. - Separate container gpbackman-export: runs the gpbackman image and copies the binary to a shared Docker volume (gpbackman_bin) for use inside the Greenplum container. * Separate container for Greenplum. The [docker-greenplum image](https://github.com/woblerr/docker-greenplum) is used to run a single-node Greenplum cluster. diff --git a/e2e_tests/scripts/prepare/prepare_gpdb_backups.sh b/e2e_tests/scripts/prepare/prepare_gpdb_backups.sh index 8710a74..655835d 100755 --- a/e2e_tests/scripts/prepare/prepare_gpdb_backups.sh +++ b/e2e_tests/scripts/prepare/prepare_gpdb_backups.sh @@ -7,13 +7,13 @@ set -Eeuo pipefail # 3. full_local_exclude_table : Full LOCAL backup excluding sch1.tbl_b # 4. metadata_only_s3 : Metadata-only S3 backup (no data) # 5. full_s3 : Full S3 backup (all tables, leaf partition data) -# 6. full_s3_include_table : Full S3 backup including only sch2.tbl_c -# 7. full_s3_exclude_table : Full S3 backup excluding sch2.tbl_d +# 6. full_s3_include_tables : Full S3 backup including sch2.tbl_c, sch2.tbl_d +# 7. full_s3_exclude_schema : Full S3 backup excluding schema sch1 # 8. (data change) : Insert into sch2.tbl_c and sch2.tbl_d # 9. incr_s3 : Incremental S3 backup -# 10. incr_s3_include_table : Incremental S3 backup including only sch2.tbl_c +# 10. incr_s3_include_tables : Incremental S3 backup including sch2.tbl_c, sch2.tbl_d # 11. (data change) : Insert more rows into sch2.tbl_c -# 12. incr_s3_exclude_table : Incremental S3 backup excluding sch2.tbl_d +# 12. incr_s3_exclude_schema : Incremental S3 backup excluding schema sch1 # 13. data_only_local : Data-only LOCAL backup (no metadata) # 14. full_local : Final full LOCAL backup (all tables) @@ -43,11 +43,11 @@ run_backup metadata_only_s3 "${COMMON_PLUGIN_FLAGS[@]}" --metadata-only # Full S3 no filters run_backup full_s3 "${COMMON_PLUGIN_FLAGS[@]}" --leaf-partition-data -# Full S3 include-table sch2.tbl_c -run_backup full_s3_include_table "${COMMON_PLUGIN_FLAGS[@]}" --include-table sch2.tbl_c --leaf-partition-data +# Full S3 include-table sch2.tbl_c, sch2.tbl_d +run_backup full_s3_include_table "${COMMON_PLUGIN_FLAGS[@]}" --include-table sch2.tbl_c --include-table sch2.tbl_d --leaf-partition-data -# Full S3 exclude-table sch2.tbl_d -run_backup full_s3_exclude_table "${COMMON_PLUGIN_FLAGS[@]}" --exclude-table sch2.tbl_d --leaf-partition-data +# Full S3 exclude-schema sch1 +run_backup full_s3_exclude_schema "${COMMON_PLUGIN_FLAGS[@]}" --exclude-schema sch1 --leaf-partition-data # Insert data psql -d demo -c "INSERT INTO sch2.tbl_c SELECT i, i FROM generate_series(1,100000) i;" @@ -56,14 +56,14 @@ psql -d demo -c "INSERT INTO sch2.tbl_d SELECT i, i FROM generate_series(1,10000 # Incremental S3 no filters run_backup incr_s3 "${COMMON_PLUGIN_FLAGS[@]}" --incremental --leaf-partition-data -# Incremental S3 include-table sch2.tbl_c -run_backup incr_s3_include_table "${COMMON_PLUGIN_FLAGS[@]}" --incremental --include-table sch2.tbl_c --leaf-partition-data +# Incremental S3 include-tables sch2.tbl_c, sch2.tbl_d +run_backup incr_s3_include_table "${COMMON_PLUGIN_FLAGS[@]}" --incremental --include-table sch2.tbl_c --include-table sch2.tbl_d --leaf-partition-data # Insert data psql -d demo -c "INSERT INTO sch2.tbl_c SELECT i, i FROM generate_series(1,100000) i;" -# Incremental S3 exclude-table sch2.tbl_d -run_backup incr_s3_exclude_table "${COMMON_PLUGIN_FLAGS[@]}" --incremental --exclude-table sch2.tbl_d --leaf-partition-data +# Incremental S3 exclude-schema sch1 +run_backup incr_s3_exclude_schema "${COMMON_PLUGIN_FLAGS[@]}" --incremental --exclude-schema sch1 --leaf-partition-data # Data-only LOCAL no filters run_backup data_only_local --data-only diff --git a/e2e_tests/scripts/run_tests/run_backup-info.sh b/e2e_tests/scripts/run_tests/run_backup-info.sh index a33ba0a..22bc267 100755 --- a/e2e_tests/scripts/run_tests/run_backup-info.sh +++ b/e2e_tests/scripts/run_tests/run_backup-info.sh @@ -5,6 +5,11 @@ source "$(dirname "${BASH_SOURCE[0]}")/common_functions.sh" COMMAND="backup-info" +get_backup_info_timestamp() { + local label="${1}"; shift + run_gpbackman "backup-info" "${label}" "$@" +} + # Test 1: Count all backups in history database test_count_all_backups() { local want=12 @@ -37,10 +42,10 @@ test_count_include_table_backups() { assert_equals "${want}" "${got}" } -# Test 5: Count backups that exclude table sch2.tbl_d -test_count_exclude_table_backups() { +# Test 5: Count backups that exclude schema sch1 +test_count_exclude_schema_backups() { local want=2 - local got=$(get_backup_info total_exclude_table_backups --history-db ${DATA_DIR}/gpbackup_history.db --table sch2.tbl_d --exclude | grep -E "${TIMESTAMP_GREP_PATTERN}" | wc -l) + local got=$(get_backup_info total_exclude_schema_backups --history-db ${DATA_DIR}/gpbackup_history.db --schema sch1 --exclude | grep -E "${TIMESTAMP_GREP_PATTERN}" | wc -l) assert_equals "${want}" "${got}" } @@ -52,19 +57,61 @@ test_count_include_table_full_backups() { assert_equals "${want}" "${got}" } -# Test 7: Count incremental backups that exclude table sch2.tbl_d -test_count_exclude_table_incremental_backups() { +# Test 7: Count incremental backups that exclude schema sch1 +test_count_exclude_schema_incremental_backups() { + local want=1 + local got=$(get_backup_info total_exclude_schema_incremental_backups --history-db ${DATA_DIR}/gpbackup_history.db --schema sch1 --exclude --type incremental | grep -E "${TIMESTAMP_GREP_PATTERN}" | wc -l) + assert_equals "${want}" "${got}" +} + +# Test 8: Check backup chain and details for include tables sch2.tbl_c, sch2.tbl_d +test_backup_chain_include_tables() { + local want=2 + local cutoff_timestamp=$(get_cutoff_timestamp 7) + local got=$(get_backup_info_timestamp backup_chain_include_tables --history-db ${DATA_DIR}/gpbackup_history.db --timestamp "${cutoff_timestamp}" | grep -E "${TIMESTAMP_GREP_PATTERN}" | wc -l) + assert_equals "${want}" "${got}" + local got_details=$(get_backup_info_timestamp backup_chain_include_tables --history-db ${DATA_DIR}/gpbackup_history.db --timestamp "${cutoff_timestamp}" | grep -E "${TIMESTAMP_GREP_PATTERN}" | awk -F'|' '{print $NF}') + if [ ! -n "${got_details}" ]; then + echo "[ERROR] Expected details column to be non-empty" + exit 1 + fi +} + +# Test 9: Check backup chain and details for incremental backup that exclude schema sch1 +# For incremental there is no backup chain, so only one backup should be returned +test_backup_chain_incremental_exclude() { + local want=1 + local cutoff_timestamp=$(get_cutoff_timestamp 3) + local got=$(get_backup_info_timestamp backup_chain_incremental_exclude --history-db ${DATA_DIR}/gpbackup_history.db --timestamp "${cutoff_timestamp}" | grep -E "${TIMESTAMP_GREP_PATTERN}" | wc -l) + assert_equals "${want}" "${got}" + local got_details=$(get_backup_info_timestamp backup_chain_incremental_exclude --history-db ${DATA_DIR}/gpbackup_history.db --timestamp "${cutoff_timestamp}" | grep -E "${TIMESTAMP_GREP_PATTERN}" | awk -F'|' '{print $NF}') + if [ ! -n "${got_details}" ]; then + echo "[ERROR] Expected details column to be non-empty" + exit 1 + fi +} + +# Test 10: Check full local backup with include table sch1.tbl_a and object filtering details +test_full_local_include_table_details() { local want=1 - local got=$(get_backup_info total_exclude_table_incremental_backups --history-db ${DATA_DIR}/gpbackup_history.db --table sch2.tbl_d --exclude --type incremental | grep -E "${TIMESTAMP_GREP_PATTERN}" | wc -l) + local got=$(get_backup_info full_local_include_table_details --history-db ${DATA_DIR}/gpbackup_history.db --table sch1.tbl_a --type full --detail | grep -E "${TIMESTAMP_GREP_PATTERN}" | wc -l) assert_equals "${want}" "${got}" + local got_details=$(get_backup_info full_local_include_table_details --history-db ${DATA_DIR}/gpbackup_history.db --table sch1.tbl_a --type full --detail| grep -E "${TIMESTAMP_GREP_PATTERN}" | awk -F'|' '{print $NF}') + if [ ! -n "${got_details}" ]; then + echo "[ERROR] Expected details column to be non-empty" + exit 1 + fi } run_test "${COMMAND}" 1 test_count_all_backups run_test "${COMMAND}" 2 test_count_full_backups run_test "${COMMAND}" 3 test_count_incremental_backups run_test "${COMMAND}" 4 test_count_include_table_backups -run_test "${COMMAND}" 5 test_count_exclude_table_backups +run_test "${COMMAND}" 5 test_count_exclude_schema_backups run_test "${COMMAND}" 6 test_count_include_table_full_backups -run_test "${COMMAND}" 7 test_count_exclude_table_incremental_backups +run_test "${COMMAND}" 7 test_count_exclude_schema_incremental_backups +run_test "${COMMAND}" 8 test_backup_chain_include_tables +run_test "${COMMAND}" 9 test_backup_chain_incremental_exclude +run_test "${COMMAND}" 10 test_full_local_include_table_details log_all_tests_passed "${COMMAND}" diff --git a/gpbckpconfig/struct.go b/gpbckpconfig/struct.go index dabfdf0..bdca9a0 100644 --- a/gpbckpconfig/struct.go +++ b/gpbckpconfig/struct.go @@ -2,6 +2,7 @@ package gpbckpconfig import ( "errors" + "strings" "time" ) @@ -148,6 +149,24 @@ func (backupConfig BackupConfig) GetObjectFilteringInfo() (string, error) { } } +// GetObjectFilteringDetails returns a comma-separated string with object filtering details +// depending on the active filtering type. If no filtering is active, it returns an empty string. +func (backupConfig BackupConfig) GetObjectFilteringDetails() string { + filter, _ := backupConfig.GetObjectFilteringInfo() + switch filter { + case objectFilteringIncludeTable: + return strings.Join(backupConfig.IncludeRelations, ", ") + case objectFilteringExcludeTable: + return strings.Join(backupConfig.ExcludeRelations, ", ") + case objectFilteringIncludeSchema: + return strings.Join(backupConfig.IncludeSchemas, ", ") + case objectFilteringExcludeSchema: + return strings.Join(backupConfig.ExcludeSchemas, ", ") + default: + return "" + } +} + // GetBackupDate Get backup date. // If an error occurs when parsing the date, the empty string and error are returned. func (backupConfig BackupConfig) GetBackupDate() (string, error) { diff --git a/gpbckpconfig/struct_test.go b/gpbckpconfig/struct_test.go index e2503f4..b099411 100644 --- a/gpbckpconfig/struct_test.go +++ b/gpbckpconfig/struct_test.go @@ -297,8 +297,58 @@ func TestGetObjectFilteringInfo(t *testing.T) { } } -func TestGetObjectFilteringInfoAllCombos(t *testing.T) { - // Removed: consolidated into TestGetObjectFilteringInfo +func TestGetObjectFilteringDetails(t *testing.T) { + tests := []struct { + name string + config BackupConfig + want string + }{ + { + name: "IncludeTable details", + config: BackupConfig{ + IncludeTableFiltered: true, + IncludeRelations: []string{"public.t1", "s.t2"}, + }, + want: "public.t1, s.t2", + }, + { + name: "ExcludeTable details", + config: BackupConfig{ + ExcludeTableFiltered: true, + ExcludeRelations: []string{"public.t3"}, + }, + want: "public.t3", + }, + { + name: "IncludeSchema details", + config: BackupConfig{ + IncludeSchemaFiltered: true, + IncludeSchemas: []string{"public", "sales"}, + }, + want: "public, sales", + }, + { + name: "ExcludeSchema details", + config: BackupConfig{ + ExcludeSchemaFiltered: true, + ExcludeSchemas: []string{"tmp"}, + }, + want: "tmp", + }, + { + name: "No filtering", + config: BackupConfig{}, + want: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.config.GetObjectFilteringDetails() + if got != tt.want { + t.Errorf("\nVariables do not match:\n%v\nwant:\n%v", got, tt.want) + } + }) + } } func TestGetBackupDate(t *testing.T) {