-
Notifications
You must be signed in to change notification settings - Fork 234
Open
Description
Hi there,
I’m experiencing a WAL archiving issue when attempting to archive WAL files to a NooBaa S3 bucket.
First I created a cluster using a YAML file and configured the barmanObjectStore section as follows:
apiVersion: postgresql.k8s.enterprisedb.io/v1
kind: Cluster
metadata:
name: smps
namespace: ibas-smps
spec:
backup:
target: prefer-standby
barmanObjectStore:
data:
compression: gzip
destinationPath: s3://s3pgbackupsbucket/ibas/ibas-smps/repo
endpointURL: https://192.168.25.55:6443
endpointCA:
key: ca.crt
name: s3-ca-secret
s3Credentials:
accessKeyId:
key: ACCESS_KEY_ID
name: aws-creds
secretAccessKey:
key: ACCESS_SECRET_KEY
name: aws-creds
wal:
compression: gzip
encryption: AES256
maxParallel: 8
...
But WAL archiving is failing with the following errors:
{"level":"info","ts":"2026-01-04T12:31:53.592515738Z","logger":"barman-cloud-check-wal-archive","msg":"2026-01-04 12:31:53,592 [736] ERROR: Barman cloud WAL archive check exception: An error occurred (403) when calling the HeadBucket operation: Forbidden","pipe":"stderr","logging_pod":"smps-1"}
{"level":"error","ts":"2026-01-04T12:31:53.652754173Z","logger":"wal-archive","msg":"Error invoking barman-cloud-check-wal-archive","logging_pod":"smps-1","options":["--endpoint-url","https://192.168.25.55:6443","--cloud-provider","aws-s3","s3://s3pgbackupsbucket/ibas/ibas-smps/repo","smps"],"exitCode":-1,"error":"exit status 4","stacktrace":"github.com/cloudnative-pg/machinery/pkg/log.(*logger).Error\n\tpkg/mod/github.com/cloudnative-pg/machinery@v0.3.1/pkg/log/log.go:125\ngithub.com/cloudnative-pg/barman-cloud/pkg/walarchive.(*BarmanArchiver).CheckWalArchiveDestination\n\tpkg/mod/github.com/cloudnative-pg/barman-cloud@v0.3.3/pkg/walarchive/cmd.go:175\ngithub.com/cloudnative-pg/barman-cloud/pkg/archiver.(*WALArchiver).CheckWalArchiveDestination\n\tpkg/mod/github.com/cloudnative-pg/barman-cloud@v0.3.3/pkg/archiver/archiver.go:131\ngithub.com/EnterpriseDB/cloud-native-postgres/pkg/management/postgres/archiver.checkWalArchive\n\tpkg/management/postgres/archiver/archiver.go:345\ngithub.com/EnterpriseDB/cloud-native-postgres/pkg/management/postgres/archiver.internalRun\n\tpkg/management/postgres/archiver/archiver.go:210\ngithub.com/EnterpriseDB/cloud-native-postgres/pkg/management/postgres/archiver.Run\n\tpkg/management/postgres/archiver/archiver.go:155\ngithub.com/EnterpriseDB/cloud-native-postgres/internal/cmd/manager/walarchive.NewCmd.func1\n\tinternal/cmd/manager/walarchive/cmd.go:68\ngithub.com/spf13/cobra.(*Command).execute\n\tpkg/mod/github.com/spf13/cobra@v1.10.1/command.go:1015\ngithub.com/spf13/cobra.(*Command).ExecuteC\n\tpkg/mod/github.com/spf13/cobra@v1.10.1/command.go:1148\ngithub.com/spf13/cobra.(*Command).Execute\n\tpkg/mod/github.com/spf13/cobra@v1.10.1/command.go:1071\nmain.main\n\tcmd/manager/main.go:75\nruntime.main\n\t/opt/hostedtoolcache/go/1.25.5/x64/src/runtime/proc.go:285"}
{"level":"error","ts":"2026-01-04T12:31:53.652965811Z","logger":"wal-archive","msg":"while barman-cloud-check-wal-archive","logging_pod":"smps-1","error":"unexpected failure invoking barman-cloud-wal-archive: exit status 4","stacktrace":"github.com/cloudnative-pg/machinery/pkg/log.(*logger).Error\n\tpkg/mod/github.com/cloudnative-pg/machinery@v0.3.1/pkg/log/log.go:125\ngithub.com/EnterpriseDB/cloud-native-postgres/pkg/management/postgres/archiver.checkWalArchive\n\tpkg/management/postgres/archiver/archiver.go:346\ngithub.com/EnterpriseDB/cloud-native-postgres/pkg/management/postgres/archiver.internalRun\n\tpkg/management/postgres/archiver/archiver.go:210\ngithub.com/EnterpriseDB/cloud-native-postgres/pkg/management/postgres/archiver.Run\n\tpkg/management/postgres/archiver/archiver.go:155\ngithub.com/EnterpriseDB/cloud-native-postgres/internal/cmd/manager/walarchive.NewCmd.func1\n\tinternal/cmd/manager/walarchive/cmd.go:68\ngithub.com/spf13/cobra.(*Command).execute\n\tpkg/mod/github.com/spf13/cobra@v1.10.1/command.go:1015\ngithub.com/spf13/cobra.(*Command).ExecuteC\n\tpkg/mod/github.com/spf13/cobra@v1.10.1/command.go:1148\ngithub.com/spf13/cobra.(*Command).Execute\n\tpkg/mod/github.com/spf13/cobra@v1.10.1/command.go:1071\nmain.main\n\tcmd/manager/main.go:75\nruntime.main\n\t/opt/hostedtoolcache/go/1.25.5/x64/src/runtime/proc.go:285"}
{"level":"error","ts":"2026-01-04T12:31:53.653035844Z","logger":"wal-archive","msg":"failed to run wal-archive command","logging_pod":"smps-1","error":"unexpected failure invoking barman-cloud-wal-archive: exit status 4","stacktrace":"github.com/cloudnative-pg/machinery/pkg/log.(*logger).Error\n\tpkg/mod/github.com/cloudnative-pg/machinery@v0.3.1/pkg/log/log.go:125\ngithub.com/EnterpriseDB/cloud-native-postgres/internal/cmd/manager/walarchive.NewCmd.func1\n\tinternal/cmd/manager/walarchive/cmd.go:73\ngithub.com/spf13/cobra.(*Command).execute\n\tpkg/mod/github.com/spf13/cobra@v1.10.1/command.go:1015\ngithub.com/spf13/cobra.(*Command).ExecuteC\n\tpkg/mod/github.com/spf13/cobra@v1.10.1/command.go:1148\ngithub.com/spf13/cobra.(*Command).Execute\n\tpkg/mod/github.com/spf13/cobra@v1.10.1/command.go:1071\nmain.main\n\tcmd/manager/main.go:75\nruntime.main\n\t/opt/hostedtoolcache/go/1.25.5/x64/src/runtime/proc.go:285"}
{"level":"info","ts":"2026-01-04T12:31:53.655453753Z","logger":"postgres","msg":"record","logging_pod":"smps-1","record":{"log_time":"2026-01-04 13:31:53.655 CET","process_id":"31","session_id":"695a5cf0.1f","session_line_num":"19","session_start_time":"2026-01-04 13:28:32 CET","transaction_id":"0","error_severity":"LOG","sql_state_code":"00000","message":"archive command failed with exit code 1","detail":"The failed archive command was: /controller/manager wal-archive --log-destination /controller/log/postgres.json pg_wal/000000010000000000000001","backend_type":"archiver","query_id":"0"}}
{"level":"info","ts":"2026-01-04T12:31:53.655494111Z","logger":"postgres","msg":"record","logging_pod":"smps-1","record":{"log_time":"2026-01-04 13:31:53.655 CET","process_id":"31","session_id":"695a5cf0.1f","session_line_num":"20","session_start_time":"2026-01-04 13:28:32 CET","transaction_id":"0","error_severity":"WARNING","sql_state_code":"01000","message":"archiving write-ahead log file \"000000010000000000000001\" failed too many times, will try again later","backend_type":"archiver","query_id":"0"}}
The S3 user has all the necessary permissions:
s3:ListBucket
s3:GetBucketLocation
s3:HeadBucket
s3:PutObject
s3:GetObject
I've checked the permissions:
[root@s3-9000-ibas certificates]# export AWS_CA_BUNDLE=/root/certificates/s3-vip-ibas.crt
[root@s3-9000-ibas certificates]# aws --endpoint-url https://192.168.25.55:6443 s3 ls s3://s3pgbackupsbucket
PRE ibas/
[root@s3-9000-ibas certificates]#
[root@s3-9000-ibas certificates]# aws s3api get-bucket-location --bucket s3pgbackupsbucket --endpoint-url https://192.168.25.55:6443
{
"LocationConstraint": null
}
[root@s3-9000-ibas certificates]#
[root@s3-9000-ibas certificates]# aws s3api head-bucket --bucket s3pgbackupsbucket --endpoint-url https://192.168.25.55:6443
[root@s3-9000-ibas certificates]# echo $?
0
[root@s3-9000-ibas certificates]#
[root@s3-9000-ibas certificates]# echo "test" > test.txt
[root@s3-9000-ibas certificates]# aws s3 cp test.txt s3://s3pgbackupsbucket/test.txt --endpoint-url https://192.168.25.55:6443
upload: ./test.txt to s3://s3pgbackupsbucket/test.txt
[root@s3-9000-ibas certificates]#
[root@s3-9000-ibas certificates]# aws s3 cp s3://s3pgbackupsbucket/test.txt ./downloaded.txt --endpoint-url https://192.168.25.55:6443
download: s3://s3pgbackupsbucket/test.txt to ./downloaded.txt
[root@s3-9000-ibas certificates]# diff test.txt downloaded.txt
[root@s3-9000-ibas certificates]#
[root@s3-9000-ibas certificates]# aws s3 rm s3://s3pgbackupsbucket/test.txt --endpoint-url https://192.168.25.55:6443
delete: s3://s3pgbackupsbucket/test.txt
[root@s3-9000-ibas certificates]# ls -l test.txt
-rw-r--r--. 1 root root 5 Jan 7 03:00 test.txt
[root@s3-9000-ibas certificates]#
[root@s3-9000-ibas certificates]# rm -rf test.txt downloaded.txt
[root@s3-9000-ibas certificates]#
[root@s3-9000-ibas ~]# systemctl --type=service | grep -iE 'minio|rgw|noobaa|object|s3'
noobaa.service loaded active running The NooBaa service.
[root@s3-9000-ibas ~]#
[root@s3-9000-ibas ~]# ss -tulnp | grep -E '9000|9001|7480|8080|443'
tcp LISTEN 0 511 *:47443 *:* users:(("java",pid=8394,fd=135))
tcp LISTEN 0 511 *:6443 *:* users:(("noobaa",pid=13055,fd=20))
tcp LISTEN 0 511 *:9443 *:* users:(("noobaa",pid=13055,fd=24))
[root@s3-9000-ibas ~]#
pgBackRest has no problems using this same bucket configuration.
Please advise if there’s anything wrong with my configuration or if a workaround exists.
Thank you.
Metadata
Metadata
Assignees
Labels
No labels