From e7d04ac299dcfb2679223d553f5b831b06c50137 Mon Sep 17 00:00:00 2001 From: sivakumar subraani Date: Wed, 19 Jul 2023 15:45:43 +0000 Subject: [PATCH] vendoring kdmp repo from master branch --- go.mod | 4 +- go.sum | 6 +- .../pkg/controllers/dataexport/reconcile.go | 79 +++++++++++++------ .../controllers/resourceexport/reconcile.go | 2 +- .../pkg/drivers/kopiabackup/kopiabackup.go | 2 +- .../pkg/drivers/kopiadelete/kopiadelete.go | 2 +- .../pkg/drivers/kopiarestore/kopiarestore.go | 2 +- .../drivers/nfscsirestore/nfscsirestore.go | 22 +++--- .../portworx/kdmp/pkg/drivers/utils/common.go | 12 +++ .../portworx/kdmp/pkg/executor/common.go | 37 ++++++++- .../portworx/kdmp/pkg/version/version.go | 2 +- vendor/modules.txt | 4 +- 12 files changed, 126 insertions(+), 48 deletions(-) diff --git a/go.mod b/go.mod index d4beeb604b..15c9e115d8 100644 --- a/go.mod +++ b/go.mod @@ -62,7 +62,7 @@ require ( sigs.k8s.io/sig-storage-lib-external-provisioner/v6 v6.3.0 ) -require github.com/portworx/kdmp v0.4.1-0.20230607162028-5246250670c5 +require github.com/portworx/kdmp v0.4.1-0.20230719084848-d9ea7eeb2c87 require ( cloud.google.com/go v0.110.0 // indirect @@ -300,7 +300,7 @@ replace ( github.com/kubernetes-incubator/external-storage v0.20.4-openstorage-rc10 => github.com/kubernetes-incubator/external-storage v0.25.1-openstorage-rc1 github.com/libopenstorage/autopilot-api => github.com/libopenstorage/autopilot-api v0.6.1-0.20210301232050-ca2633c6e114 github.com/libopenstorage/openstorage => github.com/libopenstorage/openstorage v0.0.0-20230511212757-41751b27d69f - github.com/portworx/kdmp => github.com/portworx/kdmp v0.4.1-0.20230607162028-5246250670c5 + github.com/portworx/kdmp => github.com/portworx/kdmp v0.4.1-0.20230719084848-d9ea7eeb2c87 github.com/portworx/sched-ops => github.com/portworx/sched-ops v1.20.4-rc1.0.20230426231724-8d9f2c104721 github.com/portworx/torpedo => github.com/portworx/torpedo v0.0.0-20230410110229-ad7b0df86670 gopkg.in/fsnotify.v1 v1.4.7 => github.com/fsnotify/fsnotify v1.4.7 diff --git a/go.sum b/go.sum index 06b1982b76..4e7c5bf5d6 100644 --- a/go.sum +++ b/go.sum @@ -1970,7 +1970,7 @@ github.com/libopenstorage/secrets v0.0.0-20220413195519-57d1c446c5e9 h1:pZQ5uaWk github.com/libopenstorage/secrets v0.0.0-20220413195519-57d1c446c5e9/go.mod h1:gE8rSd6lwLNXNbiW3DrRZjFMs+y4fDHy/6uiOO9cdzY= github.com/libopenstorage/stork v1.4.1-0.20230329095542-eefc6837653a/go.mod h1:9ZIMQaePJPyVsx85zoWn6W+uhjTv/A+7JLoG8j58xgI= github.com/libopenstorage/stork v1.4.1-0.20230330233319-e17ea1b3fd81/go.mod h1:9ZIMQaePJPyVsx85zoWn6W+uhjTv/A+7JLoG8j58xgI= -github.com/libopenstorage/stork v1.4.1-0.20230601053837-5dd68f026569/go.mod h1:+mKPMCPNhS/XOF2RPcNFijkr67CCCWp0o8OXVG6xxAk= +github.com/libopenstorage/stork v1.4.1-0.20230610103146-72cf75320066/go.mod h1:Yst+fnOYjWk6SA5pXZBKm19wtiinjxQ/vgYTXI3k80Q= github.com/libopenstorage/systemutils v0.0.0-20160208220149-44ac83be3ce1 h1:5vqfYYWm4b+lbkMtvvWtWBiqLbmLN6dNvWaa7wVsz/Q= github.com/libopenstorage/systemutils v0.0.0-20160208220149-44ac83be3ce1/go.mod h1:xwNGC7xiz/BQ/wbMkvHujL8Gjgseg+x41xMek7sKRRQ= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= @@ -2357,8 +2357,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/polyfloyd/go-errorlint v1.0.5/go.mod h1:APVvOesVSAnne5SClsPxPdfvZTVDojXh1/G3qb5wjGI= github.com/portworx/dcos-secrets v0.0.0-20180616013705-8e8ec3f66611/go.mod h1:4hklRW/4DQpLqkcXcjtNprbH2tz/sJaNtqinfPWl/LA= -github.com/portworx/kdmp v0.4.1-0.20230607162028-5246250670c5 h1:CSS9ddCxe4pkEQbq2OCN9PbjG62Xcf9waPoYioyZaBo= -github.com/portworx/kdmp v0.4.1-0.20230607162028-5246250670c5/go.mod h1:M9QuYoZL07Jdj8hy4XsDU1OYwu39zzTdX/axFpALVS0= +github.com/portworx/kdmp v0.4.1-0.20230719084848-d9ea7eeb2c87 h1:SKi9TgbWCGaSSnxYrKSl6ke6g+HgUhIg/1G2BiGZnQc= +github.com/portworx/kdmp v0.4.1-0.20230719084848-d9ea7eeb2c87/go.mod h1:KFeJuczRm1Jw6AdejFLcXHB6yrjfKY/X1brfWDmXgr8= github.com/portworx/kvdb v0.0.0-20190105022415-cccaa09abfc9/go.mod h1:Q8YyrNDvPp3DVF96BDcQuaC7fAYUCuUX+l58S7OnD2M= github.com/portworx/kvdb v0.0.0-20200723230726-2734b7f40194/go.mod h1:Q8YyrNDvPp3DVF96BDcQuaC7fAYUCuUX+l58S7OnD2M= github.com/portworx/kvdb v0.0.0-20200929023115-b312c7519467/go.mod h1:Q8YyrNDvPp3DVF96BDcQuaC7fAYUCuUX+l58S7OnD2M= diff --git a/vendor/github.com/portworx/kdmp/pkg/controllers/dataexport/reconcile.go b/vendor/github.com/portworx/kdmp/pkg/controllers/dataexport/reconcile.go index df55af0ef3..2de31ca602 100644 --- a/vendor/github.com/portworx/kdmp/pkg/controllers/dataexport/reconcile.go +++ b/vendor/github.com/portworx/kdmp/pkg/controllers/dataexport/reconcile.go @@ -92,6 +92,7 @@ type updateDataExportDetail struct { snapshotNamespace string removeFinalizer bool volumeSnapshot string + resetLocalSnapshotRestore bool } func (c *Controller) sync(ctx context.Context, in *kdmpapi.DataExport) (bool, error) { @@ -450,8 +451,10 @@ func (c *Controller) sync(ctx context.Context, in *kdmpapi.DataExport) (bool, er return false, c.updateStatus(dataExport, data) case kdmpapi.DataExportStageCleanup: var cleanupErr error + // Need to retain the old reason present in the dataexport CR, so passing the reason again. data := updateDataExportDetail{ stage: kdmpapi.DataExportStageFinal, + reason: dataExport.Status.Reason, } // Append the job-pod log to stork's pod log in case of failure // it is best effort approach, hence errors are ignored. @@ -777,9 +780,11 @@ func (c *Controller) stageSnapshotInProgress(ctx context.Context, dataExport *kd } if dataExport.Status.Status == kdmpapi.DataExportStatusFailed { // set to the next stage + // Need to retain the old reason present in the dataexport CR, so passing the reason again. data := updateDataExportDetail{ stage: kdmpapi.DataExportStageCleanup, status: dataExport.Status.Status, + reason: dataExport.Status.Reason, } return false, c.updateStatus(dataExport, data) } @@ -1046,6 +1051,7 @@ func (c *Controller) stageLocalSnapshotRestore(ctx context.Context, dataExport * stage: kdmpapi.DataExportStageTransferScheduled, status: kdmpapi.DataExportStatusInitial, reason: "switching to restore from objectstore bucket as restoring from local snapshot did not happen", + resetLocalSnapshotRestore: true, } return false, c.updateStatus(dataExport, data) } @@ -1198,6 +1204,7 @@ func (c *Controller) stageLocalSnapshotRestoreInProgress(ctx context.Context, da status: kdmpapi.DataExportStatusInitial, reason: "", transferID: "", // Resetting transfer id if it has been set with nfs backuplocation job + resetLocalSnapshotRestore: true, } return false, c.updateStatus(dataExport, data) } @@ -1376,16 +1383,22 @@ func (c *Controller) cleanUp(driver drivers.Interface, de *kdmpapi.DataExport) e if err := core.Instance().DeletePersistentVolumeClaim(de.Status.SnapshotPVCName, de.Status.SnapshotPVCNamespace); err != nil && !k8sErrors.IsNotFound(err) { return fmt.Errorf("delete %s/%s pvc: %s", de.Status.SnapshotPVCNamespace, de.Status.SnapshotPVCName, err) } - bl, err := checkBackupLocation(de.Spec.Destination) - if err != nil { - msg := fmt.Sprintf("backuplocation fetch error for %s: %v", de.Spec.Destination.Name, err) - logrus.Errorf(msg) - } - if err == nil && bl.Location.Type == storkapi.BackupLocationNFS { + } + bl, err := checkBackupLocation(de.Spec.Destination) + if err != nil { + msg := fmt.Sprintf("backuplocation fetch error for %s: %v", de.Spec.Destination.Name, err) + logrus.Errorf(msg) + } else { + // In the case of NFS backuplocation, we will not delete vs and vsc after volumestage as + // we will need to upload the snapshot.json during the resource stage and then we will delete it. + // But if the failure happens in the volume stage, we will not go to the resource stage. + // So in the case of failure, we will cleanup the vs and vsc in the case of NFS backuplocation in volume cleanupstage itself. + if bl.Location.Type == storkapi.BackupLocationNFS && de.Status.Status == kdmpapi.DataExportStatusSuccessful { + // In the case of success, we will delete the vs and vsc during resource stage. logrus.Infof("not deleting the vs and vsc in volume stage") } else { - err = snapshotDriver.DeleteSnapshot(de.Status.VolumeSnapshot, de.Status.SnapshotPVCNamespace, true) - msg := fmt.Sprintf("failed in removing local volume snapshot CRs for %s/%s: %v", de.Status.VolumeSnapshot, de.Status.SnapshotPVCName, err) + err = snapshotDriver.DeleteSnapshot(de.Status.VolumeSnapshot, de.Status.SnapshotNamespace, true) + msg := fmt.Sprintf("failed in removing local volume snapshot CRs for %s/%s: %v", de.Status.VolumeSnapshot, de.Status.SnapshotNamespace, err) if err != nil { logrus.Errorf(msg) return fmt.Errorf(msg) @@ -1435,6 +1448,16 @@ func (c *Controller) cleanUp(driver drivers.Interface, de *kdmpapi.DataExport) e if err := core.Instance().DeletePersistentVolume(pvName); err != nil && !k8sErrors.IsNotFound(err) { return fmt.Errorf("delete %s pv: %s", pvName, err) } + if err := utils.CleanServiceAccount(jobName, namespace); err != nil { + errMsg := fmt.Sprintf("deletion of service account %s/%s failed: %v", namespace, jobName, err) + logrus.Errorf("%s: %v", "cleanUp", errMsg) + return fmt.Errorf(errMsg) + } + if err := core.Instance().DeleteSecret(utils.GetCredSecretName(jobName), namespace); err != nil && !k8sErrors.IsNotFound(err) { + errMsg := fmt.Sprintf("deletion of backup credential secret %s failed: %v", jobName, err) + logrus.Errorf(errMsg) + return fmt.Errorf(errMsg) + } } if err := core.Instance().DeleteSecret(utils.GetCredSecretName(de.Name), namespace); err != nil && !k8sErrors.IsNotFound(err) { @@ -1472,27 +1495,32 @@ func (c *Controller) cleanupLocalRestoredSnapshotResources(de *kdmpapi.DataExpor return nil, false, fmt.Errorf("failed to get snapshot driver for %v: %v", snapshotDriverName, cleanupErr) } - pvcSpec := &corev1.PersistentVolumeClaim{} + // Get the Restore pvc spec. + rpvc, err := core.Instance().GetPersistentVolumeClaim(de.Status.RestorePVC.Name, de.Namespace) + if err != nil { + if k8sErrors.IsNotFound(err) { + return nil, false, nil; + } + logrus.Errorf("cleanupLocalRestoredSnapshotResources: failed to get restore pvc [%v] err: %v", de.Status.RestorePVC.Name, err) + return nil, false, err + } + if rpvc.Spec.DataSource != nil { + err = snapshotDriver.DeleteSnapshot(rpvc.Spec.DataSource.Name, de.Namespace, true) + if err != nil { + logrus.Errorf("cleanupLocalRestoredSnapshotResources: snapshotDriver.DeleteSnapshot failed with err: %v", err) + return nil, false, err + } + } if !ignorePVC { - pvcSpec = de.Status.RestorePVC + pvcSpec := de.Status.RestorePVC if err := cleanupJobBoundResources(pvcSpec.Name, de.Namespace); err != nil { return nil, false, fmt.Errorf("cleaning up of bound job resources failed: %v", err) } - } - - if de.Status.SnapshotPVCName != "" && de.Status.SnapshotPVCNamespace != "" { if err := core.Instance().DeletePersistentVolumeClaim(pvcSpec.Name, de.Namespace); err != nil && !k8sErrors.IsNotFound(err) { return nil, false, fmt.Errorf("delete %s/%s pvc: %s", de.Namespace, pvcSpec.Name, err) } - err := snapshotDriver.DeleteSnapshot(de.Status.VolumeSnapshot, de.Status.SnapshotPVCNamespace, true) - msg := fmt.Sprintf("failed in removing local volume snapshot CRs for %s/%s: %v", de.Status.VolumeSnapshot, de.Status.SnapshotPVCName, err) - if err != nil { - logrus.Errorf(msg) - return nil, false, fmt.Errorf(msg) - } } - return nil, false, nil } @@ -1517,6 +1545,8 @@ func (c *Controller) updateStatus(de *kdmpapi.DataExport, data updateDataExportD logrus.Infof("%v", errMsg) return "", true, fmt.Errorf("%v", errMsg) } + // Need to set the reason with out any check as in the success case, we need to set the reason to empty. + de.Status.Reason = data.reason if data.stage != "" { de.Status.Stage = data.stage } @@ -1526,9 +1556,6 @@ func (c *Controller) updateStatus(de *kdmpapi.DataExport, data updateDataExportD if data.transferID != "" { de.Status.TransferID = data.transferID } - if data.reason != "" { - de.Status.Reason = data.reason - } if data.snapshotID != "" { de.Status.SnapshotID = data.snapshotID } @@ -1556,6 +1583,9 @@ func (c *Controller) updateStatus(de *kdmpapi.DataExport, data updateDataExportD if data.volumeSnapshot != "" { de.Status.VolumeSnapshot = data.volumeSnapshot } + if data.resetLocalSnapshotRestore { + de.Status.LocalSnapshotRestore = false + } actualErr = c.client.Update(context.TODO(), de) if actualErr != nil { @@ -2187,7 +2217,8 @@ func startNfsCSIRestoreVolumeJob( bl *storkapi.BackupLocation, ) (string, error) { - err := utils.CreateNfsSecret(utils.GetCredSecretName(de.Name), bl, de.Namespace, nil) + jobName := utils.GetCsiRestoreJobName(drivers.NFSCSIRestore, de.Name) + err := utils.CreateNfsSecret(utils.GetCredSecretName(jobName), bl, de.Namespace, nil) if err != nil { logrus.Errorf("failed to create NFS cred secret: %v", err) return "", fmt.Errorf("failed to create NFS cred secret: %v", err) diff --git a/vendor/github.com/portworx/kdmp/pkg/controllers/resourceexport/reconcile.go b/vendor/github.com/portworx/kdmp/pkg/controllers/resourceexport/reconcile.go index 0604e817d7..f3be31ba76 100644 --- a/vendor/github.com/portworx/kdmp/pkg/controllers/resourceexport/reconcile.go +++ b/vendor/github.com/portworx/kdmp/pkg/controllers/resourceexport/reconcile.go @@ -239,7 +239,7 @@ func (c *Controller) cleanupResources(resourceExport *kdmpapi.ResourceExport) er // clean up resources rbNamespace, rbName, err := utils.ParseJobID(resourceExport.Status.TransferID) if err != nil { - errMsg := fmt.Sprintf("failed to parse job ID %v from ResourceeExport CR: %v: %v", + errMsg := fmt.Sprintf("failed to parse job ID %v from ResourceExport CR: %v: %v", resourceExport.Status.TransferID, resourceExport.Name, err) logrus.Errorf("%v", errMsg) return err diff --git a/vendor/github.com/portworx/kdmp/pkg/drivers/kopiabackup/kopiabackup.go b/vendor/github.com/portworx/kdmp/pkg/drivers/kopiabackup/kopiabackup.go index b7b84b0442..8e96e9cb21 100644 --- a/vendor/github.com/portworx/kdmp/pkg/drivers/kopiabackup/kopiabackup.go +++ b/vendor/github.com/portworx/kdmp/pkg/drivers/kopiabackup/kopiabackup.go @@ -211,7 +211,7 @@ func (d Driver) JobStatus(id string) (*drivers.JobStatus, error) { // Check whether mount point failure mountFailed := utils.IsJobPodMountFailed(job, namespace) if mountFailed { - errMsg := fmt.Sprintf("job [%v/%v] failed while mounting NFS mount endpoint", namespace, name) + errMsg := fmt.Sprintf("job [%v/%v] failed to mount pvc, please check job pod's description for more detail", namespace, name) return utils.ToJobStatus(0, errMsg, batchv1.JobFailed), nil } diff --git a/vendor/github.com/portworx/kdmp/pkg/drivers/kopiadelete/kopiadelete.go b/vendor/github.com/portworx/kdmp/pkg/drivers/kopiadelete/kopiadelete.go index 702e106b84..7ee70c20f4 100644 --- a/vendor/github.com/portworx/kdmp/pkg/drivers/kopiadelete/kopiadelete.go +++ b/vendor/github.com/portworx/kdmp/pkg/drivers/kopiadelete/kopiadelete.go @@ -143,7 +143,7 @@ func (d Driver) JobStatus(id string) (*drivers.JobStatus, error) { // Check whether mount point failure mountFailed := utils.IsJobPodMountFailed(job, namespace) if mountFailed { - errMsg := fmt.Sprintf("job [%v/%v] failed while mounting NFS mount endpoint", namespace, name) + errMsg := fmt.Sprintf("job [%v/%v] failed to mount pvc, please check job pod's description for more detail", namespace, name) return utils.ToJobStatus(0, errMsg, batchv1.JobFailed), nil } diff --git a/vendor/github.com/portworx/kdmp/pkg/drivers/kopiarestore/kopiarestore.go b/vendor/github.com/portworx/kdmp/pkg/drivers/kopiarestore/kopiarestore.go index 41f46d7872..ba0b7c7e30 100644 --- a/vendor/github.com/portworx/kdmp/pkg/drivers/kopiarestore/kopiarestore.go +++ b/vendor/github.com/portworx/kdmp/pkg/drivers/kopiarestore/kopiarestore.go @@ -119,7 +119,7 @@ func (d Driver) JobStatus(id string) (*drivers.JobStatus, error) { // Check whether mount point failure mountFailed := utils.IsJobPodMountFailed(job, namespace) if mountFailed { - errMsg := fmt.Sprintf("job [%v/%v] failed while mounting NFS mount endpoint", namespace, name) + errMsg := fmt.Sprintf("job [%v/%v] failed to mount pvc, please check job pod's description for more detail", namespace, name) return utils.ToJobStatus(0, errMsg, batchv1.JobFailed), nil } err = utils.JobNodeExists(job) diff --git a/vendor/github.com/portworx/kdmp/pkg/drivers/nfscsirestore/nfscsirestore.go b/vendor/github.com/portworx/kdmp/pkg/drivers/nfscsirestore/nfscsirestore.go index 37108c5668..5b34ee5721 100644 --- a/vendor/github.com/portworx/kdmp/pkg/drivers/nfscsirestore/nfscsirestore.go +++ b/vendor/github.com/portworx/kdmp/pkg/drivers/nfscsirestore/nfscsirestore.go @@ -44,7 +44,8 @@ func (d Driver) StartJob(opts ...drivers.JobOption) (id string, err error) { } // Create PV & PVC only in case of NFS. if o.NfsServer != "" { - err := utils.CreateNFSPvPvcForJob(o.DataExportName, job.ObjectMeta.Namespace, o) + jobName := utils.GetCsiRestoreJobName(drivers.NFSCSIRestore, o.DataExportName) + err := utils.CreateNFSPvPvcForJob(jobName, job.ObjectMeta.Namespace, o) if err != nil { return "", err } @@ -131,7 +132,9 @@ func buildJob( funct := "NfsbuildJob" // Setup service account using same role permission as stork role logrus.Infof("Inside %s function", funct) - if err := utils.SetupNFSServiceAccount(jobOptions.DataExportName, jobOptions.Namespace, roleFor()); err != nil { + + jobName := utils.GetCsiRestoreJobName(drivers.NFSCSIRestore, jobOptions.DataExportName) + if err := utils.SetupNFSServiceAccount(jobName, jobOptions.Namespace, roleFor()); err != nil { errMsg := fmt.Sprintf("error creating service account %s/%s: %v", jobOptions.Namespace, jobOptions.DataExportName, err) logrus.Errorf("%s: %v", funct, errMsg) return nil, fmt.Errorf(errMsg) @@ -191,11 +194,12 @@ func jobForRestoreCSISnapshot( }, " ") labels := addJobLabels(jobOption.Labels) - + // changing job name to nfcsirestore-backupUID-volumeUID instead of deName for better identification + jobName := utils.GetCsiRestoreJobName(drivers.NFSCSIRestore, jobOption.DataExportName) nfsExecutorImage, imageRegistrySecret, err := utils.GetExecutorImageAndSecret(drivers.NfsExecutorImage, jobOption.NfsImageExecutorSource, jobOption.NfsImageExecutorSourceNs, - jobOption.DataExportName, + jobName, jobOption) if err != nil { logrus.Errorf("failed to get the executor image details") @@ -208,8 +212,6 @@ func jobForRestoreCSISnapshot( return nil, fmt.Errorf("failed to get the toleration details for job [%s/%s]", jobOption.Namespace, jobOption.DataExportName) } - // changing job name to nfcsirestore-backupUID-volumeUID instead of deName for better identification - jobName := drivers.NFSCSIRestore + strings.SplitN(jobOption.DataExportName, "restore", 2)[1] job := &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ Name: jobName, @@ -227,7 +229,7 @@ func jobForRestoreCSISnapshot( }, Spec: corev1.PodSpec{ RestartPolicy: corev1.RestartPolicyOnFailure, - ServiceAccountName: jobOption.DataExportName, + ServiceAccountName: jobName, Containers: []corev1.Container{ { Name: drivers.NfsExecutorImage, @@ -255,7 +257,7 @@ func jobForRestoreCSISnapshot( Name: "cred-secret", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: utils.GetCredSecretName(jobOption.DataExportName), + SecretName: utils.GetCredSecretName(jobName), }, }, }, @@ -266,7 +268,7 @@ func jobForRestoreCSISnapshot( } // Add the image secret in job spec only if it is present in the stork deployment. if len(imageRegistrySecret) != 0 { - job.Spec.Template.Spec.ImagePullSecrets = utils.ToImagePullSecret(utils.GetImageSecretName(jobOption.DataExportName)) + job.Spec.Template.Spec.ImagePullSecrets = utils.ToImagePullSecret(utils.GetImageSecretName(jobName)) } if len(jobOption.NfsServer) != 0 { volumeMount := corev1.VolumeMount{ @@ -281,7 +283,7 @@ func jobForRestoreCSISnapshot( Name: utils.NfsVolumeName, VolumeSource: corev1.VolumeSource{ PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: utils.GetPvcNameForJob(jobOption.DataExportName), + ClaimName: utils.GetPvcNameForJob(jobName), }, }, } diff --git a/vendor/github.com/portworx/kdmp/pkg/drivers/utils/common.go b/vendor/github.com/portworx/kdmp/pkg/drivers/utils/common.go index b3990c9b48..44036c01a4 100644 --- a/vendor/github.com/portworx/kdmp/pkg/drivers/utils/common.go +++ b/vendor/github.com/portworx/kdmp/pkg/drivers/utils/common.go @@ -2,6 +2,7 @@ package utils import ( "fmt" + "strings" "time" version "github.com/hashicorp/go-version" @@ -57,6 +58,11 @@ var ( JobPodBackOffLimit = int32(10) ) +// GetCsiRestoreJobName returns a CSI restore Job name based on name of the CR +func GetCsiRestoreJobName(jobPrefix string, crName string) string { + return jobPrefix + strings.SplitN(crName, "restore", 2)[1] +} + // isServiceAccountSecretMissing returns true, if the K8s version does not support secret token for the service account. func isServiceAccountSecretMissing() (bool, error) { k8sVersion, _, err := storkversion.GetFullVersion() @@ -141,6 +147,12 @@ func CleanServiceAccount(name, namespace string) error { if err := coreops.Instance().DeleteServiceAccount(name, namespace); err != nil && !errors.IsNotFound(err) { return fmt.Errorf("delete %s/%s serviceaccount: %s", namespace, name, err) } + if err := rbacops.Instance().DeleteClusterRole(name); err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("delete %s/%s clusterrole: %s", namespace, name, err) + } + if err := rbacops.Instance().DeleteClusterRoleBinding(name); err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("delete %s/%s clusterrolebinding: %s", namespace, name, err) + } return nil } diff --git a/vendor/github.com/portworx/kdmp/pkg/executor/common.go b/vendor/github.com/portworx/kdmp/pkg/executor/common.go index 21196bc8df..01906b11eb 100644 --- a/vendor/github.com/portworx/kdmp/pkg/executor/common.go +++ b/vendor/github.com/portworx/kdmp/pkg/executor/common.go @@ -8,11 +8,13 @@ import ( "path/filepath" "reflect" "strconv" + "strings" "time" storkapi "github.com/libopenstorage/stork/pkg/apis/stork/v1alpha1" "github.com/libopenstorage/stork/pkg/crypto" "github.com/libopenstorage/stork/pkg/log" + storkutils "github.com/libopenstorage/stork/pkg/utils" kdmpapi "github.com/portworx/kdmp/pkg/apis/kdmp/v1alpha1" "github.com/portworx/kdmp/pkg/drivers" "github.com/portworx/kdmp/pkg/drivers/utils" @@ -666,6 +668,23 @@ func CreateNamespacesFromMapping( return createNamespaces(backup, restore.Spec.BackupLocation, restore.Namespace, restore) } +func getRancherProjectMapping(restore *storkapi.ApplicationRestore) map[string]string { + rancherProjectMapping := map[string]string{} + if restore.Spec.RancherProjectMapping != nil { + for key, value := range restore.Spec.RancherProjectMapping { + rancherProjectMapping[key] = value + dataKey := strings.Split(key, ":") + dataVal := strings.Split(value, ":") + if len(dataKey) == 2 && len(dataVal) == 2 { + rancherProjectMapping[dataKey[1]] = dataVal[1] + } else if len(dataKey) == 1 && len(dataVal) == 2 { + rancherProjectMapping[dataKey[0]] = dataVal[1] + } + } + } + return rancherProjectMapping +} + func createNamespaces(backup *storkapi.ApplicationBackup, backupLocation string, backupLocationNamespace string, @@ -688,6 +707,8 @@ func createNamespaces(backup *storkapi.ApplicationBackup, if err != nil { return err } + + rancherProjectMapping := getRancherProjectMapping(restore) if nsData != nil { if err = json.Unmarshal(nsData, &namespaces); err != nil { return err @@ -699,6 +720,8 @@ func createNamespaces(backup *storkapi.ApplicationBackup, // Skip namespaces we aren't restoring continue } + storkutils.ParseRancherProjectMapping(ns.Annotations, rancherProjectMapping) + storkutils.ParseRancherProjectMapping(ns.Labels, rancherProjectMapping) // create mapped restore namespace with metadata of backed up // namespace _, err := core.Instance().CreateNamespace(&v1.Namespace{ @@ -727,8 +750,11 @@ func createNamespaces(backup *storkapi.ApplicationBackup, if annotations == nil { annotations = make(map[string]string) } + // Add all annotations from Namespace.json except project annotations when not found in the namespace which is not created by px-backup. + // With retain policy, project annotations should not be applied to the namespace with no projects. Applies to a scenario where project association + // is removed by the user after taking the backup with project on a namespace for k, v := range ns.GetAnnotations() { - if _, ok := annotations[k]; !ok { + if _, ok := annotations[k]; !ok && (!strings.Contains(k, storkutils.CattleProjectPrefix) || annotations[storkutils.PxbackupAnnotationCreateByKey] != "") { annotations[k] = v } } @@ -736,12 +762,19 @@ func createNamespaces(backup *storkapi.ApplicationBackup, if labels == nil { labels = make(map[string]string) } + // Add all labels from Namespace.json except project labels when not found in the namespace which is not created by px-backup. + // With retain policy, project labels should not be applied to the namespace with no projects. Applies to a scenario where project association + // is removed by the user after taking the backup with project on a namespace for k, v := range ns.GetLabels() { - if _, ok := labels[k]; !ok { + if _, ok := labels[k]; !ok && (!strings.Contains(k, storkutils.CattleProjectPrefix) || annotations[storkutils.PxbackupAnnotationCreateByKey] != "") { labels[k] = v } } + storkutils.ParseRancherProjectMapping(annotations, rancherProjectMapping) + storkutils.ParseRancherProjectMapping(labels, rancherProjectMapping) } + // delete the px backup CreateByKey Annotation + delete(annotations, storkutils.PxbackupAnnotationCreateByKey) log.ApplicationRestoreLog(restore).Tracef("Namespace already exists, updating dest namespace %v", ns.Name) _, err = core.Instance().UpdateNamespace(&v1.Namespace{ ObjectMeta: metav1.ObjectMeta{ diff --git a/vendor/github.com/portworx/kdmp/pkg/version/version.go b/vendor/github.com/portworx/kdmp/pkg/version/version.go index 69b92e5090..b905af09cd 100644 --- a/vendor/github.com/portworx/kdmp/pkg/version/version.go +++ b/vendor/github.com/portworx/kdmp/pkg/version/version.go @@ -24,7 +24,7 @@ const ( // // These variables typically come from -ldflags settings. var ( - gitVersion = "1.2.6-dev" + gitVersion = "master-latest" gitCommit = "" // sha1 from git, output of $(git rev-parse HEAD) buildDate = "1970-01-01T00:00:00Z" // build date in ISO8601 format, output of $(date -u +'%Y-%m-%dT%H:%M:%SZ') kbVerRegex = regexp.MustCompile(`^(v\d+\.\d+\.\d+)(.*)`) diff --git a/vendor/modules.txt b/vendor/modules.txt index 820e797aba..cf5440e321 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -927,7 +927,7 @@ github.com/pkg/errors # github.com/pmezard/go-difflib v1.0.0 ## explicit github.com/pmezard/go-difflib/difflib -# github.com/portworx/kdmp v0.4.1-0.20230607162028-5246250670c5 => github.com/portworx/kdmp v0.4.1-0.20230607162028-5246250670c5 +# github.com/portworx/kdmp v0.4.1-0.20230719084848-d9ea7eeb2c87 => github.com/portworx/kdmp v0.4.1-0.20230719084848-d9ea7eeb2c87 ## explicit; go 1.19 github.com/portworx/kdmp/pkg/apis/kdmp github.com/portworx/kdmp/pkg/apis/kdmp/v1alpha1 @@ -2285,7 +2285,7 @@ sigs.k8s.io/yaml # github.com/kubernetes-incubator/external-storage v0.20.4-openstorage-rc10 => github.com/kubernetes-incubator/external-storage v0.25.1-openstorage-rc1 # github.com/libopenstorage/autopilot-api => github.com/libopenstorage/autopilot-api v0.6.1-0.20210301232050-ca2633c6e114 # github.com/libopenstorage/openstorage => github.com/libopenstorage/openstorage v0.0.0-20230511212757-41751b27d69f -# github.com/portworx/kdmp => github.com/portworx/kdmp v0.4.1-0.20230607162028-5246250670c5 +# github.com/portworx/kdmp => github.com/portworx/kdmp v0.4.1-0.20230719084848-d9ea7eeb2c87 # github.com/portworx/sched-ops => github.com/portworx/sched-ops v1.20.4-rc1.0.20230426231724-8d9f2c104721 # github.com/portworx/torpedo => github.com/portworx/torpedo v0.0.0-20230410110229-ad7b0df86670 # gopkg.in/fsnotify.v1 v1.4.7 => github.com/fsnotify/fsnotify v1.4.7