Merge pull request #6445 from Lyndon-Li/fix-concurrent-repo-ensure-problem

Fix concurrent repo ensure problem

pkg/exposer/csi_snapshot.go

@@ -202,7 +202,7 @@ func (e *csiSnapshotExposer) GetExposed(ctx context.Context, ownerObject corev1.
 	}, pod)
 	if err != nil {
 		if apierrors.IsNotFound(err) {
-			curLog.WithField("backup pod", backupPodName).Errorf("Backup pod is not running in the current node %s", exposeWaitParam.NodeName)
+			curLog.WithField("backup pod", backupPodName).Debugf("Backup pod is not running in the current node %s", exposeWaitParam.NodeName)
 			return nil, nil
 		} else {
 			return nil, errors.Wrapf(err, "error to get backup pod %s", backupPodName)

pkg/exposer/generic_restore.go

@@ -123,7 +123,7 @@ func (e *genericRestoreExposer) GetExposed(ctx context.Context, ownerObject core
 	}, pod)
 	if err != nil {
 		if apierrors.IsNotFound(err) {
-			curLog.WithField("backup pod", restorePodName).Error("Backup pod is not running in the current node")
+			curLog.WithField("backup pod", restorePodName).Debug("Backup pod is not running in the current node")
 			return nil, nil
 		} else {
 			return nil, errors.Wrapf(err, "error to get backup pod %s", restorePodName)

pkg/repository/ensurer.go

@@ -80,20 +80,18 @@ func (r *Ensurer) EnsureRepo(ctx context.Context, namespace, volumeNamespace, ba
 		log.Debug("Released lock")
 	}()
 
-	repo, err := GetBackupRepository(ctx, r.repoClient, namespace, backupRepoKey, true)
+	_, err := GetBackupRepository(ctx, r.repoClient, namespace, backupRepoKey, false)
 	if err == nil {
-		log.Debug("Ready repository found")
-		return repo, nil
-	}
+		log.Info("Founding existing repo")
+		return r.waitBackupRepository(ctx, namespace, backupRepoKey)
+	} else if isBackupRepositoryNotFoundError(err) {
+		log.Info("No repository found, creating one")
 
-	if !isBackupRepositoryNotFoundError(err) {
+		// no repo found: create one and wait for it to be ready
+		return r.createBackupRepositoryAndWait(ctx, namespace, backupRepoKey)
+	} else {
 		return nil, errors.WithStack(err)
 	}
-
-	log.Debug("No repository found, creating one")
-
-	// no repo found: create one and wait for it to be ready
-	return r.createBackupRepositoryAndWait(ctx, namespace, backupRepoKey)
 }
 
 func (r *Ensurer) repoLock(key BackupRepositoryKey) *sync.Mutex {
@@ -113,6 +111,10 @@ func (r *Ensurer) createBackupRepositoryAndWait(ctx context.Context, namespace s
 		return nil, errors.Wrap(err, "unable to create backup repository resource")
 	}
 
+	return r.waitBackupRepository(ctx, namespace, backupRepoKey)
+}
+
+func (r *Ensurer) waitBackupRepository(ctx context.Context, namespace string, backupRepoKey BackupRepositoryKey) (*velerov1api.BackupRepository, error) {
 	var repo *velerov1api.BackupRepository
 	checkFunc := func(ctx context.Context) (bool, error) {
 		found, err := GetBackupRepository(ctx, r.repoClient, namespace, backupRepoKey, true)

pkg/repository/ensurer_test.go

@@ -30,13 +30,19 @@ import (
 )
 
 func TestEnsureRepo(t *testing.T) {
-	bkRepoObj := NewBackupRepository(velerov1.DefaultNamespace, BackupRepositoryKey{
+	bkRepoObjReady := NewBackupRepository(velerov1.DefaultNamespace, BackupRepositoryKey{
 		VolumeNamespace: "fake-ns",
 		BackupLocation:  "fake-bsl",
 		RepositoryType:  "fake-repo-type",
 	})
 
-	bkRepoObj.Status.Phase = velerov1.BackupRepositoryPhaseReady
+	bkRepoObjReady.Status.Phase = velerov1.BackupRepositoryPhaseReady
+
+	bkRepoObjNotReady := NewBackupRepository(velerov1.DefaultNamespace, BackupRepositoryKey{
+		VolumeNamespace: "fake-ns",
+		BackupLocation:  "fake-bsl",
+		RepositoryType:  "fake-repo-type",
+	})
 
 	scheme := runtime.NewScheme()
 	velerov1.AddToScheme(scheme)
@@ -82,10 +88,21 @@ func TestEnsureRepo(t *testing.T) {
 			bsl:            "fake-bsl",
 			repositoryType: "fake-repo-type",
 			kubeClientObj: []runtime.Object{
-				bkRepoObj,
+				bkRepoObjReady,
 			},
 			runtimeScheme: scheme,
-			expectedRepo:  bkRepoObj,
+			expectedRepo:  bkRepoObjReady,
+		},
+		{
+			name:           "wait existing repo fail",
+			namespace:      "fake-ns",
+			bsl:            "fake-bsl",
+			repositoryType: "fake-repo-type",
+			kubeClientObj: []runtime.Object{
+				bkRepoObjNotReady,
+			},
+			runtimeScheme: scheme,
+			err:           "failed to wait BackupRepository: timed out waiting for the condition",
 		},
 		{
 			name:           "create fail",