Merge pull request #8533 from blackpiglet/refactor_migration_e2e_release_1.15

[cherry-pick] Refactor the migration cases.

pkg/cmd/cli/install/install.go

@@ -42,33 +42,31 @@ import (
 
 // Options collects all the options for installing Velero into a Kubernetes cluster.
 type Options struct {
-	Namespace                 string
-	Image                     string
-	BucketName                string
-	Prefix                    string
-	ProviderName              string
-	PodAnnotations            flag.Map
-	PodLabels                 flag.Map
-	ServiceAccountAnnotations flag.Map
-	ServiceAccountName        string
-	VeleroPodCPURequest       string
-	VeleroPodMemRequest       string
-	VeleroPodCPULimit         string
-	VeleroPodMemLimit         string
-	NodeAgentPodCPURequest    string
-	NodeAgentPodMemRequest    string
-	NodeAgentPodCPULimit      string
-	NodeAgentPodMemLimit      string
-	RestoreOnly               bool
-	SecretFile                string
-	NoSecret                  bool
-	DryRun                    bool
-	BackupStorageConfig       flag.Map
-	VolumeSnapshotConfig      flag.Map
-	UseNodeAgent              bool
-	PrivilegedNodeAgent       bool
-	//TODO remove UseRestic when migration test out of using it
-	UseRestic                       bool
+	Namespace                       string
+	Image                           string
+	BucketName                      string
+	Prefix                          string
+	ProviderName                    string
+	PodAnnotations                  flag.Map
+	PodLabels                       flag.Map
+	ServiceAccountAnnotations       flag.Map
+	ServiceAccountName              string
+	VeleroPodCPURequest             string
+	VeleroPodMemRequest             string
+	VeleroPodCPULimit               string
+	VeleroPodMemLimit               string
+	NodeAgentPodCPURequest          string
+	NodeAgentPodMemRequest          string
+	NodeAgentPodCPULimit            string
+	NodeAgentPodMemLimit            string
+	RestoreOnly                     bool
+	SecretFile                      string
+	NoSecret                        bool
+	DryRun                          bool
+	BackupStorageConfig             flag.Map
+	VolumeSnapshotConfig            flag.Map
+	UseNodeAgent                    bool
+	PrivilegedNodeAgent             bool
 	Wait                            bool
 	UseVolumeSnapshots              bool
 	DefaultRepoMaintenanceFrequency time.Duration

test/e2e/e2e_suite_test.go

@@ -575,9 +575,9 @@ var _ = Describe(
 )
 
 var _ = Describe(
-	"Migrate resources between clusters by Restic",
-	Label("Migration", "Restic"),
-	MigrationWithRestic,
+	"Migrate resources between clusters by FileSystem backup",
+	Label("Migration", "FSB"),
+	MigrationWithFS,
 )
 var _ = Describe(
 	"Migrate resources between clusters by snapshot",

test/e2e/migration/migration.go

@@ -17,388 +17,455 @@ package migration
 
 import (
 	"context"
-	"flag"
 	"fmt"
 	"strings"
 	"time"
 
-	"github.com/google/uuid"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
-	. "github.com/vmware-tanzu/velero/test"
+	"github.com/vmware-tanzu/velero/test"
+	framework "github.com/vmware-tanzu/velero/test/e2e/test"
 	util "github.com/vmware-tanzu/velero/test/util/csi"
-	. "github.com/vmware-tanzu/velero/test/util/k8s"
-	. "github.com/vmware-tanzu/velero/test/util/kibishii"
-	. "github.com/vmware-tanzu/velero/test/util/providers"
-	. "github.com/vmware-tanzu/velero/test/util/velero"
+	k8sutil "github.com/vmware-tanzu/velero/test/util/k8s"
+	"github.com/vmware-tanzu/velero/test/util/kibishii"
+	"github.com/vmware-tanzu/velero/test/util/providers"
+	veleroutil "github.com/vmware-tanzu/velero/test/util/velero"
 )
 
-var migrationNamespace string
-var veleroCfg VeleroConfig
+type migrationE2E struct {
+	framework.TestCase
+	useVolumeSnapshots bool
+	veleroCLI2Version  test.VeleroCLI2Version
+	kibishiiData       kibishii.KibishiiData
+}
 
 func MigrationWithSnapshots() {
-	veleroCfg = VeleroCfg
-	for _, veleroCLI2Version := range GetVersionList(veleroCfg.MigrateFromVeleroCLI, veleroCfg.MigrateFromVeleroVersion) {
-		MigrationTest(true, veleroCLI2Version)
+	for _, veleroCLI2Version := range veleroutil.GetVersionList(
+		test.VeleroCfg.MigrateFromVeleroCLI,
+		test.VeleroCfg.MigrateFromVeleroVersion) {
+		framework.TestFunc(
+			&migrationE2E{
+				useVolumeSnapshots: true,
+				veleroCLI2Version:  veleroCLI2Version,
+			},
+		)()
 	}
 }
 
-func MigrationWithRestic() {
-	veleroCfg = VeleroCfg
-	for _, veleroCLI2Version := range GetVersionList(veleroCfg.MigrateFromVeleroCLI, veleroCfg.MigrateFromVeleroVersion) {
-		MigrationTest(false, veleroCLI2Version)
+func MigrationWithFS() {
+	for _, veleroCLI2Version := range veleroutil.GetVersionList(
+		test.VeleroCfg.MigrateFromVeleroCLI,
+		test.VeleroCfg.MigrateFromVeleroVersion) {
+		framework.TestFunc(
+			&migrationE2E{
+				useVolumeSnapshots: false,
+				veleroCLI2Version:  veleroCLI2Version,
+			},
+		)()
 	}
 }
 
-func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) {
-	var (
-		backupName, restoreName     string
-		backupScName, restoreScName string
-		kibishiiWorkerCount         int
-		err                         error
-	)
-	BeforeEach(func() {
-		kibishiiWorkerCount = 3
-		veleroCfg = VeleroCfg
-		UUIDgen, err = uuid.NewRandom()
-		migrationNamespace = "migration-" + UUIDgen.String()
-		if useVolumeSnapshots && veleroCfg.CloudProvider == Kind {
-			Skip(fmt.Sprintf("Volume snapshots not supported on %s", Kind))
-		}
-
-		if veleroCfg.DefaultClusterContext == "" && veleroCfg.StandbyClusterContext == "" {
+func (m *migrationE2E) Init() error {
+	By("Skip check", func() {
+		if m.VeleroCfg.DefaultClusterContext == "" && m.VeleroCfg.StandbyClusterContext == "" {
 			Skip("Migration test needs 2 clusters")
 		}
-		// need to uninstall Velero first in case of the affection of the existing global velero installation
-		if InstallVelero {
-			By("Uninstall Velero", func() {
-				ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5)
-				defer ctxCancel()
-				Expect(VeleroUninstall(ctx, veleroCfg)).To(Succeed())
-			})
+
+		if m.useVolumeSnapshots && m.VeleroCfg.CloudProvider == test.Kind {
+			Skip(fmt.Sprintf("Volume snapshots not supported on %s", test.Kind))
+		}
+
+		if m.VeleroCfg.SnapshotMoveData && !m.useVolumeSnapshots {
+			Skip("FSB migration test is not needed in data mover scenario")
 		}
 	})
-	AfterEach(func() {
-		if CurrentSpecReport().Failed() && veleroCfg.FailFast {
-			fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.")
+
+	By("Call the base E2E init", func() {
+		Expect(m.TestCase.Init()).To(Succeed())
+	})
+
+	m.kibishiiData = *kibishii.DefaultKibishiiData
+	m.kibishiiData.ExpectedNodes = 3
+	m.CaseBaseName = "migration-" + m.UUIDgen
+	m.BackupName = m.CaseBaseName + "-backup"
+	m.RestoreName = m.CaseBaseName + "-restore"
+	m.NSIncluded = &[]string{m.CaseBaseName}
+
+	m.RestoreArgs = []string{
+		"create", "--namespace", m.VeleroCfg.VeleroNamespace,
+		"restore", m.RestoreName,
+		"--from-backup", m.BackupName, "--wait",
+	}
+
+	// Message output by ginkgo
+	m.TestMsg = &framework.TestMSG{
+		Desc:      "Test migration workload on two clusters",
+		FailedMSG: "Fail to test migrate between two clusters",
+		Text:      "Test back up on default cluster, restore on standby cluster",
+	}
+
+	// Need to uninstall Velero on the default cluster.
+	if test.InstallVelero {
+		ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5)
+		defer ctxCancel()
+		Expect(veleroutil.VeleroUninstall(ctx, m.VeleroCfg)).To(Succeed())
+	}
+
+	return nil
+}
+
+func (m *migrationE2E) Backup() error {
+	OriginVeleroCfg := m.VeleroCfg
+	var err error
+
+	if m.veleroCLI2Version.VeleroCLI == "" {
+		//Assume tag of velero server image is identical to velero CLI version
+		//Download velero CLI if it's empty according to velero CLI version
+		By(
+			fmt.Sprintf("Install the expected version Velero CLI %s",
+				m.veleroCLI2Version.VeleroVersion),
+			func() {
+				// "self" represents 1.14.x and future versions
+				if m.veleroCLI2Version.VeleroVersion == "self" {
+					m.veleroCLI2Version.VeleroCLI = m.VeleroCfg.VeleroCLI
+				} else {
+					OriginVeleroCfg, err = veleroutil.SetImagesToDefaultValues(
+						OriginVeleroCfg,
+						m.veleroCLI2Version.VeleroVersion,
+					)
+					Expect(err).To(Succeed(),
+						"Fail to set images for the migrate-from Velero installation.")
+
+					m.veleroCLI2Version.VeleroCLI, err = veleroutil.InstallVeleroCLI(
+						m.veleroCLI2Version.VeleroVersion)
+					Expect(err).To(Succeed())
+				}
+			},
+		)
+	}
+
+	By(fmt.Sprintf("Install Velero on default cluster (%s)", m.VeleroCfg.DefaultClusterContext),
+		func() {
+			Expect(k8sutil.KubectlConfigUseContext(
+				m.Ctx, m.VeleroCfg.DefaultClusterContext)).To(Succeed())
+			OriginVeleroCfg.MigrateFromVeleroVersion = m.veleroCLI2Version.VeleroVersion
+			OriginVeleroCfg.VeleroCLI = m.veleroCLI2Version.VeleroCLI
+			OriginVeleroCfg.ClientToInstallVelero = OriginVeleroCfg.DefaultClient
+			OriginVeleroCfg.ClusterToInstallVelero = m.VeleroCfg.DefaultClusterName
+			OriginVeleroCfg.ServiceAccountNameToInstall = m.VeleroCfg.DefaultCLSServiceAccountName
+			OriginVeleroCfg.UseVolumeSnapshots = m.useVolumeSnapshots
+			OriginVeleroCfg.UseNodeAgent = !m.useVolumeSnapshots
+
+			version, err := veleroutil.GetVeleroVersion(m.Ctx, OriginVeleroCfg.VeleroCLI, true)
+			Expect(err).To(Succeed(), "Fail to get Velero version")
+			OriginVeleroCfg.VeleroVersion = version
+
+			if OriginVeleroCfg.SnapshotMoveData {
+				OriginVeleroCfg.UseNodeAgent = true
+			}
+
+			Expect(veleroutil.VeleroInstall(m.Ctx, &OriginVeleroCfg, false)).To(Succeed())
+			if m.veleroCLI2Version.VeleroVersion != "self" {
+				Expect(veleroutil.CheckVeleroVersion(
+					m.Ctx,
+					OriginVeleroCfg.VeleroCLI,
+					OriginVeleroCfg.MigrateFromVeleroVersion,
+				)).To(Succeed())
+			}
+		},
+	)
+
+	By("Create namespace for sample workload", func() {
+		Expect(k8sutil.CreateNamespace(
+			m.Ctx,
+			*m.VeleroCfg.DefaultClient,
+			m.CaseBaseName,
+		)).To(Succeed(),
+			fmt.Sprintf("Failed to create namespace %s to install Kibishii workload",
+				m.CaseBaseName))
+	})
+
+	By("Deploy sample workload of Kibishii", func() {
+		Expect(kibishii.KibishiiPrepareBeforeBackup(
+			m.Ctx,
+			*OriginVeleroCfg.DefaultClient,
+			OriginVeleroCfg.CloudProvider,
+			m.CaseBaseName,
+			OriginVeleroCfg.RegistryCredentialFile,
+			OriginVeleroCfg.Features,
+			OriginVeleroCfg.KibishiiDirectory,
+			OriginVeleroCfg.UseVolumeSnapshots,
+			&m.kibishiiData,
+		)).To(Succeed())
+	})
+
+	By(fmt.Sprintf("Backup namespace %s", m.CaseBaseName), func() {
+		m.BackupArgs = []string{
+			"create", "--namespace", m.VeleroCfg.VeleroNamespace,
+			"backup", m.BackupName,
+			"--include-namespaces", strings.Join(*m.NSIncluded, ","),
+			"--wait",
+		}
+
+		if m.useVolumeSnapshots {
+			m.BackupArgs = append(m.BackupArgs, "--snapshot-volumes=true")
 		} else {
-			By(fmt.Sprintf("Uninstall Velero on cluster %s", veleroCfg.DefaultClusterContext), func() {
-				ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5)
-				defer ctxCancel()
-
-				Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed())
-				veleroCfg.ClientToInstallVelero = veleroCfg.DefaultClient
-				veleroCfg.ClusterToInstallVelero = veleroCfg.DefaultClusterName
-				Expect(VeleroUninstall(ctx, veleroCfg)).To(Succeed())
-
-				By(fmt.Sprintf("Delete sample workload namespace %s", migrationNamespace), func() {
-					Expect(
-						DeleteNamespace(
-							context.Background(),
-							*veleroCfg.DefaultClient,
-							migrationNamespace,
-							true),
-					).To(Succeed())
-				})
-			})
-
-			By(fmt.Sprintf("Uninstall Velero on cluster %s", veleroCfg.StandbyClusterContext), func() {
-				ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5)
-				defer ctxCancel()
-				Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyClusterContext)).To(Succeed())
-				veleroCfg.ClientToInstallVelero = veleroCfg.StandbyClient
-				veleroCfg.ClusterToInstallVelero = veleroCfg.StandbyClusterName
-
-				By("Delete StorageClasses created by E2E")
-				Expect(
-					DeleteStorageClass(
-						ctx,
-						*veleroCfg.ClientToInstallVelero,
-						StorageClassName,
-					),
-				).To(Succeed())
-				Expect(
-					DeleteStorageClass(
-						ctx,
-						*veleroCfg.ClientToInstallVelero,
-						StorageClassName2,
-					),
-				).To(Succeed())
-
-				if strings.EqualFold(veleroCfg.Features, FeatureCSI) &&
-					veleroCfg.UseVolumeSnapshots {
-					By("Delete VolumeSnapshotClass created by E2E")
-					Expect(
-						KubectlDeleteByFile(
-							ctx,
-							fmt.Sprintf("../testdata/volume-snapshot-class/%s.yaml", veleroCfg.CloudProvider),
-						),
-					).To(Succeed())
-				}
-
-				Expect(VeleroUninstall(ctx, veleroCfg)).To(Succeed())
-
-				By(fmt.Sprintf("Delete sample workload namespace %s", migrationNamespace), func() {
-					Expect(
-						DeleteNamespace(
-							context.Background(),
-							*veleroCfg.StandbyClient,
-							migrationNamespace,
-							true,
-						),
-					).To(Succeed())
-				})
-			})
-
-			By(fmt.Sprintf("Switch to default KubeConfig context %s", veleroCfg.DefaultClusterContext), func() {
-				Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed())
-				veleroCfg.ClientToInstallVelero = veleroCfg.DefaultClient
-				veleroCfg.ClusterToInstallVelero = veleroCfg.DefaultClusterName
-			})
+			m.BackupArgs = append(m.BackupArgs, "--default-volumes-to-fs-backup")
 		}
-	})
-	When("kibishii is the sample workload", func() {
-		It("should be successfully backed up and restored to the default BackupStorageLocation", func() {
-			var backupNames []string
-			if veleroCfg.SnapshotMoveData {
-				if !useVolumeSnapshots {
-					Skip("FSB migration test is not needed in data mover scenario")
-				}
-			}
-			oneHourTimeout, ctxCancel := context.WithTimeout(context.Background(), time.Minute*60)
-			defer ctxCancel()
-			flag.Parse()
-			UUIDgen, err = uuid.NewRandom()
-			Expect(err).To(Succeed())
-			supportUploaderType, err := IsSupportUploaderType(veleroCLI2Version.VeleroVersion)
-			Expect(err).To(Succeed())
 
-			OriginVeleroCfg := veleroCfg
-			if veleroCLI2Version.VeleroCLI == "" {
-				//Assume tag of velero server image is identical to velero CLI version
-				//Download velero CLI if it's empty according to velero CLI version
-				By(fmt.Sprintf("Install the expected version Velero CLI (%s) for installing Velero",
-					veleroCLI2Version.VeleroVersion), func() {
-					//"self" represents 1.14.x and future versions
-					if veleroCLI2Version.VeleroVersion == "self" {
-						veleroCLI2Version.VeleroCLI = veleroCfg.VeleroCLI
-					} else {
-						OriginVeleroCfg, err = SetImagesToDefaultValues(
-							OriginVeleroCfg,
-							veleroCLI2Version.VeleroVersion,
-						)
-						Expect(err).To(Succeed(), "Fail to set images for the migrate-from Velero installation.")
+		if OriginVeleroCfg.SnapshotMoveData {
+			m.BackupArgs = append(m.BackupArgs, "--snapshot-move-data")
+		}
 
-						veleroCLI2Version.VeleroCLI, err = InstallVeleroCLI(veleroCLI2Version.VeleroVersion)
-						Expect(err).To(Succeed())
-					}
-				})
-			}
-
-			By(fmt.Sprintf("Install Velero in cluster-A (%s) to backup workload", veleroCfg.DefaultClusterContext), func() {
-				Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed())
-				OriginVeleroCfg.MigrateFromVeleroVersion = veleroCLI2Version.VeleroVersion
-				OriginVeleroCfg.VeleroCLI = veleroCLI2Version.VeleroCLI
-				OriginVeleroCfg.ClientToInstallVelero = OriginVeleroCfg.DefaultClient
-				OriginVeleroCfg.ClusterToInstallVelero = veleroCfg.DefaultClusterName
-				OriginVeleroCfg.ServiceAccountNameToInstall = veleroCfg.DefaultCLSServiceAccountName
-				OriginVeleroCfg.UseVolumeSnapshots = useVolumeSnapshots
-				OriginVeleroCfg.UseNodeAgent = !useVolumeSnapshots
-
-				version, err := GetVeleroVersion(oneHourTimeout, OriginVeleroCfg.VeleroCLI, true)
-				Expect(err).To(Succeed(), "Fail to get Velero version")
-				OriginVeleroCfg.VeleroVersion = version
-
-				if OriginVeleroCfg.SnapshotMoveData {
-					OriginVeleroCfg.UseNodeAgent = true
-				}
-
-				Expect(VeleroInstall(context.Background(), &OriginVeleroCfg, false)).To(Succeed())
-				if veleroCLI2Version.VeleroVersion != "self" {
-					Expect(CheckVeleroVersion(context.Background(), OriginVeleroCfg.VeleroCLI,
-						OriginVeleroCfg.MigrateFromVeleroVersion)).To(Succeed())
-				}
-			})
-
-			backupName = "backup-" + UUIDgen.String()
-			backupScName = backupName + "-sc"
-			restoreName = "restore-" + UUIDgen.String()
-			restoreScName = restoreName + "-sc"
-
-			By("Create namespace for sample workload", func() {
-				Expect(CreateNamespace(oneHourTimeout, *veleroCfg.DefaultClient, migrationNamespace)).To(Succeed(),
-					fmt.Sprintf("Failed to create namespace %s to install Kibishii workload", migrationNamespace))
-			})
-
-			KibishiiData := *DefaultKibishiiData
-			By("Deploy sample workload of Kibishii", func() {
-				KibishiiData.ExpectedNodes = kibishiiWorkerCount
-				Expect(KibishiiPrepareBeforeBackup(oneHourTimeout, *veleroCfg.DefaultClient, veleroCfg.CloudProvider,
-					migrationNamespace, veleroCfg.RegistryCredentialFile, veleroCfg.Features,
-					veleroCfg.KibishiiDirectory, useVolumeSnapshots, &KibishiiData)).To(Succeed())
-			})
-
-			By(fmt.Sprintf("Backup namespace %s", migrationNamespace), func() {
-				var BackupStorageClassCfg BackupConfig
-				BackupStorageClassCfg.BackupName = backupScName
-				BackupStorageClassCfg.IncludeResources = "StorageClass"
-				BackupStorageClassCfg.IncludeClusterResources = true
-
-				//TODO Remove UseRestic parameter once minor version is 1.10 or upper
-				BackupStorageClassCfg.UseResticIfFSBackup = !supportUploaderType
-				Expect(VeleroBackupNamespace(context.Background(), OriginVeleroCfg.VeleroCLI,
-					OriginVeleroCfg.VeleroNamespace, BackupStorageClassCfg)).To(Succeed(), func() string {
-					RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, BackupStorageClassCfg.BackupName, "")
-					return "Fail to backup workload"
-				})
-				backupNames = append(backupNames, BackupStorageClassCfg.BackupName)
-
-				var BackupCfg BackupConfig
-				BackupCfg.BackupName = backupName
-				BackupCfg.Namespace = migrationNamespace
-				BackupCfg.UseVolumeSnapshots = useVolumeSnapshots
-				BackupCfg.BackupLocation = ""
-				BackupCfg.Selector = ""
-				BackupCfg.DefaultVolumesToFsBackup = !useVolumeSnapshots
-				//TODO Remove UseRestic parameter once minor version is 1.10 or upper
-				BackupCfg.UseResticIfFSBackup = !supportUploaderType
-				BackupCfg.SnapshotMoveData = OriginVeleroCfg.SnapshotMoveData
-
-				Expect(VeleroBackupNamespace(context.Background(), OriginVeleroCfg.VeleroCLI,
-					OriginVeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string {
-					RunDebug(context.Background(), OriginVeleroCfg.VeleroCLI, OriginVeleroCfg.VeleroNamespace, BackupCfg.BackupName, "")
-					return "Fail to backup workload"
-				})
-				backupNames = append(backupNames, BackupCfg.BackupName)
-			})
-
-			if useVolumeSnapshots {
-				// Only wait for the snapshots.backupdriver.cnsdp.vmware.com
-				// when the vSphere plugin is used.
-				if veleroCfg.HasVspherePlugin {
-					By("Waiting for vSphere uploads to complete", func() {
-						Expect(WaitForVSphereUploadCompletion(context.Background(), time.Hour,
-							migrationNamespace, kibishiiWorkerCount)).To(Succeed())
-					})
-				}
-
-				var snapshotCheckPoint SnapshotCheckPoint
-				snapshotCheckPoint.NamespaceBackedUp = migrationNamespace
-
-				if OriginVeleroCfg.SnapshotMoveData {
-					//VolumeSnapshotContent should be deleted after data movement
-					_, err := util.CheckVolumeSnapshotCR(*veleroCfg.DefaultClient, map[string]string{"namespace": migrationNamespace}, 0)
-					Expect(err).NotTo(HaveOccurred(), "VSC count is not as expected 0")
-				} else {
-					// the snapshots of AWS may be still in pending status when do the restore, wait for a while
-					// to avoid this https://github.com/vmware-tanzu/velero/issues/1799
-					// TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed
-					if veleroCfg.CloudProvider == Azure && strings.EqualFold(veleroCfg.Features, FeatureCSI) || veleroCfg.CloudProvider == AWS {
-						By("Sleep 5 minutes to avoid snapshot recreated by unknown reason ", func() {
-							time.Sleep(5 * time.Minute)
-						})
-					}
-
-					By("Snapshot should be created in cloud object store with retain policy", func() {
-						snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.DefaultClient, veleroCfg, kibishiiWorkerCount,
-							migrationNamespace, backupName, GetKibishiiPVCNameList(kibishiiWorkerCount))
-						Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint")
-						Expect(CheckSnapshotsInProvider(
-							veleroCfg,
-							backupName,
-							snapshotCheckPoint,
-							false,
-						)).To(Succeed())
-					})
-				}
-			}
-
-			By(fmt.Sprintf("Install Velero in cluster-B (%s) to restore workload", veleroCfg.StandbyClusterContext), func() {
-				//Ensure workload of "migrationNamespace" existed in cluster-A
-				ns, err := GetNamespace(context.Background(), *veleroCfg.DefaultClient, migrationNamespace)
-				Expect(ns.Name).To(Equal(migrationNamespace))
-				Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("get namespace in source cluster err: %v", err))
-
-				//Ensure cluster-B is the target cluster
-				Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyClusterContext)).To(Succeed())
-				_, err = GetNamespace(context.Background(), *veleroCfg.StandbyClient, migrationNamespace)
-				Expect(err).To(HaveOccurred(), fmt.Sprintf("get namespace in dst cluster successfully, it's not as expected: %s", migrationNamespace))
-				fmt.Println(err)
-				Expect(strings.Contains(fmt.Sprint(err), "namespaces \""+migrationNamespace+"\" not found")).Should(BeTrue())
-
-				veleroCfg.ClientToInstallVelero = veleroCfg.StandbyClient
-				veleroCfg.ClusterToInstallVelero = veleroCfg.StandbyClusterName
-				veleroCfg.ServiceAccountNameToInstall = veleroCfg.StandbyCLSServiceAccountName
-				veleroCfg.UseNodeAgent = !useVolumeSnapshots
-				veleroCfg.UseRestic = false
-				if veleroCfg.SnapshotMoveData {
-					veleroCfg.UseNodeAgent = true
-					// For SnapshotMoveData pipelines, we should use standby cluster setting for Velero installation
-					// In nightly CI, StandbyClusterPlugins is set properly if pipeline is for SnapshotMoveData.
-					veleroCfg.Plugins = veleroCfg.StandbyClusterPlugins
-					veleroCfg.ObjectStoreProvider = veleroCfg.StandbyClusterObjectStoreProvider
-				}
-
-				By("Install StorageClass for E2E.")
-				Expect(InstallStorageClasses(veleroCfg.StandbyClusterCloudProvider)).To(Succeed())
-
-				if strings.EqualFold(veleroCfg.Features, FeatureCSI) &&
-					veleroCfg.UseVolumeSnapshots {
-					By("Install VolumeSnapshotClass for E2E.")
-					Expect(
-						KubectlApplyByFile(
-							context.Background(),
-							fmt.Sprintf("../testdata/volume-snapshot-class/%s.yaml", veleroCfg.StandbyClusterCloudProvider),
-						),
-					).To(Succeed())
-				}
-
-				Expect(VeleroInstall(context.Background(), &veleroCfg, true)).To(Succeed())
-			})
-
-			By(fmt.Sprintf("Waiting for backups sync to Velero in cluster-B (%s)", veleroCfg.StandbyClusterContext), func() {
-				Expect(WaitForBackupToBeCreated(context.Background(), backupName, 5*time.Minute, &veleroCfg)).To(Succeed())
-				Expect(WaitForBackupToBeCreated(context.Background(), backupScName, 5*time.Minute, &veleroCfg)).To(Succeed())
-			})
-
-			By(fmt.Sprintf("Restore %s", migrationNamespace), func() {
-				if OriginVeleroCfg.SnapshotMoveData {
-					cmName := "datamover-storage-class-config"
-					labels := map[string]string{"velero.io/change-storage-class": "RestoreItemAction",
-						"velero.io/plugin-config": ""}
-					data := map[string]string{KibishiiStorageClassName: StorageClassName}
-
-					By(fmt.Sprintf("Create ConfigMap %s in namespace %s", cmName, veleroCfg.VeleroNamespace), func() {
-						_, err := CreateConfigMap(veleroCfg.StandbyClient.ClientGo, veleroCfg.VeleroNamespace, cmName, labels, data)
-						Expect(err).To(Succeed(), fmt.Sprintf("failed to create configmap in the namespace %q", veleroCfg.VeleroNamespace))
-					})
-				} else {
-					Expect(VeleroRestore(context.Background(), veleroCfg.VeleroCLI,
-						veleroCfg.VeleroNamespace, restoreScName, backupScName, "StorageClass")).To(Succeed(), func() string {
-						RunDebug(context.Background(), veleroCfg.VeleroCLI,
-							veleroCfg.VeleroNamespace, "", restoreName)
-						return "Fail to restore workload"
-					})
-				}
-				Expect(VeleroRestore(context.Background(), veleroCfg.VeleroCLI,
-					veleroCfg.VeleroNamespace, restoreName, backupName, "")).To(Succeed(), func() string {
-					RunDebug(context.Background(), veleroCfg.VeleroCLI,
-						veleroCfg.VeleroNamespace, "", restoreName)
-					return "Fail to restore workload"
-				})
-			})
-
-			By(fmt.Sprintf("Verify workload %s after restore ", migrationNamespace), func() {
-				Expect(KibishiiVerifyAfterRestore(*veleroCfg.StandbyClient, migrationNamespace,
-					oneHourTimeout, &KibishiiData, "")).To(Succeed(), "Fail to verify workload after restore")
-			})
-
-			// TODO: delete backup created by case self, not all
-			By("Clean backups after test", func() {
-				veleroCfg.ClientToInstallVelero = veleroCfg.DefaultClient
-				Expect(DeleteBackups(context.Background(), backupNames, &veleroCfg)).To(Succeed())
-			})
+		Expect(veleroutil.VeleroBackupExec(
+			m.Ctx,
+			OriginVeleroCfg.VeleroCLI,
+			OriginVeleroCfg.VeleroNamespace,
+			m.BackupName,
+			m.BackupArgs,
+		)).To(Succeed(), func() string {
+			veleroutil.RunDebug(
+				context.Background(),
+				OriginVeleroCfg.VeleroCLI,
+				OriginVeleroCfg.VeleroNamespace,
+				m.BackupName,
+				"",
+			)
+			return "Failed to backup resources"
 		})
 	})
+
+	if m.useVolumeSnapshots {
+		// Only wait for the snapshots.backupdriver.cnsdp.vmware.com
+		// when the vSphere plugin is used.
+		if OriginVeleroCfg.HasVspherePlugin {
+			By("Waiting for vSphere uploads to complete", func() {
+				Expect(
+					veleroutil.WaitForVSphereUploadCompletion(
+						context.Background(),
+						time.Hour,
+						m.CaseBaseName,
+						m.kibishiiData.ExpectedNodes,
+					),
+				).To(Succeed())
+			})
+		}
+
+		var snapshotCheckPoint test.SnapshotCheckPoint
+		snapshotCheckPoint.NamespaceBackedUp = m.CaseBaseName
+
+		if OriginVeleroCfg.SnapshotMoveData {
+			//VolumeSnapshotContent should be deleted after data movement
+			_, err := util.CheckVolumeSnapshotCR(
+				*m.VeleroCfg.DefaultClient,
+				map[string]string{"namespace": m.CaseBaseName},
+				0,
+			)
+			By("Check the VSC account", func() {
+				Expect(err).NotTo(HaveOccurred(), "VSC count is not as expected 0")
+			})
+		} else {
+			// the snapshots of AWS may be still in pending status when do the restore.
+			// wait for a while to avoid this https://github.com/vmware-tanzu/velero/issues/1799
+			if OriginVeleroCfg.CloudProvider == test.Azure &&
+				strings.EqualFold(OriginVeleroCfg.Features, test.FeatureCSI) ||
+				OriginVeleroCfg.CloudProvider == test.AWS {
+				By("Sleep 5 minutes to avoid snapshot recreated by unknown reason ", func() {
+					time.Sleep(5 * time.Minute)
+				})
+			}
+
+			By("Snapshot should be created in cloud object store with retain policy", func() {
+				snapshotCheckPoint, err = veleroutil.GetSnapshotCheckPoint(
+					*OriginVeleroCfg.DefaultClient,
+					OriginVeleroCfg,
+					m.kibishiiData.ExpectedNodes,
+					m.CaseBaseName,
+					m.BackupName,
+					kibishii.GetKibishiiPVCNameList(m.kibishiiData.ExpectedNodes),
+				)
+
+				Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint")
+				Expect(providers.CheckSnapshotsInProvider(
+					OriginVeleroCfg,
+					m.BackupName,
+					snapshotCheckPoint,
+					false,
+				)).To(Succeed())
+			})
+		}
+	}
+
+	return nil
+}
+
+func (m *migrationE2E) Restore() error {
+	StandbyVeleroCfg := m.VeleroCfg
+
+	By("Install Velero in standby cluster.", func() {
+		// Ensure cluster-B is the target cluster
+		Expect(k8sutil.KubectlConfigUseContext(
+			m.Ctx, m.VeleroCfg.StandbyClusterContext)).To(Succeed())
+
+		// Check the workload namespace not exist in standby cluster.
+		_, err := k8sutil.GetNamespace(
+			m.Ctx, *m.VeleroCfg.StandbyClient, m.CaseBaseName)
+		Expect(err).To(HaveOccurred(), fmt.Sprintf(
+			"get namespace in dst cluster successfully, it's not as expected: %s", m.CaseBaseName))
+		Expect(strings.Contains(fmt.Sprint(err), "namespaces \""+m.CaseBaseName+"\" not found")).
+			Should(BeTrue())
+
+		By("Install StorageClass for E2E.")
+		Expect(veleroutil.InstallStorageClasses(
+			m.VeleroCfg.StandbyClusterCloudProvider)).To(Succeed())
+
+		if strings.EqualFold(m.VeleroCfg.Features, test.FeatureCSI) &&
+			m.VeleroCfg.UseVolumeSnapshots {
+			By("Install VolumeSnapshotClass for E2E.")
+			Expect(
+				k8sutil.KubectlApplyByFile(
+					m.Ctx,
+					fmt.Sprintf("../testdata/volume-snapshot-class/%s.yaml",
+						m.VeleroCfg.StandbyClusterCloudProvider),
+				),
+			).To(Succeed())
+		}
+
+		StandbyVeleroCfg.ClientToInstallVelero = m.VeleroCfg.StandbyClient
+		StandbyVeleroCfg.ClusterToInstallVelero = m.VeleroCfg.StandbyClusterName
+		StandbyVeleroCfg.ServiceAccountNameToInstall = m.VeleroCfg.StandbyCLSServiceAccountName
+		StandbyVeleroCfg.UseNodeAgent = !m.useVolumeSnapshots
+		if StandbyVeleroCfg.SnapshotMoveData {
+			StandbyVeleroCfg.UseNodeAgent = true
+			// For SnapshotMoveData pipelines, we should use standby cluster setting
+			// for Velero installation.
+			// In nightly CI, StandbyClusterPlugins is set properly
+			// if pipeline is for SnapshotMoveData.
+			StandbyVeleroCfg.Plugins = m.VeleroCfg.StandbyClusterPlugins
+			StandbyVeleroCfg.ObjectStoreProvider = m.VeleroCfg.StandbyClusterObjectStoreProvider
+		}
+
+		Expect(veleroutil.VeleroInstall(
+			context.Background(), &StandbyVeleroCfg, true)).To(Succeed())
+	})
+
+	By("Waiting for backups sync to Velero in standby cluster", func() {
+		Expect(veleroutil.WaitForBackupToBeCreated(
+			m.Ctx, m.BackupName, 5*time.Minute, &StandbyVeleroCfg)).To(Succeed())
+	})
+
+	By(fmt.Sprintf("Restore %s", m.CaseBaseName), func() {
+		if m.VeleroCfg.SnapshotMoveData {
+			cmName := "datamover-storage-class-config"
+			labels := map[string]string{"velero.io/change-storage-class": "RestoreItemAction",
+				"velero.io/plugin-config": ""}
+			data := map[string]string{kibishii.KibishiiStorageClassName: test.StorageClassName}
+
+			By(fmt.Sprintf("Create ConfigMap %s in namespace %s",
+				cmName, StandbyVeleroCfg.VeleroNamespace), func() {
+				_, err := k8sutil.CreateConfigMap(
+					StandbyVeleroCfg.StandbyClient.ClientGo,
+					StandbyVeleroCfg.VeleroNamespace,
+					cmName,
+					labels,
+					data,
+				)
+				Expect(err).To(Succeed(), fmt.Sprintf(
+					"failed to create ConfigMap in the namespace %q",
+					StandbyVeleroCfg.VeleroNamespace))
+			})
+		}
+
+		Expect(veleroutil.VeleroRestore(
+			m.Ctx,
+			StandbyVeleroCfg.VeleroCLI,
+			StandbyVeleroCfg.VeleroNamespace,
+			m.RestoreName,
+			m.BackupName,
+			"",
+		)).To(Succeed(), func() string {
+			veleroutil.RunDebug(
+				m.Ctx, StandbyVeleroCfg.VeleroCLI,
+				StandbyVeleroCfg.VeleroNamespace, "", m.RestoreName)
+			return "Fail to restore workload"
+		})
+	})
+
+	return nil
+}
+
+func (m *migrationE2E) Verify() error {
+	By(fmt.Sprintf("Verify workload %s after restore on standby cluster", m.CaseBaseName), func() {
+		Expect(kibishii.KibishiiVerifyAfterRestore(
+			*m.VeleroCfg.StandbyClient,
+			m.CaseBaseName,
+			m.Ctx,
+			&m.kibishiiData,
+			"",
+		)).To(Succeed(), "Fail to verify workload after restore")
+	})
+
+	return nil
+}
+
+func (m *migrationE2E) Clean() error {
+	By("Clean resource on default cluster.", func() {
+		Expect(m.TestCase.Clean()).To(Succeed())
+	})
+
+	By("Clean resource on standby cluster.", func() {
+		Expect(k8sutil.KubectlConfigUseContext(
+			m.Ctx, m.VeleroCfg.StandbyClusterContext)).To(Succeed())
+		m.VeleroCfg.ClientToInstallVelero = m.VeleroCfg.StandbyClient
+		m.VeleroCfg.ClusterToInstallVelero = m.VeleroCfg.StandbyClusterName
+
+		By("Delete StorageClasses created by E2E")
+		Expect(
+			k8sutil.DeleteStorageClass(
+				m.Ctx,
+				*m.VeleroCfg.ClientToInstallVelero,
+				test.StorageClassName,
+			),
+		).To(Succeed())
+		Expect(
+			k8sutil.DeleteStorageClass(
+				m.Ctx,
+				*m.VeleroCfg.ClientToInstallVelero,
+				test.StorageClassName2,
+			),
+		).To(Succeed())
+
+		if strings.EqualFold(m.VeleroCfg.Features, test.FeatureCSI) &&
+			m.VeleroCfg.UseVolumeSnapshots {
+			By("Delete VolumeSnapshotClass created by E2E")
+			Expect(
+				k8sutil.KubectlDeleteByFile(
+					m.Ctx,
+					fmt.Sprintf("../testdata/volume-snapshot-class/%s.yaml",
+						m.VeleroCfg.StandbyClusterCloudProvider),
+				),
+			).To(Succeed())
+		}
+
+		Expect(veleroutil.VeleroUninstall(m.Ctx, m.VeleroCfg)).To(Succeed())
+
+		Expect(
+			k8sutil.DeleteNamespace(
+				m.Ctx,
+				*m.VeleroCfg.StandbyClient,
+				m.CaseBaseName,
+				true,
+			),
+		).To(Succeed())
+	})
+
+	By("Switch to default KubeConfig context", func() {
+		Expect(k8sutil.KubectlConfigUseContext(
+			m.Ctx,
+			m.VeleroCfg.DefaultClusterContext,
+		)).To(Succeed())
+	})
+
+	return nil
 }

test/e2e/test/test.go

@@ -91,8 +91,9 @@ func TestFuncWithMultiIt(tests []VeleroBackupRestoreTest) func() {
 }
 
 func TestIt(test VeleroBackupRestoreTest) error {
-	test.Init()
-	It(test.GetTestMsg().Text, func() {
+	It("Run E2E test case", func() {
+		Expect(test.Init()).To(Succeed())
+
 		Expect(RunTestCase(test)).To(Succeed(), test.GetTestMsg().FailedMSG)
 	})
 	return nil
@@ -213,6 +214,7 @@ func RunTestCase(test VeleroBackupRestoreTest) error {
 	if test == nil {
 		return errors.New("No case should be tested")
 	}
+	fmt.Println("Running case: ", test.GetTestMsg().Text)
 	test.Start()
 	defer test.GetTestCase().CtxCancel()
 

test/e2e/upgrade/upgrade.go

@@ -138,13 +138,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
 				tmpCfgForOldVeleroInstall.VeleroVersion = version
 				tmpCfgForOldVeleroInstall.UseVolumeSnapshots = useVolumeSnapshots
 
-				if supportUploaderType {
-					tmpCfgForOldVeleroInstall.UseRestic = false
-					tmpCfgForOldVeleroInstall.UseNodeAgent = !useVolumeSnapshots
-				} else {
-					tmpCfgForOldVeleroInstall.UseRestic = !useVolumeSnapshots
-					tmpCfgForOldVeleroInstall.UseNodeAgent = false
-				}
+				tmpCfgForOldVeleroInstall.UseNodeAgent = !useVolumeSnapshots
 
 				Expect(VeleroInstall(context.Background(), &tmpCfgForOldVeleroInstall, false)).To(Succeed())
 				Expect(CheckVeleroVersion(context.Background(), tmpCfgForOldVeleroInstall.VeleroCLI,
@@ -230,7 +224,6 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
 
 			By(fmt.Sprintf("Upgrade Velero by CLI %s", tmpCfg.VeleroCLI), func() {
 				tmpCfg.GCFrequency = ""
-				tmpCfg.UseRestic = false
 				tmpCfg.UseNodeAgent = !useVolumeSnapshots
 				Expect(err).To(Succeed())
 				if supportUploaderType {

test/util/csi/common.go

@@ -113,8 +113,8 @@ func GetVolumeSnapshotContentNameByPod(client TestClient, podName, namespace, ba
 	if len(pvList) != 1 {
 		return "", errors.New(fmt.Sprintf("Only 1 PV of PVC %s pod %s should be found under namespace %s", pvcList[0], podName, namespace))
 	}
-	pv_value, err := GetPersistentVolume(context.Background(), client, "", pvList[0])
-	fmt.Println(pv_value.Annotations["pv.kubernetes.io/provisioned-by"])
+	pvValue, err := GetPersistentVolume(context.Background(), client, "", pvList[0])
+	fmt.Println(pvValue.Annotations["pv.kubernetes.io/provisioned-by"])
 	if err != nil {
 		return "", err
 	}
@@ -148,14 +148,10 @@ func CheckVolumeSnapshotCR(client TestClient, index map[string]string, expectedC
 	if len(apiVersion) == 0 {
 		return nil, errors.New("Fail to get APIVersion")
 	}
-	// if apiVersion[0] == "v1beta1" {
-	// 	if snapshotContentNameList, err = GetCsiSnapshotHandle(client, apiVersion[0], index); err != nil {
-	// 		return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content")
-	// 	}
-	// } else
+
 	if apiVersion[0] == "v1" {
 		if snapshotContentNameList, err = GetCsiSnapshotHandle(client, apiVersion[0], index); err != nil {
-			return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content")
+			return nil, errors.Wrap(err, "Fail to get CSI snapshot content")
 		}
 	} else {
 		return nil, errors.New("API version is invalid")

test/util/velero/install.go

@@ -271,9 +271,6 @@ func installVeleroServer(ctx context.Context, cli, cloudProvider string, options
 	if len(options.Image) > 0 {
 		args = append(args, "--image", options.Image)
 	}
-	if options.UseRestic {
-		args = append(args, "--use-restic")
-	}
 	if options.UseNodeAgent {
 		args = append(args, "--use-node-agent")
 	}

test/util/velero/velero_utils.go

@@ -255,10 +255,7 @@ func getProviderVeleroInstallOptions(veleroCfg *VeleroConfig,
 	io.DefaultVolumesToFsBackup = veleroCfg.DefaultVolumesToFsBackup
 	io.UseVolumeSnapshots = veleroCfg.UseVolumeSnapshots
 
-	if !veleroCfg.UseRestic {
-		io.UseNodeAgent = veleroCfg.UseNodeAgent
-	}
-	io.UseRestic = veleroCfg.UseRestic
+	io.UseNodeAgent = veleroCfg.UseNodeAgent
 	io.Image = veleroCfg.VeleroImage
 	io.Namespace = veleroCfg.VeleroNamespace
 	io.UploaderType = veleroCfg.UploaderType