Merge pull request #6287 from qiuming-best/e2e-velero-installation

E2E velero installation optimization

test/e2e/README.md

@@ -111,5 +111,30 @@ The `-focus` flag is passed to ginkgo using the `GINKGO_FOCUS` make variable. Th
 ### API clients
 When adding a test, aim to instantiate an API client only once at the beginning of the test. There is a constructor `newTestClient` that facilitates the configuration and instantiation of clients. Also, please use the `kubebuilder` runtime controller client for any new test, as we will phase out usage of `client-go` API clients.
 
+## TestCase frame related
+TestCase frame provide a serials of interface to concatenate one complete e2e test. it's makes the testing be concise and explicit.
+### VeleroBackupRestoreTest interface 
+VeleroBackupRestoreTest interface provided a standard workflow of backup and restore, which makes the whole testing process clearer and code reusability.
+
+For programming conventions, Take ResourcePoliciesCase case for example:
+##### Init
+- It's need first call the base `TestCase` Init function to generate random number as UUIDgen and set one default timeout duration
+- Assigning CaseBaseName variable with a case related prefix, all others variables will follow the prefix
+- Assigning NamespacesTotal variable, and generating namespaces
+- Assigning values to the inner variable for specific case
+- For BackupArgs, as Velero installation is using global Velero configuration, it's NEED to specify the value of the variable snapshot-volumes or default-volumes-to-fs-backup explicitly.
+##### CreateResources
+- It's better to set a global timeout in CreateResources function which is the real beginning of one e2e test
+##### Destroy
+- It's only clean up resources in currently test namespaces, if wantting to clean up all resources including resources created which is not in currently test namespaces, it's better to override base Destroy function
+##### Clean
+- Clean function only clean resources in namespaces which has the prefix CaseBaseName. So the the names of test namespaces should start with prefix of CaseBaseName.
+- If wantting to clean up all resources including resources created which is not in currently test namespaces, it's better to override base Clean function
+#### Velero Installation
+- Velero is installed with global velero config before the E2E test start, so if the case (such as upgrade/migration, etc.) does not want to use the global velero config, it is NEED TO UNINSTALL velero, or the global velero config may affect the current test.
+### TestFunc 
+The TestFunc function concatenate all the flows in a test.
+It will reduce the frequency of velero installation by installing and checking Velero with the global Velero. It's Need to explicit reinstall Velero if the case has specicial configuration, such as API Group test case we need to enable feature EnableCSI.
+
 ### Tips
 Look for the ⛵ emoji printed at the end of each install and uninstall log. There should not be two install/unintall in a row, and there should be tests between an install and an uninstall. 

test/e2e/backups/deletion.go

@@ -47,7 +47,6 @@ func BackupDeletionWithRestic() {
 func backup_deletion_test(useVolumeSnapshots bool) {
 	var (
 		backupName string
-		err        error
 		veleroCfg  VeleroConfig
 	)
 	veleroCfg = VeleroCfg
@@ -60,11 +59,11 @@ func backup_deletion_test(useVolumeSnapshots bool) {
 		}
 		var err error
 		flag.Parse()
+		if veleroCfg.InstallVelero {
+			Expect(PrepareVelero(context.Background(), "backup deletion")).To(Succeed())
+		}
 		UUIDgen, err = uuid.NewRandom()
 		Expect(err).To(Succeed())
-		if veleroCfg.InstallVelero {
-			Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed())
-		}
 	})
 
 	AfterEach(func() {
@@ -72,10 +71,6 @@ func backup_deletion_test(useVolumeSnapshots bool) {
 			By("Clean backups after test", func() {
 				DeleteBackups(context.Background(), *veleroCfg.ClientToInstallVelero)
 			})
-			if veleroCfg.InstallVelero {
-				err = VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)
-				Expect(err).To(Succeed())
-			}
 		}
 	})
 

test/e2e/basic/storage-class-changing.go

@@ -48,8 +48,6 @@ func (s *StorageClasssChanging) Init() error {
 		Text: "Change the storage class of persistent volumes and persistent" +
 			" volume claims during restores",
 	}
-	s.BackupName = "backup-sc-" + UUIDgen.String()
-	s.RestoreName = "restore-" + UUIDgen.String()
 	s.srcStorageClass = "default"
 	s.desStorageClass = StorageClassName
 	s.labels = map[string]string{"velero.io/change-storage-class": "RestoreItemAction",
@@ -58,9 +56,6 @@ func (s *StorageClasssChanging) Init() error {
 	s.configmaptName = "change-storage-class-config"
 	s.volume = "volume-1"
 	s.podName = "pod-1"
-
-	s.BackupName = s.BackupName + "backup-" + UUIDgen.String()
-	s.RestoreName = s.RestoreName + "restore-" + UUIDgen.String()
 	s.BackupArgs = []string{
 		"create", "--namespace", VeleroCfg.VeleroNamespace, "backup", s.BackupName,
 		"--include-namespaces", s.namespace,

test/e2e/bsl-mgmt/deletion.go

@@ -70,7 +70,7 @@ func BslDeletionTest(useVolumeSnapshots bool) {
 		UUIDgen, err = uuid.NewRandom()
 		Expect(err).To(Succeed())
 		if veleroCfg.InstallVelero {
-			Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed())
+			Expect(PrepareVelero(context.Background(), "BSL Deletion")).To(Succeed())
 		}
 	})
 
@@ -84,14 +84,7 @@ func BslDeletionTest(useVolumeSnapshots bool) {
 					true)).To(Succeed(), fmt.Sprintf("failed to delete the namespace %q",
 					bslDeletionTestNs))
 			})
-			if veleroCfg.InstallVelero {
-				By("Uninstall Velero", func() {
-					Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI,
-						veleroCfg.VeleroNamespace)).To(Succeed())
-				})
-			}
 		}
-
 	})
 
 	When("kibishii is the sample workload", func() {

test/e2e/e2e_suite_test.go

@@ -43,6 +43,7 @@ import (
 	. "github.com/vmware-tanzu/velero/test/e2e/schedule"
 	. "github.com/vmware-tanzu/velero/test/e2e/upgrade"
 	. "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
+	. "github.com/vmware-tanzu/velero/test/e2e/util/velero"
 )
 
 func init() {
@@ -65,6 +66,8 @@ func init() {
 	flag.StringVar(&VeleroCfg.VSLConfig, "vsl-config", "", "configuration to use for the volume snapshot location. Format is key1=value1,key2=value2")
 	flag.StringVar(&VeleroCfg.VeleroNamespace, "velero-namespace", "velero", "namespace to install Velero into")
 	flag.BoolVar(&VeleroCfg.InstallVelero, "install-velero", true, "install/uninstall velero during the test.  Optional.")
+	flag.BoolVar(&VeleroCfg.UseNodeAgent, "use-node-agent", true, "whether deploy node agent daemonset velero during the test.  Optional.")
+	flag.BoolVar(&VeleroCfg.UseVolumeSnapshots, "use-volume-snapshots", true, "Whether or not to create snapshot location automatically. Set to false if you do not plan to create volume snapshots via a storage provider.")
 	flag.StringVar(&VeleroCfg.RegistryCredentialFile, "registry-credential-file", "", "file containing credential for the image registry, follows the same format rules as the ~/.docker/config.json file. Optional.")
 	flag.StringVar(&VeleroCfg.KibishiiDirectory, "kibishii-directory", "github.com/vmware-tanzu-experiments/distributed-data-generator/kubernetes/yaml/", "The file directory or URL path to install Kibishii. Optional.")
 	//vmware-tanzu-experiments
@@ -189,3 +192,17 @@ func TestE2e(t *testing.T) {
 	junitReporter := reporters.NewJUnitReporter("report.xml")
 	RunSpecsWithDefaultAndCustomReporters(t, "E2e Suite", []Reporter{junitReporter})
 }
+
+var _ = BeforeSuite(func() {
+	if VeleroCfg.InstallVelero {
+		By("Install test resources before testing")
+		Expect(PrepareVelero(context.Background(), "install resource before testing")).To(Succeed())
+	}
+})
+
+var _ = AfterSuite(func() {
+	if VeleroCfg.InstallVelero && !VeleroCfg.Debug {
+		By("release test resources after testing")
+		Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To(Succeed())
+	}
+})

test/e2e/migration/migration.go

@@ -69,6 +69,13 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
 		if veleroCfg.DefaultCluster == "" && veleroCfg.StandbyCluster == "" {
 			Skip("Migration test needs 2 clusters")
 		}
+		// need to uninstall Velero first in case of the affection of the existing global velero installation
+		if veleroCfg.InstallVelero {
+			By("Uninstall Velero", func() {
+				Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI,
+					veleroCfg.VeleroNamespace)).To(Succeed())
+			})
+		}
 	})
 	AfterEach(func() {
 		if !veleroCfg.Debug {

test/e2e/privilegesmgmt/ssr.go

@@ -44,15 +44,7 @@ func SSRTest() {
 		flag.Parse()
 		veleroCfg.UseVolumeSnapshots = false
 		if veleroCfg.InstallVelero {
-			Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed())
-		}
-	})
-
-	AfterEach(func() {
-		if veleroCfg.InstallVelero {
-			if !veleroCfg.Debug {
-				Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).To(Succeed())
-			}
+			Expect(PrepareVelero(context.Background(), "SSR test")).To(Succeed())
 		}
 	})
 
@@ -102,7 +94,7 @@ func SSRTest() {
 		Expect(veleroCfg.ClientToInstallVelero.Kubebuilder.List(ctx, ssrListResp, &kbclient.ListOptions{Namespace: testNS})).To(Succeed(),
 			fmt.Sprintf("Failed to list ssr object in %s namespace", testNS))
 		Expect(len(ssrListResp.Items)).To(BeNumerically("==", 1),
-			fmt.Sprintf("Count of ssr object in %s namespace is not 1", testNS))
+			fmt.Sprintf("Count of ssr object in %s namespace is not 1 but %d", testNS, len(ssrListResp.Items)))
 		Expect(ssrListResp.Items[0].Status.Phase).To(BeEmpty(),
 			fmt.Sprintf("Status of ssr object in %s namespace should be empty", testNS))
 		Expect(ssrListResp.Items[0].Status.ServerVersion).To(BeEmpty(),

test/e2e/pv-backup/pv-backup-filter.go

@@ -66,7 +66,7 @@ func (p *PVBackupFiltering) Init() error {
 }
 
 func (p *PVBackupFiltering) CreateResources() error {
-	p.Ctx, p.CtxCancel = context.WithTimeout(context.Background(), 10*time.Minute)
+	p.Ctx, p.CtxCancel = context.WithTimeout(context.Background(), 30*time.Minute)
 	err := InstallStorageClass(p.Ctx, fmt.Sprintf("testdata/storage-class/%s.yaml", VeleroCfg.CloudProvider))
 	if err != nil {
 		return errors.Wrapf(err, "failed to install storage class for pv backup filtering test")

test/e2e/resourcepolicies/resource_policies.go

@@ -59,25 +59,31 @@ type ResourcePoliciesCase struct {
 var ResourcePoliciesTest func() = TestFunc(&ResourcePoliciesCase{})
 
 func (r *ResourcePoliciesCase) Init() error {
+	// generate random number as UUIDgen and set one default timeout duration
 	r.TestCase.Init()
-	r.CaseBaseName = "resource-policies-" + r.UUIDgen
-	r.cmName = "cm-" + r.CaseBaseName
-	r.yamlConfig = yamlData
-	r.VeleroCfg = VeleroCfg
-	r.Client = *r.VeleroCfg.ClientToInstallVelero
-	r.VeleroCfg.UseVolumeSnapshots = false
-	r.VeleroCfg.UseNodeAgent = true
-	r.NamespacesTotal = 3
 
+	// generate variable names based on CaseBaseName + UUIDgen
+	r.CaseBaseName = "resource-policies-" + r.UUIDgen
+	r.BackupName = "backup-" + r.CaseBaseName
+	r.RestoreName = "restore-" + r.CaseBaseName
+	r.cmName = "cm-" + r.CaseBaseName
+
+	// generate namespaces by NamespacesTotal
+	r.NamespacesTotal = 3
 	r.NSIncluded = &[]string{}
 	for nsNum := 0; nsNum < r.NamespacesTotal; nsNum++ {
 		createNSName := fmt.Sprintf("%s-%00000d", r.CaseBaseName, nsNum)
 		*r.NSIncluded = append(*r.NSIncluded, createNSName)
 	}
 
-	r.BackupName = "backup-resource-policies-" + UUIDgen.String()
-	r.RestoreName = "restore-resource-policies-" + UUIDgen.String()
+	// assign values to the inner variable for specific case
+	r.yamlConfig = yamlData
+	r.VeleroCfg = VeleroCfg
+	r.Client = *r.VeleroCfg.ClientToInstallVelero
+	r.VeleroCfg.UseVolumeSnapshots = false
+	r.VeleroCfg.UseNodeAgent = true
 
+	// NEED explicitly specify the value of the variables for snapshot-volumes or default-volumes-to-fs-backup
 	r.BackupArgs = []string{
 		"create", "--namespace", VeleroCfg.VeleroNamespace, "backup", r.BackupName,
 		"--resource-policies-configmap", r.cmName,
@@ -91,6 +97,7 @@ func (r *ResourcePoliciesCase) Init() error {
 		"--from-backup", r.BackupName, "--wait",
 	}
 
+	// Message output by ginkgo
 	r.TestMsg = &TestMSG{
 		Desc:      "Skip backup of volume by resource policies",
 		FailedMSG: "Failed to skip backup of volume by resource policies",
@@ -100,7 +107,9 @@ func (r *ResourcePoliciesCase) Init() error {
 }
 
 func (r *ResourcePoliciesCase) CreateResources() error {
+	// It's better to set a global timeout in CreateResources function which is the real beginning of one e2e test
 	r.Ctx, r.CtxCancel = context.WithTimeout(context.Background(), 10*time.Minute)
+
 	By(("Installing storage class..."), func() {
 		Expect(r.installTestStorageClasses(fmt.Sprintf("testdata/storage-class/%s.yaml", VeleroCfg.CloudProvider))).To(Succeed(), "Failed to install storage class")
 	})
@@ -178,6 +187,7 @@ func (r *ResourcePoliciesCase) Verify() error {
 }
 
 func (r *ResourcePoliciesCase) Clean() error {
+	// If created some resources which is not in current test namespace, we NEED to override the base Clean function
 	if !r.VeleroCfg.Debug {
 		if err := r.deleteTestStorageClassList([]string{"e2e-storage-class", "e2e-storage-class-2"}); err != nil {
 			return err
@@ -187,7 +197,7 @@ func (r *ResourcePoliciesCase) Clean() error {
 			return err
 		}
 
-		return r.GetTestCase().Clean()
+		return r.GetTestCase().Clean() // only clean up resources in test namespace
 	}
 	return nil
 }

test/e2e/schedule/ordered_resources.go

@@ -128,6 +128,10 @@ func (o *OrderedResources) Backup() error {
 	return nil
 }
 
+func (o *OrderedResources) Destroy() error {
+	return nil
+}
+
 func (o *OrderedResources) Verify() error {
 	By(fmt.Sprintf("Checking resource order in %s schedule cr", o.ScheduleName), func() {
 		err := CheckScheduleWithResourceOrder(o.Ctx, o.VeleroCfg.VeleroCLI, o.VeleroCfg.VeleroNamespace, o.ScheduleName, o.OrderMap)

test/e2e/schedule/schedule-backup-creation.go

@@ -36,7 +36,7 @@ var ScheduleBackupCreationTest func() = TestFunc(&ScheduleBackupCreation{})
 func (n *ScheduleBackupCreation) Init() error {
 	n.TestCase.Init()
 	n.CaseBaseName = "schedule-backup-creation-test" + n.UUIDgen
-	n.ScheduleName = n.ScheduleName + "schedule-" + UUIDgen.String()
+	n.ScheduleName = "schedule-" + n.CaseBaseName
 	n.namespace = n.CaseBaseName
 	n.VeleroCfg = VeleroCfg
 	n.Client = *n.VeleroCfg.ClientToInstallVelero

test/e2e/schedule/schedule.go

@@ -32,7 +32,7 @@ func (n *ScheduleBackup) Init() error {
 	n.CaseBaseName = "schedule-backup-" + n.UUIDgen
 	n.NSIncluded = &[]string{n.CaseBaseName}
 	n.ScheduleName = "schedule-" + n.CaseBaseName
-
+	n.RestoreName = "restore-" + n.CaseBaseName
 	n.VeleroCfg = VeleroCfg
 	n.Client = *n.VeleroCfg.ClientToInstallVelero
 	n.Period = 3      // Unit is minute
@@ -46,6 +46,7 @@ func (n *ScheduleBackup) Init() error {
 		"--include-namespaces", strings.Join(*n.NSIncluded, ","),
 		"--schedule=*/" + fmt.Sprintf("%v", n.Period) + " * * * *",
 	}
+
 	Expect(n.Period < 30).To(Equal(true))
 	return nil
 }
@@ -61,7 +62,7 @@ func (n *ScheduleBackup) CreateResources() error {
 		_, err := CreateConfigMap(n.Client.ClientGo, ns, configmaptName, nil, nil)
 		Expect(err).To(Succeed(), fmt.Sprintf("failed to create configmap in the namespace %q", ns))
 		Expect(WaitForConfigMapComplete(n.Client.ClientGo, ns, configmaptName)).To(Succeed(),
-			fmt.Sprintf("ailed to ensure secret completion in namespace: %q", ns))
+			fmt.Sprintf("failed to ensure secret completion in namespace: %q", ns))
 	}
 	return nil
 }
@@ -139,9 +140,11 @@ func (n *ScheduleBackup) Backup() error {
 
 	n.BackupName = strings.Replace(n.randBackupName, " ", "", -1)
 
-	By("Delete all namespaces", func() {
-		Expect(CleanupNamespacesWithPoll(n.Ctx, n.Client, n.CaseBaseName)).To(Succeed(), "Could cleanup retrieve namespaces")
-	})
+	n.RestoreArgs = []string{
+		"create", "--namespace", VeleroCfg.VeleroNamespace, "restore", n.RestoreName,
+		"--from-backup", n.BackupName,
+		"--wait",
+	}
 
 	backupsInfo, err := GetScheduledBackupsCreationTime(n.Ctx, VeleroCfg.VeleroCLI, "default", n.ScheduleName)
 	Expect(err).To(Succeed(), fmt.Sprintf("Fail to get backups from schedule %s", n.ScheduleName))
@@ -205,10 +208,6 @@ func (n *ScheduleBackup) Verify() error {
 	return nil
 }
 
-func (n *ScheduleBackup) Destroy() error {
-	return nil
-}
-
 func (n *ScheduleBackup) Clean() error {
 	if !n.VeleroCfg.Debug {
 		Expect(VeleroScheduleDelete(n.Ctx, n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace, n.ScheduleName)).To(Succeed())

test/e2e/test/test.go

@@ -30,6 +30,7 @@ import (
 	"github.com/pkg/errors"
 
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
+	"github.com/vmware-tanzu/velero/test/e2e"
 	. "github.com/vmware-tanzu/velero/test/e2e"
 	. "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
 	. "github.com/vmware-tanzu/velero/test/e2e/util/velero"
@@ -80,24 +81,16 @@ type TestCase struct {
 func TestFunc(test VeleroBackupRestoreTest) func() {
 	return func() {
 		Expect(test.Init()).To(Succeed(), "Failed to instantiate test cases")
-		veleroCfg := test.GetTestCase().VeleroCfg
 		BeforeEach(func() {
 			flag.Parse()
-			veleroCfg := test.GetTestCase().VeleroCfg
+			// Using the global velero config which covered the installation for most common cases
+			veleroCfg := e2e.VeleroCfg
 			// TODO: Skip nodeport test until issue https://github.com/kubernetes/kubernetes/issues/114384 fixed
 			if veleroCfg.CloudProvider == "azure" && strings.Contains(test.GetTestCase().CaseBaseName, "nodeport") {
 				Skip("Skip due to issue https://github.com/kubernetes/kubernetes/issues/114384 on AKS")
 			}
 			if veleroCfg.InstallVelero {
-				veleroCfg.UseVolumeSnapshots = test.GetTestCase().UseVolumeSnapshots
-				Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed())
-			}
-		})
-		AfterEach(func() {
-			if !veleroCfg.Debug {
-				if veleroCfg.InstallVelero {
-					Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).To((Succeed()))
-				}
+				Expect(PrepareVelero(context.Background(), test.GetTestCase().CaseBaseName)).To(Succeed())
 			}
 		})
 		It(test.GetTestMsg().Text, func() {
@@ -108,35 +101,16 @@ func TestFunc(test VeleroBackupRestoreTest) func() {
 
 func TestFuncWithMultiIt(tests []VeleroBackupRestoreTest) func() {
 	return func() {
-		var countIt int
-		var useVolumeSnapshots bool
-		var veleroCfg VeleroConfig
+		veleroCfg := e2e.VeleroCfg
 		for k := range tests {
 			Expect(tests[k].Init()).To(Succeed(), fmt.Sprintf("Failed to instantiate test %s case", tests[k].GetTestMsg().Desc))
-			veleroCfg = tests[k].GetTestCase().VeleroCfg
-			useVolumeSnapshots = tests[k].GetTestCase().UseVolumeSnapshots
 			defer tests[k].GetTestCase().CtxCancel()
 		}
 
 		BeforeEach(func() {
 			flag.Parse()
 			if veleroCfg.InstallVelero {
-				if countIt == 0 {
-					veleroCfg.UseVolumeSnapshots = useVolumeSnapshots
-					veleroCfg.UseNodeAgent = !useVolumeSnapshots
-					Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed())
-				}
-				countIt++
-			}
-		})
-
-		AfterEach(func() {
-			if !veleroCfg.Debug {
-				if veleroCfg.InstallVelero {
-					if countIt == len(tests) && !veleroCfg.Debug {
-						Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).To((Succeed()))
-					}
-				}
+				Expect(PrepareVelero(context.Background(), tests[0].GetTestCase().CaseBaseName)).To(Succeed())
 			}
 		})
 

test/e2e/upgrade/upgrade.go

@@ -74,6 +74,13 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
 		if veleroCfg.VeleroCLI == "" {
 			Skip("VeleroCLI should be provide")
 		}
+		// need to uninstall Velero first in case of the affection of the existing global velero installation
+		if veleroCfg.InstallVelero {
+			By("Uninstall Velero", func() {
+				Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI,
+					veleroCfg.VeleroNamespace)).To(Succeed())
+			})
+		}
 	})
 	AfterEach(func() {
 		if !veleroCfg.Debug {

test/e2e/util/k8s/common.go

@@ -238,7 +238,7 @@ func GetPVByPodName(client TestClient, namespace, podName string) (string, error
 		return "", err
 	}
 	if len(pvcList) != 1 {
-		return "", errors.New(fmt.Sprintf("Only 1 PVC of pod %s should be found under namespace %s", podName, namespace))
+		return "", errors.New(fmt.Sprintf("Only 1 PVC of pod %s should be found under namespace %s but got %v", podName, namespace, pvcList))
 	}
 	pvList, err := GetPvByPvc(context.Background(), namespace, pvcList[0])
 	if err != nil {

test/e2e/util/kibishii/kibishii_utils.go

@@ -291,6 +291,18 @@ func KibishiiPrepareBeforeBackup(oneHourTimeout context.Context, client TestClie
 	providerName, kibishiiNamespace, registryCredentialFile, veleroFeatures,
 	kibishiiDirectory string, useVolumeSnapshots bool, kibishiiData *KibishiiData) error {
 	fmt.Printf("installKibishii %s\n", time.Now().Format("2006-01-02 15:04:05"))
+	serviceAccountName := "default"
+
+	// wait until the service account is created before patch the image pull secret
+	if err := WaitUntilServiceAccountCreated(oneHourTimeout, client, kibishiiNamespace, serviceAccountName, 10*time.Minute); err != nil {
+		return errors.Wrapf(err, "failed to wait the service account %q created under the namespace %q", serviceAccountName, kibishiiNamespace)
+	}
+
+	// add the image pull secret to avoid the image pull limit issue of Docker Hub
+	if err := PatchServiceAccountWithImagePullSecret(oneHourTimeout, client, kibishiiNamespace, serviceAccountName, registryCredentialFile); err != nil {
+		return errors.Wrapf(err, "failed to patch the service account %q under the namespace %q", serviceAccountName, kibishiiNamespace)
+	}
+
 	if err := installKibishii(oneHourTimeout, kibishiiNamespace, providerName, veleroFeatures,
 		kibishiiDirectory, useVolumeSnapshots); err != nil {
 		return errors.Wrap(err, "Failed to install Kibishii workload")

test/e2e/util/velero/install.go

@@ -35,6 +35,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 	clientset "k8s.io/client-go/kubernetes"
 
+	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	"github.com/vmware-tanzu/velero/pkg/cmd/cli/install"
 	velerexec "github.com/vmware-tanzu/velero/pkg/util/exec"
 	. "github.com/vmware-tanzu/velero/test/e2e"
@@ -461,6 +462,70 @@ func waitVeleroReady(ctx context.Context, namespace string, useNodeAgent bool) e
 	return nil
 }
 
+func IsVeleroReady(ctx context.Context, namespace string, useNodeAgent bool) (bool, error) {
+	if useNodeAgent {
+		stdout, stderr, err := velerexec.RunCommand(exec.CommandContext(ctx, "kubectl", "get", "daemonset/node-agent",
+			"-o", "json", "-n", namespace))
+		if err != nil {
+			return false, errors.Wrapf(err, "failed to get the node-agent daemonset, stdout=%s, stderr=%s", stdout, stderr)
+		} else {
+			daemonset := &apps.DaemonSet{}
+			if err = json.Unmarshal([]byte(stdout), daemonset); err != nil {
+				return false, errors.Wrapf(err, "failed to unmarshal the node-agent daemonset")
+			}
+			if daemonset.Status.DesiredNumberScheduled != daemonset.Status.NumberAvailable {
+				return false, fmt.Errorf("the available number pod %d in node-agent daemonset not equal to scheduled number %d", daemonset.Status.NumberAvailable, daemonset.Status.DesiredNumberScheduled)
+			}
+		}
+	}
+
+	stdout, stderr, err := velerexec.RunCommand(exec.CommandContext(ctx, "kubectl", "get", "deployment/velero",
+		"-o", "json", "-n", namespace))
+	if err != nil {
+		return false, errors.Wrapf(err, "failed to get the velero deployment stdout=%s, stderr=%s", stdout, stderr)
+	} else {
+		deployment := &apps.Deployment{}
+		if err = json.Unmarshal([]byte(stdout), deployment); err != nil {
+			return false, errors.Wrapf(err, "failed to unmarshal the velero deployment")
+		}
+		if deployment.Status.AvailableReplicas != deployment.Status.Replicas {
+			return false, fmt.Errorf("the available replicas %d in velero deployment not equal to replicas %d", deployment.Status.AvailableReplicas, deployment.Status.Replicas)
+		}
+	}
+
+	// Check BSL
+	stdout, stderr, err = velerexec.RunCommand(exec.CommandContext(ctx, "kubectl", "get", "bsl", "default",
+		"-o", "json", "-n", namespace))
+	if err != nil {
+		return false, errors.Wrapf(err, "failed to get bsl %s stdout=%s, stderr=%s", VeleroCfg.BSLBucket, stdout, stderr)
+	} else {
+		bsl := &velerov1api.BackupStorageLocation{}
+		if err = json.Unmarshal([]byte(stdout), bsl); err != nil {
+			return false, errors.Wrapf(err, "failed to unmarshal the velero bsl")
+		}
+		if bsl.Status.Phase != velerov1api.BackupStorageLocationPhaseAvailable {
+			return false, fmt.Errorf("current bsl %s is not available", VeleroCfg.BSLBucket)
+		}
+	}
+
+	return true, nil
+}
+
+func PrepareVelero(ctx context.Context, caseName string) error {
+	ready, err := IsVeleroReady(context.Background(), VeleroCfg.VeleroNamespace, VeleroCfg.UseNodeAgent)
+	if err != nil {
+		fmt.Printf("error in checking velero status with %v", err)
+		VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)
+		ready = false
+	}
+	if ready {
+		fmt.Printf("velero is ready for case %s \n", caseName)
+		return nil
+	}
+	fmt.Printf("need to install velero for case %s \n", caseName)
+	return VeleroInstall(context.Background(), &VeleroCfg)
+}
+
 func VeleroUninstall(ctx context.Context, cli, namespace string) error {
 	stdout, stderr, err := velerexec.RunCommand(exec.CommandContext(ctx, cli, "uninstall", "--force", "-n", namespace))
 	if err != nil {