Merge pull request #5524 from danfengliu/update-upgrade-test

Update upgrade test

Makefile

@@ -209,7 +209,7 @@ endif
 	--build-arg=RESTIC_VERSION=$(RESTIC_VERSION) \
 	-f $(VELERO_DOCKERFILE) .
 	@echo "container: $(IMAGE):$(VERSION)"
-ifeq ($(BUILDX_OUTPUT_TYPE), "registry")
+ifeq ($(BUILDX_OUTPUT_TYPE), registry)
 	@docker pull $(IMAGE):$(VERSION)
 	@docker save $(IMAGE):$(VERSION) -o $(BIN)-$(VERSION).tar
 	@gzip $(BIN)-$(VERSION).tar

test/e2e/Makefile

@@ -55,7 +55,7 @@ VELERO_VERSION ?= $(VERSION)
 PLUGINS ?=
 RESTORE_HELPER_IMAGE ?=
 #Released version only
-UPGRADE_FROM_VELERO_VERSION ?= v1.7.1,v1.8.1
+UPGRADE_FROM_VELERO_VERSION ?= v1.8.1,v1.9.2
 # UPGRADE_FROM_VELERO_CLI can has the same format(a list divided by comma) with UPGRADE_FROM_VELERO_VERSION
 # Upgrade tests will be executed sequently according to the list by UPGRADE_FROM_VELERO_VERSION
 # So although length of UPGRADE_FROM_VELERO_CLI list is not equal with UPGRADE_FROM_VELERO_VERSION

test/e2e/backups/deletion.go

@@ -86,7 +86,7 @@ func backup_deletion_test(useVolumeSnapshots bool) {
 }
 
 // runUpgradeTests runs upgrade test on the provider by kibishii.
-func runBackupDeletionTests(client TestClient, veleroCfg VerleroConfig, backupName, backupLocation string,
+func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupName, backupLocation string,
 	useVolumeSnapshots bool, kibishiiDirectory string) error {
 	oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60)
 	veleroCLI := VeleroCfg.VeleroCLI

test/e2e/types.go

@@ -26,9 +26,9 @@ import (
 
 var UUIDgen uuid.UUID
 
-var VeleroCfg VerleroConfig
+var VeleroCfg VeleroConfig
 
-type VerleroConfig struct {
+type VeleroConfig struct {
 	VeleroCLI                string
 	VeleroImage              string
 	VeleroVersion            string

test/e2e/upgrade/upgrade.go

@@ -29,6 +29,7 @@ import (
 	. "github.com/vmware-tanzu/velero/test/e2e"
 	. "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
 	. "github.com/vmware-tanzu/velero/test/e2e/util/kibishii"
+
 	. "github.com/vmware-tanzu/velero/test/e2e/util/providers"
 	. "github.com/vmware-tanzu/velero/test/e2e/util/velero"
 )
@@ -91,7 +92,6 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
 			UUIDgen, err = uuid.NewRandom()
 			Expect(err).To(Succeed())
 			oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60)
-
 			if veleroCLI2Version.VeleroCLI == "" {
 				//Assume tag of velero server image is identical to velero CLI version
 				//Download velero CLI if it's empty according to velero CLI version
@@ -200,7 +200,8 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
 				tmpCfg.GCFrequency = ""
 				tmpCfg.UseNodeAgent = !useVolumeSnapshots
 				tmpCfg.UseRestic = false
-				Expect(VeleroInstall(context.Background(), &tmpCfg, useVolumeSnapshots)).To(Succeed())
+				tmpCfg.UploaderType = "restic"
+				Expect(VeleroUpgrade(context.Background(), tmpCfg)).To(Succeed())
 				Expect(CheckVeleroVersion(context.Background(), tmpCfg.VeleroCLI,
 					tmpCfg.VeleroVersion)).To(Succeed())
 			})

test/e2e/util/common/common.go

@@ -13,33 +13,36 @@ type OsCommandLine struct {
 	Args []string
 }
 
-func GetListBy2Pipes(ctx context.Context, cmdline1, cmdline2, cmdline3 OsCommandLine) ([]string, error) {
+func GetListByCmdPipes(ctx context.Context, cmdlines []*OsCommandLine) ([]string, error) {
-	var b2 bytes.Buffer
+	var buf bytes.Buffer
-	var errVelero, errAwk error
+	var err error
-
+	var cmds []*exec.Cmd
-	c1 := exec.CommandContext(ctx, cmdline1.Cmd, cmdline1.Args...)
+	for _, cmdline := range cmdlines {
-	c2 := exec.Command(cmdline2.Cmd, cmdline2.Args...)
+		cmd := exec.Command(cmdline.Cmd, cmdline.Args...)
-	c3 := exec.Command(cmdline3.Cmd, cmdline3.Args...)
+		cmds = append(cmds, cmd)
-	fmt.Println(c1)
+		fmt.Println(cmd)
-	fmt.Println(c2)
-	fmt.Println(c3)
-	c2.Stdin, errVelero = c1.StdoutPipe()
-	if errVelero != nil {
-		return nil, errVelero
 	}
-	c3.Stdin, errAwk = c2.StdoutPipe()
+	for i := 0; i < len(cmds); i++ {
-	if errAwk != nil {
+		if i == len(cmds)-1 {
-		return nil, errAwk
+			break
+		}
+		cmds[i+1].Stdin, err = cmds[i].StdoutPipe()
+		if err != nil {
+			return nil, err
+		}
+	}
+	cmds[len(cmds)-1].Stdout = &buf
+	for i := len(cmds) - 1; i >= 0; i-- {
+		_ = cmds[i].Start()
+		if i == 0 {
+			_ = cmds[i].Run()
+		}
+	}
+	for i := 1; i < len(cmds); i++ {
+		_ = cmds[i].Wait()
 	}
-	c3.Stdout = &b2
-	_ = c3.Start()
-	_ = c2.Start()
-	_ = c1.Run()
-	_ = c2.Wait()
-	_ = c3.Wait()
 
-	//fmt.Println(&b2)
+	scanner := bufio.NewScanner(&buf)
-	scanner := bufio.NewScanner(&b2)
 	var ret []string
 	for scanner.Scan() {
 		fmt.Printf("line: %s\n", scanner.Text())

test/e2e/util/k8s/common.go

@@ -31,7 +31,7 @@ import (
 
 	"github.com/vmware-tanzu/velero/pkg/builder"
 	veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec"
-	common "github.com/vmware-tanzu/velero/test/e2e/util/common"
+	"github.com/vmware-tanzu/velero/test/e2e/util/common"
 )
 
 // ensureClusterExists returns whether or not a kubernetes cluster exists for tests to be run on.
@@ -86,42 +86,52 @@ func GetPvcByPodName(ctx context.Context, namespace, podName string) ([]string,
 	// Example:
 	//    NAME                                  STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS             AGE
 	//    kibishii-data-kibishii-deployment-0   Bound    pvc-94b9fdf2-c30f-4a7b-87bf-06eadca0d5b6   1Gi        RWO            kibishii-storage-class   115s
-	CmdLine1 := &common.OsCommandLine{
+	cmds := []*common.OsCommandLine{}
+	cmd := &common.OsCommandLine{
 		Cmd:  "kubectl",
 		Args: []string{"get", "pvc", "-n", namespace},
 	}
-	CmdLine2 := &common.OsCommandLine{
+	cmds = append(cmds, cmd)
+
+	cmd = &common.OsCommandLine{
 		Cmd:  "grep",
 		Args: []string{podName},
 	}
-	CmdLine3 := &common.OsCommandLine{
+	cmds = append(cmds, cmd)
+
+	cmd = &common.OsCommandLine{
 		Cmd:  "awk",
 		Args: []string{"{print $1}"},
 	}
+	cmds = append(cmds, cmd)
 
-	return common.GetListBy2Pipes(ctx, *CmdLine1, *CmdLine2, *CmdLine3)
+	return common.GetListByCmdPipes(ctx, cmds)
 }
 
 func GetPvByPvc(ctx context.Context, namespace, pvc string) ([]string, error) {
 	// Example:
 	// 	  NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM                                              STORAGECLASS             REASON   AGE
 	//    pvc-3f784366-58db-40b2-8fec-77307807e74b   1Gi        RWO            Delete           Bound    bsl-deletion/kibishii-data-kibishii-deployment-0   kibishii-storage-class            6h41m
-	CmdLine1 := &common.OsCommandLine{
+	cmds := []*common.OsCommandLine{}
+	cmd := &common.OsCommandLine{
 		Cmd:  "kubectl",
 		Args: []string{"get", "pv"},
 	}
+	cmds = append(cmds, cmd)
 
-	CmdLine2 := &common.OsCommandLine{
+	cmd = &common.OsCommandLine{
 		Cmd:  "grep",
 		Args: []string{namespace + "/" + pvc},
 	}
+	cmds = append(cmds, cmd)
 
-	CmdLine3 := &common.OsCommandLine{
+	cmd = &common.OsCommandLine{
 		Cmd:  "awk",
 		Args: []string{"{print $1}"},
 	}
+	cmds = append(cmds, cmd)
 
-	return common.GetListBy2Pipes(ctx, *CmdLine1, *CmdLine2, *CmdLine3)
+	return common.GetListByCmdPipes(ctx, cmds)
 }
 
 func CRDShouldExist(ctx context.Context, name string) error {
@@ -145,22 +155,26 @@ func CRDCountShouldBe(ctx context.Context, name string, count int) error {
 }
 
 func GetCRD(ctx context.Context, name string) ([]string, error) {
-	CmdLine1 := &common.OsCommandLine{
+	cmds := []*common.OsCommandLine{}
+	cmd := &common.OsCommandLine{
 		Cmd:  "kubectl",
 		Args: []string{"get", "crd"},
 	}
+	cmds = append(cmds, cmd)
 
-	CmdLine2 := &common.OsCommandLine{
+	cmd = &common.OsCommandLine{
 		Cmd:  "grep",
 		Args: []string{name},
 	}
+	cmds = append(cmds, cmd)
 
-	CmdLine3 := &common.OsCommandLine{
+	cmd = &common.OsCommandLine{
 		Cmd:  "awk",
 		Args: []string{"{print $1}"},
 	}
+	cmds = append(cmds, cmd)
 
-	return common.GetListBy2Pipes(ctx, *CmdLine1, *CmdLine2, *CmdLine3)
+	return common.GetListByCmdPipes(ctx, cmds)
 }
 
 func AddLabelToPv(ctx context.Context, pv, label string) error {
@@ -282,3 +296,34 @@ func ReadFileFromPodVolume(ctx context.Context, namespace, podName, volume, file
 	fmt.Print(stderr)
 	return stdout, err
 }
+
+func KubectlGetInfo(cmdName string, arg []string) {
+	cmd := exec.CommandContext(context.Background(), cmdName, arg...)
+	fmt.Printf("Kubectl exec cmd =%v\n", cmd)
+	stdout, stderr, err := veleroexec.RunCommand(cmd)
+	fmt.Println(stdout)
+	if err != nil {
+		fmt.Println(stderr)
+		fmt.Println(err)
+	}
+}
+
+func KubectlGetDsJson(veleroNamespace string) (string, error) {
+	arg := []string{"get", "ds", "-n", veleroNamespace, "-ojson"}
+	cmd := exec.CommandContext(context.Background(), "kubectl", arg...)
+	fmt.Printf("Kubectl exec cmd =%v\n", cmd)
+	stdout, stderr, err := veleroexec.RunCommand(cmd)
+	fmt.Println(stdout)
+	if err != nil {
+		fmt.Println(stderr)
+		fmt.Println(err)
+		return "", err
+	}
+	return stdout, nil
+}
+
+func DeleteVeleroDs(ctx context.Context) error {
+	args := []string{"delete", "ds", "-n", "velero", "--all", "--force", "--grace-period", "0"}
+	fmt.Println(args)
+	return exec.CommandContext(ctx, "kubectl", args...).Run()
+}

test/e2e/util/kibishii/kibishii_utils.go

@@ -51,7 +51,7 @@ var DefaultKibishiiData = &KibishiiData{2, 10, 10, 1024, 1024, 0, 2}
 var KibishiiPodNameList = []string{"kibishii-deployment-0", "kibishii-deployment-1"}
 
 // RunKibishiiTests runs kibishii tests on the provider.
-func RunKibishiiTests(client TestClient, veleroCfg VerleroConfig, backupName, restoreName, backupLocation, kibishiiNamespace string,
+func RunKibishiiTests(client TestClient, veleroCfg VeleroConfig, backupName, restoreName, backupLocation, kibishiiNamespace string,
 	useVolumeSnapshots bool) error {
 	oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60)
 	veleroCLI := VeleroCfg.VeleroCLI
@@ -174,7 +174,8 @@ func installKibishii(ctx context.Context, namespace string, cloudPlatform, veler
 }
 
 func generateData(ctx context.Context, namespace string, kibishiiData *KibishiiData) error {
-	kibishiiGenerateCmd := exec.CommandContext(ctx, "kubectl", "exec", "-n", namespace, "jump-pad", "--",
+	timeout, _ := context.WithTimeout(context.Background(), time.Minute*10)
+	kibishiiGenerateCmd := exec.CommandContext(timeout, "kubectl", "exec", "-n", namespace, "jump-pad", "--",
 		"/usr/local/bin/generate.sh", strconv.Itoa(kibishiiData.Levels), strconv.Itoa(kibishiiData.DirsPerLevel),
 		strconv.Itoa(kibishiiData.FilesPerLevel), strconv.Itoa(kibishiiData.FileLength),
 		strconv.Itoa(kibishiiData.BlockSize), strconv.Itoa(kibishiiData.PassNum), strconv.Itoa(kibishiiData.ExpectedNodes))
@@ -189,7 +190,7 @@ func generateData(ctx context.Context, namespace string, kibishiiData *KibishiiD
 }
 
 func verifyData(ctx context.Context, namespace string, kibishiiData *KibishiiData) error {
-	timeout, _ := context.WithTimeout(context.Background(), time.Minute*5)
+	timeout, _ := context.WithTimeout(context.Background(), time.Minute*10)
 	kibishiiVerifyCmd := exec.CommandContext(timeout, "kubectl", "exec", "-n", namespace, "jump-pad", "--",
 		"/usr/local/bin/verify.sh", strconv.Itoa(kibishiiData.Levels), strconv.Itoa(kibishiiData.DirsPerLevel),
 		strconv.Itoa(kibishiiData.FilesPerLevel), strconv.Itoa(kibishiiData.FileLength),

test/e2e/util/velero/install.go

@@ -49,7 +49,7 @@ type installOptions struct {
 	RestoreHelperImage     string
 }
 
-func VeleroInstall(ctx context.Context, veleroCfg *VerleroConfig, useVolumeSnapshots bool) error {
+func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, useVolumeSnapshots bool) error {
 	if veleroCfg.CloudProvider != "kind" {
 		if veleroCfg.ObjectStoreProvider != "" {
 			return errors.New("For cloud platforms, object store plugin cannot be overridden") // Can't set an object store provider that is different than your cloud
@@ -103,6 +103,7 @@ func VeleroInstall(ctx context.Context, veleroCfg *VerleroConfig, useVolumeSnaps
 		RestoreHelperImage:     veleroCfg.RestoreHelperImage,
 	})
 	if err != nil {
+		RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, "", "")
 		return errors.WithMessagef(err, "Failed to install Velero in the cluster")
 	}
 

test/e2e/util/velero/velero_utils.go

@@ -96,6 +96,13 @@ var pluginsMatrix = map[string]map[string][]string{
 		"gcp":       {"velero/velero-plugin-for-gcp:v1.5.0"},
 		"azure-csi": {"velero/velero-plugin-for-microsoft-azure:v1.5.0", "velero/velero-plugin-for-csi:v0.3.0"},
 	},
+	"v1.10": {
+		"aws":       {"velero/velero-plugin-for-aws:v1.6.0"},
+		"azure":     {"velero/velero-plugin-for-microsoft-azure:v1.6.0"},
+		"vsphere":   {"velero/velero-plugin-for-aws:v1.6.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.4.1"},
+		"gcp":       {"velero/velero-plugin-for-gcp:v1.6.0"},
+		"azure-csi": {"velero/velero-plugin-for-microsoft-azure:v1.6.0", "velero/velero-plugin-for-csi:v0.4.0"},
+	},
 	"main": {
 		"aws":       {"velero/velero-plugin-for-aws:main"},
 		"azure":     {"velero/velero-plugin-for-microsoft-azure:main"},
@@ -706,23 +713,30 @@ func CheckVeleroVersion(ctx context.Context, veleroCLI string, expectedVer strin
 }
 
 func InstallVeleroCLI(version string) (string, error) {
+	var tempVeleroCliDir string
 	name := "velero-" + version + "-" + runtime.GOOS + "-" + runtime.GOARCH
 	postfix := ".tar.gz"
 	tarball := name + postfix
-	tempFile, err := getVeleroCliTarball("https://github.com/vmware-tanzu/velero/releases/download/" + version + "/" + tarball)
+	err := wait.PollImmediate(time.Second*5, time.Minute*5, func() (bool, error) {
-	if err != nil {
+		tempFile, err := getVeleroCliTarball("https://github.com/vmware-tanzu/velero/releases/download/" + version + "/" + tarball)
-		return "", errors.WithMessagef(err, "failed to get Velero CLI tarball")
+		if err != nil {
-	}
+			return false, errors.WithMessagef(err, "failed to get Velero CLI tarball")
-	tempVeleroCliDir, err := ioutil.TempDir("", "velero-test")
+		}
-	if err != nil {
+		tempVeleroCliDir, err = ioutil.TempDir("", "velero-test")
-		return "", errors.WithMessagef(err, "failed to create temp dir for tarball extraction")
+		if err != nil {
-	}
+			return false, errors.WithMessagef(err, "failed to create temp dir for tarball extraction")
+		}
 
-	cmd := exec.Command("tar", "-xvf", tempFile.Name(), "-C", tempVeleroCliDir)
+		cmd := exec.Command("tar", "-xvf", tempFile.Name(), "-C", tempVeleroCliDir)
-	defer os.Remove(tempFile.Name())
+		defer os.Remove(tempFile.Name())
 
-	if _, err := cmd.Output(); err != nil {
+		if _, err := cmd.Output(); err != nil {
-		return "", errors.WithMessagef(err, "failed to extract file from velero CLI tarball")
+			return false, errors.WithMessagef(err, "failed to extract file from velero CLI tarball")
+		}
+		return true, nil
+	})
+	if err != nil {
+		return "", errors.WithMessagef(err, "failed to install velero CLI")
 	}
 	return tempVeleroCliDir + "/" + name + "/velero", nil
 }
@@ -866,22 +880,27 @@ func GetBackupsFromBsl(ctx context.Context, veleroCLI, bslName string) ([]string
 	if strings.TrimSpace(bslName) != "" {
 		args1 = append(args1, "-l", "velero.io/storage-location="+bslName)
 	}
-	CmdLine1 := &common.OsCommandLine{
+	cmds := []*common.OsCommandLine{}
+
+	cmd := &common.OsCommandLine{
 		Cmd:  veleroCLI,
 		Args: args1,
 	}
+	cmds = append(cmds, cmd)
 
-	CmdLine2 := &common.OsCommandLine{
+	cmd = &common.OsCommandLine{
 		Cmd:  "awk",
 		Args: []string{"{print $1}"},
 	}
+	cmds = append(cmds, cmd)
 
-	CmdLine3 := &common.OsCommandLine{
+	cmd = &common.OsCommandLine{
 		Cmd:  "tail",
 		Args: []string{"-n", "+2"},
 	}
+	cmds = append(cmds, cmd)
 
-	return common.GetListBy2Pipes(ctx, *CmdLine1, *CmdLine2, *CmdLine3)
+	return common.GetListByCmdPipes(ctx, cmds)
 }
 
 func GetScheduledBackupsCreationTime(ctx context.Context, veleroCLI, bslName, scheduleName string) ([]string, error) {
@@ -903,22 +922,27 @@ func GetBackupsCreationTime(ctx context.Context, veleroCLI, bslName string) ([]s
 	if strings.TrimSpace(bslName) != "" {
 		args1 = append(args1, "-l", "velero.io/storage-location="+bslName)
 	}
-	CmdLine1 := &common.OsCommandLine{
+	cmds := []*common.OsCommandLine{}
+
+	cmd := &common.OsCommandLine{
 		Cmd:  veleroCLI,
 		Args: args1,
 	}
+	cmds = append(cmds, cmd)
 
-	CmdLine2 := &common.OsCommandLine{
+	cmd = &common.OsCommandLine{
 		Cmd:  "awk",
 		Args: []string{"{print " + createdTime + "}"},
 	}
+	cmds = append(cmds, cmd)
 
-	CmdLine3 := &common.OsCommandLine{
+	cmd = &common.OsCommandLine{
 		Cmd:  "tail",
 		Args: []string{"-n", "+2"},
 	}
+	cmds = append(cmds, cmd)
 
-	return common.GetListBy2Pipes(ctx, *CmdLine1, *CmdLine2, *CmdLine3)
+	return common.GetListByCmdPipes(ctx, cmds)
 }
 
 func GetAllBackups(ctx context.Context, veleroCLI string) ([]string, error) {
@@ -982,25 +1006,29 @@ func BackupRepositoriesCountShouldBe(ctx context.Context, veleroNamespace, targe
 }
 
 func GetResticRepositories(ctx context.Context, veleroNamespace, targetNamespace string) ([]string, error) {
-	CmdLine1 := &common.OsCommandLine{
+	cmds := []*common.OsCommandLine{}
+	cmd := &common.OsCommandLine{
 		Cmd:  "kubectl",
 		Args: []string{"get", "-n", veleroNamespace, "BackupRepositories"},
 	}
+	cmds = append(cmds, cmd)
 
-	CmdLine2 := &common.OsCommandLine{
+	cmd = &common.OsCommandLine{
 		Cmd:  "grep",
 		Args: []string{targetNamespace},
 	}
+	cmds = append(cmds, cmd)
 
-	CmdLine3 := &common.OsCommandLine{
+	cmd = &common.OsCommandLine{
 		Cmd:  "awk",
 		Args: []string{"{print $1}"},
 	}
+	cmds = append(cmds, cmd)
 
-	return common.GetListBy2Pipes(ctx, *CmdLine1, *CmdLine2, *CmdLine3)
+	return common.GetListByCmdPipes(ctx, cmds)
 }
 
-func GetSnapshotCheckPoint(client TestClient, VeleroCfg VerleroConfig, expectCount int, namespaceBackedUp, backupName string, kibishiiPodNameList []string) (SnapshotCheckPoint, error) {
+func GetSnapshotCheckPoint(client TestClient, VeleroCfg VeleroConfig, expectCount int, namespaceBackedUp, backupName string, kibishiiPodNameList []string) (SnapshotCheckPoint, error) {
 	var snapshotCheckPoint SnapshotCheckPoint
 
 	snapshotCheckPoint.ExpectCount = expectCount
@@ -1081,3 +1109,143 @@ func GetSchedule(ctx context.Context, veleroNamespace, scheduleName string) (str
 	}
 	return stdout, err
 }
+
+func VeleroUpgrade(ctx context.Context, veleroCfg VeleroConfig) error {
+	crd, err := ApplyCRDs(ctx, veleroCfg.VeleroCLI)
+	if err != nil {
+		return errors.Wrap(err, "Fail to Apply CRDs")
+	}
+	fmt.Println(crd)
+	deploy, err := UpdateVeleroDeployment(ctx, veleroCfg)
+	if err != nil {
+		return errors.Wrap(err, "Fail to update Velero deployment")
+	}
+	fmt.Println(deploy)
+	if veleroCfg.UseNodeAgent {
+		dsjson, err := KubectlGetDsJson(veleroCfg.VeleroNamespace)
+		if err != nil {
+			return errors.Wrap(err, "Fail to update Velero deployment")
+		}
+
+		err = DeleteVeleroDs(ctx)
+		if err != nil {
+			return errors.Wrap(err, "Fail to delete Velero ds")
+		}
+		update, err := UpdateNodeAgent(ctx, veleroCfg, dsjson)
+		fmt.Println(update)
+		if err != nil {
+			return errors.Wrap(err, "Fail to update node agent")
+		}
+	}
+	return waitVeleroReady(ctx, veleroCfg.VeleroNamespace, veleroCfg.UseNodeAgent)
+}
+func ApplyCRDs(ctx context.Context, veleroCLI string) ([]string, error) {
+	cmds := []*common.OsCommandLine{}
+
+	cmd := &common.OsCommandLine{
+		Cmd:  veleroCLI,
+		Args: []string{"install", "--crds-only", "--dry-run", "-o", "yaml"},
+	}
+	cmds = append(cmds, cmd)
+
+	cmd = &common.OsCommandLine{
+		Cmd:  "kubectl",
+		Args: []string{"apply", "-f", "-"},
+	}
+	cmds = append(cmds, cmd)
+	return common.GetListByCmdPipes(ctx, cmds)
+}
+
+func UpdateVeleroDeployment(ctx context.Context, veleroCfg VeleroConfig) ([]string, error) {
+	cmds := []*common.OsCommandLine{}
+
+	cmd := &common.OsCommandLine{
+		Cmd:  "kubectl",
+		Args: []string{"get", "deploy", "-n", veleroCfg.VeleroNamespace, "-ojson"},
+	}
+	cmds = append(cmds, cmd)
+	var args string
+	if veleroCfg.CloudProvider == "vsphere" {
+		args = fmt.Sprintf("s#\\\"image\\\"\\: \\\"velero\\/velero\\:v[0-9]*.[0-9]*.[0-9]\\\"#\\\"image\\\"\\: \\\"harbor-repo.vmware.com\\/velero_ci\\/velero\\:%s\\\"#g", veleroCfg.VeleroVersion)
+	} else {
+		args = fmt.Sprintf("s#\\\"image\\\"\\: \\\"velero\\/velero\\:v[0-9]*.[0-9]*.[0-9]\\\"#\\\"image\\\"\\: \\\"velero\\/velero\\:%s\\\"#g", veleroCfg.VeleroVersion)
+	}
+	cmd = &common.OsCommandLine{
+		Cmd:  "sed",
+		Args: []string{args},
+	}
+	cmds = append(cmds, cmd)
+
+	cmd = &common.OsCommandLine{
+		Cmd:  "sed",
+		Args: []string{fmt.Sprintf("s#\\\"server\\\",#\\\"server\\\",\\\"--uploader-type=%s\\\",#g", veleroCfg.UploaderType)},
+	}
+	cmds = append(cmds, cmd)
+
+	cmd = &common.OsCommandLine{
+		Cmd:  "sed",
+		Args: []string{"s#default-volumes-to-restic#default-volumes-to-fs-backup#g"},
+	}
+	cmds = append(cmds, cmd)
+
+	cmd = &common.OsCommandLine{
+		Cmd:  "sed",
+		Args: []string{"s#default-restic-prune-frequency#default-repo-maintain-frequency#g"},
+	}
+	cmds = append(cmds, cmd)
+
+	cmd = &common.OsCommandLine{
+		Cmd:  "sed",
+		Args: []string{"s#restic-timeout#fs-backup-timeout#g"},
+	}
+	cmds = append(cmds, cmd)
+
+	cmd = &common.OsCommandLine{
+		Cmd:  "kubectl",
+		Args: []string{"apply", "-f", "-"},
+	}
+	cmds = append(cmds, cmd)
+
+	return common.GetListByCmdPipes(ctx, cmds)
+}
+
+func UpdateNodeAgent(ctx context.Context, veleroCfg VeleroConfig, dsjson string) ([]string, error) {
+	cmds := []*common.OsCommandLine{}
+
+	cmd := &common.OsCommandLine{
+		Cmd:  "echo",
+		Args: []string{dsjson},
+	}
+	cmds = append(cmds, cmd)
+	var args string
+	if veleroCfg.CloudProvider == "vsphere" {
+		args = fmt.Sprintf("s#\\\"image\\\"\\: \\\"velero\\/velero\\:v[0-9]*.[0-9]*.[0-9]\\\"#\\\"image\\\"\\: \\\"harbor-repo.vmware.com\\/velero_ci\\/velero\\:%s\\\"#g", veleroCfg.VeleroVersion)
+	} else {
+		args = fmt.Sprintf("s#\\\"image\\\"\\: \\\"velero\\/velero\\:v[0-9]*.[0-9]*.[0-9]\\\"#\\\"image\\\"\\: \\\"velero\\/velero\\:%s\\\"#g", veleroCfg.VeleroVersion)
+	}
+	cmd = &common.OsCommandLine{
+		Cmd:  "sed",
+		Args: []string{args},
+	}
+	cmds = append(cmds, cmd)
+
+	cmd = &common.OsCommandLine{
+		Cmd:  "sed",
+		Args: []string{"s#\\\"name\\\"\\: \\\"restic\\\"#\\\"name\\\"\\: \\\"node-agent\\\"#g"},
+	}
+	cmds = append(cmds, cmd)
+
+	cmd = &common.OsCommandLine{
+		Cmd:  "sed",
+		Args: []string{"s#\\\"restic\\\",#\\\"node-agent\\\",#g"},
+	}
+	cmds = append(cmds, cmd)
+
+	cmd = &common.OsCommandLine{
+		Cmd:  "kubectl",
+		Args: []string{"create", "-f", "-"},
+	}
+	cmds = append(cmds, cmd)
+
+	return common.GetListByCmdPipes(ctx, cmds)
+}