Add Kopia support for Velero installation in E2E test

Signed-off-by: danfengl <danfengl@vmware.com>

test/e2e/Makefile

@@ -95,6 +95,8 @@ DEBUG_E2E_TEST ?= false
 DEFAULT_CLUSTER ?=
 STANDBY_CLUSTER ?=
 
+UPLOADER_TYPE ?=
+
 
 .PHONY:ginkgo
 ginkgo: # Make sure ginkgo is in $GOPATH/bin
@@ -137,7 +139,8 @@ run: ginkgo
 		-kibishii-directory=$(KIBISHII_DIRECTORY) \
 		-debug-e2e-test=$(DEBUG_E2E_TEST) \
 		-default-cluster=$(DEFAULT_CLUSTER) \
-		-standby-cluster=$(STANDBY_CLUSTER)
+		-standby-cluster=$(STANDBY_CLUSTER) \
+		-uploader-type=$(UPLOADER_TYPE)
 
 build: ginkgo
 	mkdir -p $(OUTPUT_DIR)

test/e2e/backups/deletion.go

@@ -122,12 +122,14 @@ func runBackupDeletionTests(client TestClient, veleroCfg VerleroConfig, backupNa
 	BackupCfg.BackupLocation = backupLocation
 	BackupCfg.UseVolumeSnapshots = useVolumeSnapshots
 	BackupCfg.Selector = ""
-	if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, BackupCfg); err != nil {
+
-		// TODO currently, the upgrade case covers the upgrade path from 1.6 to main and the velero v1.6 doesn't support "debug" command
+	By(fmt.Sprintf("Back up workload with name %s", BackupCfg.BackupName), func() {
-		// TODO move to "runDebug" after we bump up to 1.7 in the upgrade case
+		Expect(VeleroBackupNamespace(oneHourTimeout, veleroCLI,
-		VeleroBackupLogs(context.Background(), VeleroCfg.UpgradeFromVeleroCLI, veleroNamespace, backupName)
+			veleroNamespace, BackupCfg)).To(Succeed(), func() string {
-		return errors.Wrapf(err, "Failed to backup kibishii namespace %s", deletionTest)
+			RunDebug(context.Background(), veleroCLI, veleroNamespace, BackupCfg.BackupName, "")
-	}
+			return "Fail to backup workload"
+		})
+	})
 
 	if providerName == "vsphere" && useVolumeSnapshots {
 		// Wait for uploads started by the Velero Plug-in for vSphere to complete
@@ -176,22 +178,27 @@ func runBackupDeletionTests(client TestClient, veleroCfg VerleroConfig, backupNa
 			return errors.Wrap(err, "exceed waiting for snapshot created in cloud")
 		}
 	}
-
 	backupName = "backup-1-" + UUIDgen.String()
-	if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, BackupCfg); err != nil {
+	BackupCfg.BackupName = backupName
-		// TODO currently, the upgrade case covers the upgrade path from 1.6 to main and the velero v1.6 doesn't support "debug" command
+
-		// TODO move to "runDebug" after we bump up to 1.7 in the upgrade case
+	By(fmt.Sprintf("Back up workload with name %s", BackupCfg.BackupName), func() {
-		VeleroBackupLogs(context.Background(), VeleroCfg.UpgradeFromVeleroCLI, veleroNamespace, backupName)
+		Expect(VeleroBackupNamespace(oneHourTimeout, veleroCLI,
-		return errors.Wrapf(err, "Failed to backup kibishii namespace %s", deletionTest)
+			veleroNamespace, BackupCfg)).To(Succeed(), func() string {
-	}
+			RunDebug(context.Background(), veleroCLI, veleroNamespace, BackupCfg.BackupName, "")
+			return "Fail to backup workload"
+		})
+	})
+
 	err = DeleteObjectsInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix)
 	if err != nil {
 		return err
 	}
+
 	err = ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 1)
 	if err != nil {
 		return err
 	}
+
 	err = DeleteBackupResource(context.Background(), veleroCLI, backupName)
 	if err != nil {
 		return errors.Wrapf(err, "|| UNEXPECTED || - Failed to delete backup %q", backupName)

test/e2e/backups/sync_backups.go

@@ -93,10 +93,11 @@ func BackupsSyncTest() {
 		BackupCfg.UseVolumeSnapshots = false
 		BackupCfg.Selector = ""
 		By(fmt.Sprintf("Backup the workload in %s namespace", test.testNS), func() {
-			if err = VeleroBackupNamespace(test.ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, BackupCfg); err != nil {
+			Expect(VeleroBackupNamespace(test.ctx, VeleroCfg.VeleroCLI,
+				VeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string {
 				RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.backupName, "")
-			}
+				return "Fail to backup workload"
-			Expect(err).To(Succeed(), fmt.Sprintf("Failed to backup %s namespace", test.testNS))
+			})
 		})
 
 		By("Uninstall velero", func() {
@@ -132,10 +133,11 @@ func BackupsSyncTest() {
 		BackupCfg.UseVolumeSnapshots = false
 		BackupCfg.Selector = ""
 		By(fmt.Sprintf("Backup the workload in %s namespace", test.testNS), func() {
-			if err = VeleroBackupNamespace(test.ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, BackupCfg); err != nil {
+			Expect(VeleroBackupNamespace(test.ctx, VeleroCfg.VeleroCLI,
+				VeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string {
 				RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.backupName, "")
-			}
+				return "Fail to backup workload"
-			Expect(err).To(Succeed(), fmt.Sprintf("Failed to backup %s namespace", test.testNS))
+			})
 		})
 
 		By(fmt.Sprintf("Delete %s backup files in object store", test.backupName), func() {

test/e2e/basic/enable_api_group_versions.go

@@ -125,10 +125,10 @@ func APIExtensionsVersionsTest() {
 				BackupCfg.IncludeClusterResources = true
 				BackupCfg.Selector = label
 				Expect(VeleroBackupNamespace(context.Background(), VeleroCfg.VeleroCLI,
-					VeleroCfg.VeleroNamespace, BackupCfg)).ShouldNot(HaveOccurred(), func() string {
+					VeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string {
-					VeleroBackupLogs(context.Background(), VeleroCfg.VeleroCLI,
+					RunDebug(context.Background(), VeleroCfg.VeleroCLI,
-						VeleroCfg.VeleroNamespace, backupName)
+						VeleroCfg.VeleroNamespace, backupName, "")
-					return "Get backup logs"
+					return "Fail to backup workload"
 				})
 			})
 
@@ -367,11 +367,13 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client TestClient, reso
 		BackupCfg.BackupLocation = ""
 		BackupCfg.UseVolumeSnapshots = false
 		BackupCfg.Selector = ""
-		err = VeleroBackupNamespace(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, BackupCfg)
+
-		if err != nil {
+		Expect(VeleroBackupNamespace(ctx, VeleroCfg.VeleroCLI,
-			RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backup, "")
+			VeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string {
-			return errors.Wrapf(err, "back up %s namespaces on source cluster", namespacesStr)
+			RunDebug(context.Background(), VeleroCfg.VeleroCLI,
-		}
+				VeleroCfg.VeleroNamespace, backup, "")
+			return "Fail to backup workload"
+		})
 
 		if err := deleteCRD(ctx, tc.srcCrdYaml); err != nil {
 			return errors.Wrapf(err, "delete music-system CRD from source cluster")

test/e2e/bsl-mgmt/deletion.go

@@ -189,7 +189,10 @@ func BslDeletionTest(useVolumeSnapshots bool) {
 				// TODO currently, the upgrade case covers the upgrade path from 1.6 to main and the velero v1.6 doesn't support "debug" command
 				// TODO move to "runDebug" after we bump up to 1.7 in the upgrade case
 				Expect(VeleroBackupNamespace(oneHourTimeout, VeleroCfg.VeleroCLI,
-					VeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed())
+					VeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string {
+					RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, BackupCfg.BackupName, "")
+					return "Fail to backup workload"
+				})
 			})
 
 			BackupCfg.BackupName = backupName_2
@@ -197,7 +200,10 @@ func BslDeletionTest(useVolumeSnapshots bool) {
 			BackupCfg.Selector = label_2
 			By(fmt.Sprintf("Back up the other one PV of sample workload with label-2 into the additional BSL %s", backupLocation_2), func() {
 				Expect(VeleroBackupNamespace(oneHourTimeout, VeleroCfg.VeleroCLI,
-					VeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed())
+					VeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string {
+					RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, BackupCfg.BackupName, "")
+					return "Fail to backup workload"
+				})
 			})
 
 			if useVolumeSnapshots {

test/e2e/e2e_suite_test.go

@@ -77,6 +77,7 @@ func init() {
 	flag.StringVar(&VeleroCfg.GCFrequency, "garbage-collection-frequency", "", "Frequency of garbage collection.")
 	flag.StringVar(&VeleroCfg.DefaultCluster, "default-cluster", "", "Default cluster context for migration test.")
 	flag.StringVar(&VeleroCfg.StandbyCluster, "standby-cluster", "", "Standby cluster context for migration test.")
+	flag.StringVar(&VeleroCfg.UploaderType, "uploader-type", "", "Identify persistent volume backup uploader.")
 }
 
 var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", APIGropuVersionsTest)

test/e2e/migration/migration.go

@@ -126,6 +126,8 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
 					OriginVeleroCfg.VeleroImage = ""
 					OriginVeleroCfg.ResticHelperImage = ""
 					OriginVeleroCfg.Plugins = ""
+					//TODO: Remove this once origin Velero version is 1.10 and upper
+					OriginVeleroCfg.UploaderType = ""
 				}
 				fmt.Println(OriginVeleroCfg)
 				Expect(VeleroInstall(context.Background(), &OriginVeleroCfg, useVolumeSnapshots)).To(Succeed())
@@ -156,11 +158,11 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
 				BackupStorageClassCfg.BackupName = backupScName
 				BackupStorageClassCfg.IncludeResources = "StorageClass"
 				BackupStorageClassCfg.IncludeClusterResources = true
+
 				Expect(VeleroBackupNamespace(context.Background(), VeleroCfg.VeleroCLI,
-					VeleroCfg.VeleroNamespace, BackupStorageClassCfg)).ShouldNot(HaveOccurred(), func() string {
+					VeleroCfg.VeleroNamespace, BackupStorageClassCfg)).To(Succeed(), func() string {
-					err = VeleroBackupLogs(context.Background(), VeleroCfg.VeleroCLI,
+					RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, BackupStorageClassCfg.BackupName, "")
-						VeleroCfg.VeleroNamespace, backupName)
+					return "Fail to backup workload"
-					return "Get backup logs"
 				})
 
 				var BackupCfg BackupConfig
@@ -169,14 +171,11 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
 				BackupCfg.UseVolumeSnapshots = useVolumeSnapshots
 				BackupCfg.BackupLocation = ""
 				BackupCfg.Selector = ""
-				//BackupCfg.ExcludeResources = "tierentitlementbindings,tierentitlements,tiers,capabilities,customresourcedefinitions"
 				Expect(VeleroBackupNamespace(context.Background(), VeleroCfg.VeleroCLI,
-					VeleroCfg.VeleroNamespace, BackupCfg)).ShouldNot(HaveOccurred(), func() string {
+					VeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string {
-					err = VeleroBackupLogs(context.Background(), VeleroCfg.VeleroCLI,
+					RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, BackupCfg.BackupName, "")
-						VeleroCfg.VeleroNamespace, backupName)
+					return "Fail to backup workload"
-					return "Get backup logs"
 				})
-
 			})
 
 			if useVolumeSnapshots {

test/e2e/types.go

@@ -63,6 +63,7 @@ type VerleroConfig struct {
 	ClientToInstallVelero    *TestClient
 	DefaultClient            *TestClient
 	StandbyClient            *TestClient
+	UploaderType             string
 }
 
 type SnapshotCheckPoint struct {
@@ -86,6 +87,7 @@ type BackupConfig struct {
 	ExcludeResources        string
 	IncludeClusterResources bool
 	OrderedResources        string
+	UseRestic               bool
 }
 
 type VeleroCLI2Version struct {

test/e2e/upgrade/upgrade.go

@@ -112,6 +112,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
 				tmpCfgForOldVeleroInstall.VeleroImage = ""
 				tmpCfgForOldVeleroInstall.ResticHelperImage = ""
 				tmpCfgForOldVeleroInstall.Plugins = ""
+				tmpCfgForOldVeleroInstall.UploaderType = ""
 
 				Expect(VeleroInstall(context.Background(), &tmpCfgForOldVeleroInstall,
 					useVolumeSnapshots)).To(Succeed())
@@ -143,11 +144,13 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
 				BackupCfg.BackupLocation = ""
 				BackupCfg.UseVolumeSnapshots = useVolumeSnapshots
 				BackupCfg.Selector = ""
+				//TODO: pay attention to this param
+				BackupCfg.UseRestic = true
 				Expect(VeleroBackupNamespace(oneHourTimeout, tmpCfg.UpgradeFromVeleroCLI,
-					tmpCfg.VeleroNamespace, BackupCfg)).ShouldNot(HaveOccurred(), func() string {
+					tmpCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string {
-					err = VeleroBackupLogs(context.Background(), tmpCfg.UpgradeFromVeleroCLI,
+					RunDebug(context.Background(), tmpCfg.UpgradeFromVeleroCLI, tmpCfg.VeleroNamespace,
-						tmpCfg.VeleroNamespace, backupName)
+						BackupCfg.BackupName, "")
-					return "Get backup logs"
+					return "Fail to backup workload"
 				})
 			})
 

test/e2e/util/velero/install.go

@@ -90,6 +90,7 @@ func VeleroInstall(ctx context.Context, veleroCfg *VerleroConfig, useVolumeSnaps
 	veleroInstallOptions.UseRestic = !useVolumeSnapshots
 	veleroInstallOptions.Image = veleroCfg.VeleroImage
 	veleroInstallOptions.Namespace = veleroCfg.VeleroNamespace
+	veleroInstallOptions.UploaderType = veleroCfg.UploaderType
 	GCFrequency, _ := time.ParseDuration(veleroCfg.GCFrequency)
 	veleroInstallOptions.GarbageCollectionFrequency = GCFrequency
 
@@ -213,6 +214,10 @@ func installVeleroServer(ctx context.Context, cli string, options *installOption
 		args = append(args, fmt.Sprintf("--garbage-collection-frequency=%v", options.GarbageCollectionFrequency))
 	}
 
+	if len(options.UploaderType) > 0 {
+		args = append(args, fmt.Sprintf("--uploader-type=%v", options.UploaderType))
+	}
+
 	if err := createVelereResources(ctx, cli, namespace, args, options.RegistryCredentialFile, options.ResticHelperImage); err != nil {
 		return err
 	}

test/e2e/util/velero/velero_utils.go

@@ -322,8 +322,13 @@ func VeleroBackupNamespace(ctx context.Context, veleroCLI, veleroNamespace strin
 
 	if backupCfg.UseVolumeSnapshots {
 		args = append(args, "--snapshot-volumes")
+	} else {
+		if backupCfg.UseRestic {
+			args = append(args, "--default-volumes-to-restic")
 		} else {
 			args = append(args, "--default-volumes-to-fs-backup")
+		}
+
 		// To workaround https://github.com/vmware-tanzu/velero-plugin-for-vsphere/issues/347 for vsphere plugin v1.1.1
 		// if the "--snapshot-volumes=false" isn't specified explicitly, the vSphere plugin will always take snapshots
 		// for the volumes even though the "--default-volumes-to-fs-backup" is specified