Support API Version V1 for CR volumesnapshotclass in E2E test

1. In K8S v1.27 API Version v1beta1 for CR volumesnapshotclass is deprcated, so E2E test should adapt both API versions to cover all K8S versio;
2. Support getting additional plugin from input;
3. Velero version and plugin map should not deprated version older than v1.10, because upgrade test will use them.

Signed-off-by: danfengl <danfengl@vmware.com>

test/e2e/backup/backup.go

@@ -127,7 +127,7 @@ func BackupRestoreTest(useVolumeSnapshots bool) {
 
 				Expect(VeleroInstall(context.Background(), &veleroCfg, false)).To(Succeed())
 			}
-			Expect(VeleroAddPluginsForProvider(context.TODO(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, veleroCfg.AdditionalBSLProvider)).To(Succeed())
+			Expect(VeleroAddPluginsForProvider(context.TODO(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, veleroCfg.AdditionalBSLProvider, veleroCfg.AddBSLPlugins)).To(Succeed())
 
 			// Create Secret for additional BSL
 			secretName := fmt.Sprintf("bsl-credentials-%s", UUIDgen)

test/e2e/bsl-mgmt/deletion.go

@@ -104,7 +104,7 @@ func BslDeletionTest(useVolumeSnapshots bool) {
 			}
 
 			By(fmt.Sprintf("Add an additional plugin for provider %s", veleroCfg.AdditionalBSLProvider), func() {
-				Expect(VeleroAddPluginsForProvider(context.TODO(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, veleroCfg.AdditionalBSLProvider)).To(Succeed())
+				Expect(VeleroAddPluginsForProvider(context.TODO(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, veleroCfg.AdditionalBSLProvider, veleroCfg.AddBSLPlugins)).To(Succeed())
 			})
 
 			additionalBsl := fmt.Sprintf("bsl-%s", UUIDgen)

test/e2e/migration/migration.go

@@ -145,6 +145,10 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
 				OriginVeleroCfg.UseVolumeSnapshots = useVolumeSnapshots
 				OriginVeleroCfg.UseNodeAgent = !useVolumeSnapshots
 
+				version, err := GetVeleroVersion(oneHourTimeout, OriginVeleroCfg.VeleroCLI, true)
+				Expect(err).To(Succeed(), "Fail to get Velero version")
+				OriginVeleroCfg.VeleroVersion = version
+
 				// self represents v1.12
 				if veleroCLI2Version.VeleroVersion == "self" {
 					if OriginVeleroCfg.SnapshotMoveData {

test/e2e/upgrade/upgrade.go

@@ -119,6 +119,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
 			veleroCfg.GCFrequency = ""
 			By(fmt.Sprintf("Install the expected old version Velero (%s) for upgrade",
 				veleroCLI2Version.VeleroVersion), func() {
+
 				//Set VeleroImage and RestoreHelperImage to blank
 				//VeleroImage and RestoreHelperImage should be the default value in originalCli
 				tmpCfgForOldVeleroInstall := veleroCfg
@@ -128,6 +129,10 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
 				tmpCfgForOldVeleroInstall.RestoreHelperImage = ""
 				tmpCfgForOldVeleroInstall.Plugins = ""
 				tmpCfgForOldVeleroInstall.UploaderType = ""
+				version, err := GetVeleroVersion(oneHourTimeout, tmpCfgForOldVeleroInstall.VeleroCLI, true)
+				Expect(err).To(Succeed(), "Fail to get Velero version")
+				tmpCfgForOldVeleroInstall.VeleroVersion = version
+
 				if supportUploaderType {
 					tmpCfgForOldVeleroInstall.UseRestic = false
 					tmpCfgForOldVeleroInstall.UseNodeAgent = !useVolumeSnapshots

test/e2e/util/csi/common.go

@@ -91,6 +91,43 @@ func GetCsiSnapshotHandle(client TestClient, backupName string) ([]string, error
 	}
 	return snapshotHandleList, nil
 }
+func GetCsiSnapshotHandleV1(client TestClient, backupName string) ([]string, error) {
+	_, snapshotClient, err := GetClients()
+	if err != nil {
+		return nil, err
+	}
+	vscList, err1 := snapshotClient.SnapshotV1().VolumeSnapshotContents().List(context.TODO(), metav1.ListOptions{})
+	if err1 != nil {
+		return nil, err
+	}
+
+	var snapshotHandleList []string
+	for _, i := range vscList.Items {
+		if i.Status == nil {
+			fmt.Println("SnapshotHandle Status s nil")
+			continue
+		}
+		if i.Status.SnapshotHandle == nil {
+			fmt.Println("SnapshotHandle is nil")
+			continue
+		}
+
+		if i.Labels == nil {
+			fmt.Println("VolumeSnapshotContents label is nil")
+			continue
+		}
+
+		if i.Labels["velero.io/backup-name"] == backupName {
+			tmp := strings.Split(*i.Status.SnapshotHandle, "/")
+			snapshotHandleList = append(snapshotHandleList, tmp[len(tmp)-1])
+		}
+	}
+
+	if len(snapshotHandleList) == 0 {
+		fmt.Printf("No VolumeSnapshotContent from backup %s", backupName)
+	}
+	return snapshotHandleList, nil
+}
 func GetVolumeSnapshotContentNameByPod(client TestClient, podName, namespace, backupName string) (string, error) {
 	pvcList, err := GetPvcByPodName(context.Background(), namespace, podName)
 	if err != nil {
@@ -128,11 +165,19 @@ func GetVolumeSnapshotContentNameByPod(client TestClient, podName, namespace, ba
 	return "", errors.New(fmt.Sprintf("Fail to get VolumeSnapshotContentName for pod %s under namespace %s", podName, namespace))
 }
 
-func CheckVolumeSnapshotCR(client TestClient, backupName string, expectedCount int) ([]string, error) {
+func CheckVolumeSnapshotCR(client TestClient, backupName string, expectedCount int, apiVersion string) ([]string, error) {
 	var err error
 	var snapshotContentNameList []string
-	if snapshotContentNameList, err = GetCsiSnapshotHandle(client, backupName); err != nil {
+	if apiVersion == "v1beta1" {
-		return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content")
+		if snapshotContentNameList, err = GetCsiSnapshotHandle(client, backupName); err != nil {
+			return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content")
+		}
+	} else if apiVersion == "v1" {
+		if snapshotContentNameList, err = GetCsiSnapshotHandleV1(client, backupName); err != nil {
+			return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content")
+		}
+	} else {
+		return nil, errors.New("API version is invalid")
 	}
 	if len(snapshotContentNameList) != expectedCount {
 		return nil, errors.New(fmt.Sprintf("Snapshot count %d is not as expect %d", len(snapshotContentNameList), expectedCount))

test/e2e/util/velero/install.go

@@ -111,7 +111,10 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluste
 		return errors.WithMessagef(err, "Failed to get Velero InstallOptions for plugin provider %s", veleroCfg.ObjectStoreProvider)
 	}
 	veleroInstallOptions.UseVolumeSnapshots = veleroCfg.UseVolumeSnapshots
-	veleroInstallOptions.UseNodeAgent = veleroCfg.UseNodeAgent
+	if !veleroCfg.UseRestic {
+		veleroInstallOptions.UseNodeAgent = veleroCfg.UseNodeAgent
+	}
+	veleroInstallOptions.UseRestic = veleroCfg.UseRestic
 	veleroInstallOptions.Image = veleroCfg.VeleroImage
 	veleroInstallOptions.Namespace = veleroCfg.VeleroNamespace
 	veleroInstallOptions.UploaderType = veleroCfg.UploaderType
@@ -198,6 +201,9 @@ func installVeleroServer(ctx context.Context, cli, cloudProvider string, options
 	if len(options.Image) > 0 {
 		args = append(args, "--image", options.Image)
 	}
+	if options.UseRestic {
+		args = append(args, "--use-restic")
+	}
 	if options.UseNodeAgent {
 		args = append(args, "--use-node-agent")
 	}

test/e2e/util/velero/velero_utils.go

@@ -56,6 +56,27 @@ const RestoreObjectsPrefix = "restores"
 const PluginsObjectsPrefix = "plugins"
 
 var pluginsMatrix = map[string]map[string][]string{
+	"v1.7": {
+		"aws":     {"velero/velero-plugin-for-aws:v1.3.0"},
+		"azure":   {"velero/velero-plugin-for-microsoft-azure:v1.3.0"},
+		"vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.3.0"},
+		"gcp":     {"velero/velero-plugin-for-gcp:v1.3.0"},
+		"csi":     {"velero/velero-plugin-for-csi:v0.2.0"},
+	},
+	"v1.8": {
+		"aws":     {"velero/velero-plugin-for-aws:v1.4.0"},
+		"azure":   {"velero/velero-plugin-for-microsoft-azure:v1.4.0"},
+		"vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.3.1"},
+		"gcp":     {"velero/velero-plugin-for-gcp:v1.4.0"},
+		"csi":     {"velero/velero-plugin-for-csi:v0.2.0"},
+	},
+	"v1.9": {
+		"aws":     {"velero/velero-plugin-for-aws:v1.5.0"},
+		"azure":   {"velero/velero-plugin-for-microsoft-azure:v1.5.0"},
+		"vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.4.0"},
+		"gcp":     {"velero/velero-plugin-for-gcp:v1.5.0"},
+		"csi":     {"velero/velero-plugin-for-csi:v0.3.0"},
+	},
 	"v1.10": {
 		"aws":     {"velero/velero-plugin-for-aws:v1.6.0"},
 		"azure":   {"velero/velero-plugin-for-microsoft-azure:v1.6.0"},
@@ -558,7 +579,7 @@ func getProviderPlugins(ctx context.Context, veleroCLI string, cloudProvider str
 		return []string{}, errors.New("CloudProvider should be provided")
 	}
 
-	version, err := getVeleroVersion(ctx, veleroCLI, true)
+	version, err := GetVeleroVersion(ctx, veleroCLI, true)
 	if err != nil {
 		return nil, errors.WithMessage(err, "failed to get velero version")
 	}
@@ -593,11 +614,17 @@ func getPlugins(ctx context.Context, veleroCfg VeleroConfig) ([]string, error) {
 		if objectStoreProvider == "" {
 			objectStoreProvider = cloudProvider
 		}
-		version, err := getVeleroVersion(ctx, veleroCLI, true)
-		if err != nil {
-			return nil, errors.WithMessage(err, "failed to get velero version")
-		}
 
+		var version string
+		var err error
+		if veleroCfg.VeleroVersion != "" {
+			version = veleroCfg.VeleroVersion
+		} else {
+			version, err = GetVeleroVersion(ctx, veleroCLI, true)
+			if err != nil {
+				return nil, errors.WithMessage(err, "failed to get velero version")
+			}
+		}
 		if veleroCfg.SnapshotMoveData && veleroCfg.DataMoverPlugin == "" {
 			needDataMoverPlugin = true
 		}
@@ -611,8 +638,14 @@ func getPlugins(ctx context.Context, veleroCfg VeleroConfig) ([]string, error) {
 
 // VeleroAddPluginsForProvider determines which plugins need to be installed for a provider and
 // installs them in the current Velero installation, skipping over those that are already installed.
-func VeleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNamespace string, provider string) error {
+func VeleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNamespace string, provider string, plugin string) error {
-	plugins, err := getProviderPlugins(ctx, veleroCLI, provider)
+	var err error
+	var plugins []string
+	if plugin == "" {
+		plugins, err = getProviderPlugins(ctx, veleroCLI, provider)
+	} else {
+		plugins = append(plugins, plugin)
+	}
 	fmt.Printf("provider cmd = %v\n", provider)
 	fmt.Printf("plugins cmd = %v\n", plugins)
 	if err != nil {
@@ -743,7 +776,7 @@ func GetVsphereSnapshotIDs(ctx context.Context, timeout time.Duration, namespace
 	return result, nil
 }
 
-func getVeleroVersion(ctx context.Context, veleroCLI string, clientOnly bool) (string, error) {
+func GetVeleroVersion(ctx context.Context, veleroCLI string, clientOnly bool) (string, error) {
 	args := []string{"version", "--timeout", "60s"}
 	if clientOnly {
 		args = append(args, "--client-only")
@@ -778,7 +811,7 @@ func getVeleroVersion(ctx context.Context, veleroCLI string, clientOnly bool) (s
 
 func CheckVeleroVersion(ctx context.Context, veleroCLI string, expectedVer string) error {
 	tag := expectedVer
-	tagInstalled, err := getVeleroVersion(ctx, veleroCLI, false)
+	tagInstalled, err := GetVeleroVersion(ctx, veleroCLI, false)
 	if err != nil {
 		return errors.WithMessagef(err, "failed to get Velero version")
 	}
@@ -1107,13 +1140,23 @@ func GetResticRepositories(ctx context.Context, veleroNamespace, targetNamespace
 
 func GetSnapshotCheckPoint(client TestClient, VeleroCfg VeleroConfig, expectCount int, namespaceBackedUp, backupName string, KibishiiPVCNameList []string) (SnapshotCheckPoint, error) {
 	var snapshotCheckPoint SnapshotCheckPoint
-	var err error
+
 	snapshotCheckPoint.ExpectCount = expectCount
 	snapshotCheckPoint.NamespaceBackedUp = namespaceBackedUp
 	snapshotCheckPoint.PodName = KibishiiPVCNameList
 	if VeleroCfg.CloudProvider == "azure" && strings.EqualFold(VeleroCfg.Features, "EnableCSI") {
 		snapshotCheckPoint.EnableCSI = true
-		if snapshotCheckPoint.SnapshotIDList, err = util.CheckVolumeSnapshotCR(client, backupName, expectCount); err != nil {
+		resourceName := "snapshot.storage.k8s.io"
+
+		srcVersions, err := GetAPIVersions(VeleroCfg.DefaultClient, resourceName)
+
+		if err != nil {
+			return snapshotCheckPoint, err
+		}
+		if len(srcVersions) == 0 {
+			return snapshotCheckPoint, errors.New("Fail to get APIVersion")
+		}
+		if snapshotCheckPoint.SnapshotIDList, err = util.CheckVolumeSnapshotCR(client, backupName, expectCount, srcVersions[0]); err != nil {
 			return snapshotCheckPoint, errors.Wrapf(err, "Fail to get Azure CSI snapshot content")
 		}
 	}