Merge pull request #6509 from danfengliu/adapt-k8s-1.27-api

Support AP IVersion V1  for CR volumesnapshotclass in E2E test and FIx plugin parameter issue

test/e2e/backup/backup.go

@@ -127,7 +127,7 @@ func BackupRestoreTest(useVolumeSnapshots bool) {
 
 				Expect(VeleroInstall(context.Background(), &veleroCfg, false)).To(Succeed())
 			}
-			Expect(VeleroAddPluginsForProvider(context.TODO(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, veleroCfg.AdditionalBSLProvider)).To(Succeed())
+			Expect(VeleroAddPluginsForProvider(context.TODO(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, veleroCfg.AdditionalBSLProvider, veleroCfg.AddBSLPlugins)).To(Succeed())
 
 			// Create Secret for additional BSL
 			secretName := fmt.Sprintf("bsl-credentials-%s", UUIDgen)

test/e2e/bsl-mgmt/deletion.go

@@ -104,7 +104,7 @@ func BslDeletionTest(useVolumeSnapshots bool) {
 			}
 
 			By(fmt.Sprintf("Add an additional plugin for provider %s", veleroCfg.AdditionalBSLProvider), func() {
-				Expect(VeleroAddPluginsForProvider(context.TODO(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, veleroCfg.AdditionalBSLProvider)).To(Succeed())
+				Expect(VeleroAddPluginsForProvider(context.TODO(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, veleroCfg.AdditionalBSLProvider, veleroCfg.AddBSLPlugins)).To(Succeed())
 			})
 
 			additionalBsl := fmt.Sprintf("bsl-%s", UUIDgen)

test/e2e/migration/migration.go

@@ -145,6 +145,10 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
 				OriginVeleroCfg.UseVolumeSnapshots = useVolumeSnapshots
 				OriginVeleroCfg.UseNodeAgent = !useVolumeSnapshots
 
+				version, err := GetVeleroVersion(oneHourTimeout, OriginVeleroCfg.VeleroCLI, true)
+				Expect(err).To(Succeed(), "Fail to get Velero version")
+				OriginVeleroCfg.VeleroVersion = version
+
 				// self represents v1.12
 				if veleroCLI2Version.VeleroVersion == "self" {
 					if OriginVeleroCfg.SnapshotMoveData {

test/e2e/upgrade/upgrade.go

@@ -119,6 +119,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
 			veleroCfg.GCFrequency = ""
 			By(fmt.Sprintf("Install the expected old version Velero (%s) for upgrade",
 				veleroCLI2Version.VeleroVersion), func() {
+
 				//Set VeleroImage and RestoreHelperImage to blank
 				//VeleroImage and RestoreHelperImage should be the default value in originalCli
 				tmpCfgForOldVeleroInstall := veleroCfg
@@ -128,6 +129,10 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
 				tmpCfgForOldVeleroInstall.RestoreHelperImage = ""
 				tmpCfgForOldVeleroInstall.Plugins = ""
 				tmpCfgForOldVeleroInstall.UploaderType = ""
+				version, err := GetVeleroVersion(oneHourTimeout, tmpCfgForOldVeleroInstall.VeleroCLI, true)
+				Expect(err).To(Succeed(), "Fail to get Velero version")
+				tmpCfgForOldVeleroInstall.VeleroVersion = version
+
 				if supportUploaderType {
 					tmpCfgForOldVeleroInstall.UseRestic = false
 					tmpCfgForOldVeleroInstall.UseNodeAgent = !useVolumeSnapshots

test/e2e/util/csi/common.go

@@ -91,6 +91,43 @@ func GetCsiSnapshotHandle(client TestClient, backupName string) ([]string, error
 	}
 	return snapshotHandleList, nil
 }
+func GetCsiSnapshotHandleV1(client TestClient, backupName string) ([]string, error) {
+	_, snapshotClient, err := GetClients()
+	if err != nil {
+		return nil, err
+	}
+	vscList, err1 := snapshotClient.SnapshotV1().VolumeSnapshotContents().List(context.TODO(), metav1.ListOptions{})
+	if err1 != nil {
+		return nil, err
+	}
+
+	var snapshotHandleList []string
+	for _, i := range vscList.Items {
+		if i.Status == nil {
+			fmt.Println("SnapshotHandle Status s nil")
+			continue
+		}
+		if i.Status.SnapshotHandle == nil {
+			fmt.Println("SnapshotHandle is nil")
+			continue
+		}
+
+		if i.Labels == nil {
+			fmt.Println("VolumeSnapshotContents label is nil")
+			continue
+		}
+
+		if i.Labels["velero.io/backup-name"] == backupName {
+			tmp := strings.Split(*i.Status.SnapshotHandle, "/")
+			snapshotHandleList = append(snapshotHandleList, tmp[len(tmp)-1])
+		}
+	}
+
+	if len(snapshotHandleList) == 0 {
+		fmt.Printf("No VolumeSnapshotContent from backup %s", backupName)
+	}
+	return snapshotHandleList, nil
+}
 func GetVolumeSnapshotContentNameByPod(client TestClient, podName, namespace, backupName string) (string, error) {
 	pvcList, err := GetPvcByPodName(context.Background(), namespace, podName)
 	if err != nil {
@@ -128,11 +165,19 @@ func GetVolumeSnapshotContentNameByPod(client TestClient, podName, namespace, ba
 	return "", errors.New(fmt.Sprintf("Fail to get VolumeSnapshotContentName for pod %s under namespace %s", podName, namespace))
 }
 
-func CheckVolumeSnapshotCR(client TestClient, backupName string, expectedCount int) ([]string, error) {
+func CheckVolumeSnapshotCR(client TestClient, backupName string, expectedCount int, apiVersion string) ([]string, error) {
 	var err error
 	var snapshotContentNameList []string
-	if snapshotContentNameList, err = GetCsiSnapshotHandle(client, backupName); err != nil {
+	if apiVersion == "v1beta1" {
-		return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content")
+		if snapshotContentNameList, err = GetCsiSnapshotHandle(client, backupName); err != nil {
+			return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content")
+		}
+	} else if apiVersion == "v1" {
+		if snapshotContentNameList, err = GetCsiSnapshotHandleV1(client, backupName); err != nil {
+			return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content")
+		}
+	} else {
+		return nil, errors.New("API version is invalid")
 	}
 	if len(snapshotContentNameList) != expectedCount {
 		return nil, errors.New(fmt.Sprintf("Snapshot count %d is not as expect %d", len(snapshotContentNameList), expectedCount))

test/e2e/util/velero/install.go

@@ -111,7 +111,10 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluste
 		return errors.WithMessagef(err, "Failed to get Velero InstallOptions for plugin provider %s", veleroCfg.ObjectStoreProvider)
 	}
 	veleroInstallOptions.UseVolumeSnapshots = veleroCfg.UseVolumeSnapshots
-	veleroInstallOptions.UseNodeAgent = veleroCfg.UseNodeAgent
+	if !veleroCfg.UseRestic {
+		veleroInstallOptions.UseNodeAgent = veleroCfg.UseNodeAgent
+	}
+	veleroInstallOptions.UseRestic = veleroCfg.UseRestic
 	veleroInstallOptions.Image = veleroCfg.VeleroImage
 	veleroInstallOptions.Namespace = veleroCfg.VeleroNamespace
 	veleroInstallOptions.UploaderType = veleroCfg.UploaderType
@@ -198,6 +201,9 @@ func installVeleroServer(ctx context.Context, cli, cloudProvider string, options
 	if len(options.Image) > 0 {
 		args = append(args, "--image", options.Image)
 	}
+	if options.UseRestic {
+		args = append(args, "--use-restic")
+	}
 	if options.UseNodeAgent {
 		args = append(args, "--use-node-agent")
 	}

test/e2e/util/velero/velero_utils.go

@@ -56,6 +56,27 @@ const RestoreObjectsPrefix = "restores"
 const PluginsObjectsPrefix = "plugins"
 
 var pluginsMatrix = map[string]map[string][]string{
+	"v1.7": {
+		"aws":     {"velero/velero-plugin-for-aws:v1.3.0"},
+		"azure":   {"velero/velero-plugin-for-microsoft-azure:v1.3.0"},
+		"vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.3.0"},
+		"gcp":     {"velero/velero-plugin-for-gcp:v1.3.0"},
+		"csi":     {"velero/velero-plugin-for-csi:v0.2.0"},
+	},
+	"v1.8": {
+		"aws":     {"velero/velero-plugin-for-aws:v1.4.0"},
+		"azure":   {"velero/velero-plugin-for-microsoft-azure:v1.4.0"},
+		"vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.3.1"},
+		"gcp":     {"velero/velero-plugin-for-gcp:v1.4.0"},
+		"csi":     {"velero/velero-plugin-for-csi:v0.2.0"},
+	},
+	"v1.9": {
+		"aws":     {"velero/velero-plugin-for-aws:v1.5.0"},
+		"azure":   {"velero/velero-plugin-for-microsoft-azure:v1.5.0"},
+		"vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.4.0"},
+		"gcp":     {"velero/velero-plugin-for-gcp:v1.5.0"},
+		"csi":     {"velero/velero-plugin-for-csi:v0.3.0"},
+	},
 	"v1.10": {
 		"aws":     {"velero/velero-plugin-for-aws:v1.6.0"},
 		"azure":   {"velero/velero-plugin-for-microsoft-azure:v1.6.0"},
@@ -558,7 +579,7 @@ func getProviderPlugins(ctx context.Context, veleroCLI string, cloudProvider str
 		return []string{}, errors.New("CloudProvider should be provided")
 	}
 
-	version, err := getVeleroVersion(ctx, veleroCLI, true)
+	version, err := GetVeleroVersion(ctx, veleroCLI, true)
 	if err != nil {
 		return nil, errors.WithMessage(err, "failed to get velero version")
 	}
@@ -593,11 +614,17 @@ func getPlugins(ctx context.Context, veleroCfg VeleroConfig) ([]string, error) {
 		if objectStoreProvider == "" {
 			objectStoreProvider = cloudProvider
 		}
-		version, err := getVeleroVersion(ctx, veleroCLI, true)
-		if err != nil {
-			return nil, errors.WithMessage(err, "failed to get velero version")
-		}
 
+		var version string
+		var err error
+		if veleroCfg.VeleroVersion != "" {
+			version = veleroCfg.VeleroVersion
+		} else {
+			version, err = GetVeleroVersion(ctx, veleroCLI, true)
+			if err != nil {
+				return nil, errors.WithMessage(err, "failed to get velero version")
+			}
+		}
 		if veleroCfg.SnapshotMoveData && veleroCfg.DataMoverPlugin == "" {
 			needDataMoverPlugin = true
 		}
@@ -611,8 +638,14 @@ func getPlugins(ctx context.Context, veleroCfg VeleroConfig) ([]string, error) {
 
 // VeleroAddPluginsForProvider determines which plugins need to be installed for a provider and
 // installs them in the current Velero installation, skipping over those that are already installed.
-func VeleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNamespace string, provider string) error {
+func VeleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNamespace string, provider string, plugin string) error {
-	plugins, err := getProviderPlugins(ctx, veleroCLI, provider)
+	var err error
+	var plugins []string
+	if plugin == "" {
+		plugins, err = getProviderPlugins(ctx, veleroCLI, provider)
+	} else {
+		plugins = append(plugins, plugin)
+	}
 	fmt.Printf("provider cmd = %v\n", provider)
 	fmt.Printf("plugins cmd = %v\n", plugins)
 	if err != nil {
@@ -743,7 +776,7 @@ func GetVsphereSnapshotIDs(ctx context.Context, timeout time.Duration, namespace
 	return result, nil
 }
 
-func getVeleroVersion(ctx context.Context, veleroCLI string, clientOnly bool) (string, error) {
+func GetVeleroVersion(ctx context.Context, veleroCLI string, clientOnly bool) (string, error) {
 	args := []string{"version", "--timeout", "60s"}
 	if clientOnly {
 		args = append(args, "--client-only")
@@ -778,7 +811,7 @@ func getVeleroVersion(ctx context.Context, veleroCLI string, clientOnly bool) (s
 
 func CheckVeleroVersion(ctx context.Context, veleroCLI string, expectedVer string) error {
 	tag := expectedVer
-	tagInstalled, err := getVeleroVersion(ctx, veleroCLI, false)
+	tagInstalled, err := GetVeleroVersion(ctx, veleroCLI, false)
 	if err != nil {
 		return errors.WithMessagef(err, "failed to get Velero version")
 	}
@@ -1107,13 +1140,23 @@ func GetResticRepositories(ctx context.Context, veleroNamespace, targetNamespace
 
 func GetSnapshotCheckPoint(client TestClient, VeleroCfg VeleroConfig, expectCount int, namespaceBackedUp, backupName string, KibishiiPVCNameList []string) (SnapshotCheckPoint, error) {
 	var snapshotCheckPoint SnapshotCheckPoint
-	var err error
+
 	snapshotCheckPoint.ExpectCount = expectCount
 	snapshotCheckPoint.NamespaceBackedUp = namespaceBackedUp
 	snapshotCheckPoint.PodName = KibishiiPVCNameList
 	if VeleroCfg.CloudProvider == "azure" && strings.EqualFold(VeleroCfg.Features, "EnableCSI") {
 		snapshotCheckPoint.EnableCSI = true
-		if snapshotCheckPoint.SnapshotIDList, err = util.CheckVolumeSnapshotCR(client, backupName, expectCount); err != nil {
+		resourceName := "snapshot.storage.k8s.io"
+
+		srcVersions, err := GetAPIVersions(VeleroCfg.DefaultClient, resourceName)
+
+		if err != nil {
+			return snapshotCheckPoint, err
+		}
+		if len(srcVersions) == 0 {
+			return snapshotCheckPoint, errors.New("Fail to get APIVersion")
+		}
+		if snapshotCheckPoint.SnapshotIDList, err = util.CheckVolumeSnapshotCR(client, backupName, expectCount, srcVersions[0]); err != nil {
 			return snapshotCheckPoint, errors.Wrapf(err, "Fail to get Azure CSI snapshot content")
 		}
 	}