From 35697a9509092db83dceb82c651a27a6ce72d398 Mon Sep 17 00:00:00 2001 From: danfengl Date: Mon, 17 Jul 2023 09:57:34 +0000 Subject: [PATCH] Support API Version V1 for CR volumesnapshotclass in E2E test 1. In K8S v1.27 API Version v1beta1 for CR volumesnapshotclass is deprcated, so E2E test should adapt both API versions to cover all K8S versio; 2. Support getting additional plugin from input; 3. Velero version and plugin map should not deprated version older than v1.10, because upgrade test will use them. Signed-off-by: danfengl --- test/e2e/backup/backup.go | 2 +- test/e2e/bsl-mgmt/deletion.go | 2 +- test/e2e/migration/migration.go | 4 ++ test/e2e/upgrade/upgrade.go | 5 +++ test/e2e/util/csi/common.go | 51 ++++++++++++++++++++-- test/e2e/util/velero/install.go | 8 +++- test/e2e/util/velero/velero_utils.go | 65 +++++++++++++++++++++++----- 7 files changed, 120 insertions(+), 17 deletions(-) diff --git a/test/e2e/backup/backup.go b/test/e2e/backup/backup.go index 8a1f2d40e..8312d4df7 100644 --- a/test/e2e/backup/backup.go +++ b/test/e2e/backup/backup.go @@ -127,7 +127,7 @@ func BackupRestoreTest(useVolumeSnapshots bool) { Expect(VeleroInstall(context.Background(), &veleroCfg, false)).To(Succeed()) } - Expect(VeleroAddPluginsForProvider(context.TODO(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, veleroCfg.AdditionalBSLProvider)).To(Succeed()) + Expect(VeleroAddPluginsForProvider(context.TODO(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, veleroCfg.AdditionalBSLProvider, veleroCfg.AddBSLPlugins)).To(Succeed()) // Create Secret for additional BSL secretName := fmt.Sprintf("bsl-credentials-%s", UUIDgen) diff --git a/test/e2e/bsl-mgmt/deletion.go b/test/e2e/bsl-mgmt/deletion.go index 874cf53c7..834ab5ca5 100644 --- a/test/e2e/bsl-mgmt/deletion.go +++ b/test/e2e/bsl-mgmt/deletion.go @@ -104,7 +104,7 @@ func BslDeletionTest(useVolumeSnapshots bool) { } By(fmt.Sprintf("Add an additional plugin for provider %s", veleroCfg.AdditionalBSLProvider), func() { - Expect(VeleroAddPluginsForProvider(context.TODO(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, veleroCfg.AdditionalBSLProvider)).To(Succeed()) + Expect(VeleroAddPluginsForProvider(context.TODO(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, veleroCfg.AdditionalBSLProvider, veleroCfg.AddBSLPlugins)).To(Succeed()) }) additionalBsl := fmt.Sprintf("bsl-%s", UUIDgen) diff --git a/test/e2e/migration/migration.go b/test/e2e/migration/migration.go index 6b10ba925..0a35e00cb 100644 --- a/test/e2e/migration/migration.go +++ b/test/e2e/migration/migration.go @@ -145,6 +145,10 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) OriginVeleroCfg.UseVolumeSnapshots = useVolumeSnapshots OriginVeleroCfg.UseNodeAgent = !useVolumeSnapshots + version, err := GetVeleroVersion(oneHourTimeout, OriginVeleroCfg.VeleroCLI, true) + Expect(err).To(Succeed(), "Fail to get Velero version") + OriginVeleroCfg.VeleroVersion = version + // self represents v1.12 if veleroCLI2Version.VeleroVersion == "self" { if OriginVeleroCfg.SnapshotMoveData { diff --git a/test/e2e/upgrade/upgrade.go b/test/e2e/upgrade/upgrade.go index 58b892255..f5102b79a 100644 --- a/test/e2e/upgrade/upgrade.go +++ b/test/e2e/upgrade/upgrade.go @@ -119,6 +119,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC veleroCfg.GCFrequency = "" By(fmt.Sprintf("Install the expected old version Velero (%s) for upgrade", veleroCLI2Version.VeleroVersion), func() { + //Set VeleroImage and RestoreHelperImage to blank //VeleroImage and RestoreHelperImage should be the default value in originalCli tmpCfgForOldVeleroInstall := veleroCfg @@ -128,6 +129,10 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC tmpCfgForOldVeleroInstall.RestoreHelperImage = "" tmpCfgForOldVeleroInstall.Plugins = "" tmpCfgForOldVeleroInstall.UploaderType = "" + version, err := GetVeleroVersion(oneHourTimeout, tmpCfgForOldVeleroInstall.VeleroCLI, true) + Expect(err).To(Succeed(), "Fail to get Velero version") + tmpCfgForOldVeleroInstall.VeleroVersion = version + if supportUploaderType { tmpCfgForOldVeleroInstall.UseRestic = false tmpCfgForOldVeleroInstall.UseNodeAgent = !useVolumeSnapshots diff --git a/test/e2e/util/csi/common.go b/test/e2e/util/csi/common.go index 677de4127..56af1ef58 100644 --- a/test/e2e/util/csi/common.go +++ b/test/e2e/util/csi/common.go @@ -91,6 +91,43 @@ func GetCsiSnapshotHandle(client TestClient, backupName string) ([]string, error } return snapshotHandleList, nil } +func GetCsiSnapshotHandleV1(client TestClient, backupName string) ([]string, error) { + _, snapshotClient, err := GetClients() + if err != nil { + return nil, err + } + vscList, err1 := snapshotClient.SnapshotV1().VolumeSnapshotContents().List(context.TODO(), metav1.ListOptions{}) + if err1 != nil { + return nil, err + } + + var snapshotHandleList []string + for _, i := range vscList.Items { + if i.Status == nil { + fmt.Println("SnapshotHandle Status s nil") + continue + } + if i.Status.SnapshotHandle == nil { + fmt.Println("SnapshotHandle is nil") + continue + } + + if i.Labels == nil { + fmt.Println("VolumeSnapshotContents label is nil") + continue + } + + if i.Labels["velero.io/backup-name"] == backupName { + tmp := strings.Split(*i.Status.SnapshotHandle, "/") + snapshotHandleList = append(snapshotHandleList, tmp[len(tmp)-1]) + } + } + + if len(snapshotHandleList) == 0 { + fmt.Printf("No VolumeSnapshotContent from backup %s", backupName) + } + return snapshotHandleList, nil +} func GetVolumeSnapshotContentNameByPod(client TestClient, podName, namespace, backupName string) (string, error) { pvcList, err := GetPvcByPodName(context.Background(), namespace, podName) if err != nil { @@ -128,11 +165,19 @@ func GetVolumeSnapshotContentNameByPod(client TestClient, podName, namespace, ba return "", errors.New(fmt.Sprintf("Fail to get VolumeSnapshotContentName for pod %s under namespace %s", podName, namespace)) } -func CheckVolumeSnapshotCR(client TestClient, backupName string, expectedCount int) ([]string, error) { +func CheckVolumeSnapshotCR(client TestClient, backupName string, expectedCount int, apiVersion string) ([]string, error) { var err error var snapshotContentNameList []string - if snapshotContentNameList, err = GetCsiSnapshotHandle(client, backupName); err != nil { - return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content") + if apiVersion == "v1beta1" { + if snapshotContentNameList, err = GetCsiSnapshotHandle(client, backupName); err != nil { + return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content") + } + } else if apiVersion == "v1" { + if snapshotContentNameList, err = GetCsiSnapshotHandleV1(client, backupName); err != nil { + return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content") + } + } else { + return nil, errors.New("API version is invalid") } if len(snapshotContentNameList) != expectedCount { return nil, errors.New(fmt.Sprintf("Snapshot count %d is not as expect %d", len(snapshotContentNameList), expectedCount)) diff --git a/test/e2e/util/velero/install.go b/test/e2e/util/velero/install.go index 1830b15d3..be8727b54 100644 --- a/test/e2e/util/velero/install.go +++ b/test/e2e/util/velero/install.go @@ -111,7 +111,10 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluste return errors.WithMessagef(err, "Failed to get Velero InstallOptions for plugin provider %s", veleroCfg.ObjectStoreProvider) } veleroInstallOptions.UseVolumeSnapshots = veleroCfg.UseVolumeSnapshots - veleroInstallOptions.UseNodeAgent = veleroCfg.UseNodeAgent + if !veleroCfg.UseRestic { + veleroInstallOptions.UseNodeAgent = veleroCfg.UseNodeAgent + } + veleroInstallOptions.UseRestic = veleroCfg.UseRestic veleroInstallOptions.Image = veleroCfg.VeleroImage veleroInstallOptions.Namespace = veleroCfg.VeleroNamespace veleroInstallOptions.UploaderType = veleroCfg.UploaderType @@ -198,6 +201,9 @@ func installVeleroServer(ctx context.Context, cli, cloudProvider string, options if len(options.Image) > 0 { args = append(args, "--image", options.Image) } + if options.UseRestic { + args = append(args, "--use-restic") + } if options.UseNodeAgent { args = append(args, "--use-node-agent") } diff --git a/test/e2e/util/velero/velero_utils.go b/test/e2e/util/velero/velero_utils.go index c5d1834c1..ad5359260 100644 --- a/test/e2e/util/velero/velero_utils.go +++ b/test/e2e/util/velero/velero_utils.go @@ -56,6 +56,27 @@ const RestoreObjectsPrefix = "restores" const PluginsObjectsPrefix = "plugins" var pluginsMatrix = map[string]map[string][]string{ + "v1.7": { + "aws": {"velero/velero-plugin-for-aws:v1.3.0"}, + "azure": {"velero/velero-plugin-for-microsoft-azure:v1.3.0"}, + "vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.3.0"}, + "gcp": {"velero/velero-plugin-for-gcp:v1.3.0"}, + "csi": {"velero/velero-plugin-for-csi:v0.2.0"}, + }, + "v1.8": { + "aws": {"velero/velero-plugin-for-aws:v1.4.0"}, + "azure": {"velero/velero-plugin-for-microsoft-azure:v1.4.0"}, + "vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.3.1"}, + "gcp": {"velero/velero-plugin-for-gcp:v1.4.0"}, + "csi": {"velero/velero-plugin-for-csi:v0.2.0"}, + }, + "v1.9": { + "aws": {"velero/velero-plugin-for-aws:v1.5.0"}, + "azure": {"velero/velero-plugin-for-microsoft-azure:v1.5.0"}, + "vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.4.0"}, + "gcp": {"velero/velero-plugin-for-gcp:v1.5.0"}, + "csi": {"velero/velero-plugin-for-csi:v0.3.0"}, + }, "v1.10": { "aws": {"velero/velero-plugin-for-aws:v1.6.0"}, "azure": {"velero/velero-plugin-for-microsoft-azure:v1.6.0"}, @@ -558,7 +579,7 @@ func getProviderPlugins(ctx context.Context, veleroCLI string, cloudProvider str return []string{}, errors.New("CloudProvider should be provided") } - version, err := getVeleroVersion(ctx, veleroCLI, true) + version, err := GetVeleroVersion(ctx, veleroCLI, true) if err != nil { return nil, errors.WithMessage(err, "failed to get velero version") } @@ -593,11 +614,17 @@ func getPlugins(ctx context.Context, veleroCfg VeleroConfig) ([]string, error) { if objectStoreProvider == "" { objectStoreProvider = cloudProvider } - version, err := getVeleroVersion(ctx, veleroCLI, true) - if err != nil { - return nil, errors.WithMessage(err, "failed to get velero version") - } + var version string + var err error + if veleroCfg.VeleroVersion != "" { + version = veleroCfg.VeleroVersion + } else { + version, err = GetVeleroVersion(ctx, veleroCLI, true) + if err != nil { + return nil, errors.WithMessage(err, "failed to get velero version") + } + } if veleroCfg.SnapshotMoveData && veleroCfg.DataMoverPlugin == "" { needDataMoverPlugin = true } @@ -611,8 +638,14 @@ func getPlugins(ctx context.Context, veleroCfg VeleroConfig) ([]string, error) { // VeleroAddPluginsForProvider determines which plugins need to be installed for a provider and // installs them in the current Velero installation, skipping over those that are already installed. -func VeleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNamespace string, provider string) error { - plugins, err := getProviderPlugins(ctx, veleroCLI, provider) +func VeleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNamespace string, provider string, plugin string) error { + var err error + var plugins []string + if plugin == "" { + plugins, err = getProviderPlugins(ctx, veleroCLI, provider) + } else { + plugins = append(plugins, plugin) + } fmt.Printf("provider cmd = %v\n", provider) fmt.Printf("plugins cmd = %v\n", plugins) if err != nil { @@ -743,7 +776,7 @@ func GetVsphereSnapshotIDs(ctx context.Context, timeout time.Duration, namespace return result, nil } -func getVeleroVersion(ctx context.Context, veleroCLI string, clientOnly bool) (string, error) { +func GetVeleroVersion(ctx context.Context, veleroCLI string, clientOnly bool) (string, error) { args := []string{"version", "--timeout", "60s"} if clientOnly { args = append(args, "--client-only") @@ -778,7 +811,7 @@ func getVeleroVersion(ctx context.Context, veleroCLI string, clientOnly bool) (s func CheckVeleroVersion(ctx context.Context, veleroCLI string, expectedVer string) error { tag := expectedVer - tagInstalled, err := getVeleroVersion(ctx, veleroCLI, false) + tagInstalled, err := GetVeleroVersion(ctx, veleroCLI, false) if err != nil { return errors.WithMessagef(err, "failed to get Velero version") } @@ -1107,13 +1140,23 @@ func GetResticRepositories(ctx context.Context, veleroNamespace, targetNamespace func GetSnapshotCheckPoint(client TestClient, VeleroCfg VeleroConfig, expectCount int, namespaceBackedUp, backupName string, KibishiiPVCNameList []string) (SnapshotCheckPoint, error) { var snapshotCheckPoint SnapshotCheckPoint - var err error + snapshotCheckPoint.ExpectCount = expectCount snapshotCheckPoint.NamespaceBackedUp = namespaceBackedUp snapshotCheckPoint.PodName = KibishiiPVCNameList if VeleroCfg.CloudProvider == "azure" && strings.EqualFold(VeleroCfg.Features, "EnableCSI") { snapshotCheckPoint.EnableCSI = true - if snapshotCheckPoint.SnapshotIDList, err = util.CheckVolumeSnapshotCR(client, backupName, expectCount); err != nil { + resourceName := "snapshot.storage.k8s.io" + + srcVersions, err := GetAPIVersions(VeleroCfg.DefaultClient, resourceName) + + if err != nil { + return snapshotCheckPoint, err + } + if len(srcVersions) == 0 { + return snapshotCheckPoint, errors.New("Fail to get APIVersion") + } + if snapshotCheckPoint.SnapshotIDList, err = util.CheckVolumeSnapshotCR(client, backupName, expectCount, srcVersions[0]); err != nil { return snapshotCheckPoint, errors.Wrapf(err, "Fail to get Azure CSI snapshot content") } }