diff --git a/.github/workflows/e2e-test-kind.yaml b/.github/workflows/e2e-test-kind.yaml index 38fcd1d2a..f889f9e7c 100644 --- a/.github/workflows/e2e-test-kind.yaml +++ b/.github/workflows/e2e-test-kind.yaml @@ -121,7 +121,7 @@ jobs: CREDS_FILE=/tmp/credential BSL_BUCKET=bucket \ ADDITIONAL_OBJECT_STORE_PROVIDER=aws ADDITIONAL_BSL_CONFIG=region=minio,s3ForcePathStyle="true",s3Url=http://$(hostname -i):9000 \ ADDITIONAL_CREDS_FILE=/tmp/credential ADDITIONAL_BSL_BUCKET=additional-bucket \ - GINKGO_FOCUS=Basic VELERO_IMAGE=velero:pr-test \ + GINKGO_FOCUS='Basic\].+\[ClusterResource' VELERO_IMAGE=velero:pr-test \ make -C test/e2e run timeout-minutes: 30 - name: Upload debug bundle diff --git a/test/e2e/Makefile b/test/e2e/Makefile index f84ca3b53..79af0d571 100644 --- a/test/e2e/Makefile +++ b/test/e2e/Makefile @@ -72,7 +72,8 @@ CLOUD_PROVIDER ?= OBJECT_STORE_PROVIDER ?= INSTALL_VELERO ?= true REGISTRY_CREDENTIAL_FILE ?= -KIBISHII_DIRECTORY ?= +KIBISHII_DIRECTORY ?= github.com/vmware-tanzu-experiments/distributed-data-generator/kubernetes/yaml/ + # Flags to create an additional BSL for multiple credentials tests ADDITIONAL_BSL_PLUGINS ?= @@ -82,6 +83,8 @@ ADDITIONAL_BSL_BUCKET ?= ADDITIONAL_BSL_PREFIX ?= ADDITIONAL_BSL_CONFIG ?= +FEATURES ?= + .PHONY:ginkgo ginkgo: # Make sure ginkgo is in $GOPATH/bin go get github.com/onsi/ginkgo/ginkgo @@ -115,6 +118,7 @@ run: ginkgo -additional-bsl-bucket=$(ADDITIONAL_BSL_BUCKET) \ -additional-bsl-prefix=$(ADDITIONAL_BSL_PREFIX) \ -additional-bsl-config=$(ADDITIONAL_BSL_CONFIG) \ + -features=$(FEATURES) \ -install-velero=$(INSTALL_VELERO) \ -registry-credential-file=$(REGISTRY_CREDENTIAL_FILE) \ -kibishii-directory=$(KIBISHII_DIRECTORY) diff --git a/test/e2e/backup/backup.go b/test/e2e/backup/backup.go index 7bf79b9a1..730f6abf9 100644 --- a/test/e2e/backup/backup.go +++ b/test/e2e/backup/backup.go @@ -42,7 +42,7 @@ func BackupRestoreTest(useVolumeSnapshots bool) { var ( backupName, restoreName string ) - + kibishiiNamespace := "kibishii-workload" client, err := NewTestClient() Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests") @@ -55,11 +55,14 @@ func BackupRestoreTest(useVolumeSnapshots bool) { UUIDgen, err = uuid.NewRandom() Expect(err).To(Succeed()) if VeleroCfg.InstallVelero { - Expect(VeleroInstall(context.Background(), &VeleroCfg, "", useVolumeSnapshots)).To(Succeed()) + Expect(VeleroInstall(context.Background(), &VeleroCfg, useVolumeSnapshots)).To(Succeed()) } }) AfterEach(func() { + DeleteNamespace(context.Background(), client, kibishiiNamespace, true) + Expect(err).To(Succeed(), fmt.Sprintf("failed to delete the namespace %q", kibishiiNamespace)) + if VeleroCfg.InstallVelero { err = VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace) Expect(err).To(Succeed()) @@ -72,7 +75,7 @@ func BackupRestoreTest(useVolumeSnapshots bool) { restoreName = "restore-" + UUIDgen.String() // Even though we are using Velero's CloudProvider plugin for object storage, the kubernetes cluster is running on // KinD. So use the kind installation for Kibishii. - Expect(RunKibishiiTests(client, VeleroCfg.CloudProvider, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backupName, restoreName, "", useVolumeSnapshots, VeleroCfg.RegistryCredentialFile, VeleroCfg.KibishiiDirectory)).To(Succeed(), + Expect(RunKibishiiTests(client, VeleroCfg, backupName, restoreName, "", kibishiiNamespace, useVolumeSnapshots)).To(Succeed(), "Failed to successfully backup and restore Kibishii namespace") }) @@ -89,7 +92,9 @@ func BackupRestoreTest(useVolumeSnapshots bool) { Skip("no additional BSL credentials given, not running multiple BackupStorageLocation with unique credentials tests") } - Expect(VeleroAddPluginsForProvider(context.TODO(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, VeleroCfg.AdditionalBSLProvider, VeleroCfg.AddBSLPlugins)).To(Succeed()) + Expect(VeleroAddPluginsForProvider(context.TODO(), VeleroCfg.VeleroCLI, + VeleroCfg.VeleroNamespace, VeleroCfg.AdditionalBSLProvider, + VeleroCfg.AddBSLPlugins, VeleroCfg.Features)).To(Succeed()) // Create Secret for additional BSL secretName := fmt.Sprintf("bsl-credentials-%s", UUIDgen) @@ -125,7 +130,7 @@ func BackupRestoreTest(useVolumeSnapshots bool) { backupName = fmt.Sprintf("%s-%s", backupName, UUIDgen) restoreName = fmt.Sprintf("%s-%s", restoreName, UUIDgen) } - Expect(RunKibishiiTests(client, VeleroCfg.CloudProvider, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backupName, restoreName, bsl, useVolumeSnapshots, VeleroCfg.RegistryCredentialFile, VeleroCfg.KibishiiDirectory)).To(Succeed(), + Expect(RunKibishiiTests(client, VeleroCfg, backupName, restoreName, bsl, kibishiiNamespace, useVolumeSnapshots)).To(Succeed(), "Failed to successfully backup and restore Kibishii namespace using BSL %s", bsl) } }) diff --git a/test/e2e/backups/deletion.go b/test/e2e/backups/deletion.go index 0f02ef33e..a1d7a0cc3 100644 --- a/test/e2e/backups/deletion.go +++ b/test/e2e/backups/deletion.go @@ -61,7 +61,7 @@ func backup_deletion_test(useVolumeSnapshots bool) { UUIDgen, err = uuid.NewRandom() Expect(err).To(Succeed()) if VeleroCfg.InstallVelero { - Expect(VeleroInstall(context.Background(), &VeleroCfg, "", useVolumeSnapshots)).To(Succeed()) + Expect(VeleroInstall(context.Background(), &VeleroCfg, useVolumeSnapshots)).To(Succeed()) } }) @@ -75,16 +75,23 @@ func backup_deletion_test(useVolumeSnapshots bool) { When("kibishii is the sample workload", func() { It("Deleted backups are deleted from object storage and backups deleted from object storage can be deleted locally", func() { backupName = "backup-" + UUIDgen.String() - Expect(runBackupDeletionTests(client, VeleroCfg.VeleroCLI, VeleroCfg.CloudProvider, VeleroCfg.VeleroNamespace, backupName, "", useVolumeSnapshots, VeleroCfg.RegistryCredentialFile, VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, VeleroCfg.KibishiiDirectory)).To(Succeed(), + Expect(runBackupDeletionTests(client, VeleroCfg, backupName, "", useVolumeSnapshots, VeleroCfg.KibishiiDirectory)).To(Succeed(), "Failed to run backup deletion test") }) }) } // runUpgradeTests runs upgrade test on the provider by kibishii. -func runBackupDeletionTests(client TestClient, veleroCLI, providerName, veleroNamespace, backupName, backupLocation string, - useVolumeSnapshots bool, registryCredentialFile, bslPrefix, bslConfig, kibishiiDirectory string) error { +func runBackupDeletionTests(client TestClient, veleroCfg VerleroConfig, backupName, backupLocation string, + useVolumeSnapshots bool, kibishiiDirectory string) error { oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60) + veleroCLI := VeleroCfg.VeleroCLI + providerName := VeleroCfg.CloudProvider + veleroNamespace := VeleroCfg.VeleroNamespace + registryCredentialFile := VeleroCfg.RegistryCredentialFile + bslPrefix := VeleroCfg.BSLPrefix + bslConfig := VeleroCfg.BSLConfig + veleroFeatures := VeleroCfg.Features if err := CreateNamespace(oneHourTimeout, client, deletionTest); err != nil { return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", deletionTest) @@ -95,7 +102,8 @@ func runBackupDeletionTests(client TestClient, veleroCLI, providerName, veleroNa } }() - if err := KibishiiPrepareBeforeBackup(oneHourTimeout, client, providerName, deletionTest, registryCredentialFile, kibishiiDirectory); err != nil { + if err := KibishiiPrepareBeforeBackup(oneHourTimeout, client, providerName, deletionTest, + registryCredentialFile, veleroFeatures, kibishiiDirectory, useVolumeSnapshots); err != nil { return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", deletionTest) } err := ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, backupName, BackupObjectsPrefix, 1) diff --git a/test/e2e/backups/sync_backups.go b/test/e2e/backups/sync_backups.go index ae0756ff9..a329b5114 100644 --- a/test/e2e/backups/sync_backups.go +++ b/test/e2e/backups/sync_backups.go @@ -61,7 +61,7 @@ func BackupsSyncTest() { BeforeEach(func() { flag.Parse() if VeleroCfg.InstallVelero { - Expect(VeleroInstall(context.Background(), &VeleroCfg, "", false)).To(Succeed()) + Expect(VeleroInstall(context.Background(), &VeleroCfg, false)).To(Succeed()) } }) @@ -94,7 +94,7 @@ func BackupsSyncTest() { By("Install velero", func() { VeleroCfg.ObjectStoreProvider = "" - Expect(VeleroInstall(test.ctx, &VeleroCfg, "", false)).To(Succeed()) + Expect(VeleroInstall(test.ctx, &VeleroCfg, false)).To(Succeed()) }) By("Check all backups in object storage are synced to Velero", func() { diff --git a/test/e2e/basic/enable_api_group_versions.go b/test/e2e/basic/enable_api_group_versions.go index a251b278d..46a7aeb26 100644 --- a/test/e2e/basic/enable_api_group_versions.go +++ b/test/e2e/basic/enable_api_group_versions.go @@ -59,7 +59,8 @@ func APIGropuVersionsTest() { // TODO: install Velero once for the test suite once feature flag is // removed and velero installation becomes the same as other e2e tests. if VeleroCfg.InstallVelero { - err = VeleroInstall(context.Background(), &VeleroCfg, "EnableAPIGroupVersions", false) + VeleroCfg.Features = "EnableAPIGroupVersions" + err = VeleroInstall(context.Background(), &VeleroCfg, false) Expect(err).NotTo(HaveOccurred()) } }) diff --git a/test/e2e/bsl-mgmt/deletion.go b/test/e2e/bsl-mgmt/deletion.go index 92d85123a..5bc6d3595 100644 --- a/test/e2e/bsl-mgmt/deletion.go +++ b/test/e2e/bsl-mgmt/deletion.go @@ -64,7 +64,7 @@ func BslDeletionTest(useVolumeSnapshots bool) { UUIDgen, err = uuid.NewRandom() Expect(err).To(Succeed()) if VeleroCfg.InstallVelero { - Expect(VeleroInstall(context.Background(), &VeleroCfg, "", useVolumeSnapshots)).To(Succeed()) + Expect(VeleroInstall(context.Background(), &VeleroCfg, useVolumeSnapshots)).To(Succeed()) } }) @@ -93,7 +93,9 @@ func BslDeletionTest(useVolumeSnapshots bool) { } By(fmt.Sprintf("Add an additional plugin for provider %s", VeleroCfg.AdditionalBSLProvider), func() { - Expect(VeleroAddPluginsForProvider(context.TODO(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, VeleroCfg.AdditionalBSLProvider, VeleroCfg.AddBSLPlugins)).To(Succeed()) + Expect(VeleroAddPluginsForProvider(context.TODO(), VeleroCfg.VeleroCLI, + VeleroCfg.VeleroNamespace, VeleroCfg.AdditionalBSLProvider, + VeleroCfg.AddBSLPlugins, VeleroCfg.Features)).To(Succeed()) }) additionalBsl := fmt.Sprintf("bsl-%s", UUIDgen) @@ -140,7 +142,8 @@ func BslDeletionTest(useVolumeSnapshots bool) { By("Deploy sample workload of Kibishii", func() { Expect(KibishiiPrepareBeforeBackup(oneHourTimeout, client, VeleroCfg.CloudProvider, - bslDeletionTestNs, VeleroCfg.RegistryCredentialFile, VeleroCfg.KibishiiDirectory)).To(Succeed()) + bslDeletionTestNs, VeleroCfg.RegistryCredentialFile, VeleroCfg.Features, + VeleroCfg.KibishiiDirectory, useVolumeSnapshots)).To(Succeed()) }) // Restic can not backup PV only, so pod need to be labeled also diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 588641b0b..9186ca3aa 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -56,23 +56,24 @@ func init() { flag.BoolVar(&VeleroCfg.InstallVelero, "install-velero", true, "install/uninstall velero during the test. Optional.") flag.StringVar(&VeleroCfg.RegistryCredentialFile, "registry-credential-file", "", "file containing credential for the image registry, follows the same format rules as the ~/.docker/config.json file. Optional.") flag.StringVar(&VeleroCfg.KibishiiDirectory, "kibishii-directory", "github.com/vmware-tanzu-experiments/distributed-data-generator/kubernetes/yaml/", "The file directory or URL path to install Kibishii. Optional.") - + //vmware-tanzu-experiments // Flags to create an additional BSL for multiple credentials test flag.StringVar(&VeleroCfg.AdditionalBSLProvider, "additional-bsl-object-store-provider", "", "Provider of object store plugin for additional backup storage location. Required if testing multiple credentials support.") flag.StringVar(&VeleroCfg.AdditionalBSLBucket, "additional-bsl-bucket", "", "name of the object storage bucket for additional backup storage location. Required if testing multiple credentials support.") flag.StringVar(&VeleroCfg.AdditionalBSLPrefix, "additional-bsl-prefix", "", "prefix under which all Velero data should be stored within the bucket for additional backup storage location. Optional.") flag.StringVar(&VeleroCfg.AdditionalBSLConfig, "additional-bsl-config", "", "configuration to use for the additional backup storage location. Format is key1=value1,key2=value2") flag.StringVar(&VeleroCfg.AdditionalBSLCredentials, "additional-bsl-credentials-file", "", "file containing credentials for additional backup storage location provider. Required if testing multiple credentials support.") + flag.StringVar(&VeleroCfg.Features, "features", "", "Comma-separated list of features to enable for this Velero process.") } var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", APIGropuVersionsTest) // Test backup and restore of Kibishi using restic -var _ = Describe("[Restic] Velero tests on cluster using the plugin provider for object storage and Restic for volume backups", BackupRestoreWithRestic) +var _ = Describe("[Basic][Restic] Velero tests on cluster using the plugin provider for object storage and Restic for volume backups", BackupRestoreWithRestic) -var _ = Describe("[Snapshot] Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupRestoreWithSnapshots) +var _ = Describe("[Basic][Snapshot] Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupRestoreWithSnapshots) -var _ = Describe("[Basic] Backup/restore of cluster resources", ResourcesCheckTest) +var _ = Describe("[Basic][ClusterResource] Backup/restore of cluster resources", ResourcesCheckTest) var _ = Describe("[Scale] Backup/restore of 2500 namespaces", MultiNSBackupRestore) diff --git a/test/e2e/privilegesmgmt/ssr.go b/test/e2e/privilegesmgmt/ssr.go index 91af0f8b8..40be8c158 100644 --- a/test/e2e/privilegesmgmt/ssr.go +++ b/test/e2e/privilegesmgmt/ssr.go @@ -45,7 +45,7 @@ func SSRTest() { BeforeEach(func() { flag.Parse() if VeleroCfg.InstallVelero { - Expect(VeleroInstall(context.Background(), &VeleroCfg, "", false)).To(Succeed()) + Expect(VeleroInstall(context.Background(), &VeleroCfg, false)).To(Succeed()) } }) diff --git a/test/e2e/test/test.go b/test/e2e/test/test.go index 5648c06de..de27e2bb6 100644 --- a/test/e2e/test/test.go +++ b/test/e2e/test/test.go @@ -79,7 +79,7 @@ func TestFunc(test VeleroBackupRestoreTest) func() { BeforeEach(func() { flag.Parse() if VeleroCfg.InstallVelero { - Expect(VeleroInstall(context.Background(), &VeleroCfg, "", false)).To(Succeed()) + Expect(VeleroInstall(context.Background(), &VeleroCfg, false)).To(Succeed()) } }) AfterEach(func() { @@ -107,7 +107,7 @@ func TestFuncWithMultiIt(tests []VeleroBackupRestoreTest) func() { flag.Parse() if VeleroCfg.InstallVelero { if countIt == 0 { - Expect(VeleroInstall(context.Background(), &VeleroCfg, "", false)).To(Succeed()) + Expect(VeleroInstall(context.Background(), &VeleroCfg, false)).To(Succeed()) } countIt++ } diff --git a/test/e2e/types.go b/test/e2e/types.go index 253ea067b..3e4cd7168 100644 --- a/test/e2e/types.go +++ b/test/e2e/types.go @@ -49,6 +49,7 @@ type VerleroConfig struct { AddBSLPlugins string InstallVelero bool KibishiiDirectory string + Features string } type SnapshotCheckPoint struct { diff --git a/test/e2e/upgrade/upgrade.go b/test/e2e/upgrade/upgrade.go index 9354d15cc..5bbb7c21f 100644 --- a/test/e2e/upgrade/upgrade.go +++ b/test/e2e/upgrade/upgrade.go @@ -118,6 +118,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, upgradeFromVelero Upgrade Expect(err).To(Succeed()) }) } + By(fmt.Sprintf("Install the expected old version Velero (%s) for upgrade", upgradeFromVelero.UpgradeFromVeleroVersion), func() { //Set VeleroImage and ResticHelperImage to blank @@ -129,17 +130,17 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, upgradeFromVelero Upgrade tmpCfgForOldVeleroInstall.ResticHelperImage = "" tmpCfgForOldVeleroInstall.Plugins = "" - Expect(VeleroInstall(context.Background(), &tmpCfgForOldVeleroInstall, "", - false)).To(Succeed()) + Expect(VeleroInstall(context.Background(), &tmpCfgForOldVeleroInstall, + useVolumeSnapshots)).To(Succeed()) Expect(CheckVeleroVersion(context.Background(), tmpCfgForOldVeleroInstall.VeleroCLI, tmpCfgForOldVeleroInstall.UpgradeFromVeleroVersion)).To(Succeed()) }) backupName = "backup-" + UUIDgen.String() restoreName = "restore-" + UUIDgen.String() - tmpCfg1 := VeleroCfg - tmpCfg1.UpgradeFromVeleroCLI = upgradeFromVelero.UpgradeFromVeleroCLI - tmpCfg1.UpgradeFromVeleroVersion = upgradeFromVelero.UpgradeFromVeleroVersion + tmpCfg := VeleroCfg + tmpCfg.UpgradeFromVeleroCLI = upgradeFromVelero.UpgradeFromVeleroCLI + tmpCfg.UpgradeFromVeleroVersion = upgradeFromVelero.UpgradeFromVeleroVersion oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60) By("Create namespace for sample workload", func() { @@ -148,16 +149,17 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, upgradeFromVelero Upgrade }) By("Deploy sample workload of Kibishii", func() { - Expect(KibishiiPrepareBeforeBackup(oneHourTimeout, client, tmpCfg1.CloudProvider, - upgradeNamespace, tmpCfg1.RegistryCredentialFile, tmpCfg1.KibishiiDirectory)).To(Succeed()) + Expect(KibishiiPrepareBeforeBackup(oneHourTimeout, client, tmpCfg.CloudProvider, + upgradeNamespace, tmpCfg.RegistryCredentialFile, tmpCfg.Features, + tmpCfg.KibishiiDirectory, useVolumeSnapshots)).To(Succeed()) }) By(fmt.Sprintf("Backup namespace %s", upgradeNamespace), func() { - Expect(VeleroBackupNamespace(oneHourTimeout, tmpCfg1.UpgradeFromVeleroCLI, - tmpCfg1.VeleroNamespace, backupName, upgradeNamespace, "", + Expect(VeleroBackupNamespace(oneHourTimeout, tmpCfg.UpgradeFromVeleroCLI, + tmpCfg.VeleroNamespace, backupName, upgradeNamespace, "", useVolumeSnapshots, "")).ShouldNot(HaveOccurred(), func() string { - err = VeleroBackupLogs(context.Background(), tmpCfg1.UpgradeFromVeleroCLI, - tmpCfg1.VeleroNamespace, backupName) + err = VeleroBackupLogs(context.Background(), tmpCfg.UpgradeFromVeleroCLI, + tmpCfg.VeleroNamespace, backupName) return "Get backup logs" }) }) @@ -178,22 +180,22 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, upgradeFromVelero Upgrade // the snapshots of AWS may be still in pending status when do the restore, wait for a while // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed - if tmpCfg1.CloudProvider == "aws" && useVolumeSnapshots { + if tmpCfg.CloudProvider == "aws" && useVolumeSnapshots { fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...") time.Sleep(5 * time.Minute) } - By(fmt.Sprintf("Upgrade Velero by CLI %s", tmpCfg1.VeleroCLI), func() { - Expect(VeleroInstall(context.Background(), &tmpCfg1, "", useVolumeSnapshots)).To(Succeed()) - Expect(CheckVeleroVersion(context.Background(), tmpCfg1.VeleroCLI, - tmpCfg1.VeleroVersion)).To(Succeed()) + By(fmt.Sprintf("Upgrade Velero by CLI %s", tmpCfg.VeleroCLI), func() { + Expect(VeleroInstall(context.Background(), &tmpCfg, useVolumeSnapshots)).To(Succeed()) + Expect(CheckVeleroVersion(context.Background(), tmpCfg.VeleroCLI, + tmpCfg.VeleroVersion)).To(Succeed()) }) By(fmt.Sprintf("Restore %s", upgradeNamespace), func() { - Expect(VeleroRestore(oneHourTimeout, tmpCfg1.VeleroCLI, - tmpCfg1.VeleroNamespace, restoreName, backupName)).To(Succeed(), func() string { - RunDebug(context.Background(), tmpCfg1.VeleroCLI, - tmpCfg1.VeleroNamespace, "", restoreName) + Expect(VeleroRestore(oneHourTimeout, tmpCfg.VeleroCLI, + tmpCfg.VeleroNamespace, restoreName, backupName)).To(Succeed(), func() string { + RunDebug(context.Background(), tmpCfg.VeleroCLI, + tmpCfg.VeleroNamespace, "", restoreName) return "Fail to restore workload" }) }) diff --git a/test/e2e/util/csi/AzureVolumeSnapshotClass.yaml b/test/e2e/util/csi/AzureVolumeSnapshotClass.yaml new file mode 100644 index 000000000..4e3ca494f --- /dev/null +++ b/test/e2e/util/csi/AzureVolumeSnapshotClass.yaml @@ -0,0 +1,8 @@ +apiVersion: snapshot.storage.k8s.io/v1 +kind: VolumeSnapshotClass +metadata: + name: velero + labels: + velero.io/csi-volumesnapshot-class: "true" +driver: disk.csi.azure.com +deletionPolicy: Delete \ No newline at end of file diff --git a/test/e2e/util/k8s/common.go b/test/e2e/util/k8s/common.go index 688084b76..06d7cdaea 100644 --- a/test/e2e/util/k8s/common.go +++ b/test/e2e/util/k8s/common.go @@ -136,3 +136,8 @@ func AddLabelToPod(ctx context.Context, podName, namespace, label string) error fmt.Println(args) return exec.CommandContext(ctx, "kubectl", args...).Run() } + +func KubectlApplyByFile(ctx context.Context, file string) error { + args := []string{"apply", "-f", file, "--force=true"} + return exec.CommandContext(ctx, "kubectl", args...).Run() +} diff --git a/test/e2e/util/kibishii/kibishii_utils.go b/test/e2e/util/kibishii/kibishii_utils.go index 8d5c49040..7ad137cbc 100644 --- a/test/e2e/util/kibishii/kibishii_utils.go +++ b/test/e2e/util/kibishii/kibishii_utils.go @@ -20,38 +20,45 @@ import ( "fmt" "os/exec" "strconv" + "strings" "time" "github.com/pkg/errors" "golang.org/x/net/context" veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" + . "github.com/vmware-tanzu/velero/test/e2e" . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" . "github.com/vmware-tanzu/velero/test/e2e/util/velero" ) const ( - kibishiiNamespace = "kibishii-workload" - jumpPadPod = "jump-pad" + jumpPadPod = "jump-pad" ) // RunKibishiiTests runs kibishii tests on the provider. -func RunKibishiiTests(client TestClient, providerName, veleroCLI, veleroNamespace, backupName, restoreName, backupLocation string, - useVolumeSnapshots bool, registryCredentialFile string, kibishiiDirectory string) error { +func RunKibishiiTests(client TestClient, veleroCfg VerleroConfig, backupName, restoreName, backupLocation, kibishiiNamespace string, + useVolumeSnapshots bool) error { oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60) + veleroCLI := VeleroCfg.VeleroCLI + providerName := VeleroCfg.CloudProvider + veleroNamespace := VeleroCfg.VeleroNamespace + registryCredentialFile := VeleroCfg.RegistryCredentialFile + veleroFeatures := VeleroCfg.Features + kibishiiDirectory := VeleroCfg.KibishiiDirectory + if err := CreateNamespace(oneHourTimeout, client, kibishiiNamespace); err != nil { return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", kibishiiNamespace) } - defer func() { - if err := DeleteNamespace(context.Background(), client, kibishiiNamespace, true); err != nil { - fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", kibishiiNamespace)) - } - }() - if err := KibishiiPrepareBeforeBackup(oneHourTimeout, client, providerName, kibishiiNamespace, registryCredentialFile, kibishiiDirectory); err != nil { + + if err := KibishiiPrepareBeforeBackup(oneHourTimeout, client, providerName, + kibishiiNamespace, registryCredentialFile, veleroFeatures, + kibishiiDirectory, useVolumeSnapshots); err != nil { return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", kibishiiNamespace) } - if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, kibishiiNamespace, backupLocation, useVolumeSnapshots, ""); err != nil { + if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, + kibishiiNamespace, backupLocation, useVolumeSnapshots, ""); err != nil { RunDebug(context.Background(), veleroCLI, veleroNamespace, backupName, "") return errors.Wrapf(err, "Failed to backup kibishii namespace %s", kibishiiNamespace) } @@ -90,11 +97,19 @@ func RunKibishiiTests(client TestClient, providerName, veleroCLI, veleroNamespac return nil } -func installKibishii(ctx context.Context, namespace string, cloudPlatform string, kibishiiDirectory string) error { +func installKibishii(ctx context.Context, namespace string, cloudPlatform, veleroFeatures, + kibishiiDirectory string, useVolumeSnapshots bool) error { + if strings.EqualFold(cloudPlatform, "azure") && + strings.EqualFold(veleroFeatures, "EnableCSI") && + useVolumeSnapshots { + cloudPlatform = "azure-csi" + } // We use kustomize to generate YAML for Kibishii from the checked-in yaml directories kibishiiInstallCmd := exec.CommandContext(ctx, "kubectl", "apply", "-n", namespace, "-k", kibishiiDirectory+cloudPlatform) + _, stderr, err := veleroexec.RunCommand(kibishiiInstallCmd) + fmt.Printf("Install Kibishii cmd: %s\n", kibishiiInstallCmd) if err != nil { return errors.Wrapf(err, "failed to install kibishii, stderr=%s", stderr) } @@ -149,7 +164,9 @@ func waitForKibishiiPods(ctx context.Context, client TestClient, kibishiiNamespa return WaitForPods(ctx, client, kibishiiNamespace, []string{"jump-pad", "etcd0", "etcd1", "etcd2", "kibishii-deployment-0", "kibishii-deployment-1"}) } -func KibishiiPrepareBeforeBackup(oneHourTimeout context.Context, client TestClient, providerName, kibishiiNamespace, registryCredentialFile, kibishiiDirectory string) error { +func KibishiiPrepareBeforeBackup(oneHourTimeout context.Context, client TestClient, + providerName, kibishiiNamespace, registryCredentialFile, veleroFeatures, + kibishiiDirectory string, useVolumeSnapshots bool) error { serviceAccountName := "default" // wait until the service account is created before patch the image pull secret @@ -161,7 +178,8 @@ func KibishiiPrepareBeforeBackup(oneHourTimeout context.Context, client TestClie return errors.Wrapf(err, "failed to patch the service account %q under the namespace %q", serviceAccountName, kibishiiNamespace) } - if err := installKibishii(oneHourTimeout, kibishiiNamespace, providerName, kibishiiDirectory); err != nil { + if err := installKibishii(oneHourTimeout, kibishiiNamespace, providerName, veleroFeatures, + kibishiiDirectory, useVolumeSnapshots); err != nil { return errors.Wrap(err, "Failed to install Kibishii workload") } diff --git a/test/e2e/util/velero/install.go b/test/e2e/util/velero/install.go index 9c5c30c4a..30b1b1c42 100644 --- a/test/e2e/util/velero/install.go +++ b/test/e2e/util/velero/install.go @@ -24,6 +24,7 @@ import ( "io/ioutil" "os" "os/exec" + "strings" "time" "github.com/pkg/errors" @@ -48,7 +49,7 @@ type installOptions struct { ResticHelperImage string } -func VeleroInstall(ctx context.Context, veleroCfg *VerleroConfig, features string, useVolumeSnapshots bool) error { +func VeleroInstall(ctx context.Context, veleroCfg *VerleroConfig, useVolumeSnapshots bool) error { if veleroCfg.CloudProvider != "kind" { if veleroCfg.ObjectStoreProvider != "" { return errors.New("For cloud platforms, object store plugin cannot be overridden") // Can't set an object store provider that is different than your cloud @@ -60,7 +61,7 @@ func VeleroInstall(ctx context.Context, veleroCfg *VerleroConfig, features strin } } - providerPluginsTmp, err := getProviderPlugins(ctx, veleroCfg.VeleroCLI, veleroCfg.ObjectStoreProvider, veleroCfg.Plugins) + providerPluginsTmp, err := getProviderPlugins(ctx, veleroCfg.VeleroCLI, veleroCfg.ObjectStoreProvider, veleroCfg.Plugins, veleroCfg.Features) if err != nil { return errors.WithMessage(err, "Failed to get provider plugins") } @@ -81,7 +82,7 @@ func VeleroInstall(ctx context.Context, veleroCfg *VerleroConfig, features strin } veleroInstallOptions, err := getProviderVeleroInstallOptions(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, - veleroCfg.BSLPrefix, veleroCfg.BSLConfig, veleroCfg.VSLConfig, providerPluginsTmp, features) + veleroCfg.BSLPrefix, veleroCfg.BSLConfig, veleroCfg.VSLConfig, providerPluginsTmp, veleroCfg.Features) if err != nil { return errors.WithMessagef(err, "Failed to get Velero InstallOptions for plugin provider %s", veleroCfg.ObjectStoreProvider) } @@ -199,6 +200,14 @@ func installVeleroServer(ctx context.Context, cli string, options *installOption } if len(options.Features) > 0 { args = append(args, "--features", options.Features) + if strings.EqualFold(options.Features, "EnableCSI") && options.UseVolumeSnapshots { + if strings.EqualFold(options.ProviderName, "Azure") { + if err := KubectlApplyByFile(ctx, "util/csi/AzureVolumeSnapshotClass.yaml"); err != nil { + return err + } + } + + } } if err := createVelereResources(ctx, cli, namespace, args, options.RegistryCredentialFile, options.ResticHelperImage); err != nil { diff --git a/test/e2e/util/velero/velero_utils.go b/test/e2e/util/velero/velero_utils.go index 807bf4029..4ca48d415 100644 --- a/test/e2e/util/velero/velero_utils.go +++ b/test/e2e/util/velero/velero_utils.go @@ -48,44 +48,50 @@ const PluginsObjectsPrefix = "plugins" var pluginsMatrix = map[string]map[string][]string{ "v1.4": { - "aws": {"velero/velero-plugin-for-aws:v1.1.0"}, - "azure": {"velero/velero-plugin-for-microsoft-azure:v1.1.2"}, - "vsphere": {"velero/velero-plugin-for-aws:v1.1.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.0.2"}, - "gcp": {"velero/velero-plugin-for-gcp:v1.1.0"}, + "aws": {"velero/velero-plugin-for-aws:v1.1.0"}, + "azure": {"velero/velero-plugin-for-microsoft-azure:v1.1.2"}, + "vsphere": {"velero/velero-plugin-for-aws:v1.1.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.0.2"}, + "gcp": {"velero/velero-plugin-for-gcp:v1.1.0"}, + "azure-csi": {"velero/velero-plugin-for-microsoft-azure:v1.1.2", "velero/velero-plugin-for-csi:v0.1.1 "}, }, "v1.5": { - "aws": {"velero/velero-plugin-for-aws:v1.1.0"}, - "azure": {"velero/velero-plugin-for-microsoft-azure:v1.1.2"}, - "vsphere": {"velero/velero-plugin-for-aws:v1.1.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.1.1"}, - "gcp": {"velero/velero-plugin-for-gcp:v1.1.0"}, + "aws": {"velero/velero-plugin-for-aws:v1.1.0"}, + "azure": {"velero/velero-plugin-for-microsoft-azure:v1.1.2"}, + "vsphere": {"velero/velero-plugin-for-aws:v1.1.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.1.1"}, + "gcp": {"velero/velero-plugin-for-gcp:v1.1.0"}, + "azure-csi": {"velero/velero-plugin-for-microsoft-azure:v1.1.2", "velero/velero-plugin-for-csi:v0.1.2 "}, }, "v1.6": { - "aws": {"velero/velero-plugin-for-aws:v1.2.1"}, - "azure": {"velero/velero-plugin-for-microsoft-azure:v1.2.1"}, - "vsphere": {"velero/velero-plugin-for-aws:v1.2.1", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.1.1"}, - "gcp": {"velero/velero-plugin-for-gcp:v1.2.1"}, + "aws": {"velero/velero-plugin-for-aws:v1.2.1"}, + "azure": {"velero/velero-plugin-for-microsoft-azure:v1.2.1"}, + "vsphere": {"velero/velero-plugin-for-aws:v1.2.1", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.1.1"}, + "gcp": {"velero/velero-plugin-for-gcp:v1.2.1"}, + "azure-csi": {"velero/velero-plugin-for-microsoft-azure:v1.3.0", "velero/velero-plugin-for-csi:v0.1.2 "}, }, "v1.7": { - "aws": {"velero/velero-plugin-for-aws:v1.3.0"}, - "azure": {"velero/velero-plugin-for-microsoft-azure:v1.3.0"}, - "vsphere": {"velero/velero-plugin-for-aws:v1.3.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.3.0"}, - "gcp": {"velero/velero-plugin-for-gcp:v1.3.0"}, + "aws": {"velero/velero-plugin-for-aws:v1.3.0"}, + "azure": {"velero/velero-plugin-for-microsoft-azure:v1.3.0"}, + "vsphere": {"velero/velero-plugin-for-aws:v1.3.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.3.0"}, + "gcp": {"velero/velero-plugin-for-gcp:v1.3.0"}, + "azure-csi": {"velero/velero-plugin-for-microsoft-azure:v1.3.0", "velero/velero-plugin-for-csi:v0.2.0"}, }, "v1.8": { - "aws": {"velero/velero-plugin-for-aws:v1.4.0"}, - "azure": {"velero/velero-plugin-for-microsoft-azure:v1.4.0"}, - "vsphere": {"velero/velero-plugin-for-aws:v1.4.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.3.1"}, - "gcp": {"velero/velero-plugin-for-gcp:v1.4.0"}, + "aws": {"velero/velero-plugin-for-aws:v1.4.0"}, + "azure": {"velero/velero-plugin-for-microsoft-azure:v1.4.0"}, + "vsphere": {"velero/velero-plugin-for-aws:v1.4.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.3.1"}, + "gcp": {"velero/velero-plugin-for-gcp:v1.4.0"}, + "azure-csi": {"velero/velero-plugin-for-microsoft-azure:v1.4.0", "velero/velero-plugin-for-csi:v0.2.0"}, }, "main": { - "aws": {"velero/velero-plugin-for-aws:main"}, - "azure": {"velero/velero-plugin-for-microsoft-azure:main"}, - "vsphere": {"velero/velero-plugin-for-aws:main", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.3.1"}, - "gcp": {"velero/velero-plugin-for-gcp:main"}, + "aws": {"velero/velero-plugin-for-aws:main"}, + "azure": {"velero/velero-plugin-for-microsoft-azure:main"}, + "vsphere": {"velero/velero-plugin-for-aws:main", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.3.1"}, + "gcp": {"velero/velero-plugin-for-gcp:main"}, + "azure-csi": {"velero/velero-plugin-for-microsoft-azure:main", "velero/velero-plugin-for-csi:main"}, }, } -func getProviderPluginsByVersion(version, providerName string) ([]string, error) { +func GetProviderPluginsByVersion(version, providerName, feature string) ([]string, error) { var cloudMap map[string][]string arr := strings.Split(version, ".") if len(arr) >= 3 { @@ -97,6 +103,9 @@ func getProviderPluginsByVersion(version, providerName string) ([]string, error) return nil, errors.Errorf("fail to get plugins by version: main") } } + if strings.EqualFold(providerName, "azure") && strings.EqualFold(feature, "EnableCSI") { + providerName = "azure-csi" + } plugins, ok := cloudMap[providerName] if !ok { return nil, errors.Errorf("fail to get plugins by version: %s and provider %s", version, providerName) @@ -380,7 +389,7 @@ func VeleroCreateBackupLocation(ctx context.Context, return VeleroCmdExec(ctx, veleroCLI, args) } -func getProviderPlugins(ctx context.Context, veleroCLI, objectStoreProvider, providerPlugins string) ([]string, error) { +func getProviderPlugins(ctx context.Context, veleroCLI, objectStoreProvider, providerPlugins, feature string) ([]string, error) { // Fetch the plugins for the provider before checking for the object store provider below. var plugins []string if len(providerPlugins) > 0 { @@ -390,7 +399,7 @@ func getProviderPlugins(ctx context.Context, veleroCLI, objectStoreProvider, pro if err != nil { return nil, errors.WithMessage(err, "failed to get velero version") } - plugins, err = getProviderPluginsByVersion(version, objectStoreProvider) + plugins, err = GetProviderPluginsByVersion(version, objectStoreProvider, feature) if err != nil { return nil, errors.WithMessagef(err, "Fail to get plugin by provider %s and version %s", objectStoreProvider, version) } @@ -400,8 +409,8 @@ func getProviderPlugins(ctx context.Context, veleroCLI, objectStoreProvider, pro // VeleroAddPluginsForProvider determines which plugins need to be installed for a provider and // installs them in the current Velero installation, skipping over those that are already installed. -func VeleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNamespace string, provider string, addPlugins string) error { - plugins, err := getProviderPlugins(ctx, veleroCLI, provider, addPlugins) +func VeleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNamespace string, provider string, addPlugins, feature string) error { + plugins, err := getProviderPlugins(ctx, veleroCLI, provider, addPlugins, feature) fmt.Printf("addPlugins cmd =%v\n", addPlugins) fmt.Printf("provider cmd = %v\n", provider) fmt.Printf("plugins cmd = %v\n", plugins)