diff --git a/changelogs/unreleased/4386-mqiu b/changelogs/unreleased/4386-mqiu new file mode 100644 index 000000000..568a1af77 --- /dev/null +++ b/changelogs/unreleased/4386-mqiu @@ -0,0 +1 @@ +Adjust structure of e2e test codes diff --git a/test/e2e/backup_test.go b/test/e2e/backup/backup.go similarity index 51% rename from test/e2e/backup_test.go rename to test/e2e/backup/backup.go index 9d5f92baf..e5ff9bf26 100644 --- a/test/e2e/backup_test.go +++ b/test/e2e/backup/backup.go @@ -13,7 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -package e2e +package backup import ( "context" @@ -23,98 +23,93 @@ import ( "github.com/google/uuid" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + + . "github.com/vmware-tanzu/velero/test/e2e" + k8sutils "github.com/vmware-tanzu/velero/test/e2e/util/k8s" + kibishiiutils "github.com/vmware-tanzu/velero/test/e2e/util/kibishii" + veleroutils "github.com/vmware-tanzu/velero/test/e2e/util/velero" ) -var ( - uuidgen uuid.UUID -) - -// Test backup and restore of Kibishi using restic -var _ = Describe("[Restic] Velero tests on cluster using the plugin provider for object storage and Restic for volume backups", backup_restore_with_restic) - -var _ = Describe("[Snapshot] Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", backup_restore_with_snapshots) - -func backup_restore_with_snapshots() { - backup_restore_test(true) +func BackupRestoreWithSnapshots() { + BackupRestoreTest(true) } -func backup_restore_with_restic() { - backup_restore_test(false) +func BackupRestoreWithRestic() { + BackupRestoreTest(false) } -func backup_restore_test(useVolumeSnapshots bool) { +func BackupRestoreTest(useVolumeSnapshots bool) { var ( backupName, restoreName string ) - client, err := newTestClient() + client, err := k8sutils.NewTestClient() Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests") BeforeEach(func() { - if useVolumeSnapshots && cloudProvider == "kind" { + if useVolumeSnapshots && VeleroCfg.CloudProvider == "kind" { Skip("Volume snapshots not supported on kind") } var err error flag.Parse() - uuidgen, err = uuid.NewRandom() + UUIDgen, err = uuid.NewRandom() Expect(err).To(Succeed()) - if installVelero { - Expect(veleroInstall(context.Background(), veleroCLI, veleroImage, resticHelperImage, plugins, veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots, - cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, crdsVersion, "", registryCredentialFile)).To(Succeed()) + if VeleroCfg.InstallVelero { + Expect(veleroutils.VeleroInstall(context.Background(), &VeleroCfg, "", useVolumeSnapshots)).To(Succeed()) } }) AfterEach(func() { - if installVelero { - err = veleroUninstall(context.Background(), veleroCLI, veleroNamespace) + if VeleroCfg.InstallVelero { + err = veleroutils.VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace) Expect(err).To(Succeed()) } }) When("kibishii is the sample workload", func() { It("should be successfully backed up and restored to the default BackupStorageLocation", func() { - backupName = "backup-" + uuidgen.String() - restoreName = "restore-" + uuidgen.String() + backupName = "backup-" + UUIDgen.String() + restoreName = "restore-" + UUIDgen.String() // Even though we are using Velero's CloudProvider plugin for object storage, the kubernetes cluster is running on // KinD. So use the kind installation for Kibishii. - Expect(runKibishiiTests(client, cloudProvider, veleroCLI, veleroNamespace, backupName, restoreName, "", useVolumeSnapshots, registryCredentialFile)).To(Succeed(), + Expect(kibishiiutils.RunKibishiiTests(client, VeleroCfg.CloudProvider, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backupName, restoreName, "", useVolumeSnapshots, VeleroCfg.RegistryCredentialFile)).To(Succeed(), "Failed to successfully backup and restore Kibishii namespace") }) It("should successfully back up and restore to an additional BackupStorageLocation with unique credentials", func() { - if additionalBSLProvider == "" { + if VeleroCfg.AdditionalBSLProvider == "" { Skip("no additional BSL provider given, not running multiple BackupStorageLocation with unique credentials tests") } - if additionalBSLBucket == "" { + if VeleroCfg.AdditionalBSLBucket == "" { Skip("no additional BSL bucket given, not running multiple BackupStorageLocation with unique credentials tests") } - if additionalBSLCredentials == "" { + if VeleroCfg.AdditionalBSLCredentials == "" { Skip("no additional BSL credentials given, not running multiple BackupStorageLocation with unique credentials tests") } - Expect(veleroAddPluginsForProvider(context.TODO(), veleroCLI, veleroNamespace, additionalBSLProvider, addBSLPlugins)).To(Succeed()) + Expect(veleroutils.VeleroAddPluginsForProvider(context.TODO(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, VeleroCfg.AdditionalBSLProvider, VeleroCfg.AddBSLPlugins)).To(Succeed()) // Create Secret for additional BSL - secretName := fmt.Sprintf("bsl-credentials-%s", uuidgen) - secretKey := fmt.Sprintf("creds-%s", additionalBSLProvider) + secretName := fmt.Sprintf("bsl-credentials-%s", UUIDgen) + secretKey := fmt.Sprintf("creds-%s", VeleroCfg.AdditionalBSLProvider) files := map[string]string{ - secretKey: additionalBSLCredentials, + secretKey: VeleroCfg.AdditionalBSLCredentials, } - Expect(createSecretFromFiles(context.TODO(), client, veleroNamespace, secretName, files)).To(Succeed()) + Expect(k8sutils.CreateSecretFromFiles(context.TODO(), client, VeleroCfg.VeleroNamespace, secretName, files)).To(Succeed()) // Create additional BSL using credential - additionalBsl := fmt.Sprintf("bsl-%s", uuidgen) - Expect(veleroCreateBackupLocation(context.TODO(), - veleroCLI, - veleroNamespace, + additionalBsl := fmt.Sprintf("bsl-%s", UUIDgen) + Expect(veleroutils.VeleroCreateBackupLocation(context.TODO(), + VeleroCfg.VeleroCLI, + VeleroCfg.VeleroNamespace, additionalBsl, - additionalBSLProvider, - additionalBSLBucket, - additionalBSLPrefix, - additionalBSLConfig, + VeleroCfg.AdditionalBSLProvider, + VeleroCfg.AdditionalBSLBucket, + VeleroCfg.AdditionalBSLPrefix, + VeleroCfg.AdditionalBSLConfig, secretName, secretKey, )).To(Succeed()) @@ -127,11 +122,10 @@ func backup_restore_test(useVolumeSnapshots bool) { // We limit the length of backup name here to avoid the issue of vsphere plugin https://github.com/vmware-tanzu/velero-plugin-for-vsphere/issues/370 // We can remove the logic once the issue is fixed if bsl == "default" { - backupName = fmt.Sprintf("%s-%s", backupName, uuidgen) - restoreName = fmt.Sprintf("%s-%s", restoreName, uuidgen) + backupName = fmt.Sprintf("%s-%s", backupName, UUIDgen) + restoreName = fmt.Sprintf("%s-%s", restoreName, UUIDgen) } - - Expect(runKibishiiTests(client, cloudProvider, veleroCLI, veleroNamespace, backupName, restoreName, bsl, useVolumeSnapshots, registryCredentialFile)).To(Succeed(), + Expect(kibishiiutils.RunKibishiiTests(client, VeleroCfg.CloudProvider, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backupName, restoreName, bsl, useVolumeSnapshots, VeleroCfg.RegistryCredentialFile)).To(Succeed(), "Failed to successfully backup and restore Kibishii namespace using BSL %s", bsl) } }) diff --git a/test/e2e/enable_api_group_versions_test.go b/test/e2e/basic/enable_api_group_versions.go similarity index 86% rename from test/e2e/enable_api_group_versions_test.go rename to test/e2e/basic/enable_api_group_versions.go index 68014f184..d9c13e280 100644 --- a/test/e2e/enable_api_group_versions_test.go +++ b/test/e2e/basic/enable_api_group_versions.go @@ -14,11 +14,12 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e +package basic import ( "context" "encoding/json" + "flag" "fmt" "os/exec" "strconv" @@ -34,46 +35,31 @@ import ( "github.com/vmware-tanzu/velero/pkg/builder" veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" + . "github.com/vmware-tanzu/velero/test/e2e" + k8sutils "github.com/vmware-tanzu/velero/test/e2e/util/k8s" + veleroutils "github.com/vmware-tanzu/velero/test/e2e/util/velero" ) -var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", func() { +func APIGropuVersionsTest() { var ( resource, group string err error ctx = context.Background() ) - client, err := newTestClient() + client, err := k8sutils.NewTestClient() Expect(err).To(Succeed(), "Failed to instantiate cluster client for group version tests") BeforeEach(func() { resource = "rockbands" group = "music.example.io" - - uuidgen, err = uuid.NewRandom() + UUIDgen, err = uuid.NewRandom() Expect(err).NotTo(HaveOccurred()) - + flag.Parse() // TODO: install Velero once for the test suite once feature flag is // removed and velero installation becomes the same as other e2e tests. - if installVelero { - err = veleroInstall( - context.Background(), - veleroCLI, - veleroImage, - resticHelperImage, - plugins, - veleroNamespace, - cloudProvider, - objectStoreProvider, - false, - cloudCredentialsFile, - bslBucket, - bslPrefix, - bslConfig, - vslConfig, - crdsVersion, - "EnableAPIGroupVersions", // TODO: remove when feature flag is removed - registryCredentialFile) + if VeleroCfg.InstallVelero { + err = veleroutils.VeleroInstall(context.Background(), &VeleroCfg, "EnableAPIGroupVersions", false) Expect(err).NotTo(HaveOccurred()) } }) @@ -88,8 +74,8 @@ var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", } Expect(err).NotTo(HaveOccurred()) - if installVelero { - err = veleroUninstall(ctx, veleroCLI, veleroNamespace) + if VeleroCfg.InstallVelero { + err = veleroutils.VeleroUninstall(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace) Expect(err).NotTo(HaveOccurred()) } @@ -105,9 +91,9 @@ var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", )).To(Succeed(), "Failed to successfully backup and restore multiple API Groups") }) }) -}) +} -func runEnableAPIGroupVersionsTests(ctx context.Context, client testClient, resource, group string) error { +func runEnableAPIGroupVersionsTests(ctx context.Context, client k8sutils.TestClient, resource, group string) error { tests := []struct { name string namespaces []string @@ -180,7 +166,7 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client testClient, reso }, tgtCrdYaml: "testdata/enable_api_group_versions/case-d-target-manually-added-mutations.yaml", tgtVer: "v2beta1", - cm: builder.ForConfigMap(veleroNamespace, "enableapigroupversions").Data( + cm: builder.ForConfigMap(VeleroCfg.VeleroNamespace, "enableapigroupversions").Data( "restoreResourcesVersionPriority", `rockbands.music.example.io=v2beta1,v2beta2,v2`, ).Result(), @@ -224,11 +210,11 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client testClient, reso for version, cr := range tc.srcCRs { ns := resource + "-src-" + version - if err := createNamespace(ctx, client, ns); err != nil { + if err := k8sutils.CreateNamespace(ctx, client, ns); err != nil { return errors.Wrapf(err, "create %s namespace", ns) } defer func(namespace string) { - if err = deleteNamespace(ctx, client, namespace, true); err != nil { + if err = k8sutils.DeleteNamespace(ctx, client, namespace, true); err != nil { fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", ns)) } }(ns) @@ -243,16 +229,16 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client testClient, reso // Restart Velero pods in order to recognize music-system CRD right away // instead of waiting for discovery helper to refresh. See // https://github.com/vmware-tanzu/velero/issues/3471. - if err := restartPods(ctx, veleroNamespace); err != nil { + if err := restartPods(ctx, VeleroCfg.VeleroNamespace); err != nil { return errors.Wrapf(err, "restart Velero pods") } - backup := "backup-rockbands-" + uuidgen.String() + "-" + strconv.Itoa(i) + backup := "backup-rockbands-" + UUIDgen.String() + "-" + strconv.Itoa(i) namespacesStr := strings.Join(tc.namespaces, ",") - err = veleroBackupNamespace(ctx, veleroCLI, veleroNamespace, backup, namespacesStr, "", false) + err = veleroutils.VeleroBackupNamespace(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backup, namespacesStr, "", false) if err != nil { - runDebug(context.Background(), veleroCLI, veleroNamespace, backup, "") + veleroutils.RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backup, "") return errors.Wrapf(err, "back up %s namespaces on source cluster", namespacesStr) } @@ -261,7 +247,7 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client testClient, reso } for _, ns := range tc.namespaces { - if err := deleteNamespace(ctx, client, ns, true); err != nil { + if err := k8sutils.DeleteNamespace(ctx, client, ns, true); err != nil { return errors.Wrapf(err, "delete %s namespace from source cluster", ns) } } @@ -275,23 +261,23 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client testClient, reso // Apply config map if there is one. if tc.cm != nil { - _, err := client.clientGo.CoreV1().ConfigMaps(veleroNamespace).Create(ctx, tc.cm, metav1.CreateOptions{}) + _, err := client.ClientGo.CoreV1().ConfigMaps(VeleroCfg.VeleroNamespace).Create(ctx, tc.cm, metav1.CreateOptions{}) if err != nil { return errors.Wrap(err, "create config map with user version priorities") } } // Reset Velero to recognize music-system CRD. - if err := restartPods(ctx, veleroNamespace); err != nil { + if err := restartPods(ctx, VeleroCfg.VeleroNamespace); err != nil { return errors.Wrapf(err, "restart Velero pods") } // Restore rockbands namespaces. - restore := "restore-rockbands-" + uuidgen.String() + "-" + strconv.Itoa(i) + restore := "restore-rockbands-" + UUIDgen.String() + "-" + strconv.Itoa(i) if tc.want != nil { - if err := veleroRestore(ctx, veleroCLI, veleroNamespace, restore, backup); err != nil { - runDebug(context.Background(), veleroCLI, veleroNamespace, "", restore) + if err := veleroutils.VeleroRestore(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, restore, backup); err != nil { + veleroutils.RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, "", restore) return errors.Wrapf(err, "restore %s namespaces on target cluster", namespacesStr) } @@ -329,7 +315,7 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client testClient, reso } else { // No custom resource should have been restored. Expect "no resource found" // error during restore. - err := veleroRestore(ctx, veleroCLI, veleroNamespace, restore, backup) + err := veleroutils.VeleroRestore(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, restore, backup) if err.Error() != "Unexpected restore phase got PartiallyFailed, expecting Completed" { return errors.New("expected error but not none") diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 37e111703..825ce2938 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e +package e2e_test import ( "flag" @@ -23,44 +23,59 @@ import ( . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" . "github.com/onsi/gomega" -) -var ( - veleroCLI, veleroImage, veleroVersion, cloudCredentialsFile, bslConfig, bslBucket, bslPrefix, vslConfig, cloudProvider, objectStoreProvider, veleroNamespace, crdsVersion string - additionalBSLProvider, additionalBSLBucket, additionalBSLPrefix, additionalBSLConfig, additionalBSLCredentials, registryCredentialFile, resticHelperImage string - upgradeFromVeleroVersion, upgradeFromVeleroCLI, plugins, addBSLPlugins string - installVelero bool + . "github.com/vmware-tanzu/velero/test/e2e" + . "github.com/vmware-tanzu/velero/test/e2e/backup" + . "github.com/vmware-tanzu/velero/test/e2e/basic" + . "github.com/vmware-tanzu/velero/test/e2e/scale" + . "github.com/vmware-tanzu/velero/test/e2e/upgrade" ) func init() { - flag.StringVar(&cloudProvider, "cloud-provider", "", "cloud that Velero will be installed into. Required.") - flag.StringVar(&objectStoreProvider, "object-store-provider", "", "provider of object store plugin. Required if cloud-provider is kind, otherwise ignored.") - flag.StringVar(&bslBucket, "bucket", "", "name of the object storage bucket where backups from e2e tests should be stored. Required.") - flag.StringVar(&cloudCredentialsFile, "credentials-file", "", "file containing credentials for backup and volume provider. Required.") - flag.StringVar(&veleroCLI, "velerocli", "velero", "path to the velero application to use.") - flag.StringVar(&veleroImage, "velero-image", "velero/velero:main", "image for the velero server to be tested.") - flag.StringVar(&plugins, "plugins", "", "provider plugins to be tested.") - flag.StringVar(&addBSLPlugins, "additional-bsl-plugins", "", "additional plugins to be tested.") - flag.StringVar(&veleroVersion, "velero-version", "main", "image version for the velero server to be tested with.") - flag.StringVar(&resticHelperImage, "restic-helper-image", "", "image for the velero restic restore helper to be tested.") - flag.StringVar(&upgradeFromVeleroCLI, "upgrade-from-velero-cli", "", "path to the pre-upgrade velero application to use.") - flag.StringVar(&upgradeFromVeleroVersion, "upgrade-from-velero-version", "v1.6.3", "image for the pre-upgrade velero server to be tested.") - flag.StringVar(&bslConfig, "bsl-config", "", "configuration to use for the backup storage location. Format is key1=value1,key2=value2") - flag.StringVar(&bslPrefix, "prefix", "", "prefix under which all Velero data should be stored within the bucket. Optional.") - flag.StringVar(&vslConfig, "vsl-config", "", "configuration to use for the volume snapshot location. Format is key1=value1,key2=value2") - flag.StringVar(&veleroNamespace, "velero-namespace", "velero", "namespace to install Velero into") - flag.BoolVar(&installVelero, "install-velero", true, "install/uninstall velero during the test. Optional.") - flag.StringVar(®istryCredentialFile, "registry-credential-file", "", "file containing credential for the image registry, follows the same format rules as the ~/.docker/config.json file. Optional.") + flag.StringVar(&VeleroCfg.CloudProvider, "cloud-provider", "", "cloud that Velero will be installed into. Required.") + flag.StringVar(&VeleroCfg.ObjectStoreProvider, "object-store-provider", "", "provider of object store plugin. Required if cloud-provider is kind, otherwise ignored.") + flag.StringVar(&VeleroCfg.BSLBucket, "bucket", "", "name of the object storage bucket where backups from e2e tests should be stored. Required.") + flag.StringVar(&VeleroCfg.CloudCredentialsFile, "credentials-file", "", "file containing credentials for backup and volume provider. Required.") + flag.StringVar(&VeleroCfg.VeleroCLI, "velerocli", "velero", "path to the velero application to use.") + flag.StringVar(&VeleroCfg.VeleroImage, "velero-image", "velero/velero:main", "image for the velero server to be tested.") + flag.StringVar(&VeleroCfg.Plugins, "plugins", "", "provider plugins to be tested.") + flag.StringVar(&VeleroCfg.AddBSLPlugins, "additional-bsl-plugins", "", "additional plugins to be tested.") + flag.StringVar(&VeleroCfg.VeleroVersion, "velero-version", "main", "image version for the velero server to be tested with.") + flag.StringVar(&VeleroCfg.ResticHelperImage, "restic-helper-image", "", "image for the velero restic restore helper to be tested.") + flag.StringVar(&VeleroCfg.UpgradeFromVeleroCLI, "upgrade-from-velero-cli", "", "path to the pre-upgrade velero application to use.") + flag.StringVar(&VeleroCfg.UpgradeFromVeleroVersion, "upgrade-from-velero-version", "v1.6.3", "image for the pre-upgrade velero server to be tested.") + flag.StringVar(&VeleroCfg.BSLConfig, "bsl-config", "", "configuration to use for the backup storage location. Format is key1=value1,key2=value2") + flag.StringVar(&VeleroCfg.BSLPrefix, "prefix", "", "prefix under which all Velero data should be stored within the bucket. Optional.") + flag.StringVar(&VeleroCfg.VSLConfig, "vsl-config", "", "configuration to use for the volume snapshot location. Format is key1=value1,key2=value2") + flag.StringVar(&VeleroCfg.VeleroNamespace, "velero-namespace", "velero", "namespace to install Velero into") + flag.BoolVar(&VeleroCfg.InstallVelero, "install-velero", true, "install/uninstall velero during the test. Optional.") + flag.StringVar(&VeleroCfg.RegistryCredentialFile, "registry-credential-file", "", "file containing credential for the image registry, follows the same format rules as the ~/.docker/config.json file. Optional.") // Flags to create an additional BSL for multiple credentials test - flag.StringVar(&additionalBSLProvider, "additional-bsl-object-store-provider", "", "Provider of object store plugin for additional backup storage location. Required if testing multiple credentials support.") - flag.StringVar(&additionalBSLBucket, "additional-bsl-bucket", "", "name of the object storage bucket for additional backup storage location. Required if testing multiple credentials support.") - flag.StringVar(&additionalBSLPrefix, "additional-bsl-prefix", "", "prefix under which all Velero data should be stored within the bucket for additional backup storage location. Optional.") - flag.StringVar(&additionalBSLConfig, "additional-bsl-config", "", "configuration to use for the additional backup storage location. Format is key1=value1,key2=value2") - flag.StringVar(&additionalBSLCredentials, "additional-bsl-credentials-file", "", "file containing credentials for additional backup storage location provider. Required if testing multiple credentials support.") - flag.StringVar(&crdsVersion, "crds-version", "v1", "CRD apiVersion for velero CRD creation.") + flag.StringVar(&VeleroCfg.AdditionalBSLProvider, "additional-bsl-object-store-provider", "", "Provider of object store plugin for additional backup storage location. Required if testing multiple credentials support.") + flag.StringVar(&VeleroCfg.AdditionalBSLBucket, "additional-bsl-bucket", "", "name of the object storage bucket for additional backup storage location. Required if testing multiple credentials support.") + flag.StringVar(&VeleroCfg.AdditionalBSLPrefix, "additional-bsl-prefix", "", "prefix under which all Velero data should be stored within the bucket for additional backup storage location. Optional.") + flag.StringVar(&VeleroCfg.AdditionalBSLConfig, "additional-bsl-config", "", "configuration to use for the additional backup storage location. Format is key1=value1,key2=value2") + flag.StringVar(&VeleroCfg.AdditionalBSLCredentials, "additional-bsl-credentials-file", "", "file containing credentials for additional backup storage location provider. Required if testing multiple credentials support.") + flag.StringVar(&VeleroCfg.CRDsVersion, "crds-version", "v1", "CRD apiVersion for velero CRD creation.") } +var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", APIGropuVersionsTest) + +// Test backup and restore of Kibishi using restic +var _ = Describe("[Restic] Velero tests on cluster using the plugin provider for object storage and Restic for volume backups", BackupRestoreWithRestic) + +var _ = Describe("[Snapshot] Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupRestoreWithSnapshots) + +var _ = Describe("[Basic] Backup/restore of 2 namespaces", BasicBackupRestore) + +var _ = Describe("[Scale] Backup/restore of 2500 namespaces", MultiNSBackupRestore) + +// Upgrade test by Kibishi using restic +var _ = Describe("[Upgrade][Restic] Velero upgrade tests on cluster using the plugin provider for object storage and Restic for volume backups", BackupUpgradeRestoreWithRestic) + +var _ = Describe("[Upgrade][Snapshot] Velero upgrade tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupUpgradeRestoreWithSnapshots) + func TestE2e(t *testing.T) { // Skip running E2E tests when running only "short" tests because: // 1. E2E tests are long running tests involving installation of Velero and performing backup and restore operations. diff --git a/test/e2e/multiple_namespaces_test.go b/test/e2e/scale/multiple_namespaces.go similarity index 50% rename from test/e2e/multiple_namespaces_test.go rename to test/e2e/scale/multiple_namespaces.go index bb6b71466..dd7832d3b 100644 --- a/test/e2e/multiple_namespaces_test.go +++ b/test/e2e/scale/multiple_namespaces.go @@ -1,40 +1,58 @@ -package e2e +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package scale import ( "context" "flag" "fmt" - "strings" "time" "github.com/google/uuid" + k8sutils "github.com/vmware-tanzu/velero/test/e2e/util/k8s" + veleroutils "github.com/vmware-tanzu/velero/test/e2e/util/velero" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/pkg/errors" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + . "github.com/vmware-tanzu/velero/test/e2e" ) -var _ = Describe("[Basic] Backup/restore of 2 namespaces", func() { +func BasicBackupRestore() { - client, err := newTestClient() + client, err := k8sutils.NewTestClient() Expect(err).To(Succeed(), "Failed to instantiate cluster client for multiple namespace tests") BeforeEach(func() { var err error flag.Parse() - uuidgen, err = uuid.NewRandom() + UUIDgen, err = uuid.NewRandom() Expect(err).To(Succeed()) - if installVelero { - Expect(veleroInstall(context.Background(), veleroCLI, veleroImage, resticHelperImage, plugins, veleroNamespace, cloudProvider, objectStoreProvider, false, - cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, crdsVersion, "", registryCredentialFile)).To(Succeed()) + if VeleroCfg.InstallVelero { + Expect(veleroutils.VeleroInstall(context.Background(), &VeleroCfg, "", false)).To(Succeed()) } }) AfterEach(func() { - if installVelero { - err := veleroUninstall(context.Background(), veleroCLI, veleroNamespace) + if VeleroCfg.InstallVelero { + err := veleroutils.VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace) Expect(err).To(Succeed()) } @@ -42,34 +60,33 @@ var _ = Describe("[Basic] Backup/restore of 2 namespaces", func() { Context("When I create 2 namespaces", func() { It("should be successfully backed up and restored", func() { - backupName := "backup-" + uuidgen.String() - restoreName := "restore-" + uuidgen.String() + backupName := "backup-" + UUIDgen.String() + restoreName := "restore-" + UUIDgen.String() fiveMinTimeout, _ := context.WithTimeout(context.Background(), 5*time.Minute) - Expect(RunMultipleNamespaceTest(fiveMinTimeout, client, "nstest-"+uuidgen.String(), 2, + Expect(RunMultipleNamespaceTest(fiveMinTimeout, client, "nstest-"+UUIDgen.String(), 2, backupName, restoreName)).To(Succeed(), "Failed to successfully backup and restore multiple namespaces") }) }) -}) +} -var _ = Describe("[Scale] Backup/restore of 2500 namespaces", func() { +func MultiNSBackupRestore() { - client, err := newTestClient() + client, err := k8sutils.NewTestClient() Expect(err).To(Succeed(), "Failed to instantiate cluster client for multiple namespace tests") BeforeEach(func() { var err error flag.Parse() - uuidgen, err = uuid.NewRandom() + UUIDgen, err = uuid.NewRandom() Expect(err).To(Succeed()) - if installVelero { - Expect(veleroInstall(context.Background(), veleroCLI, veleroImage, resticHelperImage, plugins, veleroNamespace, cloudProvider, objectStoreProvider, false, - cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, crdsVersion, "", registryCredentialFile)).To(Succeed()) + if VeleroCfg.InstallVelero { + Expect(veleroutils.VeleroInstall(context.Background(), &VeleroCfg, "", false)).To(Succeed()) } }) AfterEach(func() { - if installVelero { - err := veleroUninstall(context.Background(), veleroCLI, veleroNamespace) + if VeleroCfg.InstallVelero { + err := veleroutils.VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace) Expect(err).To(Succeed()) } @@ -77,22 +94,22 @@ var _ = Describe("[Scale] Backup/restore of 2500 namespaces", func() { Context("When I create 2500 namespaces", func() { It("should be successfully backed up and restored", func() { - backupName := "backup-" + uuidgen.String() - restoreName := "restore-" + uuidgen.String() + backupName := "backup-" + UUIDgen.String() + restoreName := "restore-" + UUIDgen.String() twoHourTimeout, _ := context.WithTimeout(context.Background(), 2*time.Hour) - Expect(RunMultipleNamespaceTest(twoHourTimeout, client, "nstest-"+uuidgen.String(), 2500, + Expect(RunMultipleNamespaceTest(twoHourTimeout, client, "nstest-"+UUIDgen.String(), 2500, backupName, restoreName)).To(Succeed(), "Failed to successfully backup and restore multiple namespaces") }) }) -}) +} -func RunMultipleNamespaceTest(ctx context.Context, client testClient, nsBaseName string, numberOfNamespaces int, backupName string, restoreName string) error { - defer cleanupNamespaces(context.Background(), client, nsBaseName) // Run at exit for final cleanup +func RunMultipleNamespaceTest(ctx context.Context, client k8sutils.TestClient, nsBaseName string, numberOfNamespaces int, backupName string, restoreName string) error { + defer k8sutils.CleanupNamespaces(context.Background(), client, nsBaseName) // Run at exit for final cleanup var excludeNamespaces []string // Currently it's hard to build a large list of namespaces to include and wildcards do not work so instead // we will exclude all of the namespaces that existed prior to the test from the backup - namespaces, err := client.clientGo.CoreV1().Namespaces().List(ctx, v1.ListOptions{}) + namespaces, err := client.ClientGo.CoreV1().Namespaces().List(ctx, v1.ListOptions{}) if err != nil { return errors.Wrap(err, "Could not retrieve namespaces") } @@ -104,30 +121,30 @@ func RunMultipleNamespaceTest(ctx context.Context, client testClient, nsBaseName fmt.Printf("Creating namespaces ...\n") for nsNum := 0; nsNum < numberOfNamespaces; nsNum++ { createNSName := fmt.Sprintf("%s-%00000d", nsBaseName, nsNum) - if err := createNamespace(ctx, client, createNSName); err != nil { + if err := k8sutils.CreateNamespace(ctx, client, createNSName); err != nil { return errors.Wrapf(err, "Failed to create namespace %s", createNSName) } } - if err := veleroBackupExcludeNamespaces(ctx, veleroCLI, veleroNamespace, backupName, excludeNamespaces); err != nil { - runDebug(context.Background(), veleroCLI, veleroNamespace, backupName, "") + if err := veleroutils.VeleroBackupExcludeNamespaces(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backupName, excludeNamespaces); err != nil { + veleroutils.RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backupName, "") return errors.Wrapf(err, "Failed to backup backup namespaces %s-*", nsBaseName) } - err = cleanupNamespaces(ctx, client, nsBaseName) + err = k8sutils.CleanupNamespaces(ctx, client, nsBaseName) if err != nil { return errors.Wrap(err, "Could cleanup retrieve namespaces") } - err = veleroRestore(ctx, veleroCLI, veleroNamespace, restoreName, backupName) + err = veleroutils.VeleroRestore(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, restoreName, backupName) if err != nil { - runDebug(context.Background(), veleroCLI, veleroNamespace, "", restoreName) + veleroutils.RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, "", restoreName) return errors.Wrap(err, "Restore failed") } // Verify that we got back all of the namespaces we created for nsNum := 0; nsNum < numberOfNamespaces; nsNum++ { checkNSName := fmt.Sprintf("%s-%00000d", nsBaseName, nsNum) - checkNS, err := getNamespace(ctx, client, checkNSName) + checkNS, err := k8sutils.GetNamespace(ctx, client, checkNSName) if err != nil { return errors.Wrapf(err, "Could not retrieve test namespace %s", checkNSName) } @@ -138,21 +155,3 @@ func RunMultipleNamespaceTest(ctx context.Context, client testClient, nsBaseName // Cleanup is automatic on the way out return nil } - -func cleanupNamespaces(ctx context.Context, client testClient, nsBaseName string) error { - namespaces, err := client.clientGo.CoreV1().Namespaces().List(ctx, v1.ListOptions{}) - if err != nil { - return errors.Wrap(err, "Could not retrieve namespaces") - } - - fmt.Printf("Cleaning up namespaces ...\n") - for _, checkNamespace := range namespaces.Items { - if strings.HasPrefix(checkNamespace.Name, nsBaseName) { - err = client.clientGo.CoreV1().Namespaces().Delete(ctx, checkNamespace.Name, v1.DeleteOptions{}) - if err != nil { - return errors.Wrapf(err, "Could not delete namespace %s", checkNamespace.Name) - } - } - } - return nil -} diff --git a/test/e2e/types.go b/test/e2e/types.go new file mode 100644 index 000000000..40e3ed05e --- /dev/null +++ b/test/e2e/types.go @@ -0,0 +1,52 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "github.com/google/uuid" +) + +var UUIDgen uuid.UUID + +var VeleroCfg VerleroConfig + +type VerleroConfig struct { + VeleroCLI string + VeleroImage string + VeleroVersion string + CloudCredentialsFile string + BSLConfig string + BSLBucket string + BSLPrefix string + VSLConfig string + CloudProvider string + ObjectStoreProvider string + VeleroNamespace string + CRDsVersion string + AdditionalBSLProvider string + AdditionalBSLBucket string + AdditionalBSLPrefix string + AdditionalBSLConfig string + AdditionalBSLCredentials string + RegistryCredentialFile string + ResticHelperImage string + UpgradeFromVeleroVersion string + UpgradeFromVeleroCLI string + Plugins string + AddBSLPlugins string + InstallVelero bool +} diff --git a/test/e2e/upgrade/upgrade.go b/test/e2e/upgrade/upgrade.go new file mode 100644 index 000000000..e5825b79d --- /dev/null +++ b/test/e2e/upgrade/upgrade.go @@ -0,0 +1,170 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package upgrade + +import ( + "context" + "flag" + "fmt" + "time" + + "github.com/google/uuid" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/pkg/errors" + + . "github.com/vmware-tanzu/velero/test/e2e" + k8sutils "github.com/vmware-tanzu/velero/test/e2e/util/k8s" + kibishiiutils "github.com/vmware-tanzu/velero/test/e2e/util/kibishii" + veleroutils "github.com/vmware-tanzu/velero/test/e2e/util/velero" +) + +const ( + upgradeNamespace = "upgrade-workload" +) + +func BackupUpgradeRestoreWithSnapshots() { + BackupUpgradeRestoreTest(true) +} + +func BackupUpgradeRestoreWithRestic() { + BackupUpgradeRestoreTest(false) +} + +func BackupUpgradeRestoreTest(useVolumeSnapshots bool) { + var ( + backupName, restoreName, upgradeFromVeleroCLI string + ) + + client, err := k8sutils.NewTestClient() + Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests") + + BeforeEach(func() { + if (len(VeleroCfg.UpgradeFromVeleroVersion)) == 0 { + Skip("An original velero version is required to run upgrade test, please run test with upgrade-from-velero-version=") + } + if useVolumeSnapshots && VeleroCfg.CloudProvider == "kind" { + Skip("Volume snapshots not supported on kind") + } + //Assume tag of velero server image is identical to velero CLI version + //Download velero CLI if it's empty according to velero CLI version + if (len(VeleroCfg.UpgradeFromVeleroCLI)) == 0 { + upgradeFromVeleroCLI, err = veleroutils.InstallVeleroCLI(VeleroCfg.UpgradeFromVeleroVersion) + Expect(err).To(Succeed()) + } + + var err error + flag.Parse() + UUIDgen, err = uuid.NewRandom() + Expect(err).To(Succeed()) + if VeleroCfg.InstallVelero { + //Set VeleroImage and ResticHelperImage to blank + //VeleroImage and ResticHelperImage should be the default value in originalCli + tmpCfg := VeleroCfg + tmpCfg.VeleroCLI = upgradeFromVeleroCLI + tmpCfg.VeleroImage = "" + tmpCfg.ResticHelperImage = "" + tmpCfg.Plugins = "" + tmpCfg.CRDsVersion = "" + Expect(veleroutils.VeleroInstall(context.Background(), &tmpCfg, "", useVolumeSnapshots)).To(Succeed()) + Expect(veleroutils.CheckVeleroVersion(context.Background(), upgradeFromVeleroCLI, tmpCfg.UpgradeFromVeleroVersion)).To(Succeed()) + } else { + Skip("Upgrade test is skipped since user don't want to install any other velero") + } + }) + + AfterEach(func() { + if VeleroCfg.InstallVelero { + err = veleroutils.VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace) + Expect(err).To(Succeed()) + } + }) + + When("kibishii is the sample workload", func() { + It("should be successfully backed up and restored to the default BackupStorageLocation", func() { + backupName = "backup-" + UUIDgen.String() + restoreName = "restore-" + UUIDgen.String() + Expect(runUpgradeTests(client, &VeleroCfg, upgradeFromVeleroCLI, backupName, restoreName, "", useVolumeSnapshots)).To(Succeed(), + "Failed to successfully backup and restore Kibishii namespace") + }) + }) +} + +// runUpgradeTests runs upgrade test on the provider by kibishii. +func runUpgradeTests(client k8sutils.TestClient, veleroCfg *VerleroConfig, upgradeFromVeleroCLI, backupName, restoreName, backupLocation string, + useVolumeSnapshots bool) error { + if veleroCfg.VeleroCLI == "" { + return errors.New("empty") + } + oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60) + if err := k8sutils.CreateNamespace(oneHourTimeout, client, upgradeNamespace); err != nil { + return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", upgradeNamespace) + } + defer func() { + if err := k8sutils.DeleteNamespace(context.Background(), client, upgradeNamespace, true); err != nil { + fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", upgradeNamespace)) + } + }() + if err := kibishiiutils.KibishiiPrepareBeforeBackup(oneHourTimeout, client, veleroCfg.CloudProvider, upgradeNamespace, veleroCfg.RegistryCredentialFile); err != nil { + return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", upgradeNamespace) + } + + if err := veleroutils.VeleroBackupNamespace(oneHourTimeout, upgradeFromVeleroCLI, veleroCfg.VeleroNamespace, backupName, upgradeNamespace, backupLocation, useVolumeSnapshots); err != nil { + // TODO currently, the upgrade case covers the upgrade path from 1.6 to main and the velero v1.6 doesn't support "debug" command + // TODO move to "veleroutils.RunDebug" after we bump up to 1.7 in the upgrade case + veleroutils.VeleroBackupLogs(context.Background(), upgradeFromVeleroCLI, veleroCfg.VeleroNamespace, backupName) + return errors.Wrapf(err, "Failed to backup kibishii namespace %s", upgradeNamespace) + } + + if veleroCfg.CloudProvider == "vsphere" && useVolumeSnapshots { + // Wait for uploads started by the Velero Plug-in for vSphere to complete + // TODO - remove after upload progress monitoring is implemented + fmt.Println("Waiting for vSphere uploads to complete") + if err := veleroutils.WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, upgradeNamespace); err != nil { + return errors.Wrapf(err, "Error waiting for uploads to complete") + } + } + fmt.Printf("Simulating a disaster by removing namespace %s\n", upgradeNamespace) + if err := k8sutils.DeleteNamespace(oneHourTimeout, client, upgradeNamespace, true); err != nil { + return errors.Wrapf(err, "failed to delete namespace %s", upgradeNamespace) + } + + // the snapshots of AWS may be still in pending status when do the restore, wait for a while + // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 + // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed + if veleroCfg.CloudProvider == "aws" && useVolumeSnapshots { + fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...") + time.Sleep(5 * time.Minute) + } + + if err := veleroutils.VeleroInstall(context.Background(), veleroCfg, "", useVolumeSnapshots); err != nil { + return errors.Wrapf(err, "Failed to install velero from image %s", veleroCfg.VeleroImage) + } + if err := veleroutils.CheckVeleroVersion(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroVersion); err != nil { + return errors.Wrapf(err, "Velero install version mismatch.") + } + if err := veleroutils.VeleroRestore(oneHourTimeout, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, restoreName, backupName); err != nil { + veleroutils.RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, "", restoreName) + return errors.Wrapf(err, "Restore %s failed from backup %s", restoreName, backupName) + } + + if err := kibishiiutils.KibishiiVerifyAfterRestore(client, upgradeNamespace, oneHourTimeout); err != nil { + return errors.Wrapf(err, "Error verifying kibishii after restore") + } + + fmt.Printf("Upgrade test completed successfully\n") + return nil +} diff --git a/test/e2e/upgrade_test.go b/test/e2e/upgrade_test.go deleted file mode 100644 index c647e7b31..000000000 --- a/test/e2e/upgrade_test.go +++ /dev/null @@ -1,164 +0,0 @@ -/* -Copyright the Velero contributors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -package e2e - -import ( - "context" - "flag" - "fmt" - "time" - - "github.com/google/uuid" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/pkg/errors" -) - -const ( - upgradeNamespace = "upgrade-workload" -) - -// Upgrade test by Kibishi using restic -var _ = Describe("[Upgrade][Restic] Velero upgrade tests on cluster using the plugin provider for object storage and Restic for volume backups", backup_upgrade_restore_with_restic) - -var _ = Describe("[Upgrade][Snapshot] Velero upgrade tests on cluster using the plugin provider for object storage and snapshots for volume backups", backup_upgrade_restore_with_snapshots) - -func backup_upgrade_restore_with_snapshots() { - backup_upgrade_restore_test(true) -} - -func backup_upgrade_restore_with_restic() { - backup_upgrade_restore_test(false) -} - -func backup_upgrade_restore_test(useVolumeSnapshots bool) { - var ( - backupName, restoreName string - ) - upgradeFromVeleroCLI := upgradeFromVeleroCLI - - client, err := newTestClient() - Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests") - - BeforeEach(func() { - if (len(upgradeFromVeleroVersion)) == 0 { - Skip("An original velero version is required to run upgrade test, please run test with upgrade-from-velero-version=") - } - if useVolumeSnapshots && cloudProvider == "kind" { - Skip("Volume snapshots not supported on kind") - } - //Assume tag of velero server image is identical to velero CLI version - //Download velero CLI if it's empty according to velero CLI version - if (len(upgradeFromVeleroCLI)) == 0 { - upgradeFromVeleroCLI, err = installVeleroCLI(upgradeFromVeleroVersion) - Expect(err).To(Succeed()) - } - - var err error - flag.Parse() - uuidgen, err = uuid.NewRandom() - Expect(err).To(Succeed()) - if installVelero { - //Set veleroImage and resticHelperImage to blank - //veleroImage and resticHelperImage should be the default value in originalCli - Expect(veleroInstall(context.Background(), upgradeFromVeleroCLI, "", "", "", veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots, - cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, "", "", registryCredentialFile)).To(Succeed()) - Expect(checkVeleroVersion(context.Background(), upgradeFromVeleroCLI, upgradeFromVeleroVersion)).To(Succeed()) - } else { - Skip("Upgrade test is skipped since user don't want to install any other velero") - } - }) - - AfterEach(func() { - if installVelero { - err = veleroUninstall(context.Background(), veleroCLI, veleroNamespace) - Expect(err).To(Succeed()) - } - }) - - When("kibishii is the sample workload", func() { - It("should be successfully backed up and restored to the default BackupStorageLocation", func() { - backupName = "backup-" + uuidgen.String() - restoreName = "restore-" + uuidgen.String() - Expect(runUpgradeTests(client, veleroImage, veleroVersion, cloudProvider, upgradeFromVeleroCLI, veleroNamespace, backupName, restoreName, "", useVolumeSnapshots, registryCredentialFile)).To(Succeed(), - "Failed to successfully backup and restore Kibishii namespace") - }) - }) -} - -// runUpgradeTests runs upgrade test on the provider by kibishii. -func runUpgradeTests(client testClient, upgradeToVeleroImage, upgradeToVeleroVersion, providerName, upgradeFromVeleroCLI, veleroNamespace, backupName, restoreName, backupLocation string, - useVolumeSnapshots bool, registryCredentialFile string) error { - oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60) - if err := createNamespace(oneHourTimeout, client, upgradeNamespace); err != nil { - return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", upgradeNamespace) - } - defer func() { - if err := deleteNamespace(context.Background(), client, upgradeNamespace, true); err != nil { - fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", upgradeNamespace)) - } - }() - if err := kibishiiPrepareBeforeBackup(oneHourTimeout, client, providerName, upgradeNamespace, registryCredentialFile); err != nil { - return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", upgradeNamespace) - } - - if err := veleroBackupNamespace(oneHourTimeout, upgradeFromVeleroCLI, veleroNamespace, backupName, upgradeNamespace, backupLocation, useVolumeSnapshots); err != nil { - // TODO currently, the upgrade case covers the upgrade path from 1.6 to main and the velero v1.6 doesn't support "debug" command - // TODO move to "runDebug" after we bump up to 1.7 in the upgrade case - veleroBackupLogs(context.Background(), upgradeFromVeleroCLI, veleroNamespace, backupName) - return errors.Wrapf(err, "Failed to backup kibishii namespace %s", upgradeNamespace) - } - - if providerName == "vsphere" && useVolumeSnapshots { - // Wait for uploads started by the Velero Plug-in for vSphere to complete - // TODO - remove after upload progress monitoring is implemented - fmt.Println("Waiting for vSphere uploads to complete") - if err := waitForVSphereUploadCompletion(oneHourTimeout, time.Hour, upgradeNamespace); err != nil { - return errors.Wrapf(err, "Error waiting for uploads to complete") - } - } - fmt.Printf("Simulating a disaster by removing namespace %s\n", upgradeNamespace) - if err := deleteNamespace(oneHourTimeout, client, upgradeNamespace, true); err != nil { - return errors.Wrapf(err, "failed to delete namespace %s", upgradeNamespace) - } - - // the snapshots of AWS may be still in pending status when do the restore, wait for a while - // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 - // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed - if providerName == "aws" && useVolumeSnapshots { - fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...") - time.Sleep(5 * time.Minute) - } - - if err := veleroInstall(context.Background(), veleroCLI, upgradeToVeleroImage, resticHelperImage, plugins, veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots, - cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, crdsVersion, "", registryCredentialFile); err != nil { - return errors.Wrapf(err, "Failed to install velero from image %s", upgradeToVeleroImage) - } - if err := checkVeleroVersion(context.Background(), veleroCLI, upgradeToVeleroVersion); err != nil { - return errors.Wrapf(err, "Velero install version mismatch.") - } - if err := veleroRestore(oneHourTimeout, veleroCLI, veleroNamespace, restoreName, backupName); err != nil { - runDebug(context.Background(), veleroCLI, veleroNamespace, "", restoreName) - return errors.Wrapf(err, "Restore %s failed from backup %s", restoreName, backupName) - } - - if err := kibishiiVerifyAfterRestore(client, upgradeNamespace, oneHourTimeout); err != nil { - return errors.Wrapf(err, "Error verifying kibishii after restore") - } - - fmt.Printf("Upgrade test completed successfully\n") - return nil -} diff --git a/test/e2e/client.go b/test/e2e/util/k8s/client.go similarity index 82% rename from test/e2e/client.go rename to test/e2e/util/k8s/client.go index 0b1993934..839be2b83 100644 --- a/test/e2e/client.go +++ b/test/e2e/util/k8s/client.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e +package k8s import ( "k8s.io/client-go/kubernetes" @@ -23,10 +23,10 @@ import ( "github.com/vmware-tanzu/velero/pkg/client" ) -// testClient contains different API clients that are in use throughout +// TestClient contains different API clients that are in use throughout // the e2e tests. -type testClient struct { +type TestClient struct { kubebuilder kbclient.Client // clientGo returns a client-go API client. @@ -34,7 +34,7 @@ type testClient struct { // Deprecated, TODO(2.0): presuming all controllers and resources are converted to the // controller runtime framework by v2.0, it is the intent to remove all // client-go API clients. Please use the controller runtime to make API calls for tests. - clientGo kubernetes.Interface + ClientGo kubernetes.Interface // dynamicFactory returns a client-go API client for retrieving dynamic clients // for GroupVersionResources and GroupVersionKinds. @@ -45,35 +45,35 @@ type testClient struct { dynamicFactory client.DynamicFactory } -// newTestClient returns a set of ready-to-use API clients. -func newTestClient() (testClient, error) { +// k8sutils.NewTestClient returns a set of ready-to-use API clients. +func NewTestClient() (TestClient, error) { config, err := client.LoadConfig() if err != nil { - return testClient{}, err + return TestClient{}, err } f := client.NewFactory("e2e", config) clientGo, err := f.KubeClient() if err != nil { - return testClient{}, err + return TestClient{}, err } kb, err := f.KubebuilderClient() if err != nil { - return testClient{}, err + return TestClient{}, err } dynamicClient, err := f.DynamicClient() if err != nil { - return testClient{}, err + return TestClient{}, err } factory := client.NewDynamicFactory(dynamicClient) - return testClient{ + return TestClient{ kubebuilder: kb, - clientGo: clientGo, + ClientGo: clientGo, dynamicFactory: factory, }, nil } diff --git a/test/e2e/common.go b/test/e2e/util/k8s/common.go similarity index 85% rename from test/e2e/common.go rename to test/e2e/util/k8s/common.go index 25a2bd3ed..1e4a5eb91 100644 --- a/test/e2e/common.go +++ b/test/e2e/util/k8s/common.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e +package k8s import ( "fmt" @@ -32,11 +32,11 @@ import ( ) // ensureClusterExists returns whether or not a kubernetes cluster exists for tests to be run on. -func ensureClusterExists(ctx context.Context) error { +func EnsureClusterExists(ctx context.Context) error { return exec.CommandContext(ctx, "kubectl", "cluster-info").Run() } -func createSecretFromFiles(ctx context.Context, client testClient, namespace string, name string, files map[string]string) error { +func CreateSecretFromFiles(ctx context.Context, client TestClient, namespace string, name string, files map[string]string) error { data := make(map[string][]byte) for key, filePath := range files { @@ -49,17 +49,17 @@ func createSecretFromFiles(ctx context.Context, client testClient, namespace str } secret := builder.ForSecret(namespace, name).Data(data).Result() - _, err := client.clientGo.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) + _, err := client.ClientGo.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) return err } -// waitForPods waits until all of the pods have gone to PodRunning state -func waitForPods(ctx context.Context, client testClient, namespace string, pods []string) error { +// WaitForPods waits until all of the pods have gone to PodRunning state +func WaitForPods(ctx context.Context, client TestClient, namespace string, pods []string) error { timeout := 10 * time.Minute interval := 5 * time.Second err := wait.PollImmediate(interval, timeout, func() (bool, error) { for _, podName := range pods { - checkPod, err := client.clientGo.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{}) + checkPod, err := client.ClientGo.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{}) if err != nil { return false, errors.WithMessage(err, fmt.Sprintf("Failed to verify pod %s/%s is %s", namespace, podName, corev1api.PodRunning)) } diff --git a/test/e2e/namespace.go b/test/e2e/util/k8s/namespace.go similarity index 59% rename from test/e2e/namespace.go rename to test/e2e/util/k8s/namespace.go index 7ac2efdac..2c33e32d5 100644 --- a/test/e2e/namespace.go +++ b/test/e2e/util/k8s/namespace.go @@ -14,11 +14,12 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e +package k8s import ( "context" "fmt" + "strings" "time" "github.com/pkg/errors" @@ -27,26 +28,27 @@ import ( corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" waitutil "k8s.io/apimachinery/pkg/util/wait" "github.com/vmware-tanzu/velero/pkg/builder" ) -func createNamespace(ctx context.Context, client testClient, namespace string) error { +func CreateNamespace(ctx context.Context, client TestClient, namespace string) error { ns := builder.ForNamespace(namespace).Result() - _, err := client.clientGo.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) + _, err := client.ClientGo.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) if apierrors.IsAlreadyExists(err) { return nil } return err } -func getNamespace(ctx context.Context, client testClient, namespace string) (*corev1api.Namespace, error) { - return client.clientGo.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}) +func GetNamespace(ctx context.Context, client TestClient, namespace string) (*corev1api.Namespace, error) { + return client.ClientGo.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}) } -func deleteNamespace(ctx context.Context, client testClient, namespace string, wait bool) error { - if err := client.clientGo.CoreV1().Namespaces().Delete(ctx, namespace, metav1.DeleteOptions{}); err != nil { +func DeleteNamespace(ctx context.Context, client TestClient, namespace string, wait bool) error { + if err := client.ClientGo.CoreV1().Namespaces().Delete(ctx, namespace, metav1.DeleteOptions{}); err != nil { return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", namespace)) } if !wait { @@ -55,7 +57,7 @@ func deleteNamespace(ctx context.Context, client testClient, namespace string, w return waitutil.PollImmediateInfinite(5*time.Second, func() (bool, error) { - if _, err := client.clientGo.CoreV1().Namespaces().Get(context.TODO(), namespace, metav1.GetOptions{}); err != nil { + if _, err := client.ClientGo.CoreV1().Namespaces().Get(context.TODO(), namespace, metav1.GetOptions{}); err != nil { if apierrors.IsNotFound(err) { return true, nil } @@ -65,3 +67,19 @@ func deleteNamespace(ctx context.Context, client testClient, namespace string, w return false, nil }) } + +func CleanupNamespaces(ctx context.Context, client TestClient, nsBaseName string) error { + namespaces, err := client.ClientGo.CoreV1().Namespaces().List(ctx, v1.ListOptions{}) + if err != nil { + return errors.Wrap(err, "Could not retrieve namespaces") + } + for _, checkNamespace := range namespaces.Items { + if strings.HasPrefix(checkNamespace.Name, nsBaseName) { + err = client.ClientGo.CoreV1().Namespaces().Delete(ctx, checkNamespace.Name, v1.DeleteOptions{}) + if err != nil { + return errors.Wrapf(err, "Could not delete namespace %s", checkNamespace.Name) + } + } + } + return nil +} diff --git a/test/e2e/serviceaccount.go b/test/e2e/util/k8s/serviceaccount.go similarity index 81% rename from test/e2e/serviceaccount.go rename to test/e2e/util/k8s/serviceaccount.go index 30e46a85c..c7274e864 100644 --- a/test/e2e/serviceaccount.go +++ b/test/e2e/util/k8s/serviceaccount.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e +package k8s import ( "context" @@ -33,10 +33,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func waitUntilServiceAccountCreated(ctx context.Context, client testClient, namespace, serviceAccount string, timeout time.Duration) error { +func WaitUntilServiceAccountCreated(ctx context.Context, client TestClient, namespace, serviceAccount string, timeout time.Duration) error { return wait.PollImmediate(5*time.Second, timeout, func() (bool, error) { - if _, err := client.clientGo.CoreV1().ServiceAccounts(namespace).Get(ctx, serviceAccount, metav1.GetOptions{}); err != nil { + if _, err := client.ClientGo.CoreV1().ServiceAccounts(namespace).Get(ctx, serviceAccount, metav1.GetOptions{}); err != nil { if !apierrors.IsNotFound(err) { return false, err } @@ -46,7 +46,7 @@ func waitUntilServiceAccountCreated(ctx context.Context, client testClient, name }) } -func patchServiceAccountWithImagePullSecret(ctx context.Context, client testClient, namespace, serviceAccount, dockerCredentialFile string) error { +func PatchServiceAccountWithImagePullSecret(ctx context.Context, client TestClient, namespace, serviceAccount, dockerCredentialFile string) error { credential, err := ioutil.ReadFile(dockerCredentialFile) if err != nil { return errors.Wrapf(err, "failed to read the docker credential file %q", dockerCredentialFile) @@ -54,11 +54,11 @@ func patchServiceAccountWithImagePullSecret(ctx context.Context, client testClie secretName := "image-pull-secret" secret := builder.ForSecret(namespace, secretName).Data(map[string][]byte{".dockerconfigjson": credential}).Result() secret.Type = corev1.SecretTypeDockerConfigJson - if _, err = client.clientGo.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}); err != nil { + if _, err = client.ClientGo.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}); err != nil { return errors.Wrapf(err, "failed to create secret %q under namespace %q", secretName, namespace) } - if _, err = client.clientGo.CoreV1().ServiceAccounts(namespace).Patch(ctx, serviceAccount, types.StrategicMergePatchType, + if _, err = client.ClientGo.CoreV1().ServiceAccounts(namespace).Patch(ctx, serviceAccount, types.StrategicMergePatchType, []byte(fmt.Sprintf(`{"imagePullSecrets": [{"name": "%s"}]}`, secretName)), metav1.PatchOptions{}); err != nil { return errors.Wrapf(err, "failed to patch the service account %q under the namespace %q", serviceAccount, namespace) } diff --git a/test/e2e/kibishii_utils.go b/test/e2e/util/kibishii/kibishii_utils.go similarity index 76% rename from test/e2e/kibishii_utils.go rename to test/e2e/util/kibishii/kibishii_utils.go index b8efc35c1..3221f6da8 100644 --- a/test/e2e/kibishii_utils.go +++ b/test/e2e/util/kibishii/kibishii_utils.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e +package kibishii import ( "fmt" @@ -26,6 +26,8 @@ import ( "golang.org/x/net/context" veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" + k8sutils "github.com/vmware-tanzu/velero/test/e2e/util/k8s" + veleroutils "github.com/vmware-tanzu/velero/test/e2e/util/velero" ) const ( @@ -33,25 +35,24 @@ const ( jumpPadPod = "jump-pad" ) -// runKibishiiTests runs kibishii tests on the provider. -func runKibishiiTests(client testClient, providerName, veleroCLI, veleroNamespace, backupName, restoreName, backupLocation string, +// RunKibishiiTests runs kibishii tests on the provider. +func RunKibishiiTests(client k8sutils.TestClient, providerName, veleroCLI, veleroNamespace, backupName, restoreName, backupLocation string, useVolumeSnapshots bool, registryCredentialFile string) error { oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60) - - if err := createNamespace(oneHourTimeout, client, kibishiiNamespace); err != nil { + if err := k8sutils.CreateNamespace(oneHourTimeout, client, kibishiiNamespace); err != nil { return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", kibishiiNamespace) } defer func() { - if err := deleteNamespace(context.Background(), client, kibishiiNamespace, true); err != nil { + if err := k8sutils.DeleteNamespace(context.Background(), client, kibishiiNamespace, true); err != nil { fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", kibishiiNamespace)) } }() - if err := kibishiiPrepareBeforeBackup(oneHourTimeout, client, providerName, kibishiiNamespace, registryCredentialFile); err != nil { + if err := KibishiiPrepareBeforeBackup(oneHourTimeout, client, providerName, kibishiiNamespace, registryCredentialFile); err != nil { return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", kibishiiNamespace) } - if err := veleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, kibishiiNamespace, backupLocation, useVolumeSnapshots); err != nil { - runDebug(context.Background(), veleroCLI, veleroNamespace, backupName, "") + if err := veleroutils.VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, kibishiiNamespace, backupLocation, useVolumeSnapshots); err != nil { + veleroutils.RunDebug(context.Background(), veleroCLI, veleroNamespace, backupName, "") return errors.Wrapf(err, "Failed to backup kibishii namespace %s", kibishiiNamespace) } @@ -59,12 +60,12 @@ func runKibishiiTests(client testClient, providerName, veleroCLI, veleroNamespac // Wait for uploads started by the Velero Plug-in for vSphere to complete // TODO - remove after upload progress monitoring is implemented fmt.Println("Waiting for vSphere uploads to complete") - if err := waitForVSphereUploadCompletion(oneHourTimeout, time.Hour, kibishiiNamespace); err != nil { + if err := veleroutils.WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, kibishiiNamespace); err != nil { return errors.Wrapf(err, "Error waiting for uploads to complete") } } fmt.Printf("Simulating a disaster by removing namespace %s\n", kibishiiNamespace) - if err := deleteNamespace(oneHourTimeout, client, kibishiiNamespace, true); err != nil { + if err := k8sutils.DeleteNamespace(oneHourTimeout, client, kibishiiNamespace, true); err != nil { return errors.Wrapf(err, "failed to delete namespace %s", kibishiiNamespace) } @@ -76,12 +77,12 @@ func runKibishiiTests(client testClient, providerName, veleroCLI, veleroNamespac time.Sleep(5 * time.Minute) } - if err := veleroRestore(oneHourTimeout, veleroCLI, veleroNamespace, restoreName, backupName); err != nil { - runDebug(context.Background(), veleroCLI, veleroNamespace, "", restoreName) + if err := veleroutils.VeleroRestore(oneHourTimeout, veleroCLI, veleroNamespace, restoreName, backupName); err != nil { + veleroutils.RunDebug(context.Background(), veleroCLI, veleroNamespace, "", restoreName) return errors.Wrapf(err, "Restore %s failed from backup %s", restoreName, backupName) } - if err := kibishiiVerifyAfterRestore(client, kibishiiNamespace, oneHourTimeout); err != nil { + if err := KibishiiVerifyAfterRestore(client, kibishiiNamespace, oneHourTimeout); err != nil { return errors.Wrapf(err, "Error verifying kibishii after restore") } @@ -144,19 +145,19 @@ func verifyData(ctx context.Context, namespace string, levels int, filesPerLevel return nil } -func waitForKibishiiPods(ctx context.Context, client testClient, kibishiiNamespace string) error { - return waitForPods(ctx, client, kibishiiNamespace, []string{"jump-pad", "etcd0", "etcd1", "etcd2", "kibishii-deployment-0", "kibishii-deployment-1"}) +func waitForKibishiiPods(ctx context.Context, client k8sutils.TestClient, kibishiiNamespace string) error { + return k8sutils.WaitForPods(ctx, client, kibishiiNamespace, []string{"jump-pad", "etcd0", "etcd1", "etcd2", "kibishii-deployment-0", "kibishii-deployment-1"}) } -func kibishiiPrepareBeforeBackup(oneHourTimeout context.Context, client testClient, providerName, kibishiiNamespace, registryCredentialFile string) error { +func KibishiiPrepareBeforeBackup(oneHourTimeout context.Context, client k8sutils.TestClient, providerName, kibishiiNamespace, registryCredentialFile string) error { serviceAccountName := "default" // wait until the service account is created before patch the image pull secret - if err := waitUntilServiceAccountCreated(oneHourTimeout, client, kibishiiNamespace, serviceAccountName, 10*time.Minute); err != nil { + if err := k8sutils.WaitUntilServiceAccountCreated(oneHourTimeout, client, kibishiiNamespace, serviceAccountName, 10*time.Minute); err != nil { return errors.Wrapf(err, "failed to wait the service account %q created under the namespace %q", serviceAccountName, kibishiiNamespace) } // add the image pull secret to avoid the image pull limit issue of Docker Hub - if err := patchServiceAccountWithImagePullSecret(oneHourTimeout, client, kibishiiNamespace, serviceAccountName, registryCredentialFile); err != nil { + if err := k8sutils.PatchServiceAccountWithImagePullSecret(oneHourTimeout, client, kibishiiNamespace, serviceAccountName, registryCredentialFile); err != nil { return errors.Wrapf(err, "failed to patch the service account %q under the namespace %q", serviceAccountName, kibishiiNamespace) } @@ -177,7 +178,7 @@ func kibishiiPrepareBeforeBackup(oneHourTimeout context.Context, client testClie return nil } -func kibishiiVerifyAfterRestore(client testClient, kibishiiNamespace string, oneHourTimeout context.Context) error { +func KibishiiVerifyAfterRestore(client k8sutils.TestClient, kibishiiNamespace string, oneHourTimeout context.Context) error { // wait for kibishii pod startup // TODO - Fix kibishii so we can check that it is ready to go fmt.Printf("Waiting for kibishii pods to be ready\n") diff --git a/test/e2e/install.go b/test/e2e/util/velero/install.go similarity index 86% rename from test/e2e/install.go rename to test/e2e/util/velero/install.go index 80c344c44..38c58f18a 100644 --- a/test/e2e/install.go +++ b/test/e2e/util/velero/install.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e +package velero import ( "bytes" @@ -35,6 +35,8 @@ import ( "github.com/vmware-tanzu/velero/pkg/cmd/cli/install" velerexec "github.com/vmware-tanzu/velero/pkg/util/exec" + . "github.com/vmware-tanzu/velero/test/e2e" + k8sutils "github.com/vmware-tanzu/velero/test/e2e/util/k8s" ) // we provide more install options other than the standard install.InstallOptions in E2E test @@ -44,53 +46,49 @@ type installOptions struct { ResticHelperImage string } -// TODO too many parameters for this function, better to make it a structure, we can introduces a structure `config` for the E2E to hold all configuration items -func veleroInstall(ctx context.Context, cli, veleroImage, resticHelperImage, providerPlugins, veleroNamespace, cloudProvider, objectStoreProvider string, useVolumeSnapshots bool, - cloudCredentialsFile string, bslBucket string, bslPrefix string, bslConfig string, vslConfig string, - crdsVersion string, features string, registryCredentialFile string) error { - - if cloudProvider != "kind" { - if objectStoreProvider != "" { +func VeleroInstall(ctx context.Context, veleroCfg *VerleroConfig, features string, useVolumeSnapshots bool) error { + if veleroCfg.CloudProvider != "kind" { + if veleroCfg.ObjectStoreProvider != "" { return errors.New("For cloud platforms, object store plugin cannot be overridden") // Can't set an object store provider that is different than your cloud } - objectStoreProvider = cloudProvider + veleroCfg.ObjectStoreProvider = veleroCfg.CloudProvider } else { - if objectStoreProvider == "" { + if veleroCfg.ObjectStoreProvider == "" { return errors.New("No object store provider specified - must be specified when using kind as the cloud provider") // Gotta have an object store provider } } - providerPluginsTmp, err := getProviderPlugins(ctx, cli, objectStoreProvider, providerPlugins) + providerPluginsTmp, err := getProviderPlugins(ctx, veleroCfg.VeleroCLI, veleroCfg.ObjectStoreProvider, veleroCfg.Plugins) if err != nil { return errors.WithMessage(err, "Failed to get provider plugins") } // TODO - handle this better - if cloudProvider == "vsphere" { - // We overrider the objectStoreProvider here for vSphere because we want to use the aws plugin for the + if veleroCfg.CloudProvider == "vsphere" { + // We overrider the ObjectStoreProvider here for vSphere because we want to use the aws plugin for the // backup, but needed to pick up the provider plugins earlier. vSphere plugin no longer needs a Volume // Snapshot location specified - objectStoreProvider = "aws" + veleroCfg.ObjectStoreProvider = "aws" } - err = ensureClusterExists(ctx) + err = k8sutils.EnsureClusterExists(ctx) if err != nil { return errors.WithMessage(err, "Failed to ensure Kubernetes cluster exists") } - veleroInstallOptions, err := getProviderVeleroInstallOptions(objectStoreProvider, cloudCredentialsFile, bslBucket, - bslPrefix, bslConfig, vslConfig, providerPluginsTmp, features) + veleroInstallOptions, err := getProviderVeleroInstallOptions(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, + veleroCfg.BSLPrefix, veleroCfg.BSLConfig, veleroCfg.VSLConfig, providerPluginsTmp, features) if err != nil { - return errors.WithMessagef(err, "Failed to get Velero InstallOptions for plugin provider %s", objectStoreProvider) + return errors.WithMessagef(err, "Failed to get Velero InstallOptions for plugin provider %s", veleroCfg.ObjectStoreProvider) } veleroInstallOptions.UseVolumeSnapshots = useVolumeSnapshots veleroInstallOptions.UseRestic = !useVolumeSnapshots - veleroInstallOptions.Image = veleroImage - veleroInstallOptions.CRDsVersion = crdsVersion - veleroInstallOptions.Namespace = veleroNamespace + veleroInstallOptions.Image = veleroCfg.VeleroImage + veleroInstallOptions.CRDsVersion = veleroCfg.CRDsVersion + veleroInstallOptions.Namespace = veleroCfg.VeleroNamespace - err = installVeleroServer(ctx, cli, &installOptions{ + err = installVeleroServer(ctx, veleroCfg.VeleroCLI, &installOptions{ InstallOptions: veleroInstallOptions, - RegistryCredentialFile: registryCredentialFile, - ResticHelperImage: resticHelperImage, + RegistryCredentialFile: veleroCfg.RegistryCredentialFile, + ResticHelperImage: veleroCfg.ResticHelperImage, }) if err != nil { return errors.WithMessagef(err, "Failed to install Velero in the cluster") @@ -195,7 +193,7 @@ func createVelereResources(ctx context.Context, cli, namespace string, args []st return errors.Wrapf(err, "failed to unmarshal the resources: %s", string(stdout)) } - if err = patchResources(ctx, resources, namespace, registryCredentialFile, resticHelperImage); err != nil { + if err = patchResources(ctx, resources, namespace, registryCredentialFile, VeleroCfg.ResticHelperImage); err != nil { return errors.Wrapf(err, "failed to patch resources") } @@ -261,7 +259,7 @@ func patchResources(ctx context.Context, resources *unstructured.UnstructuredLis } // customize the restic restore helper image - if len(resticHelperImage) > 0 { + if len(VeleroCfg.ResticHelperImage) > 0 { restoreActionConfig := corev1.ConfigMap{ TypeMeta: metav1.TypeMeta{ Kind: "ConfigMap", @@ -276,7 +274,7 @@ func patchResources(ctx context.Context, resources *unstructured.UnstructuredLis }, }, Data: map[string]string{ - "image": resticHelperImage, + "image": VeleroCfg.ResticHelperImage, }, } @@ -337,7 +335,7 @@ func waitVeleroReady(ctx context.Context, namespace string, useRestic bool) erro return nil } -func veleroUninstall(ctx context.Context, cli, namespace string) error { +func VeleroUninstall(ctx context.Context, cli, namespace string) error { stdout, stderr, err := velerexec.RunCommand(exec.CommandContext(ctx, cli, "uninstall", "--force", "-n", namespace)) if err != nil { return errors.Wrapf(err, "failed to uninstall velero, stdout=%s, stderr=%s", stdout, stderr) diff --git a/test/e2e/velero_utils.go b/test/e2e/util/velero/velero_utils.go similarity index 82% rename from test/e2e/velero_utils.go rename to test/e2e/util/velero/velero_utils.go index 09ef1407f..d32c9f577 100644 --- a/test/e2e/velero_utils.go +++ b/test/e2e/util/velero/velero_utils.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e +package velero import ( "bytes" @@ -98,7 +98,7 @@ func getProviderVeleroInstallOptions( pluginProvider, credentialsFile, objectStoreBucket, - objectStorePrefix string, + objectStorePrefix, bslConfig, vslConfig string, plugins []string, @@ -134,7 +134,7 @@ func getProviderVeleroInstallOptions( return io, nil } -// checkBackupPhase uses veleroCLI to inspect the phase of a Velero backup. +// checkBackupPhase uses VeleroCLI to inspect the phase of a Velero backup. func checkBackupPhase(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, expectedPhase velerov1api.BackupPhase) error { checkCMD := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "backup", "get", "-o", "json", @@ -178,7 +178,7 @@ func checkBackupPhase(ctx context.Context, veleroCLI string, veleroNamespace str return nil } -// checkRestorePhase uses veleroCLI to inspect the phase of a Velero restore. +// checkRestorePhase uses VeleroCLI to inspect the phase of a Velero restore. func checkRestorePhase(ctx context.Context, veleroCLI string, veleroNamespace string, restoreName string, expectedPhase velerov1api.RestorePhase) error { checkCMD := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "restore", "get", "-o", "json", @@ -222,8 +222,8 @@ func checkRestorePhase(ctx context.Context, veleroCLI string, veleroNamespace st return nil } -// veleroBackupNamespace uses the veleroCLI to backup a namespace. -func veleroBackupNamespace(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, namespace string, backupLocation string, +// VeleroBackupNamespace uses the veleroCLI to backup a namespace. +func VeleroBackupNamespace(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, namespace string, backupLocation string, useVolumeSnapshots bool) error { args := []string{ "--namespace", veleroNamespace, @@ -246,71 +246,80 @@ func veleroBackupNamespace(ctx context.Context, veleroCLI string, veleroNamespac args = append(args, "--storage-location", backupLocation) } - backupCmd := exec.CommandContext(ctx, veleroCLI, args...) - backupCmd.Stdout = os.Stdout - backupCmd.Stderr = os.Stderr - fmt.Printf("backup cmd =%v\n", backupCmd) - err := backupCmd.Run() - if err != nil { - return err - } - err = checkBackupPhase(ctx, veleroCLI, veleroNamespace, backupName, velerov1api.BackupPhaseCompleted) - - return err + return VeleroBackupExec(ctx, veleroCLI, veleroNamespace, backupName, args) } -// veleroBackupExcludeNamespaces uses the veleroCLI to backup a namespace. -func veleroBackupExcludeNamespaces(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, excludeNamespaces []string) error { +// VeleroBackupExcludeNamespaces uses the veleroCLI to backup a namespace. +func VeleroBackupExcludeNamespaces(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, excludeNamespaces []string) error { namespaces := strings.Join(excludeNamespaces, ",") - backupCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "create", "backup", backupName, + args := []string{ + "--namespace", veleroNamespace, "create", "backup", backupName, "--exclude-namespaces", namespaces, - "--default-volumes-to-restic", "--wait") - backupCmd.Stdout = os.Stdout - backupCmd.Stderr = os.Stderr - fmt.Printf("backup cmd =%v\n", backupCmd) - err := backupCmd.Run() - if err != nil { - return err + "--default-volumes-to-restic", "--wait", } - err = checkBackupPhase(ctx, veleroCLI, veleroNamespace, backupName, velerov1api.BackupPhaseCompleted) - - return err + return VeleroBackupExec(ctx, veleroCLI, veleroNamespace, backupName, args) } -// veleroRestore uses the veleroCLI to restore from a Velero backup. -func veleroRestore(ctx context.Context, veleroCLI string, veleroNamespace string, restoreName string, backupName string) error { - restoreCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "create", "restore", restoreName, - "--from-backup", backupName, "--wait") +// VeleroBackupIncludeNamespaces uses the veleroCLI to backup a namespace. +func VeleroBackupIncludeNamespaces(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, includeNamespaces []string) error { + namespaces := strings.Join(includeNamespaces, ",") + args := []string{ + "--namespace", veleroNamespace, "create", "backup", backupName, + "--include-namespaces", namespaces, + "--default-volumes-to-restic", "--wait", + } + return VeleroBackupExec(ctx, veleroCLI, veleroNamespace, backupName, args) +} - restoreCmd.Stdout = os.Stdout - restoreCmd.Stderr = os.Stderr - fmt.Printf("restore cmd =%v\n", restoreCmd) - err := restoreCmd.Run() - if err != nil { +// VeleroRestore uses the VeleroCLI to restore from a Velero backup. +func VeleroRestore(ctx context.Context, veleroCLI string, veleroNamespace string, restoreName string, backupName string) error { + args := []string{ + "--namespace", veleroNamespace, "create", "restore", restoreName, + "--from-backup", backupName, "--wait", + } + return VeleroRestoreExec(ctx, veleroCLI, veleroNamespace, restoreName, args) +} + +func VeleroRestoreExec(ctx context.Context, veleroCLI, veleroNamespace, restoreName string, args []string) error { + if err := VeleroCmdExec(ctx, veleroCLI, args); err != nil { return err } return checkRestorePhase(ctx, veleroCLI, veleroNamespace, restoreName, velerov1api.RestorePhaseCompleted) } -func veleroBackupLogs(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string) error { - describeCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "backup", "describe", backupName) - describeCmd.Stdout = os.Stdout - describeCmd.Stderr = os.Stderr - err := describeCmd.Run() - if err != nil { +func VeleroBackupExec(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, args []string) error { + if err := VeleroCmdExec(ctx, veleroCLI, args); err != nil { return err } - logCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "backup", "logs", backupName) - logCmd.Stdout = os.Stdout - logCmd.Stderr = os.Stderr - err = logCmd.Run() - if err != nil { - return err - } - return nil + return checkBackupPhase(ctx, veleroCLI, veleroNamespace, backupName, velerov1api.BackupPhaseCompleted) } -func runDebug(ctx context.Context, veleroCLI, veleroNamespace, backup, restore string) { +func VeleroCmdExec(ctx context.Context, veleroCLI string, args []string) error { + cmd := exec.CommandContext(ctx, veleroCLI, args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + fmt.Printf("velero cmd =%v\n", cmd) + err := cmd.Run() + if err != nil { + return err + } + return err +} + +func VeleroBackupLogs(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string) error { + args := []string{ + "--namespace", veleroNamespace, "backup", "describe", backupName, + } + if err := VeleroCmdExec(ctx, veleroCLI, args); err != nil { + return err + } + args = []string{ + "--namespace", veleroNamespace, "backup", "logs", backupName, + } + return VeleroCmdExec(ctx, veleroCLI, args) +} + +func RunDebug(ctx context.Context, veleroCLI, veleroNamespace, backup, restore string) { output := fmt.Sprintf("debug-bundle-%d.tar.gz", time.Now().UnixNano()) args := []string{"debug", "--namespace", veleroNamespace, "--output", output, "--verbose"} if len(backup) > 0 { @@ -319,26 +328,21 @@ func runDebug(ctx context.Context, veleroCLI, veleroNamespace, backup, restore s if len(restore) > 0 { args = append(args, "--restore", restore) } - - cmd := exec.CommandContext(ctx, veleroCLI, args...) - cmd.Stdout = os.Stdout - cmd.Stdin = os.Stdin - fmt.Printf("debug cmd=%s\n", cmd.String()) fmt.Printf("Generating the debug tarball at %s\n", output) - if err := cmd.Run(); err != nil { + if err := VeleroCmdExec(ctx, veleroCLI, args); err != nil { fmt.Println(errors.Wrapf(err, "failed to run the debug command")) } } -func veleroCreateBackupLocation(ctx context.Context, - veleroCLI string, - veleroNamespace string, - name string, - objectStoreProvider string, - bucket string, - prefix string, - config string, - secretName string, +func VeleroCreateBackupLocation(ctx context.Context, + veleroCLI, + veleroNamespace, + name, + objectStoreProvider, + bucket, + prefix, + config, + secretName, secretKey string, ) error { args := []string{ @@ -359,12 +363,7 @@ func veleroCreateBackupLocation(ctx context.Context, if secretName != "" && secretKey != "" { args = append(args, "--credential", fmt.Sprintf("%s=%s", secretName, secretKey)) } - - bslCreateCmd := exec.CommandContext(ctx, veleroCLI, args...) - bslCreateCmd.Stdout = os.Stdout - bslCreateCmd.Stderr = os.Stderr - - return bslCreateCmd.Run() + return VeleroCmdExec(ctx, veleroCLI, args) } func getProviderPlugins(ctx context.Context, veleroCLI, objectStoreProvider, providerPlugins string) ([]string, error) { @@ -385,9 +384,9 @@ func getProviderPlugins(ctx context.Context, veleroCLI, objectStoreProvider, pro return plugins, nil } -// veleroAddPluginsForProvider determines which plugins need to be installed for a provider and +// VeleroAddPluginsForProvider determines which plugins need to be installed for a provider and // installs them in the current Velero installation, skipping over those that are already installed. -func veleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNamespace string, provider string, addPlugins string) error { +func VeleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNamespace string, provider string, addPlugins string) error { plugins, err := getProviderPlugins(ctx, veleroCLI, provider, addPlugins) if err != nil { return errors.WithMessage(err, "Failed to get plugins") @@ -417,9 +416,9 @@ func veleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNa return nil } -// waitForVSphereUploadCompletion waits for uploads started by the Velero Plug-in for vSphere to complete +// WaitForVSphereUploadCompletion waits for uploads started by the Velero Plug-in for vSphere to complete // TODO - remove after upload progress monitoring is implemented -func waitForVSphereUploadCompletion(ctx context.Context, timeout time.Duration, namespace string) error { +func WaitForVSphereUploadCompletion(ctx context.Context, timeout time.Duration, namespace string) error { err := wait.PollImmediate(time.Minute, timeout, func() (bool, error) { checkSnapshotCmd := exec.CommandContext(ctx, "kubectl", "get", "-n", namespace, "snapshots.backupdriver.cnsdp.vmware.com", "-o=jsonpath='{range .items[*]}{.spec.resourceHandle.name}{\"=\"}{.status.phase}{\"\\n\"}{end}'") @@ -447,6 +446,7 @@ func waitForVSphereUploadCompletion(ctx context.Context, timeout time.Duration, // status will move to Canceling. The snapshot ID will be removed from the status status if has been filled in // and the snapshot ID will not longer be valid for a Clone operation // Canceled - the operation was canceled, the snapshot ID is not valid + // Canceled - the operation was canceled, the snapshot ID is not valid if len(comps) == 2 { phase := comps[1] switch phase { @@ -497,7 +497,7 @@ func getVeleroVersion(ctx context.Context, veleroCLI string, clientOnly bool) (s return versionMatches[1], nil } -func checkVeleroVersion(ctx context.Context, veleroCLI string, expectedVer string) error { +func CheckVeleroVersion(ctx context.Context, veleroCLI string, expectedVer string) error { tag := expectedVer tagInstalled, err := getVeleroVersion(ctx, veleroCLI, false) if err != nil { @@ -510,7 +510,7 @@ func checkVeleroVersion(ctx context.Context, veleroCLI string, expectedVer strin return nil } -func installVeleroCLI(version string) (string, error) { +func InstallVeleroCLI(version string) (string, error) { name := "velero-" + version + "-" + runtime.GOOS + "-" + runtime.GOARCH postfix := ".tar.gz" tarball := name + postfix