vmware-tanzu
/
velero
mirror of https://github.com/vmware-tanzu/velero.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #4386 from redenval/e2e-restructure 

Adjust structure of e2e test codes
pull/4392/head
Wenkai Yin(尹文开) 2021-11-22 16:23:34 +08:00 committed by GitHub
parent 474fd61283 58325050ec
commit 04cfadfb14
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 571 additions and 501 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
changelogs/unreleased/4386-mqiu Normal file
View File

@ -0,0 +1 @@
Adjust structure of e2e test codes

86
test/e2e/backup_test.go → test/e2e/backup/backup.go
View File

@ -13,7 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e
package backup
import (
"context"
@ -23,98 +23,93 @@ import (
"github.com/google/uuid"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
. "github.com/vmware-tanzu/velero/test/e2e"
k8sutils "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
kibishiiutils "github.com/vmware-tanzu/velero/test/e2e/util/kibishii"
veleroutils "github.com/vmware-tanzu/velero/test/e2e/util/velero"
)
var (
uuidgen uuid.UUID
)
// Test backup and restore of Kibishi using restic
var _ = Describe("[Restic] Velero tests on cluster using the plugin provider for object storage and Restic for volume backups", backup_restore_with_restic)
var _ = Describe("[Snapshot] Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", backup_restore_with_snapshots)
func backup_restore_with_snapshots() {
backup_restore_test(true)
func BackupRestoreWithSnapshots() {
BackupRestoreTest(true)
}
func backup_restore_with_restic() {
backup_restore_test(false)
func BackupRestoreWithRestic() {
BackupRestoreTest(false)
}
func backup_restore_test(useVolumeSnapshots bool) {
func BackupRestoreTest(useVolumeSnapshots bool) {
var (
backupName, restoreName string
)
client, err := newTestClient()
client, err := k8sutils.NewTestClient()
Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests")
BeforeEach(func() {
if useVolumeSnapshots && cloudProvider == "kind" {
if useVolumeSnapshots && VeleroCfg.CloudProvider == "kind" {
Skip("Volume snapshots not supported on kind")
}
var err error
flag.Parse()
uuidgen, err = uuid.NewRandom()
UUIDgen, err = uuid.NewRandom()
Expect(err).To(Succeed())
if installVelero {
Expect(veleroInstall(context.Background(), veleroCLI, veleroImage, resticHelperImage, plugins, veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots,
cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, crdsVersion, "", registryCredentialFile)).To(Succeed())
if VeleroCfg.InstallVelero {
Expect(veleroutils.VeleroInstall(context.Background(), &VeleroCfg, "", useVolumeSnapshots)).To(Succeed())
}
})
AfterEach(func() {
if installVelero {
err = veleroUninstall(context.Background(), veleroCLI, veleroNamespace)
if VeleroCfg.InstallVelero {
err = veleroutils.VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)
Expect(err).To(Succeed())
}
})
When("kibishii is the sample workload", func() {
It("should be successfully backed up and restored to the default BackupStorageLocation", func() {
backupName = "backup-" + uuidgen.String()
restoreName = "restore-" + uuidgen.String()
backupName = "backup-" + UUIDgen.String()
restoreName = "restore-" + UUIDgen.String()
// Even though we are using Velero's CloudProvider plugin for object storage, the kubernetes cluster is running on
// KinD. So use the kind installation for Kibishii.
Expect(runKibishiiTests(client, cloudProvider, veleroCLI, veleroNamespace, backupName, restoreName, "", useVolumeSnapshots, registryCredentialFile)).To(Succeed(),
Expect(kibishiiutils.RunKibishiiTests(client, VeleroCfg.CloudProvider, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backupName, restoreName, "", useVolumeSnapshots, VeleroCfg.RegistryCredentialFile)).To(Succeed(),
"Failed to successfully backup and restore Kibishii namespace")
})
It("should successfully back up and restore to an additional BackupStorageLocation with unique credentials", func() {
if additionalBSLProvider == "" {
if VeleroCfg.AdditionalBSLProvider == "" {
Skip("no additional BSL provider given, not running multiple BackupStorageLocation with unique credentials tests")
}
if additionalBSLBucket == "" {
if VeleroCfg.AdditionalBSLBucket == "" {
Skip("no additional BSL bucket given, not running multiple BackupStorageLocation with unique credentials tests")
}
if additionalBSLCredentials == "" {
if VeleroCfg.AdditionalBSLCredentials == "" {
Skip("no additional BSL credentials given, not running multiple BackupStorageLocation with unique credentials tests")
}
Expect(veleroAddPluginsForProvider(context.TODO(), veleroCLI, veleroNamespace, additionalBSLProvider, addBSLPlugins)).To(Succeed())
Expect(veleroutils.VeleroAddPluginsForProvider(context.TODO(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, VeleroCfg.AdditionalBSLProvider, VeleroCfg.AddBSLPlugins)).To(Succeed())
// Create Secret for additional BSL
secretName := fmt.Sprintf("bsl-credentials-%s", uuidgen)
secretKey := fmt.Sprintf("creds-%s", additionalBSLProvider)
secretName := fmt.Sprintf("bsl-credentials-%s", UUIDgen)
secretKey := fmt.Sprintf("creds-%s", VeleroCfg.AdditionalBSLProvider)
files := map[string]string{
secretKey: additionalBSLCredentials,
secretKey: VeleroCfg.AdditionalBSLCredentials,
}
Expect(createSecretFromFiles(context.TODO(), client, veleroNamespace, secretName, files)).To(Succeed())
Expect(k8sutils.CreateSecretFromFiles(context.TODO(), client, VeleroCfg.VeleroNamespace, secretName, files)).To(Succeed())
// Create additional BSL using credential
additionalBsl := fmt.Sprintf("bsl-%s", uuidgen)
Expect(veleroCreateBackupLocation(context.TODO(),
veleroCLI,
veleroNamespace,
additionalBsl := fmt.Sprintf("bsl-%s", UUIDgen)
Expect(veleroutils.VeleroCreateBackupLocation(context.TODO(),
VeleroCfg.VeleroCLI,
VeleroCfg.VeleroNamespace,
additionalBsl,
additionalBSLProvider,
additionalBSLBucket,
additionalBSLPrefix,
additionalBSLConfig,
VeleroCfg.AdditionalBSLProvider,
VeleroCfg.AdditionalBSLBucket,
VeleroCfg.AdditionalBSLPrefix,
VeleroCfg.AdditionalBSLConfig,
secretName,
secretKey,
)).To(Succeed())
@ -127,11 +122,10 @@ func backup_restore_test(useVolumeSnapshots bool) {
// We limit the length of backup name here to avoid the issue of vsphere plugin https://github.com/vmware-tanzu/velero-plugin-for-vsphere/issues/370
// We can remove the logic once the issue is fixed
if bsl == "default" {
backupName = fmt.Sprintf("%s-%s", backupName, uuidgen)
restoreName = fmt.Sprintf("%s-%s", restoreName, uuidgen)
backupName = fmt.Sprintf("%s-%s", backupName, UUIDgen)
restoreName = fmt.Sprintf("%s-%s", restoreName, UUIDgen)
}
Expect(runKibishiiTests(client, cloudProvider, veleroCLI, veleroNamespace, backupName, restoreName, bsl, useVolumeSnapshots, registryCredentialFile)).To(Succeed(),
Expect(kibishiiutils.RunKibishiiTests(client, VeleroCfg.CloudProvider, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backupName, restoreName, bsl, useVolumeSnapshots, VeleroCfg.RegistryCredentialFile)).To(Succeed(),
"Failed to successfully backup and restore Kibishii namespace using BSL %s", bsl)
}
})

72
test/e2e/enable_api_group_versions_test.go → test/e2e/basic/enable_api_group_versions.go
View File

@ -14,11 +14,12 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e
package basic
import (
"context"
"encoding/json"
"flag"
"fmt"
"os/exec"
"strconv"
@ -34,46 +35,31 @@ import (
"github.com/vmware-tanzu/velero/pkg/builder"
veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec"
. "github.com/vmware-tanzu/velero/test/e2e"
k8sutils "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
veleroutils "github.com/vmware-tanzu/velero/test/e2e/util/velero"
)
var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", func() {
func APIGropuVersionsTest() {
var (
resource, group string
err error
ctx = context.Background()
)
client, err := newTestClient()
client, err := k8sutils.NewTestClient()
Expect(err).To(Succeed(), "Failed to instantiate cluster client for group version tests")
BeforeEach(func() {
resource = "rockbands"
group = "music.example.io"
uuidgen, err = uuid.NewRandom()
UUIDgen, err = uuid.NewRandom()
Expect(err).NotTo(HaveOccurred())
flag.Parse()
// TODO: install Velero once for the test suite once feature flag is
// removed and velero installation becomes the same as other e2e tests.
if installVelero {
err = veleroInstall(
context.Background(),
veleroCLI,
veleroImage,
resticHelperImage,
plugins,
veleroNamespace,
cloudProvider,
objectStoreProvider,
false,
cloudCredentialsFile,
bslBucket,
bslPrefix,
bslConfig,
vslConfig,
crdsVersion,
"EnableAPIGroupVersions", // TODO: remove when feature flag is removed
registryCredentialFile)
if VeleroCfg.InstallVelero {
err = veleroutils.VeleroInstall(context.Background(), &VeleroCfg, "EnableAPIGroupVersions", false)
Expect(err).NotTo(HaveOccurred())
}
})
@ -88,8 +74,8 @@ var _ = Describe("[APIGroup] Velero tests with various CRD API group versions",
}
Expect(err).NotTo(HaveOccurred())
if installVelero {
err = veleroUninstall(ctx, veleroCLI, veleroNamespace)
if VeleroCfg.InstallVelero {
err = veleroutils.VeleroUninstall(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)
Expect(err).NotTo(HaveOccurred())
}
@ -105,9 +91,9 @@ var _ = Describe("[APIGroup] Velero tests with various CRD API group versions",
)).To(Succeed(), "Failed to successfully backup and restore multiple API Groups")
})
})
})
}
func runEnableAPIGroupVersionsTests(ctx context.Context, client testClient, resource, group string) error {
func runEnableAPIGroupVersionsTests(ctx context.Context, client k8sutils.TestClient, resource, group string) error {
tests := []struct {
name string
namespaces []string
@ -180,7 +166,7 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client testClient, reso
},
tgtCrdYaml: "testdata/enable_api_group_versions/case-d-target-manually-added-mutations.yaml",
tgtVer: "v2beta1",
cm: builder.ForConfigMap(veleroNamespace, "enableapigroupversions").Data(
cm: builder.ForConfigMap(VeleroCfg.VeleroNamespace, "enableapigroupversions").Data(
"restoreResourcesVersionPriority",
`rockbands.music.example.io=v2beta1,v2beta2,v2`,
).Result(),
@ -224,11 +210,11 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client testClient, reso
for version, cr := range tc.srcCRs {
ns := resource + "-src-" + version
if err := createNamespace(ctx, client, ns); err != nil {
if err := k8sutils.CreateNamespace(ctx, client, ns); err != nil {
return errors.Wrapf(err, "create %s namespace", ns)
}
defer func(namespace string) {
if err = deleteNamespace(ctx, client, namespace, true); err != nil {
if err = k8sutils.DeleteNamespace(ctx, client, namespace, true); err != nil {
fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", ns))
}
}(ns)
@ -243,16 +229,16 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client testClient, reso
// Restart Velero pods in order to recognize music-system CRD right away
// instead of waiting for discovery helper to refresh. See
// https://github.com/vmware-tanzu/velero/issues/3471.
if err := restartPods(ctx, veleroNamespace); err != nil {
if err := restartPods(ctx, VeleroCfg.VeleroNamespace); err != nil {
return errors.Wrapf(err, "restart Velero pods")
}
backup := "backup-rockbands-" + uuidgen.String() + "-" + strconv.Itoa(i)
backup := "backup-rockbands-" + UUIDgen.String() + "-" + strconv.Itoa(i)
namespacesStr := strings.Join(tc.namespaces, ",")
err = veleroBackupNamespace(ctx, veleroCLI, veleroNamespace, backup, namespacesStr, "", false)
err = veleroutils.VeleroBackupNamespace(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backup, namespacesStr, "", false)
if err != nil {
runDebug(context.Background(), veleroCLI, veleroNamespace, backup, "")
veleroutils.RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backup, "")
return errors.Wrapf(err, "back up %s namespaces on source cluster", namespacesStr)
}
@ -261,7 +247,7 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client testClient, reso
}
for _, ns := range tc.namespaces {
if err := deleteNamespace(ctx, client, ns, true); err != nil {
if err := k8sutils.DeleteNamespace(ctx, client, ns, true); err != nil {
return errors.Wrapf(err, "delete %s namespace from source cluster", ns)
}
}
@ -275,23 +261,23 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client testClient, reso
// Apply config map if there is one.
if tc.cm != nil {
_, err := client.clientGo.CoreV1().ConfigMaps(veleroNamespace).Create(ctx, tc.cm, metav1.CreateOptions{})
_, err := client.ClientGo.CoreV1().ConfigMaps(VeleroCfg.VeleroNamespace).Create(ctx, tc.cm, metav1.CreateOptions{})
if err != nil {
return errors.Wrap(err, "create config map with user version priorities")
}
}
// Reset Velero to recognize music-system CRD.
if err := restartPods(ctx, veleroNamespace); err != nil {
if err := restartPods(ctx, VeleroCfg.VeleroNamespace); err != nil {
return errors.Wrapf(err, "restart Velero pods")
}
// Restore rockbands namespaces.
restore := "restore-rockbands-" + uuidgen.String() + "-" + strconv.Itoa(i)
restore := "restore-rockbands-" + UUIDgen.String() + "-" + strconv.Itoa(i)
if tc.want != nil {
if err := veleroRestore(ctx, veleroCLI, veleroNamespace, restore, backup); err != nil {
runDebug(context.Background(), veleroCLI, veleroNamespace, "", restore)
if err := veleroutils.VeleroRestore(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, restore, backup); err != nil {
veleroutils.RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, "", restore)
return errors.Wrapf(err, "restore %s namespaces on target cluster", namespacesStr)
}
@ -329,7 +315,7 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client testClient, reso
} else {
// No custom resource should have been restored. Expect "no resource found"
// error during restore.
err := veleroRestore(ctx, veleroCLI, veleroNamespace, restore, backup)
err := veleroutils.VeleroRestore(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, restore, backup)
if err.Error() != "Unexpected restore phase got PartiallyFailed, expecting Completed" {
return errors.New("expected error but not none")

77
test/e2e/e2e_suite_test.go
View File

@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e
package e2e_test
import (
"flag"
@ -23,44 +23,59 @@ import (
. "github.com/onsi/ginkgo"
"github.com/onsi/ginkgo/reporters"
. "github.com/onsi/gomega"
)
var (
veleroCLI, veleroImage, veleroVersion, cloudCredentialsFile, bslConfig, bslBucket, bslPrefix, vslConfig, cloudProvider, objectStoreProvider, veleroNamespace, crdsVersion string
additionalBSLProvider, additionalBSLBucket, additionalBSLPrefix, additionalBSLConfig, additionalBSLCredentials, registryCredentialFile, resticHelperImage string
upgradeFromVeleroVersion, upgradeFromVeleroCLI, plugins, addBSLPlugins string
installVelero bool
. "github.com/vmware-tanzu/velero/test/e2e"
. "github.com/vmware-tanzu/velero/test/e2e/backup"
. "github.com/vmware-tanzu/velero/test/e2e/basic"
. "github.com/vmware-tanzu/velero/test/e2e/scale"
. "github.com/vmware-tanzu/velero/test/e2e/upgrade"
)
func init() {
flag.StringVar(&cloudProvider, "cloud-provider", "", "cloud that Velero will be installed into. Required.")
flag.StringVar(&objectStoreProvider, "object-store-provider", "", "provider of object store plugin. Required if cloud-provider is kind, otherwise ignored.")
flag.StringVar(&bslBucket, "bucket", "", "name of the object storage bucket where backups from e2e tests should be stored. Required.")
flag.StringVar(&cloudCredentialsFile, "credentials-file", "", "file containing credentials for backup and volume provider. Required.")
flag.StringVar(&veleroCLI, "velerocli", "velero", "path to the velero application to use.")
flag.StringVar(&veleroImage, "velero-image", "velero/velero:main", "image for the velero server to be tested.")
flag.StringVar(&plugins, "plugins", "", "provider plugins to be tested.")
flag.StringVar(&addBSLPlugins, "additional-bsl-plugins", "", "additional plugins to be tested.")
flag.StringVar(&veleroVersion, "velero-version", "main", "image version for the velero server to be tested with.")
flag.StringVar(&resticHelperImage, "restic-helper-image", "", "image for the velero restic restore helper to be tested.")
flag.StringVar(&upgradeFromVeleroCLI, "upgrade-from-velero-cli", "", "path to the pre-upgrade velero application to use.")
flag.StringVar(&upgradeFromVeleroVersion, "upgrade-from-velero-version", "v1.6.3", "image for the pre-upgrade velero server to be tested.")
flag.StringVar(&bslConfig, "bsl-config", "", "configuration to use for the backup storage location. Format is key1=value1,key2=value2")
flag.StringVar(&bslPrefix, "prefix", "", "prefix under which all Velero data should be stored within the bucket. Optional.")
flag.StringVar(&vslConfig, "vsl-config", "", "configuration to use for the volume snapshot location. Format is key1=value1,key2=value2")
flag.StringVar(&veleroNamespace, "velero-namespace", "velero", "namespace to install Velero into")
flag.BoolVar(&installVelero, "install-velero", true, "install/uninstall velero during the test. Optional.")
flag.StringVar(&registryCredentialFile, "registry-credential-file", "", "file containing credential for the image registry, follows the same format rules as the ~/.docker/config.json file. Optional.")
flag.StringVar(&VeleroCfg.CloudProvider, "cloud-provider", "", "cloud that Velero will be installed into. Required.")
flag.StringVar(&VeleroCfg.ObjectStoreProvider, "object-store-provider", "", "provider of object store plugin. Required if cloud-provider is kind, otherwise ignored.")
flag.StringVar(&VeleroCfg.BSLBucket, "bucket", "", "name of the object storage bucket where backups from e2e tests should be stored. Required.")
flag.StringVar(&VeleroCfg.CloudCredentialsFile, "credentials-file", "", "file containing credentials for backup and volume provider. Required.")
flag.StringVar(&VeleroCfg.VeleroCLI, "velerocli", "velero", "path to the velero application to use.")
flag.StringVar(&VeleroCfg.VeleroImage, "velero-image", "velero/velero:main", "image for the velero server to be tested.")
flag.StringVar(&VeleroCfg.Plugins, "plugins", "", "provider plugins to be tested.")
flag.StringVar(&VeleroCfg.AddBSLPlugins, "additional-bsl-plugins", "", "additional plugins to be tested.")
flag.StringVar(&VeleroCfg.VeleroVersion, "velero-version", "main", "image version for the velero server to be tested with.")
flag.StringVar(&VeleroCfg.ResticHelperImage, "restic-helper-image", "", "image for the velero restic restore helper to be tested.")
flag.StringVar(&VeleroCfg.UpgradeFromVeleroCLI, "upgrade-from-velero-cli", "", "path to the pre-upgrade velero application to use.")
flag.StringVar(&VeleroCfg.UpgradeFromVeleroVersion, "upgrade-from-velero-version", "v1.6.3", "image for the pre-upgrade velero server to be tested.")
flag.StringVar(&VeleroCfg.BSLConfig, "bsl-config", "", "configuration to use for the backup storage location. Format is key1=value1,key2=value2")
flag.StringVar(&VeleroCfg.BSLPrefix, "prefix", "", "prefix under which all Velero data should be stored within the bucket. Optional.")
flag.StringVar(&VeleroCfg.VSLConfig, "vsl-config", "", "configuration to use for the volume snapshot location. Format is key1=value1,key2=value2")
flag.StringVar(&VeleroCfg.VeleroNamespace, "velero-namespace", "velero", "namespace to install Velero into")
flag.BoolVar(&VeleroCfg.InstallVelero, "install-velero", true, "install/uninstall velero during the test. Optional.")
flag.StringVar(&VeleroCfg.RegistryCredentialFile, "registry-credential-file", "", "file containing credential for the image registry, follows the same format rules as the ~/.docker/config.json file. Optional.")
// Flags to create an additional BSL for multiple credentials test
flag.StringVar(&additionalBSLProvider, "additional-bsl-object-store-provider", "", "Provider of object store plugin for additional backup storage location. Required if testing multiple credentials support.")
flag.StringVar(&additionalBSLBucket, "additional-bsl-bucket", "", "name of the object storage bucket for additional backup storage location. Required if testing multiple credentials support.")
flag.StringVar(&additionalBSLPrefix, "additional-bsl-prefix", "", "prefix under which all Velero data should be stored within the bucket for additional backup storage location. Optional.")
flag.StringVar(&additionalBSLConfig, "additional-bsl-config", "", "configuration to use for the additional backup storage location. Format is key1=value1,key2=value2")
flag.StringVar(&additionalBSLCredentials, "additional-bsl-credentials-file", "", "file containing credentials for additional backup storage location provider. Required if testing multiple credentials support.")
flag.StringVar(&crdsVersion, "crds-version", "v1", "CRD apiVersion for velero CRD creation.")
flag.StringVar(&VeleroCfg.AdditionalBSLProvider, "additional-bsl-object-store-provider", "", "Provider of object store plugin for additional backup storage location. Required if testing multiple credentials support.")
flag.StringVar(&VeleroCfg.AdditionalBSLBucket, "additional-bsl-bucket", "", "name of the object storage bucket for additional backup storage location. Required if testing multiple credentials support.")
flag.StringVar(&VeleroCfg.AdditionalBSLPrefix, "additional-bsl-prefix", "", "prefix under which all Velero data should be stored within the bucket for additional backup storage location. Optional.")
flag.StringVar(&VeleroCfg.AdditionalBSLConfig, "additional-bsl-config", "", "configuration to use for the additional backup storage location. Format is key1=value1,key2=value2")
flag.StringVar(&VeleroCfg.AdditionalBSLCredentials, "additional-bsl-credentials-file", "", "file containing credentials for additional backup storage location provider. Required if testing multiple credentials support.")
flag.StringVar(&VeleroCfg.CRDsVersion, "crds-version", "v1", "CRD apiVersion for velero CRD creation.")
}
var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", APIGropuVersionsTest)
// Test backup and restore of Kibishi using restic
var _ = Describe("[Restic] Velero tests on cluster using the plugin provider for object storage and Restic for volume backups", BackupRestoreWithRestic)
var _ = Describe("[Snapshot] Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupRestoreWithSnapshots)
var _ = Describe("[Basic] Backup/restore of 2 namespaces", BasicBackupRestore)
var _ = Describe("[Scale] Backup/restore of 2500 namespaces", MultiNSBackupRestore)
// Upgrade test by Kibishi using restic
var _ = Describe("[Upgrade][Restic] Velero upgrade tests on cluster using the plugin provider for object storage and Restic for volume backups", BackupUpgradeRestoreWithRestic)
var _ = Describe("[Upgrade][Snapshot] Velero upgrade tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupUpgradeRestoreWithSnapshots)
func TestE2e(t *testing.T) {
// Skip running E2E tests when running only "short" tests because:
// 1. E2E tests are long running tests involving installation of Velero and performing backup and restore operations.

109
test/e2e/multiple_namespaces_test.go → test/e2e/scale/multiple_namespaces.go
View File

@ -1,40 +1,58 @@
package e2e
/*
Copyright the Velero contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package scale
import (
"context"
"flag"
"fmt"
"strings"
"time"
"github.com/google/uuid"
k8sutils "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
veleroutils "github.com/vmware-tanzu/velero/test/e2e/util/velero"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/pkg/errors"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
. "github.com/vmware-tanzu/velero/test/e2e"
)
var _ = Describe("[Basic] Backup/restore of 2 namespaces", func() {
func BasicBackupRestore() {
client, err := newTestClient()
client, err := k8sutils.NewTestClient()
Expect(err).To(Succeed(), "Failed to instantiate cluster client for multiple namespace tests")
BeforeEach(func() {
var err error
flag.Parse()
uuidgen, err = uuid.NewRandom()
UUIDgen, err = uuid.NewRandom()
Expect(err).To(Succeed())
if installVelero {
Expect(veleroInstall(context.Background(), veleroCLI, veleroImage, resticHelperImage, plugins, veleroNamespace, cloudProvider, objectStoreProvider, false,
cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, crdsVersion, "", registryCredentialFile)).To(Succeed())
if VeleroCfg.InstallVelero {
Expect(veleroutils.VeleroInstall(context.Background(), &VeleroCfg, "", false)).To(Succeed())
}
})
AfterEach(func() {
if installVelero {
err := veleroUninstall(context.Background(), veleroCLI, veleroNamespace)
if VeleroCfg.InstallVelero {
err := veleroutils.VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)
Expect(err).To(Succeed())
}
@ -42,34 +60,33 @@ var _ = Describe("[Basic] Backup/restore of 2 namespaces", func() {
Context("When I create 2 namespaces", func() {
It("should be successfully backed up and restored", func() {
backupName := "backup-" + uuidgen.String()
restoreName := "restore-" + uuidgen.String()
backupName := "backup-" + UUIDgen.String()
restoreName := "restore-" + UUIDgen.String()
fiveMinTimeout, _ := context.WithTimeout(context.Background(), 5*time.Minute)
Expect(RunMultipleNamespaceTest(fiveMinTimeout, client, "nstest-"+uuidgen.String(), 2,
Expect(RunMultipleNamespaceTest(fiveMinTimeout, client, "nstest-"+UUIDgen.String(), 2,
backupName, restoreName)).To(Succeed(), "Failed to successfully backup and restore multiple namespaces")
})
})
})
}
var _ = Describe("[Scale] Backup/restore of 2500 namespaces", func() {
func MultiNSBackupRestore() {
client, err := newTestClient()
client, err := k8sutils.NewTestClient()
Expect(err).To(Succeed(), "Failed to instantiate cluster client for multiple namespace tests")
BeforeEach(func() {
var err error
flag.Parse()
uuidgen, err = uuid.NewRandom()
UUIDgen, err = uuid.NewRandom()
Expect(err).To(Succeed())
if installVelero {
Expect(veleroInstall(context.Background(), veleroCLI, veleroImage, resticHelperImage, plugins, veleroNamespace, cloudProvider, objectStoreProvider, false,
cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, crdsVersion, "", registryCredentialFile)).To(Succeed())
if VeleroCfg.InstallVelero {
Expect(veleroutils.VeleroInstall(context.Background(), &VeleroCfg, "", false)).To(Succeed())
}
})
AfterEach(func() {
if installVelero {
err := veleroUninstall(context.Background(), veleroCLI, veleroNamespace)
if VeleroCfg.InstallVelero {
err := veleroutils.VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)
Expect(err).To(Succeed())
}
@ -77,22 +94,22 @@ var _ = Describe("[Scale] Backup/restore of 2500 namespaces", func() {
Context("When I create 2500 namespaces", func() {
It("should be successfully backed up and restored", func() {
backupName := "backup-" + uuidgen.String()
restoreName := "restore-" + uuidgen.String()
backupName := "backup-" + UUIDgen.String()
restoreName := "restore-" + UUIDgen.String()
twoHourTimeout, _ := context.WithTimeout(context.Background(), 2*time.Hour)
Expect(RunMultipleNamespaceTest(twoHourTimeout, client, "nstest-"+uuidgen.String(), 2500,
Expect(RunMultipleNamespaceTest(twoHourTimeout, client, "nstest-"+UUIDgen.String(), 2500,
backupName, restoreName)).To(Succeed(), "Failed to successfully backup and restore multiple namespaces")
})
})
})
}
func RunMultipleNamespaceTest(ctx context.Context, client testClient, nsBaseName string, numberOfNamespaces int, backupName string, restoreName string) error {
defer cleanupNamespaces(context.Background(), client, nsBaseName) // Run at exit for final cleanup
func RunMultipleNamespaceTest(ctx context.Context, client k8sutils.TestClient, nsBaseName string, numberOfNamespaces int, backupName string, restoreName string) error {
defer k8sutils.CleanupNamespaces(context.Background(), client, nsBaseName) // Run at exit for final cleanup
var excludeNamespaces []string
// Currently it's hard to build a large list of namespaces to include and wildcards do not work so instead
// we will exclude all of the namespaces that existed prior to the test from the backup
namespaces, err := client.clientGo.CoreV1().Namespaces().List(ctx, v1.ListOptions{})
namespaces, err := client.ClientGo.CoreV1().Namespaces().List(ctx, v1.ListOptions{})
if err != nil {
return errors.Wrap(err, "Could not retrieve namespaces")
}
@ -104,30 +121,30 @@ func RunMultipleNamespaceTest(ctx context.Context, client testClient, nsBaseName
fmt.Printf("Creating namespaces ...\n")
for nsNum := 0; nsNum < numberOfNamespaces; nsNum++ {
createNSName := fmt.Sprintf("%s-%00000d", nsBaseName, nsNum)
if err := createNamespace(ctx, client, createNSName); err != nil {
if err := k8sutils.CreateNamespace(ctx, client, createNSName); err != nil {
return errors.Wrapf(err, "Failed to create namespace %s", createNSName)
}
}
if err := veleroBackupExcludeNamespaces(ctx, veleroCLI, veleroNamespace, backupName, excludeNamespaces); err != nil {
runDebug(context.Background(), veleroCLI, veleroNamespace, backupName, "")
if err := veleroutils.VeleroBackupExcludeNamespaces(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backupName, excludeNamespaces); err != nil {
veleroutils.RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backupName, "")
return errors.Wrapf(err, "Failed to backup backup namespaces %s-*", nsBaseName)
}
err = cleanupNamespaces(ctx, client, nsBaseName)
err = k8sutils.CleanupNamespaces(ctx, client, nsBaseName)
if err != nil {
return errors.Wrap(err, "Could cleanup retrieve namespaces")
}
err = veleroRestore(ctx, veleroCLI, veleroNamespace, restoreName, backupName)
err = veleroutils.VeleroRestore(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, restoreName, backupName)
if err != nil {
runDebug(context.Background(), veleroCLI, veleroNamespace, "", restoreName)
veleroutils.RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, "", restoreName)
return errors.Wrap(err, "Restore failed")
}
// Verify that we got back all of the namespaces we created
for nsNum := 0; nsNum < numberOfNamespaces; nsNum++ {
checkNSName := fmt.Sprintf("%s-%00000d", nsBaseName, nsNum)
checkNS, err := getNamespace(ctx, client, checkNSName)
checkNS, err := k8sutils.GetNamespace(ctx, client, checkNSName)
if err != nil {
return errors.Wrapf(err, "Could not retrieve test namespace %s", checkNSName)
}
@ -138,21 +155,3 @@ func RunMultipleNamespaceTest(ctx context.Context, client testClient, nsBaseName
// Cleanup is automatic on the way out
return nil
}
func cleanupNamespaces(ctx context.Context, client testClient, nsBaseName string) error {
namespaces, err := client.clientGo.CoreV1().Namespaces().List(ctx, v1.ListOptions{})
if err != nil {
return errors.Wrap(err, "Could not retrieve namespaces")
}
fmt.Printf("Cleaning up namespaces ...\n")
for _, checkNamespace := range namespaces.Items {
if strings.HasPrefix(checkNamespace.Name, nsBaseName) {
err = client.clientGo.CoreV1().Namespaces().Delete(ctx, checkNamespace.Name, v1.DeleteOptions{})
if err != nil {
return errors.Wrapf(err, "Could not delete namespace %s", checkNamespace.Name)
}
}
}
return nil
}

52
test/e2e/types.go Normal file
View File

@ -0,0 +1,52 @@
/*
Copyright the Velero contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e
import (
"github.com/google/uuid"
)
var UUIDgen uuid.UUID
var VeleroCfg VerleroConfig
type VerleroConfig struct {
VeleroCLI string
VeleroImage string
VeleroVersion string
CloudCredentialsFile string
BSLConfig string
BSLBucket string
BSLPrefix string
VSLConfig string
CloudProvider string
ObjectStoreProvider string
VeleroNamespace string
CRDsVersion string
AdditionalBSLProvider string
AdditionalBSLBucket string
AdditionalBSLPrefix string
AdditionalBSLConfig string
AdditionalBSLCredentials string
RegistryCredentialFile string
ResticHelperImage string
UpgradeFromVeleroVersion string
UpgradeFromVeleroCLI string
Plugins string
AddBSLPlugins string
InstallVelero bool
}

170
test/e2e/upgrade/upgrade.go Normal file
View File

@ -0,0 +1,170 @@
/*
Copyright the Velero contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package upgrade
import (
"context"
"flag"
"fmt"
"time"
"github.com/google/uuid"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/pkg/errors"
. "github.com/vmware-tanzu/velero/test/e2e"
k8sutils "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
kibishiiutils "github.com/vmware-tanzu/velero/test/e2e/util/kibishii"
veleroutils "github.com/vmware-tanzu/velero/test/e2e/util/velero"
)
const (
upgradeNamespace = "upgrade-workload"
)
func BackupUpgradeRestoreWithSnapshots() {
BackupUpgradeRestoreTest(true)
}
func BackupUpgradeRestoreWithRestic() {
BackupUpgradeRestoreTest(false)
}
func BackupUpgradeRestoreTest(useVolumeSnapshots bool) {
var (
backupName, restoreName, upgradeFromVeleroCLI string
)
client, err := k8sutils.NewTestClient()
Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests")
BeforeEach(func() {
if (len(VeleroCfg.UpgradeFromVeleroVersion)) == 0 {
Skip("An original velero version is required to run upgrade test, please run test with upgrade-from-velero-version=<version>")
}
if useVolumeSnapshots && VeleroCfg.CloudProvider == "kind" {
Skip("Volume snapshots not supported on kind")
}
//Assume tag of velero server image is identical to velero CLI version
//Download velero CLI if it's empty according to velero CLI version
if (len(VeleroCfg.UpgradeFromVeleroCLI)) == 0 {
upgradeFromVeleroCLI, err = veleroutils.InstallVeleroCLI(VeleroCfg.UpgradeFromVeleroVersion)
Expect(err).To(Succeed())
}
var err error
flag.Parse()
UUIDgen, err = uuid.NewRandom()
Expect(err).To(Succeed())
if VeleroCfg.InstallVelero {
//Set VeleroImage and ResticHelperImage to blank
//VeleroImage and ResticHelperImage should be the default value in originalCli
tmpCfg := VeleroCfg
tmpCfg.VeleroCLI = upgradeFromVeleroCLI
tmpCfg.VeleroImage = ""
tmpCfg.ResticHelperImage = ""
tmpCfg.Plugins = ""
tmpCfg.CRDsVersion = ""
Expect(veleroutils.VeleroInstall(context.Background(), &tmpCfg, "", useVolumeSnapshots)).To(Succeed())
Expect(veleroutils.CheckVeleroVersion(context.Background(), upgradeFromVeleroCLI, tmpCfg.UpgradeFromVeleroVersion)).To(Succeed())
} else {
Skip("Upgrade test is skipped since user don't want to install any other velero")
}
})
AfterEach(func() {
if VeleroCfg.InstallVelero {
err = veleroutils.VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)
Expect(err).To(Succeed())
}
})
When("kibishii is the sample workload", func() {
It("should be successfully backed up and restored to the default BackupStorageLocation", func() {
backupName = "backup-" + UUIDgen.String()
restoreName = "restore-" + UUIDgen.String()
Expect(runUpgradeTests(client, &VeleroCfg, upgradeFromVeleroCLI, backupName, restoreName, "", useVolumeSnapshots)).To(Succeed(),
"Failed to successfully backup and restore Kibishii namespace")
})
})
}
// runUpgradeTests runs upgrade test on the provider by kibishii.
func runUpgradeTests(client k8sutils.TestClient, veleroCfg *VerleroConfig, upgradeFromVeleroCLI, backupName, restoreName, backupLocation string,
useVolumeSnapshots bool) error {
if veleroCfg.VeleroCLI == "" {
return errors.New("empty")
}
oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60)
if err := k8sutils.CreateNamespace(oneHourTimeout, client, upgradeNamespace); err != nil {
return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", upgradeNamespace)
}
defer func() {
if err := k8sutils.DeleteNamespace(context.Background(), client, upgradeNamespace, true); err != nil {
fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", upgradeNamespace))
}
}()
if err := kibishiiutils.KibishiiPrepareBeforeBackup(oneHourTimeout, client, veleroCfg.CloudProvider, upgradeNamespace, veleroCfg.RegistryCredentialFile); err != nil {
return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", upgradeNamespace)
}
if err := veleroutils.VeleroBackupNamespace(oneHourTimeout, upgradeFromVeleroCLI, veleroCfg.VeleroNamespace, backupName, upgradeNamespace, backupLocation, useVolumeSnapshots); err != nil {
// TODO currently, the upgrade case covers the upgrade path from 1.6 to main and the velero v1.6 doesn't support "debug" command
// TODO move to "veleroutils.RunDebug" after we bump up to 1.7 in the upgrade case
veleroutils.VeleroBackupLogs(context.Background(), upgradeFromVeleroCLI, veleroCfg.VeleroNamespace, backupName)
return errors.Wrapf(err, "Failed to backup kibishii namespace %s", upgradeNamespace)
}
if veleroCfg.CloudProvider == "vsphere" && useVolumeSnapshots {
// Wait for uploads started by the Velero Plug-in for vSphere to complete
// TODO - remove after upload progress monitoring is implemented
fmt.Println("Waiting for vSphere uploads to complete")
if err := veleroutils.WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, upgradeNamespace); err != nil {
return errors.Wrapf(err, "Error waiting for uploads to complete")
}
}
fmt.Printf("Simulating a disaster by removing namespace %s\n", upgradeNamespace)
if err := k8sutils.DeleteNamespace(oneHourTimeout, client, upgradeNamespace, true); err != nil {
return errors.Wrapf(err, "failed to delete namespace %s", upgradeNamespace)
}
// the snapshots of AWS may be still in pending status when do the restore, wait for a while
// to avoid this https://github.com/vmware-tanzu/velero/issues/1799
// TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed
if veleroCfg.CloudProvider == "aws" && useVolumeSnapshots {
fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...")
time.Sleep(5 * time.Minute)
}
if err := veleroutils.VeleroInstall(context.Background(), veleroCfg, "", useVolumeSnapshots); err != nil {
return errors.Wrapf(err, "Failed to install velero from image %s", veleroCfg.VeleroImage)
}
if err := veleroutils.CheckVeleroVersion(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroVersion); err != nil {
return errors.Wrapf(err, "Velero install version mismatch.")
}
if err := veleroutils.VeleroRestore(oneHourTimeout, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, restoreName, backupName); err != nil {
veleroutils.RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, "", restoreName)
return errors.Wrapf(err, "Restore %s failed from backup %s", restoreName, backupName)
}
if err := kibishiiutils.KibishiiVerifyAfterRestore(client, upgradeNamespace, oneHourTimeout); err != nil {
return errors.Wrapf(err, "Error verifying kibishii after restore")
}
fmt.Printf("Upgrade test completed successfully\n")
return nil
}

164
test/e2e/upgrade_test.go
View File

@ -1,164 +0,0 @@
/*
Copyright the Velero contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e
import (
"context"
"flag"
"fmt"
"time"
"github.com/google/uuid"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/pkg/errors"
)
const (
upgradeNamespace = "upgrade-workload"
)
// Upgrade test by Kibishi using restic
var _ = Describe("[Upgrade][Restic] Velero upgrade tests on cluster using the plugin provider for object storage and Restic for volume backups", backup_upgrade_restore_with_restic)
var _ = Describe("[Upgrade][Snapshot] Velero upgrade tests on cluster using the plugin provider for object storage and snapshots for volume backups", backup_upgrade_restore_with_snapshots)
func backup_upgrade_restore_with_snapshots() {
backup_upgrade_restore_test(true)
}
func backup_upgrade_restore_with_restic() {
backup_upgrade_restore_test(false)
}
func backup_upgrade_restore_test(useVolumeSnapshots bool) {
var (
backupName, restoreName string
)
upgradeFromVeleroCLI := upgradeFromVeleroCLI
client, err := newTestClient()
Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests")
BeforeEach(func() {
if (len(upgradeFromVeleroVersion)) == 0 {
Skip("An original velero version is required to run upgrade test, please run test with upgrade-from-velero-version=<version>")
}
if useVolumeSnapshots && cloudProvider == "kind" {
Skip("Volume snapshots not supported on kind")
}
//Assume tag of velero server image is identical to velero CLI version
//Download velero CLI if it's empty according to velero CLI version
if (len(upgradeFromVeleroCLI)) == 0 {
upgradeFromVeleroCLI, err = installVeleroCLI(upgradeFromVeleroVersion)
Expect(err).To(Succeed())
}
var err error
flag.Parse()
uuidgen, err = uuid.NewRandom()
Expect(err).To(Succeed())
if installVelero {
//Set veleroImage and resticHelperImage to blank
//veleroImage and resticHelperImage should be the default value in originalCli
Expect(veleroInstall(context.Background(), upgradeFromVeleroCLI, "", "", "", veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots,
cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, "", "", registryCredentialFile)).To(Succeed())
Expect(checkVeleroVersion(context.Background(), upgradeFromVeleroCLI, upgradeFromVeleroVersion)).To(Succeed())
} else {
Skip("Upgrade test is skipped since user don't want to install any other velero")
}
})
AfterEach(func() {
if installVelero {
err = veleroUninstall(context.Background(), veleroCLI, veleroNamespace)
Expect(err).To(Succeed())
}
})
When("kibishii is the sample workload", func() {
It("should be successfully backed up and restored to the default BackupStorageLocation", func() {
backupName = "backup-" + uuidgen.String()
restoreName = "restore-" + uuidgen.String()
Expect(runUpgradeTests(client, veleroImage, veleroVersion, cloudProvider, upgradeFromVeleroCLI, veleroNamespace, backupName, restoreName, "", useVolumeSnapshots, registryCredentialFile)).To(Succeed(),
"Failed to successfully backup and restore Kibishii namespace")
})
})
}
// runUpgradeTests runs upgrade test on the provider by kibishii.
func runUpgradeTests(client testClient, upgradeToVeleroImage, upgradeToVeleroVersion, providerName, upgradeFromVeleroCLI, veleroNamespace, backupName, restoreName, backupLocation string,
useVolumeSnapshots bool, registryCredentialFile string) error {
oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60)
if err := createNamespace(oneHourTimeout, client, upgradeNamespace); err != nil {
return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", upgradeNamespace)
}
defer func() {
if err := deleteNamespace(context.Background(), client, upgradeNamespace, true); err != nil {
fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", upgradeNamespace))
}
}()
if err := kibishiiPrepareBeforeBackup(oneHourTimeout, client, providerName, upgradeNamespace, registryCredentialFile); err != nil {
return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", upgradeNamespace)
}
if err := veleroBackupNamespace(oneHourTimeout, upgradeFromVeleroCLI, veleroNamespace, backupName, upgradeNamespace, backupLocation, useVolumeSnapshots); err != nil {
// TODO currently, the upgrade case covers the upgrade path from 1.6 to main and the velero v1.6 doesn't support "debug" command
// TODO move to "runDebug" after we bump up to 1.7 in the upgrade case
veleroBackupLogs(context.Background(), upgradeFromVeleroCLI, veleroNamespace, backupName)
return errors.Wrapf(err, "Failed to backup kibishii namespace %s", upgradeNamespace)
}
if providerName == "vsphere" && useVolumeSnapshots {
// Wait for uploads started by the Velero Plug-in for vSphere to complete
// TODO - remove after upload progress monitoring is implemented
fmt.Println("Waiting for vSphere uploads to complete")
if err := waitForVSphereUploadCompletion(oneHourTimeout, time.Hour, upgradeNamespace); err != nil {
return errors.Wrapf(err, "Error waiting for uploads to complete")
}
}
fmt.Printf("Simulating a disaster by removing namespace %s\n", upgradeNamespace)
if err := deleteNamespace(oneHourTimeout, client, upgradeNamespace, true); err != nil {
return errors.Wrapf(err, "failed to delete namespace %s", upgradeNamespace)
}
// the snapshots of AWS may be still in pending status when do the restore, wait for a while
// to avoid this https://github.com/vmware-tanzu/velero/issues/1799
// TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed
if providerName == "aws" && useVolumeSnapshots {
fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...")
time.Sleep(5 * time.Minute)
}
if err := veleroInstall(context.Background(), veleroCLI, upgradeToVeleroImage, resticHelperImage, plugins, veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots,
cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, crdsVersion, "", registryCredentialFile); err != nil {
return errors.Wrapf(err, "Failed to install velero from image %s", upgradeToVeleroImage)
}
if err := checkVeleroVersion(context.Background(), veleroCLI, upgradeToVeleroVersion); err != nil {
return errors.Wrapf(err, "Velero install version mismatch.")
}
if err := veleroRestore(oneHourTimeout, veleroCLI, veleroNamespace, restoreName, backupName); err != nil {
runDebug(context.Background(), veleroCLI, veleroNamespace, "", restoreName)
return errors.Wrapf(err, "Restore %s failed from backup %s", restoreName, backupName)
}
if err := kibishiiVerifyAfterRestore(client, upgradeNamespace, oneHourTimeout); err != nil {
return errors.Wrapf(err, "Error verifying kibishii after restore")
}
fmt.Printf("Upgrade test completed successfully\n")
return nil
}

24
test/e2e/client.go → test/e2e/util/k8s/client.go
View File

@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e
package k8s
import (
"k8s.io/client-go/kubernetes"
@ -23,10 +23,10 @@ import (
"github.com/vmware-tanzu/velero/pkg/client"
)
// testClient contains different API clients that are in use throughout
// TestClient contains different API clients that are in use throughout
// the e2e tests.
type testClient struct {
type TestClient struct {
kubebuilder kbclient.Client
// clientGo returns a client-go API client.
@ -34,7 +34,7 @@ type testClient struct {
// Deprecated, TODO(2.0): presuming all controllers and resources are converted to the
// controller runtime framework by v2.0, it is the intent to remove all
// client-go API clients. Please use the controller runtime to make API calls for tests.
clientGo kubernetes.Interface
ClientGo kubernetes.Interface
// dynamicFactory returns a client-go API client for retrieving dynamic clients
// for GroupVersionResources and GroupVersionKinds.
@ -45,35 +45,35 @@ type testClient struct {
dynamicFactory client.DynamicFactory
}
// newTestClient returns a set of ready-to-use API clients.
func newTestClient() (testClient, error) {
// k8sutils.NewTestClient returns a set of ready-to-use API clients.
func NewTestClient() (TestClient, error) {
config, err := client.LoadConfig()
if err != nil {
return testClient{}, err
return TestClient{}, err
}
f := client.NewFactory("e2e", config)
clientGo, err := f.KubeClient()
if err != nil {
return testClient{}, err
return TestClient{}, err
}
kb, err := f.KubebuilderClient()
if err != nil {
return testClient{}, err
return TestClient{}, err
}
dynamicClient, err := f.DynamicClient()
if err != nil {
return testClient{}, err
return TestClient{}, err
}
factory := client.NewDynamicFactory(dynamicClient)
return testClient{
return TestClient{
kubebuilder: kb,
clientGo: clientGo,
ClientGo: clientGo,
dynamicFactory: factory,
}, nil
}

14
test/e2e/common.go → test/e2e/util/k8s/common.go
View File

@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e
package k8s
import (
"fmt"
@ -32,11 +32,11 @@ import (
)
// ensureClusterExists returns whether or not a kubernetes cluster exists for tests to be run on.
func ensureClusterExists(ctx context.Context) error {
func EnsureClusterExists(ctx context.Context) error {
return exec.CommandContext(ctx, "kubectl", "cluster-info").Run()
}
func createSecretFromFiles(ctx context.Context, client testClient, namespace string, name string, files map[string]string) error {
func CreateSecretFromFiles(ctx context.Context, client TestClient, namespace string, name string, files map[string]string) error {
data := make(map[string][]byte)
for key, filePath := range files {
@ -49,17 +49,17 @@ func createSecretFromFiles(ctx context.Context, client testClient, namespace str
}
secret := builder.ForSecret(namespace, name).Data(data).Result()
_, err := client.clientGo.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{})
_, err := client.ClientGo.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{})
return err
}
// waitForPods waits until all of the pods have gone to PodRunning state
func waitForPods(ctx context.Context, client testClient, namespace string, pods []string) error {
// WaitForPods waits until all of the pods have gone to PodRunning state
func WaitForPods(ctx context.Context, client TestClient, namespace string, pods []string) error {
timeout := 10 * time.Minute
interval := 5 * time.Second
err := wait.PollImmediate(interval, timeout, func() (bool, error) {
for _, podName := range pods {
checkPod, err := client.clientGo.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{})
checkPod, err := client.ClientGo.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{})
if err != nil {
return false, errors.WithMessage(err, fmt.Sprintf("Failed to verify pod %s/%s is %s", namespace, podName, corev1api.PodRunning))
}

34
test/e2e/namespace.go → test/e2e/util/k8s/namespace.go
View File

@ -14,11 +14,12 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e
package k8s
import (
"context"
"fmt"
"strings"
"time"
"github.com/pkg/errors"
@ -27,26 +28,27 @@ import (
corev1api "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
waitutil "k8s.io/apimachinery/pkg/util/wait"
"github.com/vmware-tanzu/velero/pkg/builder"
)
func createNamespace(ctx context.Context, client testClient, namespace string) error {
func CreateNamespace(ctx context.Context, client TestClient, namespace string) error {
ns := builder.ForNamespace(namespace).Result()
_, err := client.clientGo.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{})
_, err := client.ClientGo.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{})
if apierrors.IsAlreadyExists(err) {
return nil
}
return err
}
func getNamespace(ctx context.Context, client testClient, namespace string) (*corev1api.Namespace, error) {
return client.clientGo.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{})
func GetNamespace(ctx context.Context, client TestClient, namespace string) (*corev1api.Namespace, error) {
return client.ClientGo.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{})
}
func deleteNamespace(ctx context.Context, client testClient, namespace string, wait bool) error {
if err := client.clientGo.CoreV1().Namespaces().Delete(ctx, namespace, metav1.DeleteOptions{}); err != nil {
func DeleteNamespace(ctx context.Context, client TestClient, namespace string, wait bool) error {
if err := client.ClientGo.CoreV1().Namespaces().Delete(ctx, namespace, metav1.DeleteOptions{}); err != nil {
return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", namespace))
}
if !wait {
@ -55,7 +57,7 @@ func deleteNamespace(ctx context.Context, client testClient, namespace string, w
return waitutil.PollImmediateInfinite(5*time.Second,
func() (bool, error) {
if _, err := client.clientGo.CoreV1().Namespaces().Get(context.TODO(), namespace, metav1.GetOptions{}); err != nil {
if _, err := client.ClientGo.CoreV1().Namespaces().Get(context.TODO(), namespace, metav1.GetOptions{}); err != nil {
if apierrors.IsNotFound(err) {
return true, nil
}
@ -65,3 +67,19 @@ func deleteNamespace(ctx context.Context, client testClient, namespace string, w
return false, nil
})
}
func CleanupNamespaces(ctx context.Context, client TestClient, nsBaseName string) error {
namespaces, err := client.ClientGo.CoreV1().Namespaces().List(ctx, v1.ListOptions{})
if err != nil {
return errors.Wrap(err, "Could not retrieve namespaces")
}
for _, checkNamespace := range namespaces.Items {
if strings.HasPrefix(checkNamespace.Name, nsBaseName) {
err = client.ClientGo.CoreV1().Namespaces().Delete(ctx, checkNamespace.Name, v1.DeleteOptions{})
if err != nil {
return errors.Wrapf(err, "Could not delete namespace %s", checkNamespace.Name)
}
}
}
return nil
}

12
test/e2e/serviceaccount.go → test/e2e/util/k8s/serviceaccount.go
View File

@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e
package k8s
import (
"context"
@ -33,10 +33,10 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func waitUntilServiceAccountCreated(ctx context.Context, client testClient, namespace, serviceAccount string, timeout time.Duration) error {
func WaitUntilServiceAccountCreated(ctx context.Context, client TestClient, namespace, serviceAccount string, timeout time.Duration) error {
return wait.PollImmediate(5*time.Second, timeout,
func() (bool, error) {
if _, err := client.clientGo.CoreV1().ServiceAccounts(namespace).Get(ctx, serviceAccount, metav1.GetOptions{}); err != nil {
if _, err := client.ClientGo.CoreV1().ServiceAccounts(namespace).Get(ctx, serviceAccount, metav1.GetOptions{}); err != nil {
if !apierrors.IsNotFound(err) {
return false, err
}
@ -46,7 +46,7 @@ func waitUntilServiceAccountCreated(ctx context.Context, client testClient, name
})
}
func patchServiceAccountWithImagePullSecret(ctx context.Context, client testClient, namespace, serviceAccount, dockerCredentialFile string) error {
func PatchServiceAccountWithImagePullSecret(ctx context.Context, client TestClient, namespace, serviceAccount, dockerCredentialFile string) error {
credential, err := ioutil.ReadFile(dockerCredentialFile)
if err != nil {
return errors.Wrapf(err, "failed to read the docker credential file %q", dockerCredentialFile)
@ -54,11 +54,11 @@ func patchServiceAccountWithImagePullSecret(ctx context.Context, client testClie
secretName := "image-pull-secret"
secret := builder.ForSecret(namespace, secretName).Data(map[string][]byte{".dockerconfigjson": credential}).Result()
secret.Type = corev1.SecretTypeDockerConfigJson
if _, err = client.clientGo.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}); err != nil {
if _, err = client.ClientGo.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}); err != nil {
return errors.Wrapf(err, "failed to create secret %q under namespace %q", secretName, namespace)
}
if _, err = client.clientGo.CoreV1().ServiceAccounts(namespace).Patch(ctx, serviceAccount, types.StrategicMergePatchType,
if _, err = client.ClientGo.CoreV1().ServiceAccounts(namespace).Patch(ctx, serviceAccount, types.StrategicMergePatchType,
[]byte(fmt.Sprintf(`{"imagePullSecrets": [{"name": "%s"}]}`, secretName)), metav1.PatchOptions{}); err != nil {
return errors.Wrapf(err, "failed to patch the service account %q under the namespace %q", serviceAccount, namespace)
}

41
test/e2e/kibishii_utils.go → test/e2e/util/kibishii/kibishii_utils.go
View File

@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e
package kibishii
import (
"fmt"
@ -26,6 +26,8 @@ import (
"golang.org/x/net/context"
veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec"
k8sutils "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
veleroutils "github.com/vmware-tanzu/velero/test/e2e/util/velero"
)
const (
@ -33,25 +35,24 @@ const (
jumpPadPod = "jump-pad"
)
// runKibishiiTests runs kibishii tests on the provider.
func runKibishiiTests(client testClient, providerName, veleroCLI, veleroNamespace, backupName, restoreName, backupLocation string,
// RunKibishiiTests runs kibishii tests on the provider.
func RunKibishiiTests(client k8sutils.TestClient, providerName, veleroCLI, veleroNamespace, backupName, restoreName, backupLocation string,
useVolumeSnapshots bool, registryCredentialFile string) error {
oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60)
if err := createNamespace(oneHourTimeout, client, kibishiiNamespace); err != nil {
if err := k8sutils.CreateNamespace(oneHourTimeout, client, kibishiiNamespace); err != nil {
return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", kibishiiNamespace)
}
defer func() {
if err := deleteNamespace(context.Background(), client, kibishiiNamespace, true); err != nil {
if err := k8sutils.DeleteNamespace(context.Background(), client, kibishiiNamespace, true); err != nil {
fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", kibishiiNamespace))
}
}()
if err := kibishiiPrepareBeforeBackup(oneHourTimeout, client, providerName, kibishiiNamespace, registryCredentialFile); err != nil {
if err := KibishiiPrepareBeforeBackup(oneHourTimeout, client, providerName, kibishiiNamespace, registryCredentialFile); err != nil {
return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", kibishiiNamespace)
}
if err := veleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, kibishiiNamespace, backupLocation, useVolumeSnapshots); err != nil {
runDebug(context.Background(), veleroCLI, veleroNamespace, backupName, "")
if err := veleroutils.VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, kibishiiNamespace, backupLocation, useVolumeSnapshots); err != nil {
veleroutils.RunDebug(context.Background(), veleroCLI, veleroNamespace, backupName, "")
return errors.Wrapf(err, "Failed to backup kibishii namespace %s", kibishiiNamespace)
}
@ -59,12 +60,12 @@ func runKibishiiTests(client testClient, providerName, veleroCLI, veleroNamespac
// Wait for uploads started by the Velero Plug-in for vSphere to complete
// TODO - remove after upload progress monitoring is implemented
fmt.Println("Waiting for vSphere uploads to complete")
if err := waitForVSphereUploadCompletion(oneHourTimeout, time.Hour, kibishiiNamespace); err != nil {
if err := veleroutils.WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, kibishiiNamespace); err != nil {
return errors.Wrapf(err, "Error waiting for uploads to complete")
}
}
fmt.Printf("Simulating a disaster by removing namespace %s\n", kibishiiNamespace)
if err := deleteNamespace(oneHourTimeout, client, kibishiiNamespace, true); err != nil {
if err := k8sutils.DeleteNamespace(oneHourTimeout, client, kibishiiNamespace, true); err != nil {
return errors.Wrapf(err, "failed to delete namespace %s", kibishiiNamespace)
}
@ -76,12 +77,12 @@ func runKibishiiTests(client testClient, providerName, veleroCLI, veleroNamespac
time.Sleep(5 * time.Minute)
}
if err := veleroRestore(oneHourTimeout, veleroCLI, veleroNamespace, restoreName, backupName); err != nil {
runDebug(context.Background(), veleroCLI, veleroNamespace, "", restoreName)
if err := veleroutils.VeleroRestore(oneHourTimeout, veleroCLI, veleroNamespace, restoreName, backupName); err != nil {
veleroutils.RunDebug(context.Background(), veleroCLI, veleroNamespace, "", restoreName)
return errors.Wrapf(err, "Restore %s failed from backup %s", restoreName, backupName)
}
if err := kibishiiVerifyAfterRestore(client, kibishiiNamespace, oneHourTimeout); err != nil {
if err := KibishiiVerifyAfterRestore(client, kibishiiNamespace, oneHourTimeout); err != nil {
return errors.Wrapf(err, "Error verifying kibishii after restore")
}
@ -144,19 +145,19 @@ func verifyData(ctx context.Context, namespace string, levels int, filesPerLevel
return nil
}
func waitForKibishiiPods(ctx context.Context, client testClient, kibishiiNamespace string) error {
return waitForPods(ctx, client, kibishiiNamespace, []string{"jump-pad", "etcd0", "etcd1", "etcd2", "kibishii-deployment-0", "kibishii-deployment-1"})
func waitForKibishiiPods(ctx context.Context, client k8sutils.TestClient, kibishiiNamespace string) error {
return k8sutils.WaitForPods(ctx, client, kibishiiNamespace, []string{"jump-pad", "etcd0", "etcd1", "etcd2", "kibishii-deployment-0", "kibishii-deployment-1"})
}
func kibishiiPrepareBeforeBackup(oneHourTimeout context.Context, client testClient, providerName, kibishiiNamespace, registryCredentialFile string) error {
func KibishiiPrepareBeforeBackup(oneHourTimeout context.Context, client k8sutils.TestClient, providerName, kibishiiNamespace, registryCredentialFile string) error {
serviceAccountName := "default"
// wait until the service account is created before patch the image pull secret
if err := waitUntilServiceAccountCreated(oneHourTimeout, client, kibishiiNamespace, serviceAccountName, 10*time.Minute); err != nil {
if err := k8sutils.WaitUntilServiceAccountCreated(oneHourTimeout, client, kibishiiNamespace, serviceAccountName, 10*time.Minute); err != nil {
return errors.Wrapf(err, "failed to wait the service account %q created under the namespace %q", serviceAccountName, kibishiiNamespace)
}
// add the image pull secret to avoid the image pull limit issue of Docker Hub
if err := patchServiceAccountWithImagePullSecret(oneHourTimeout, client, kibishiiNamespace, serviceAccountName, registryCredentialFile); err != nil {
if err := k8sutils.PatchServiceAccountWithImagePullSecret(oneHourTimeout, client, kibishiiNamespace, serviceAccountName, registryCredentialFile); err != nil {
return errors.Wrapf(err, "failed to patch the service account %q under the namespace %q", serviceAccountName, kibishiiNamespace)
}
@ -177,7 +178,7 @@ func kibishiiPrepareBeforeBackup(oneHourTimeout context.Context, client testClie
return nil
}
func kibishiiVerifyAfterRestore(client testClient, kibishiiNamespace string, oneHourTimeout context.Context) error {
func KibishiiVerifyAfterRestore(client k8sutils.TestClient, kibishiiNamespace string, oneHourTimeout context.Context) error {
// wait for kibishii pod startup
// TODO - Fix kibishii so we can check that it is ready to go
fmt.Printf("Waiting for kibishii pods to be ready\n")

54
test/e2e/install.go → test/e2e/util/velero/install.go
View File

@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e
package velero
import (
"bytes"
@ -35,6 +35,8 @@ import (
"github.com/vmware-tanzu/velero/pkg/cmd/cli/install"
velerexec "github.com/vmware-tanzu/velero/pkg/util/exec"
. "github.com/vmware-tanzu/velero/test/e2e"
k8sutils "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
)
// we provide more install options other than the standard install.InstallOptions in E2E test
@ -44,53 +46,49 @@ type installOptions struct {
ResticHelperImage string
}
// TODO too many parameters for this function, better to make it a structure, we can introduces a structure `config` for the E2E to hold all configuration items
func veleroInstall(ctx context.Context, cli, veleroImage, resticHelperImage, providerPlugins, veleroNamespace, cloudProvider, objectStoreProvider string, useVolumeSnapshots bool,
cloudCredentialsFile string, bslBucket string, bslPrefix string, bslConfig string, vslConfig string,
crdsVersion string, features string, registryCredentialFile string) error {
if cloudProvider != "kind" {
if objectStoreProvider != "" {
func VeleroInstall(ctx context.Context, veleroCfg *VerleroConfig, features string, useVolumeSnapshots bool) error {
if veleroCfg.CloudProvider != "kind" {
if veleroCfg.ObjectStoreProvider != "" {
return errors.New("For cloud platforms, object store plugin cannot be overridden") // Can't set an object store provider that is different than your cloud
}
objectStoreProvider = cloudProvider
veleroCfg.ObjectStoreProvider = veleroCfg.CloudProvider
} else {
if objectStoreProvider == "" {
if veleroCfg.ObjectStoreProvider == "" {
return errors.New("No object store provider specified - must be specified when using kind as the cloud provider") // Gotta have an object store provider
}
}
providerPluginsTmp, err := getProviderPlugins(ctx, cli, objectStoreProvider, providerPlugins)
providerPluginsTmp, err := getProviderPlugins(ctx, veleroCfg.VeleroCLI, veleroCfg.ObjectStoreProvider, veleroCfg.Plugins)
if err != nil {
return errors.WithMessage(err, "Failed to get provider plugins")
}
// TODO - handle this better
if cloudProvider == "vsphere" {
// We overrider the objectStoreProvider here for vSphere because we want to use the aws plugin for the
if veleroCfg.CloudProvider == "vsphere" {
// We overrider the ObjectStoreProvider here for vSphere because we want to use the aws plugin for the
// backup, but needed to pick up the provider plugins earlier. vSphere plugin no longer needs a Volume
// Snapshot location specified
objectStoreProvider = "aws"
veleroCfg.ObjectStoreProvider = "aws"
}
err = ensureClusterExists(ctx)
err = k8sutils.EnsureClusterExists(ctx)
if err != nil {
return errors.WithMessage(err, "Failed to ensure Kubernetes cluster exists")
}
veleroInstallOptions, err := getProviderVeleroInstallOptions(objectStoreProvider, cloudCredentialsFile, bslBucket,
bslPrefix, bslConfig, vslConfig, providerPluginsTmp, features)
veleroInstallOptions, err := getProviderVeleroInstallOptions(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,
veleroCfg.BSLPrefix, veleroCfg.BSLConfig, veleroCfg.VSLConfig, providerPluginsTmp, features)
if err != nil {
return errors.WithMessagef(err, "Failed to get Velero InstallOptions for plugin provider %s", objectStoreProvider)
return errors.WithMessagef(err, "Failed to get Velero InstallOptions for plugin provider %s", veleroCfg.ObjectStoreProvider)
}
veleroInstallOptions.UseVolumeSnapshots = useVolumeSnapshots
veleroInstallOptions.UseRestic = !useVolumeSnapshots
veleroInstallOptions.Image = veleroImage
veleroInstallOptions.CRDsVersion = crdsVersion
veleroInstallOptions.Namespace = veleroNamespace
veleroInstallOptions.Image = veleroCfg.VeleroImage
veleroInstallOptions.CRDsVersion = veleroCfg.CRDsVersion
veleroInstallOptions.Namespace = veleroCfg.VeleroNamespace
err = installVeleroServer(ctx, cli, &installOptions{
err = installVeleroServer(ctx, veleroCfg.VeleroCLI, &installOptions{
InstallOptions: veleroInstallOptions,
RegistryCredentialFile: registryCredentialFile,
ResticHelperImage: resticHelperImage,
RegistryCredentialFile: veleroCfg.RegistryCredentialFile,
ResticHelperImage: veleroCfg.ResticHelperImage,
})
if err != nil {
return errors.WithMessagef(err, "Failed to install Velero in the cluster")
@ -195,7 +193,7 @@ func createVelereResources(ctx context.Context, cli, namespace string, args []st
return errors.Wrapf(err, "failed to unmarshal the resources: %s", string(stdout))
}
if err = patchResources(ctx, resources, namespace, registryCredentialFile, resticHelperImage); err != nil {
if err = patchResources(ctx, resources, namespace, registryCredentialFile, VeleroCfg.ResticHelperImage); err != nil {
return errors.Wrapf(err, "failed to patch resources")
}
@ -261,7 +259,7 @@ func patchResources(ctx context.Context, resources *unstructured.UnstructuredLis
}
// customize the restic restore helper image
if len(resticHelperImage) > 0 {
if len(VeleroCfg.ResticHelperImage) > 0 {
restoreActionConfig := corev1.ConfigMap{
TypeMeta: metav1.TypeMeta{
Kind: "ConfigMap",
@ -276,7 +274,7 @@ func patchResources(ctx context.Context, resources *unstructured.UnstructuredLis
},
},
Data: map[string]string{
"image": resticHelperImage,
"image": VeleroCfg.ResticHelperImage,
},
}
@ -337,7 +335,7 @@ func waitVeleroReady(ctx context.Context, namespace string, useRestic bool) erro
return nil
}
func veleroUninstall(ctx context.Context, cli, namespace string) error {
func VeleroUninstall(ctx context.Context, cli, namespace string) error {
stdout, stderr, err := velerexec.RunCommand(exec.CommandContext(ctx, cli, "uninstall", "--force", "-n", namespace))
if err != nil {
return errors.Wrapf(err, "failed to uninstall velero, stdout=%s, stderr=%s", stdout, stderr)

162
test/e2e/velero_utils.go → test/e2e/util/velero/velero_utils.go
View File

@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e
package velero
import (
"bytes"
@ -98,7 +98,7 @@ func getProviderVeleroInstallOptions(
pluginProvider,
credentialsFile,
objectStoreBucket,
objectStorePrefix string,
objectStorePrefix,
bslConfig,
vslConfig string,
plugins []string,
@ -134,7 +134,7 @@ func getProviderVeleroInstallOptions(
return io, nil
}
// checkBackupPhase uses veleroCLI to inspect the phase of a Velero backup.
// checkBackupPhase uses VeleroCLI to inspect the phase of a Velero backup.
func checkBackupPhase(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string,
expectedPhase velerov1api.BackupPhase) error {
checkCMD := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "backup", "get", "-o", "json",
@ -178,7 +178,7 @@ func checkBackupPhase(ctx context.Context, veleroCLI string, veleroNamespace str
return nil
}
// checkRestorePhase uses veleroCLI to inspect the phase of a Velero restore.
// checkRestorePhase uses VeleroCLI to inspect the phase of a Velero restore.
func checkRestorePhase(ctx context.Context, veleroCLI string, veleroNamespace string, restoreName string,
expectedPhase velerov1api.RestorePhase) error {
checkCMD := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "restore", "get", "-o", "json",
@ -222,8 +222,8 @@ func checkRestorePhase(ctx context.Context, veleroCLI string, veleroNamespace st
return nil
}
// veleroBackupNamespace uses the veleroCLI to backup a namespace.
func veleroBackupNamespace(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, namespace string, backupLocation string,
// VeleroBackupNamespace uses the veleroCLI to backup a namespace.
func VeleroBackupNamespace(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, namespace string, backupLocation string,
useVolumeSnapshots bool) error {
args := []string{
"--namespace", veleroNamespace,
@ -246,71 +246,80 @@ func veleroBackupNamespace(ctx context.Context, veleroCLI string, veleroNamespac
args = append(args, "--storage-location", backupLocation)
}
backupCmd := exec.CommandContext(ctx, veleroCLI, args...)
backupCmd.Stdout = os.Stdout
backupCmd.Stderr = os.Stderr
fmt.Printf("backup cmd =%v\n", backupCmd)
err := backupCmd.Run()
if err != nil {
return err
}
err = checkBackupPhase(ctx, veleroCLI, veleroNamespace, backupName, velerov1api.BackupPhaseCompleted)
return err
return VeleroBackupExec(ctx, veleroCLI, veleroNamespace, backupName, args)
}
// veleroBackupExcludeNamespaces uses the veleroCLI to backup a namespace.
func veleroBackupExcludeNamespaces(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, excludeNamespaces []string) error {
// VeleroBackupExcludeNamespaces uses the veleroCLI to backup a namespace.
func VeleroBackupExcludeNamespaces(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, excludeNamespaces []string) error {
namespaces := strings.Join(excludeNamespaces, ",")
backupCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "create", "backup", backupName,
args := []string{
"--namespace", veleroNamespace, "create", "backup", backupName,
"--exclude-namespaces", namespaces,
"--default-volumes-to-restic", "--wait")
backupCmd.Stdout = os.Stdout
backupCmd.Stderr = os.Stderr
fmt.Printf("backup cmd =%v\n", backupCmd)
err := backupCmd.Run()
if err != nil {
return err
"--default-volumes-to-restic", "--wait",
}
err = checkBackupPhase(ctx, veleroCLI, veleroNamespace, backupName, velerov1api.BackupPhaseCompleted)
return err
return VeleroBackupExec(ctx, veleroCLI, veleroNamespace, backupName, args)
}
// veleroRestore uses the veleroCLI to restore from a Velero backup.
func veleroRestore(ctx context.Context, veleroCLI string, veleroNamespace string, restoreName string, backupName string) error {
restoreCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "create", "restore", restoreName,
"--from-backup", backupName, "--wait")
// VeleroBackupIncludeNamespaces uses the veleroCLI to backup a namespace.
func VeleroBackupIncludeNamespaces(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, includeNamespaces []string) error {
namespaces := strings.Join(includeNamespaces, ",")
args := []string{
"--namespace", veleroNamespace, "create", "backup", backupName,
"--include-namespaces", namespaces,
"--default-volumes-to-restic", "--wait",
}
return VeleroBackupExec(ctx, veleroCLI, veleroNamespace, backupName, args)
}
restoreCmd.Stdout = os.Stdout
restoreCmd.Stderr = os.Stderr
fmt.Printf("restore cmd =%v\n", restoreCmd)
err := restoreCmd.Run()
if err != nil {
// VeleroRestore uses the VeleroCLI to restore from a Velero backup.
func VeleroRestore(ctx context.Context, veleroCLI string, veleroNamespace string, restoreName string, backupName string) error {
args := []string{
"--namespace", veleroNamespace, "create", "restore", restoreName,
"--from-backup", backupName, "--wait",
}
return VeleroRestoreExec(ctx, veleroCLI, veleroNamespace, restoreName, args)
}
func VeleroRestoreExec(ctx context.Context, veleroCLI, veleroNamespace, restoreName string, args []string) error {
if err := VeleroCmdExec(ctx, veleroCLI, args); err != nil {
return err
}
return checkRestorePhase(ctx, veleroCLI, veleroNamespace, restoreName, velerov1api.RestorePhaseCompleted)
}
func veleroBackupLogs(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string) error {
describeCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "backup", "describe", backupName)
describeCmd.Stdout = os.Stdout
describeCmd.Stderr = os.Stderr
err := describeCmd.Run()
if err != nil {
func VeleroBackupExec(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, args []string) error {
if err := VeleroCmdExec(ctx, veleroCLI, args); err != nil {
return err
}
logCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "backup", "logs", backupName)
logCmd.Stdout = os.Stdout
logCmd.Stderr = os.Stderr
err = logCmd.Run()
if err != nil {
return err
}
return nil
return checkBackupPhase(ctx, veleroCLI, veleroNamespace, backupName, velerov1api.BackupPhaseCompleted)
}
func runDebug(ctx context.Context, veleroCLI, veleroNamespace, backup, restore string) {
func VeleroCmdExec(ctx context.Context, veleroCLI string, args []string) error {
cmd := exec.CommandContext(ctx, veleroCLI, args...)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
fmt.Printf("velero cmd =%v\n", cmd)
err := cmd.Run()
if err != nil {
return err
}
return err
}
func VeleroBackupLogs(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string) error {
args := []string{
"--namespace", veleroNamespace, "backup", "describe", backupName,
}
if err := VeleroCmdExec(ctx, veleroCLI, args); err != nil {
return err
}
args = []string{
"--namespace", veleroNamespace, "backup", "logs", backupName,
}
return VeleroCmdExec(ctx, veleroCLI, args)
}
func RunDebug(ctx context.Context, veleroCLI, veleroNamespace, backup, restore string) {
output := fmt.Sprintf("debug-bundle-%d.tar.gz", time.Now().UnixNano())
args := []string{"debug", "--namespace", veleroNamespace, "--output", output, "--verbose"}
if len(backup) > 0 {
@ -319,26 +328,21 @@ func runDebug(ctx context.Context, veleroCLI, veleroNamespace, backup, restore s
if len(restore) > 0 {
args = append(args, "--restore", restore)
}
cmd := exec.CommandContext(ctx, veleroCLI, args...)
cmd.Stdout = os.Stdout
cmd.Stdin = os.Stdin
fmt.Printf("debug cmd=%s\n", cmd.String())
fmt.Printf("Generating the debug tarball at %s\n", output)
if err := cmd.Run(); err != nil {
if err := VeleroCmdExec(ctx, veleroCLI, args); err != nil {
fmt.Println(errors.Wrapf(err, "failed to run the debug command"))
}
}
func veleroCreateBackupLocation(ctx context.Context,
veleroCLI string,
veleroNamespace string,
name string,
objectStoreProvider string,
bucket string,
prefix string,
config string,
secretName string,
func VeleroCreateBackupLocation(ctx context.Context,
veleroCLI,
veleroNamespace,
name,
objectStoreProvider,
bucket,
prefix,
config,
secretName,
secretKey string,
) error {
args := []string{
@ -359,12 +363,7 @@ func veleroCreateBackupLocation(ctx context.Context,
if secretName != "" && secretKey != "" {
args = append(args, "--credential", fmt.Sprintf("%s=%s", secretName, secretKey))
}
bslCreateCmd := exec.CommandContext(ctx, veleroCLI, args...)
bslCreateCmd.Stdout = os.Stdout
bslCreateCmd.Stderr = os.Stderr
return bslCreateCmd.Run()
return VeleroCmdExec(ctx, veleroCLI, args)
}
func getProviderPlugins(ctx context.Context, veleroCLI, objectStoreProvider, providerPlugins string) ([]string, error) {
@ -385,9 +384,9 @@ func getProviderPlugins(ctx context.Context, veleroCLI, objectStoreProvider, pro
return plugins, nil
}
// veleroAddPluginsForProvider determines which plugins need to be installed for a provider and
// VeleroAddPluginsForProvider determines which plugins need to be installed for a provider and
// installs them in the current Velero installation, skipping over those that are already installed.
func veleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNamespace string, provider string, addPlugins string) error {
func VeleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNamespace string, provider string, addPlugins string) error {
plugins, err := getProviderPlugins(ctx, veleroCLI, provider, addPlugins)
if err != nil {
return errors.WithMessage(err, "Failed to get plugins")
@ -417,9 +416,9 @@ func veleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNa
return nil
}
// waitForVSphereUploadCompletion waits for uploads started by the Velero Plug-in for vSphere to complete
// WaitForVSphereUploadCompletion waits for uploads started by the Velero Plug-in for vSphere to complete
// TODO - remove after upload progress monitoring is implemented
func waitForVSphereUploadCompletion(ctx context.Context, timeout time.Duration, namespace string) error {
func WaitForVSphereUploadCompletion(ctx context.Context, timeout time.Duration, namespace string) error {
err := wait.PollImmediate(time.Minute, timeout, func() (bool, error) {
checkSnapshotCmd := exec.CommandContext(ctx, "kubectl",
"get", "-n", namespace, "snapshots.backupdriver.cnsdp.vmware.com", "-o=jsonpath='{range .items[*]}{.spec.resourceHandle.name}{\"=\"}{.status.phase}{\"\\n\"}{end}'")
@ -447,6 +446,7 @@ func waitForVSphereUploadCompletion(ctx context.Context, timeout time.Duration,
// status will move to Canceling. The snapshot ID will be removed from the status status if has been filled in
// and the snapshot ID will not longer be valid for a Clone operation
// Canceled - the operation was canceled, the snapshot ID is not valid
// Canceled - the operation was canceled, the snapshot ID is not valid
if len(comps) == 2 {
phase := comps[1]
switch phase {
@ -497,7 +497,7 @@ func getVeleroVersion(ctx context.Context, veleroCLI string, clientOnly bool) (s
return versionMatches[1], nil
}
func checkVeleroVersion(ctx context.Context, veleroCLI string, expectedVer string) error {
func CheckVeleroVersion(ctx context.Context, veleroCLI string, expectedVer string) error {
tag := expectedVer
tagInstalled, err := getVeleroVersion(ctx, veleroCLI, false)
if err != nil {
@ -510,7 +510,7 @@ func checkVeleroVersion(ctx context.Context, veleroCLI string, expectedVer strin
return nil
}
func installVeleroCLI(version string) (string, error) {
func InstallVeleroCLI(version string) (string, error) {
name := "velero-" + version + "-" + runtime.GOOS + "-" + runtime.GOARCH
postfix := ".tar.gz"
tarball := name + postfix