vmware-tanzu
/
velero
mirror of https://github.com/vmware-tanzu/velero.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #4789 from danfengliu/add-csi-plugin-in-e2e-test 

Add csi plugin in e2e test
pull/4810/head
Xun Jiang/Bruce Jiang 2022-04-04 09:56:12 +08:00 committed by GitHub
parent 7c40949992 658699636d
commit 4f358af902
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 165 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/e2e-test-kind.yaml vendored
View File

@ -121,7 +121,7 @@ jobs:
CREDS_FILE=/tmp/credential BSL_BUCKET=bucket \
ADDITIONAL_OBJECT_STORE_PROVIDER=aws ADDITIONAL_BSL_CONFIG=region=minio,s3ForcePathStyle="true",s3Url=http://$(hostname -i):9000 \
ADDITIONAL_CREDS_FILE=/tmp/credential ADDITIONAL_BSL_BUCKET=additional-bucket \
GINKGO_FOCUS=Basic VELERO_IMAGE=velero:pr-test \
GINKGO_FOCUS='Basic\].+\[ClusterResource' VELERO_IMAGE=velero:pr-test \
make -C test/e2e run
timeout-minutes: 30
- name: Upload debug bundle

6
test/e2e/Makefile
View File

@ -72,7 +72,8 @@ CLOUD_PROVIDER ?=
OBJECT_STORE_PROVIDER ?=
INSTALL_VELERO ?= true
REGISTRY_CREDENTIAL_FILE ?=
KIBISHII_DIRECTORY ?=
KIBISHII_DIRECTORY ?= github.com/vmware-tanzu-experiments/distributed-data-generator/kubernetes/yaml/
# Flags to create an additional BSL for multiple credentials tests
ADDITIONAL_BSL_PLUGINS ?=
@ -82,6 +83,8 @@ ADDITIONAL_BSL_BUCKET ?=
ADDITIONAL_BSL_PREFIX ?=
ADDITIONAL_BSL_CONFIG ?=
FEATURES ?=
.PHONY:ginkgo
ginkgo: # Make sure ginkgo is in $GOPATH/bin
go get github.com/onsi/ginkgo/ginkgo
@ -115,6 +118,7 @@ run: ginkgo
-additional-bsl-bucket=$(ADDITIONAL_BSL_BUCKET) \
-additional-bsl-prefix=$(ADDITIONAL_BSL_PREFIX) \
-additional-bsl-config=$(ADDITIONAL_BSL_CONFIG) \
-features=$(FEATURES) \
-install-velero=$(INSTALL_VELERO) \
-registry-credential-file=$(REGISTRY_CREDENTIAL_FILE) \
-kibishii-directory=$(KIBISHII_DIRECTORY)

15
test/e2e/backup/backup.go
View File

@ -42,7 +42,7 @@ func BackupRestoreTest(useVolumeSnapshots bool) {
var (
backupName, restoreName string
)
kibishiiNamespace := "kibishii-workload"
client, err := NewTestClient()
Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests")
@ -55,11 +55,14 @@ func BackupRestoreTest(useVolumeSnapshots bool) {
UUIDgen, err = uuid.NewRandom()
Expect(err).To(Succeed())
if VeleroCfg.InstallVelero {
Expect(VeleroInstall(context.Background(), &VeleroCfg, "", useVolumeSnapshots)).To(Succeed())
Expect(VeleroInstall(context.Background(), &VeleroCfg, useVolumeSnapshots)).To(Succeed())
}
})
AfterEach(func() {
DeleteNamespace(context.Background(), client, kibishiiNamespace, true)
Expect(err).To(Succeed(), fmt.Sprintf("failed to delete the namespace %q", kibishiiNamespace))
if VeleroCfg.InstallVelero {
err = VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)
Expect(err).To(Succeed())
@ -72,7 +75,7 @@ func BackupRestoreTest(useVolumeSnapshots bool) {
restoreName = "restore-" + UUIDgen.String()
// Even though we are using Velero's CloudProvider plugin for object storage, the kubernetes cluster is running on
// KinD. So use the kind installation for Kibishii.
Expect(RunKibishiiTests(client, VeleroCfg.CloudProvider, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backupName, restoreName, "", useVolumeSnapshots, VeleroCfg.RegistryCredentialFile, VeleroCfg.KibishiiDirectory)).To(Succeed(),
Expect(RunKibishiiTests(client, VeleroCfg, backupName, restoreName, "", kibishiiNamespace, useVolumeSnapshots)).To(Succeed(),
"Failed to successfully backup and restore Kibishii namespace")
})
@ -89,7 +92,9 @@ func BackupRestoreTest(useVolumeSnapshots bool) {
Skip("no additional BSL credentials given, not running multiple BackupStorageLocation with unique credentials tests")
}
Expect(VeleroAddPluginsForProvider(context.TODO(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, VeleroCfg.AdditionalBSLProvider, VeleroCfg.AddBSLPlugins)).To(Succeed())
Expect(VeleroAddPluginsForProvider(context.TODO(), VeleroCfg.VeleroCLI,
VeleroCfg.VeleroNamespace, VeleroCfg.AdditionalBSLProvider,
VeleroCfg.AddBSLPlugins, VeleroCfg.Features)).To(Succeed())
// Create Secret for additional BSL
secretName := fmt.Sprintf("bsl-credentials-%s", UUIDgen)
@ -125,7 +130,7 @@ func BackupRestoreTest(useVolumeSnapshots bool) {
backupName = fmt.Sprintf("%s-%s", backupName, UUIDgen)
restoreName = fmt.Sprintf("%s-%s", restoreName, UUIDgen)
}
Expect(RunKibishiiTests(client, VeleroCfg.CloudProvider, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backupName, restoreName, bsl, useVolumeSnapshots, VeleroCfg.RegistryCredentialFile, VeleroCfg.KibishiiDirectory)).To(Succeed(),
Expect(RunKibishiiTests(client, VeleroCfg, backupName, restoreName, bsl, kibishiiNamespace, useVolumeSnapshots)).To(Succeed(),
"Failed to successfully backup and restore Kibishii namespace using BSL %s", bsl)
}
})

18
test/e2e/backups/deletion.go
View File

@ -61,7 +61,7 @@ func backup_deletion_test(useVolumeSnapshots bool) {
UUIDgen, err = uuid.NewRandom()
Expect(err).To(Succeed())
if VeleroCfg.InstallVelero {
Expect(VeleroInstall(context.Background(), &VeleroCfg, "", useVolumeSnapshots)).To(Succeed())
Expect(VeleroInstall(context.Background(), &VeleroCfg, useVolumeSnapshots)).To(Succeed())
}
})
@ -75,16 +75,23 @@ func backup_deletion_test(useVolumeSnapshots bool) {
When("kibishii is the sample workload", func() {
It("Deleted backups are deleted from object storage and backups deleted from object storage can be deleted locally", func() {
backupName = "backup-" + UUIDgen.String()
Expect(runBackupDeletionTests(client, VeleroCfg.VeleroCLI, VeleroCfg.CloudProvider, VeleroCfg.VeleroNamespace, backupName, "", useVolumeSnapshots, VeleroCfg.RegistryCredentialFile, VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, VeleroCfg.KibishiiDirectory)).To(Succeed(),
Expect(runBackupDeletionTests(client, VeleroCfg, backupName, "", useVolumeSnapshots, VeleroCfg.KibishiiDirectory)).To(Succeed(),
"Failed to run backup deletion test")
})
})
}
// runUpgradeTests runs upgrade test on the provider by kibishii.
func runBackupDeletionTests(client TestClient, veleroCLI, providerName, veleroNamespace, backupName, backupLocation string,
useVolumeSnapshots bool, registryCredentialFile, bslPrefix, bslConfig, kibishiiDirectory string) error {
func runBackupDeletionTests(client TestClient, veleroCfg VerleroConfig, backupName, backupLocation string,
useVolumeSnapshots bool, kibishiiDirectory string) error {
oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60)
veleroCLI := VeleroCfg.VeleroCLI
providerName := VeleroCfg.CloudProvider
veleroNamespace := VeleroCfg.VeleroNamespace
registryCredentialFile := VeleroCfg.RegistryCredentialFile
bslPrefix := VeleroCfg.BSLPrefix
bslConfig := VeleroCfg.BSLConfig
veleroFeatures := VeleroCfg.Features
if err := CreateNamespace(oneHourTimeout, client, deletionTest); err != nil {
return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", deletionTest)
@ -95,7 +102,8 @@ func runBackupDeletionTests(client TestClient, veleroCLI, providerName, veleroNa
}
}()
if err := KibishiiPrepareBeforeBackup(oneHourTimeout, client, providerName, deletionTest, registryCredentialFile, kibishiiDirectory); err != nil {
if err := KibishiiPrepareBeforeBackup(oneHourTimeout, client, providerName, deletionTest,
registryCredentialFile, veleroFeatures, kibishiiDirectory, useVolumeSnapshots); err != nil {
return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", deletionTest)
}
err := ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, backupName, BackupObjectsPrefix, 1)

4
test/e2e/backups/sync_backups.go
View File

@ -61,7 +61,7 @@ func BackupsSyncTest() {
BeforeEach(func() {
flag.Parse()
if VeleroCfg.InstallVelero {
Expect(VeleroInstall(context.Background(), &VeleroCfg, "", false)).To(Succeed())
Expect(VeleroInstall(context.Background(), &VeleroCfg, false)).To(Succeed())
}
})
@ -94,7 +94,7 @@ func BackupsSyncTest() {
By("Install velero", func() {
VeleroCfg.ObjectStoreProvider = ""
Expect(VeleroInstall(test.ctx, &VeleroCfg, "", false)).To(Succeed())
Expect(VeleroInstall(test.ctx, &VeleroCfg, false)).To(Succeed())
})
By("Check all backups in object storage are synced to Velero", func() {

3
test/e2e/basic/enable_api_group_versions.go
View File

@ -59,7 +59,8 @@ func APIGropuVersionsTest() {
// TODO: install Velero once for the test suite once feature flag is
// removed and velero installation becomes the same as other e2e tests.
if VeleroCfg.InstallVelero {
err = VeleroInstall(context.Background(), &VeleroCfg, "EnableAPIGroupVersions", false)
VeleroCfg.Features = "EnableAPIGroupVersions"
err = VeleroInstall(context.Background(), &VeleroCfg, false)
Expect(err).NotTo(HaveOccurred())
}
})

9
test/e2e/bsl-mgmt/deletion.go
View File

@ -64,7 +64,7 @@ func BslDeletionTest(useVolumeSnapshots bool) {
UUIDgen, err = uuid.NewRandom()
Expect(err).To(Succeed())
if VeleroCfg.InstallVelero {
Expect(VeleroInstall(context.Background(), &VeleroCfg, "", useVolumeSnapshots)).To(Succeed())
Expect(VeleroInstall(context.Background(), &VeleroCfg, useVolumeSnapshots)).To(Succeed())
}
})
@ -93,7 +93,9 @@ func BslDeletionTest(useVolumeSnapshots bool) {
}
By(fmt.Sprintf("Add an additional plugin for provider %s", VeleroCfg.AdditionalBSLProvider), func() {
Expect(VeleroAddPluginsForProvider(context.TODO(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, VeleroCfg.AdditionalBSLProvider, VeleroCfg.AddBSLPlugins)).To(Succeed())
Expect(VeleroAddPluginsForProvider(context.TODO(), VeleroCfg.VeleroCLI,
VeleroCfg.VeleroNamespace, VeleroCfg.AdditionalBSLProvider,
VeleroCfg.AddBSLPlugins, VeleroCfg.Features)).To(Succeed())
})
additionalBsl := fmt.Sprintf("bsl-%s", UUIDgen)
@ -140,7 +142,8 @@ func BslDeletionTest(useVolumeSnapshots bool) {
By("Deploy sample workload of Kibishii", func() {
Expect(KibishiiPrepareBeforeBackup(oneHourTimeout, client, VeleroCfg.CloudProvider,
bslDeletionTestNs, VeleroCfg.RegistryCredentialFile, VeleroCfg.KibishiiDirectory)).To(Succeed())
bslDeletionTestNs, VeleroCfg.RegistryCredentialFile, VeleroCfg.Features,
VeleroCfg.KibishiiDirectory, useVolumeSnapshots)).To(Succeed())
})
// Restic can not backup PV only, so pod need to be labeled also

9
test/e2e/e2e_suite_test.go
View File

@ -56,23 +56,24 @@ func init() {
flag.BoolVar(&VeleroCfg.InstallVelero, "install-velero", true, "install/uninstall velero during the test. Optional.")
flag.StringVar(&VeleroCfg.RegistryCredentialFile, "registry-credential-file", "", "file containing credential for the image registry, follows the same format rules as the ~/.docker/config.json file. Optional.")
flag.StringVar(&VeleroCfg.KibishiiDirectory, "kibishii-directory", "github.com/vmware-tanzu-experiments/distributed-data-generator/kubernetes/yaml/", "The file directory or URL path to install Kibishii. Optional.")
//vmware-tanzu-experiments
// Flags to create an additional BSL for multiple credentials test
flag.StringVar(&VeleroCfg.AdditionalBSLProvider, "additional-bsl-object-store-provider", "", "Provider of object store plugin for additional backup storage location. Required if testing multiple credentials support.")
flag.StringVar(&VeleroCfg.AdditionalBSLBucket, "additional-bsl-bucket", "", "name of the object storage bucket for additional backup storage location. Required if testing multiple credentials support.")
flag.StringVar(&VeleroCfg.AdditionalBSLPrefix, "additional-bsl-prefix", "", "prefix under which all Velero data should be stored within the bucket for additional backup storage location. Optional.")
flag.StringVar(&VeleroCfg.AdditionalBSLConfig, "additional-bsl-config", "", "configuration to use for the additional backup storage location. Format is key1=value1,key2=value2")
flag.StringVar(&VeleroCfg.AdditionalBSLCredentials, "additional-bsl-credentials-file", "", "file containing credentials for additional backup storage location provider. Required if testing multiple credentials support.")
flag.StringVar(&VeleroCfg.Features, "features", "", "Comma-separated list of features to enable for this Velero process.")
}
var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", APIGropuVersionsTest)
// Test backup and restore of Kibishi using restic
var _ = Describe("[Restic] Velero tests on cluster using the plugin provider for object storage and Restic for volume backups", BackupRestoreWithRestic)
var _ = Describe("[Basic][Restic] Velero tests on cluster using the plugin provider for object storage and Restic for volume backups", BackupRestoreWithRestic)
var _ = Describe("[Snapshot] Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupRestoreWithSnapshots)
var _ = Describe("[Basic][Snapshot] Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupRestoreWithSnapshots)
var _ = Describe("[Basic] Backup/restore of cluster resources", ResourcesCheckTest)
var _ = Describe("[Basic][ClusterResource] Backup/restore of cluster resources", ResourcesCheckTest)
var _ = Describe("[Scale] Backup/restore of 2500 namespaces", MultiNSBackupRestore)

2
test/e2e/privilegesmgmt/ssr.go
View File

@ -45,7 +45,7 @@ func SSRTest() {
BeforeEach(func() {
flag.Parse()
if VeleroCfg.InstallVelero {
Expect(VeleroInstall(context.Background(), &VeleroCfg, "", false)).To(Succeed())
Expect(VeleroInstall(context.Background(), &VeleroCfg, false)).To(Succeed())
}
})

4
test/e2e/test/test.go
View File

@ -79,7 +79,7 @@ func TestFunc(test VeleroBackupRestoreTest) func() {
BeforeEach(func() {
flag.Parse()
if VeleroCfg.InstallVelero {
Expect(VeleroInstall(context.Background(), &VeleroCfg, "", false)).To(Succeed())
Expect(VeleroInstall(context.Background(), &VeleroCfg, false)).To(Succeed())
}
})
AfterEach(func() {
@ -107,7 +107,7 @@ func TestFuncWithMultiIt(tests []VeleroBackupRestoreTest) func() {
flag.Parse()
if VeleroCfg.InstallVelero {
if countIt == 0 {
Expect(VeleroInstall(context.Background(), &VeleroCfg, "", false)).To(Succeed())
Expect(VeleroInstall(context.Background(), &VeleroCfg, false)).To(Succeed())
}
countIt++
}

1
test/e2e/types.go
View File

@ -49,6 +49,7 @@ type VerleroConfig struct {
AddBSLPlugins string
InstallVelero bool
KibishiiDirectory string
Features string
}
type SnapshotCheckPoint struct {

42
test/e2e/upgrade/upgrade.go
View File

@ -118,6 +118,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, upgradeFromVelero Upgrade
Expect(err).To(Succeed())
})
}
By(fmt.Sprintf("Install the expected old version Velero (%s) for upgrade",
upgradeFromVelero.UpgradeFromVeleroVersion), func() {
//Set VeleroImage and ResticHelperImage to blank
@ -129,17 +130,17 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, upgradeFromVelero Upgrade
tmpCfgForOldVeleroInstall.ResticHelperImage = ""
tmpCfgForOldVeleroInstall.Plugins = ""
Expect(VeleroInstall(context.Background(), &tmpCfgForOldVeleroInstall, "",
false)).To(Succeed())
Expect(VeleroInstall(context.Background(), &tmpCfgForOldVeleroInstall,
useVolumeSnapshots)).To(Succeed())
Expect(CheckVeleroVersion(context.Background(), tmpCfgForOldVeleroInstall.VeleroCLI,
tmpCfgForOldVeleroInstall.UpgradeFromVeleroVersion)).To(Succeed())
})
backupName = "backup-" + UUIDgen.String()
restoreName = "restore-" + UUIDgen.String()
tmpCfg1 := VeleroCfg
tmpCfg1.UpgradeFromVeleroCLI = upgradeFromVelero.UpgradeFromVeleroCLI
tmpCfg1.UpgradeFromVeleroVersion = upgradeFromVelero.UpgradeFromVeleroVersion
tmpCfg := VeleroCfg
tmpCfg.UpgradeFromVeleroCLI = upgradeFromVelero.UpgradeFromVeleroCLI
tmpCfg.UpgradeFromVeleroVersion = upgradeFromVelero.UpgradeFromVeleroVersion
oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60)
By("Create namespace for sample workload", func() {
@ -148,16 +149,17 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, upgradeFromVelero Upgrade
})
By("Deploy sample workload of Kibishii", func() {
Expect(KibishiiPrepareBeforeBackup(oneHourTimeout, client, tmpCfg1.CloudProvider,
upgradeNamespace, tmpCfg1.RegistryCredentialFile, tmpCfg1.KibishiiDirectory)).To(Succeed())
Expect(KibishiiPrepareBeforeBackup(oneHourTimeout, client, tmpCfg.CloudProvider,
upgradeNamespace, tmpCfg.RegistryCredentialFile, tmpCfg.Features,
tmpCfg.KibishiiDirectory, useVolumeSnapshots)).To(Succeed())
})
By(fmt.Sprintf("Backup namespace %s", upgradeNamespace), func() {
Expect(VeleroBackupNamespace(oneHourTimeout, tmpCfg1.UpgradeFromVeleroCLI,
tmpCfg1.VeleroNamespace, backupName, upgradeNamespace, "",
Expect(VeleroBackupNamespace(oneHourTimeout, tmpCfg.UpgradeFromVeleroCLI,
tmpCfg.VeleroNamespace, backupName, upgradeNamespace, "",
useVolumeSnapshots, "")).ShouldNot(HaveOccurred(), func() string {
err = VeleroBackupLogs(context.Background(), tmpCfg1.UpgradeFromVeleroCLI,
tmpCfg1.VeleroNamespace, backupName)
err = VeleroBackupLogs(context.Background(), tmpCfg.UpgradeFromVeleroCLI,
tmpCfg.VeleroNamespace, backupName)
return "Get backup logs"
})
})
@ -178,22 +180,22 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, upgradeFromVelero Upgrade
// the snapshots of AWS may be still in pending status when do the restore, wait for a while
// to avoid this https://github.com/vmware-tanzu/velero/issues/1799
// TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed
if tmpCfg1.CloudProvider == "aws" && useVolumeSnapshots {
if tmpCfg.CloudProvider == "aws" && useVolumeSnapshots {
fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...")
time.Sleep(5 * time.Minute)
}
By(fmt.Sprintf("Upgrade Velero by CLI %s", tmpCfg1.VeleroCLI), func() {
Expect(VeleroInstall(context.Background(), &tmpCfg1, "", useVolumeSnapshots)).To(Succeed())
Expect(CheckVeleroVersion(context.Background(), tmpCfg1.VeleroCLI,
tmpCfg1.VeleroVersion)).To(Succeed())
By(fmt.Sprintf("Upgrade Velero by CLI %s", tmpCfg.VeleroCLI), func() {
Expect(VeleroInstall(context.Background(), &tmpCfg, useVolumeSnapshots)).To(Succeed())
Expect(CheckVeleroVersion(context.Background(), tmpCfg.VeleroCLI,
tmpCfg.VeleroVersion)).To(Succeed())
})
By(fmt.Sprintf("Restore %s", upgradeNamespace), func() {
Expect(VeleroRestore(oneHourTimeout, tmpCfg1.VeleroCLI,
tmpCfg1.VeleroNamespace, restoreName, backupName)).To(Succeed(), func() string {
RunDebug(context.Background(), tmpCfg1.VeleroCLI,
tmpCfg1.VeleroNamespace, "", restoreName)
Expect(VeleroRestore(oneHourTimeout, tmpCfg.VeleroCLI,
tmpCfg.VeleroNamespace, restoreName, backupName)).To(Succeed(), func() string {
RunDebug(context.Background(), tmpCfg.VeleroCLI,
tmpCfg.VeleroNamespace, "", restoreName)
return "Fail to restore workload"
})
})

8
test/e2e/util/csi/AzureVolumeSnapshotClass.yaml Normal file
View File

@ -0,0 +1,8 @@
apiVersion: snapshot.storage.k8s.io/v1
kind: VolumeSnapshotClass
metadata:
name: velero
labels:
velero.io/csi-volumesnapshot-class: "true"
driver: disk.csi.azure.com
deletionPolicy: Delete

5
test/e2e/util/k8s/common.go
View File

@ -136,3 +136,8 @@ func AddLabelToPod(ctx context.Context, podName, namespace, label string) error
fmt.Println(args)
return exec.CommandContext(ctx, "kubectl", args...).Run()
}
func KubectlApplyByFile(ctx context.Context, file string) error {
args := []string{"apply", "-f", file, "--force=true"}
return exec.CommandContext(ctx, "kubectl", args...).Run()
}

46
test/e2e/util/kibishii/kibishii_utils.go
View File

@ -20,38 +20,45 @@ import (
"fmt"
"os/exec"
"strconv"
"strings"
"time"
"github.com/pkg/errors"
"golang.org/x/net/context"
veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec"
. "github.com/vmware-tanzu/velero/test/e2e"
. "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
. "github.com/vmware-tanzu/velero/test/e2e/util/velero"
)
const (
kibishiiNamespace = "kibishii-workload"
jumpPadPod = "jump-pad"
jumpPadPod = "jump-pad"
)
// RunKibishiiTests runs kibishii tests on the provider.
func RunKibishiiTests(client TestClient, providerName, veleroCLI, veleroNamespace, backupName, restoreName, backupLocation string,
useVolumeSnapshots bool, registryCredentialFile string, kibishiiDirectory string) error {
func RunKibishiiTests(client TestClient, veleroCfg VerleroConfig, backupName, restoreName, backupLocation, kibishiiNamespace string,
useVolumeSnapshots bool) error {
oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60)
veleroCLI := VeleroCfg.VeleroCLI
providerName := VeleroCfg.CloudProvider
veleroNamespace := VeleroCfg.VeleroNamespace
registryCredentialFile := VeleroCfg.RegistryCredentialFile
veleroFeatures := VeleroCfg.Features
kibishiiDirectory := VeleroCfg.KibishiiDirectory
if err := CreateNamespace(oneHourTimeout, client, kibishiiNamespace); err != nil {
return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", kibishiiNamespace)
}
defer func() {
if err := DeleteNamespace(context.Background(), client, kibishiiNamespace, true); err != nil {
fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", kibishiiNamespace))
}
}()
if err := KibishiiPrepareBeforeBackup(oneHourTimeout, client, providerName, kibishiiNamespace, registryCredentialFile, kibishiiDirectory); err != nil {
if err := KibishiiPrepareBeforeBackup(oneHourTimeout, client, providerName,
kibishiiNamespace, registryCredentialFile, veleroFeatures,
kibishiiDirectory, useVolumeSnapshots); err != nil {
return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", kibishiiNamespace)
}
if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, kibishiiNamespace, backupLocation, useVolumeSnapshots, ""); err != nil {
if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName,
kibishiiNamespace, backupLocation, useVolumeSnapshots, ""); err != nil {
RunDebug(context.Background(), veleroCLI, veleroNamespace, backupName, "")
return errors.Wrapf(err, "Failed to backup kibishii namespace %s", kibishiiNamespace)
}
@ -90,11 +97,19 @@ func RunKibishiiTests(client TestClient, providerName, veleroCLI, veleroNamespac
return nil
}
func installKibishii(ctx context.Context, namespace string, cloudPlatform string, kibishiiDirectory string) error {
func installKibishii(ctx context.Context, namespace string, cloudPlatform, veleroFeatures,
kibishiiDirectory string, useVolumeSnapshots bool) error {
if strings.EqualFold(cloudPlatform, "azure") &&
strings.EqualFold(veleroFeatures, "EnableCSI") &&
useVolumeSnapshots {
cloudPlatform = "azure-csi"
}
// We use kustomize to generate YAML for Kibishii from the checked-in yaml directories
kibishiiInstallCmd := exec.CommandContext(ctx, "kubectl", "apply", "-n", namespace, "-k",
kibishiiDirectory+cloudPlatform)
_, stderr, err := veleroexec.RunCommand(kibishiiInstallCmd)
fmt.Printf("Install Kibishii cmd: %s\n", kibishiiInstallCmd)
if err != nil {
return errors.Wrapf(err, "failed to install kibishii, stderr=%s", stderr)
}
@ -149,7 +164,9 @@ func waitForKibishiiPods(ctx context.Context, client TestClient, kibishiiNamespa
return WaitForPods(ctx, client, kibishiiNamespace, []string{"jump-pad", "etcd0", "etcd1", "etcd2", "kibishii-deployment-0", "kibishii-deployment-1"})
}
func KibishiiPrepareBeforeBackup(oneHourTimeout context.Context, client TestClient, providerName, kibishiiNamespace, registryCredentialFile, kibishiiDirectory string) error {
func KibishiiPrepareBeforeBackup(oneHourTimeout context.Context, client TestClient,
providerName, kibishiiNamespace, registryCredentialFile, veleroFeatures,
kibishiiDirectory string, useVolumeSnapshots bool) error {
serviceAccountName := "default"
// wait until the service account is created before patch the image pull secret
@ -161,7 +178,8 @@ func KibishiiPrepareBeforeBackup(oneHourTimeout context.Context, client TestClie
return errors.Wrapf(err, "failed to patch the service account %q under the namespace %q", serviceAccountName, kibishiiNamespace)
}
if err := installKibishii(oneHourTimeout, kibishiiNamespace, providerName, kibishiiDirectory); err != nil {
if err := installKibishii(oneHourTimeout, kibishiiNamespace, providerName, veleroFeatures,
kibishiiDirectory, useVolumeSnapshots); err != nil {
return errors.Wrap(err, "Failed to install Kibishii workload")
}

15
test/e2e/util/velero/install.go
View File

@ -24,6 +24,7 @@ import (
"io/ioutil"
"os"
"os/exec"
"strings"
"time"
"github.com/pkg/errors"
@ -48,7 +49,7 @@ type installOptions struct {
ResticHelperImage string
}
func VeleroInstall(ctx context.Context, veleroCfg *VerleroConfig, features string, useVolumeSnapshots bool) error {
func VeleroInstall(ctx context.Context, veleroCfg *VerleroConfig, useVolumeSnapshots bool) error {
if veleroCfg.CloudProvider != "kind" {
if veleroCfg.ObjectStoreProvider != "" {
return errors.New("For cloud platforms, object store plugin cannot be overridden") // Can't set an object store provider that is different than your cloud
@ -60,7 +61,7 @@ func VeleroInstall(ctx context.Context, veleroCfg *VerleroConfig, features strin
}
}
providerPluginsTmp, err := getProviderPlugins(ctx, veleroCfg.VeleroCLI, veleroCfg.ObjectStoreProvider, veleroCfg.Plugins)
providerPluginsTmp, err := getProviderPlugins(ctx, veleroCfg.VeleroCLI, veleroCfg.ObjectStoreProvider, veleroCfg.Plugins, veleroCfg.Features)
if err != nil {
return errors.WithMessage(err, "Failed to get provider plugins")
}
@ -81,7 +82,7 @@ func VeleroInstall(ctx context.Context, veleroCfg *VerleroConfig, features strin
}
veleroInstallOptions, err := getProviderVeleroInstallOptions(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,
veleroCfg.BSLPrefix, veleroCfg.BSLConfig, veleroCfg.VSLConfig, providerPluginsTmp, features)
veleroCfg.BSLPrefix, veleroCfg.BSLConfig, veleroCfg.VSLConfig, providerPluginsTmp, veleroCfg.Features)
if err != nil {
return errors.WithMessagef(err, "Failed to get Velero InstallOptions for plugin provider %s", veleroCfg.ObjectStoreProvider)
}
@ -199,6 +200,14 @@ func installVeleroServer(ctx context.Context, cli string, options *installOption
}
if len(options.Features) > 0 {
args = append(args, "--features", options.Features)
if strings.EqualFold(options.Features, "EnableCSI") && options.UseVolumeSnapshots {
if strings.EqualFold(options.ProviderName, "Azure") {
if err := KubectlApplyByFile(ctx, "util/csi/AzureVolumeSnapshotClass.yaml"); err != nil {
return err
}
}
}
}
if err := createVelereResources(ctx, cli, namespace, args, options.RegistryCredentialFile, options.ResticHelperImage); err != nil {

67
test/e2e/util/velero/velero_utils.go
View File

@ -48,44 +48,50 @@ const PluginsObjectsPrefix = "plugins"
var pluginsMatrix = map[string]map[string][]string{
"v1.4": {
"aws": {"velero/velero-plugin-for-aws:v1.1.0"},
"azure": {"velero/velero-plugin-for-microsoft-azure:v1.1.2"},
"vsphere": {"velero/velero-plugin-for-aws:v1.1.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.0.2"},
"gcp": {"velero/velero-plugin-for-gcp:v1.1.0"},
"aws": {"velero/velero-plugin-for-aws:v1.1.0"},
"azure": {"velero/velero-plugin-for-microsoft-azure:v1.1.2"},
"vsphere": {"velero/velero-plugin-for-aws:v1.1.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.0.2"},
"gcp": {"velero/velero-plugin-for-gcp:v1.1.0"},
"azure-csi": {"velero/velero-plugin-for-microsoft-azure:v1.1.2", "velero/velero-plugin-for-csi:v0.1.1 "},
},
"v1.5": {
"aws": {"velero/velero-plugin-for-aws:v1.1.0"},
"azure": {"velero/velero-plugin-for-microsoft-azure:v1.1.2"},
"vsphere": {"velero/velero-plugin-for-aws:v1.1.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.1.1"},
"gcp": {"velero/velero-plugin-for-gcp:v1.1.0"},
"aws": {"velero/velero-plugin-for-aws:v1.1.0"},
"azure": {"velero/velero-plugin-for-microsoft-azure:v1.1.2"},
"vsphere": {"velero/velero-plugin-for-aws:v1.1.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.1.1"},
"gcp": {"velero/velero-plugin-for-gcp:v1.1.0"},
"azure-csi": {"velero/velero-plugin-for-microsoft-azure:v1.1.2", "velero/velero-plugin-for-csi:v0.1.2 "},
},
"v1.6": {
"aws": {"velero/velero-plugin-for-aws:v1.2.1"},
"azure": {"velero/velero-plugin-for-microsoft-azure:v1.2.1"},
"vsphere": {"velero/velero-plugin-for-aws:v1.2.1", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.1.1"},
"gcp": {"velero/velero-plugin-for-gcp:v1.2.1"},
"aws": {"velero/velero-plugin-for-aws:v1.2.1"},
"azure": {"velero/velero-plugin-for-microsoft-azure:v1.2.1"},
"vsphere": {"velero/velero-plugin-for-aws:v1.2.1", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.1.1"},
"gcp": {"velero/velero-plugin-for-gcp:v1.2.1"},
"azure-csi": {"velero/velero-plugin-for-microsoft-azure:v1.3.0", "velero/velero-plugin-for-csi:v0.1.2 "},
},
"v1.7": {
"aws": {"velero/velero-plugin-for-aws:v1.3.0"},
"azure": {"velero/velero-plugin-for-microsoft-azure:v1.3.0"},
"vsphere": {"velero/velero-plugin-for-aws:v1.3.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.3.0"},
"gcp": {"velero/velero-plugin-for-gcp:v1.3.0"},
"aws": {"velero/velero-plugin-for-aws:v1.3.0"},
"azure": {"velero/velero-plugin-for-microsoft-azure:v1.3.0"},
"vsphere": {"velero/velero-plugin-for-aws:v1.3.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.3.0"},
"gcp": {"velero/velero-plugin-for-gcp:v1.3.0"},
"azure-csi": {"velero/velero-plugin-for-microsoft-azure:v1.3.0", "velero/velero-plugin-for-csi:v0.2.0"},
},
"v1.8": {
"aws": {"velero/velero-plugin-for-aws:v1.4.0"},
"azure": {"velero/velero-plugin-for-microsoft-azure:v1.4.0"},
"vsphere": {"velero/velero-plugin-for-aws:v1.4.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.3.1"},
"gcp": {"velero/velero-plugin-for-gcp:v1.4.0"},
"aws": {"velero/velero-plugin-for-aws:v1.4.0"},
"azure": {"velero/velero-plugin-for-microsoft-azure:v1.4.0"},
"vsphere": {"velero/velero-plugin-for-aws:v1.4.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.3.1"},
"gcp": {"velero/velero-plugin-for-gcp:v1.4.0"},
"azure-csi": {"velero/velero-plugin-for-microsoft-azure:v1.4.0", "velero/velero-plugin-for-csi:v0.2.0"},
},
"main": {
"aws": {"velero/velero-plugin-for-aws:main"},
"azure": {"velero/velero-plugin-for-microsoft-azure:main"},
"vsphere": {"velero/velero-plugin-for-aws:main", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.3.1"},
"gcp": {"velero/velero-plugin-for-gcp:main"},
"aws": {"velero/velero-plugin-for-aws:main"},
"azure": {"velero/velero-plugin-for-microsoft-azure:main"},
"vsphere": {"velero/velero-plugin-for-aws:main", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.3.1"},
"gcp": {"velero/velero-plugin-for-gcp:main"},
"azure-csi": {"velero/velero-plugin-for-microsoft-azure:main", "velero/velero-plugin-for-csi:main"},
},
}
func getProviderPluginsByVersion(version, providerName string) ([]string, error) {
func GetProviderPluginsByVersion(version, providerName, feature string) ([]string, error) {
var cloudMap map[string][]string
arr := strings.Split(version, ".")
if len(arr) >= 3 {
@ -97,6 +103,9 @@ func getProviderPluginsByVersion(version, providerName string) ([]string, error)
return nil, errors.Errorf("fail to get plugins by version: main")
}
}
if strings.EqualFold(providerName, "azure") && strings.EqualFold(feature, "EnableCSI") {
providerName = "azure-csi"
}
plugins, ok := cloudMap[providerName]
if !ok {
return nil, errors.Errorf("fail to get plugins by version: %s and provider %s", version, providerName)
@ -380,7 +389,7 @@ func VeleroCreateBackupLocation(ctx context.Context,
return VeleroCmdExec(ctx, veleroCLI, args)
}
func getProviderPlugins(ctx context.Context, veleroCLI, objectStoreProvider, providerPlugins string) ([]string, error) {
func getProviderPlugins(ctx context.Context, veleroCLI, objectStoreProvider, providerPlugins, feature string) ([]string, error) {
// Fetch the plugins for the provider before checking for the object store provider below.
var plugins []string
if len(providerPlugins) > 0 {
@ -390,7 +399,7 @@ func getProviderPlugins(ctx context.Context, veleroCLI, objectStoreProvider, pro
if err != nil {
return nil, errors.WithMessage(err, "failed to get velero version")
}
plugins, err = getProviderPluginsByVersion(version, objectStoreProvider)
plugins, err = GetProviderPluginsByVersion(version, objectStoreProvider, feature)
if err != nil {
return nil, errors.WithMessagef(err, "Fail to get plugin by provider %s and version %s", objectStoreProvider, version)
}
@ -400,8 +409,8 @@ func getProviderPlugins(ctx context.Context, veleroCLI, objectStoreProvider, pro
// VeleroAddPluginsForProvider determines which plugins need to be installed for a provider and
// installs them in the current Velero installation, skipping over those that are already installed.
func VeleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNamespace string, provider string, addPlugins string) error {
plugins, err := getProviderPlugins(ctx, veleroCLI, provider, addPlugins)
func VeleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNamespace string, provider string, addPlugins, feature string) error {
plugins, err := getProviderPlugins(ctx, veleroCLI, provider, addPlugins, feature)
fmt.Printf("addPlugins cmd =%v\n", addPlugins)
fmt.Printf("provider cmd = %v\n", provider)
fmt.Printf("plugins cmd = %v\n", plugins)