Add CSI plugin in E2E test

Signed-off-by: danfengl <danfengl@vmware.com>

test/e2e/Makefile

@@ -82,6 +82,8 @@ ADDITIONAL_BSL_BUCKET ?=
 ADDITIONAL_BSL_PREFIX ?=
 ADDITIONAL_BSL_CONFIG ?=
 
+FEATURES ?=
+
 .PHONY:ginkgo
 ginkgo: # Make sure ginkgo is in $GOPATH/bin
 	go get github.com/onsi/ginkgo/ginkgo
@@ -115,6 +117,7 @@ run: ginkgo
 		-additional-bsl-bucket=$(ADDITIONAL_BSL_BUCKET) \
 		-additional-bsl-prefix=$(ADDITIONAL_BSL_PREFIX) \
 		-additional-bsl-config=$(ADDITIONAL_BSL_CONFIG) \
+		-features=$(FEATURES) \
 		-install-velero=$(INSTALL_VELERO) \
 		-registry-credential-file=$(REGISTRY_CREDENTIAL_FILE) \
 		-kibishii-directory=$(KIBISHII_DIRECTORY)

test/e2e/backup/backup.go

@@ -42,7 +42,7 @@ func BackupRestoreTest(useVolumeSnapshots bool) {
 	var (
 		backupName, restoreName string
 	)
-
+	kibishiiNamespace := "kibishii-workload"
 	client, err := NewTestClient()
 	Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests")
 
@@ -55,11 +55,14 @@ func BackupRestoreTest(useVolumeSnapshots bool) {
 		UUIDgen, err = uuid.NewRandom()
 		Expect(err).To(Succeed())
 		if VeleroCfg.InstallVelero {
-			Expect(VeleroInstall(context.Background(), &VeleroCfg, "", useVolumeSnapshots)).To(Succeed())
+			Expect(VeleroInstall(context.Background(), &VeleroCfg, "EnableCSI", useVolumeSnapshots)).To(Succeed())
 		}
 	})
 
 	AfterEach(func() {
+		DeleteNamespace(context.Background(), client, kibishiiNamespace, true)
+		Expect(err).To(Succeed(), fmt.Sprintf("failed to delete the namespace %q", kibishiiNamespace))
+
 		if VeleroCfg.InstallVelero {
 			err = VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)
 			Expect(err).To(Succeed())
@@ -72,11 +75,11 @@ func BackupRestoreTest(useVolumeSnapshots bool) {
 			restoreName = "restore-" + UUIDgen.String()
 			// Even though we are using Velero's CloudProvider plugin for object storage, the kubernetes cluster is running on
 			// KinD. So use the kind installation for Kibishii.
-			Expect(RunKibishiiTests(client, VeleroCfg.CloudProvider, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backupName, restoreName, "", useVolumeSnapshots, VeleroCfg.RegistryCredentialFile, VeleroCfg.KibishiiDirectory)).To(Succeed(),
+			Expect(RunKibishiiTests(client, VeleroCfg, backupName, restoreName, "", kibishiiNamespace, useVolumeSnapshots)).To(Succeed(),
 				"Failed to successfully backup and restore Kibishii namespace")
 		})
 
-		It("should successfully back up and restore to an additional BackupStorageLocation with unique credentials", func() {
+		XIt("should successfully back up and restore to an additional BackupStorageLocation with unique credentials", func() {
 			if VeleroCfg.AdditionalBSLProvider == "" {
 				Skip("no additional BSL provider given, not running multiple BackupStorageLocation with unique credentials tests")
 			}
@@ -125,7 +128,7 @@ func BackupRestoreTest(useVolumeSnapshots bool) {
 					backupName = fmt.Sprintf("%s-%s", backupName, UUIDgen)
 					restoreName = fmt.Sprintf("%s-%s", restoreName, UUIDgen)
 				}
-				Expect(RunKibishiiTests(client, VeleroCfg.CloudProvider, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backupName, restoreName, bsl, useVolumeSnapshots, VeleroCfg.RegistryCredentialFile, VeleroCfg.KibishiiDirectory)).To(Succeed(),
+				Expect(RunKibishiiTests(client, VeleroCfg, backupName, restoreName, bsl, kibishiiNamespace, useVolumeSnapshots)).To(Succeed(),
 					"Failed to successfully backup and restore Kibishii namespace using BSL %s", bsl)
 			}
 		})

test/e2e/backups/deletion.go

@@ -75,16 +75,23 @@ func backup_deletion_test(useVolumeSnapshots bool) {
 	When("kibishii is the sample workload", func() {
 		It("Deleted backups are deleted from object storage and backups deleted from object storage can be deleted locally", func() {
 			backupName = "backup-" + UUIDgen.String()
-			Expect(runBackupDeletionTests(client, VeleroCfg.VeleroCLI, VeleroCfg.CloudProvider, VeleroCfg.VeleroNamespace, backupName, "", useVolumeSnapshots, VeleroCfg.RegistryCredentialFile, VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, VeleroCfg.KibishiiDirectory)).To(Succeed(),
+			Expect(runBackupDeletionTests(client, VeleroCfg, backupName, "", useVolumeSnapshots, VeleroCfg.KibishiiDirectory)).To(Succeed(),
 				"Failed to run backup deletion test")
 		})
 	})
 }
 
 // runUpgradeTests runs upgrade test on the provider by kibishii.
-func runBackupDeletionTests(client TestClient, veleroCLI, providerName, veleroNamespace, backupName, backupLocation string,
-	useVolumeSnapshots bool, registryCredentialFile, bslPrefix, bslConfig, kibishiiDirectory string) error {
+func runBackupDeletionTests(client TestClient, veleroCfg VerleroConfig, backupName, backupLocation string,
+	useVolumeSnapshots bool, kibishiiDirectory string) error {
 	oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60)
+	veleroCLI := VeleroCfg.VeleroCLI
+	providerName := VeleroCfg.CloudProvider
+	veleroNamespace := VeleroCfg.VeleroNamespace
+	registryCredentialFile := VeleroCfg.RegistryCredentialFile
+	bslPrefix := VeleroCfg.BSLPrefix
+	bslConfig := VeleroCfg.BSLConfig
+	veleroFeatures := VeleroCfg.Features
 
 	if err := CreateNamespace(oneHourTimeout, client, deletionTest); err != nil {
 		return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", deletionTest)
@@ -95,7 +102,7 @@ func runBackupDeletionTests(client TestClient, veleroCLI, providerName, veleroNa
 		}
 	}()
 
-	if err := KibishiiPrepareBeforeBackup(oneHourTimeout, client, providerName, deletionTest, registryCredentialFile, kibishiiDirectory); err != nil {
+	if err := KibishiiPrepareBeforeBackup(oneHourTimeout, client, providerName, deletionTest, registryCredentialFile, veleroFeatures, kibishiiDirectory); err != nil {
 		return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", deletionTest)
 	}
 	err := ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, backupName, BackupObjectsPrefix, 1)

test/e2e/bsl-mgmt/deletion.go

@@ -140,7 +140,7 @@ func BslDeletionTest(useVolumeSnapshots bool) {
 
 			By("Deploy sample workload of Kibishii", func() {
 				Expect(KibishiiPrepareBeforeBackup(oneHourTimeout, client, VeleroCfg.CloudProvider,
-					bslDeletionTestNs, VeleroCfg.RegistryCredentialFile, VeleroCfg.KibishiiDirectory)).To(Succeed())
+					bslDeletionTestNs, VeleroCfg.RegistryCredentialFile, VeleroCfg.Features, VeleroCfg.KibishiiDirectory)).To(Succeed())
 			})
 
 			// Restic can not backup PV only, so pod need to be labeled also

test/e2e/e2e_suite_test.go

@@ -63,14 +63,15 @@ func init() {
 	flag.StringVar(&VeleroCfg.AdditionalBSLPrefix, "additional-bsl-prefix", "", "prefix under which all Velero data should be stored within the bucket for additional backup storage location. Optional.")
 	flag.StringVar(&VeleroCfg.AdditionalBSLConfig, "additional-bsl-config", "", "configuration to use for the additional backup storage location. Format is key1=value1,key2=value2")
 	flag.StringVar(&VeleroCfg.AdditionalBSLCredentials, "additional-bsl-credentials-file", "", "file containing credentials for additional backup storage location provider. Required if testing multiple credentials support.")
+	flag.StringVar(&VeleroCfg.Features, "features", "", "Comma-separated list of features to enable for this Velero process.")
 }
 
 var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", APIGropuVersionsTest)
 
 // Test backup and restore of Kibishi using restic
-var _ = Describe("[Restic] Velero tests on cluster using the plugin provider for object storage and Restic for volume backups", BackupRestoreWithRestic)
+var _ = Describe("[Basic][Restic] Velero tests on cluster using the plugin provider for object storage and Restic for volume backups", BackupRestoreWithRestic)
 
-var _ = Describe("[Snapshot] Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupRestoreWithSnapshots)
+var _ = Describe("[Basic][Snapshot] Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupRestoreWithSnapshots)
 
 var _ = Describe("[Basic] Backup/restore of cluster resources", ResourcesCheckTest)
 

test/e2e/types.go

@@ -49,6 +49,7 @@ type VerleroConfig struct {
 	AddBSLPlugins            string
 	InstallVelero            bool
 	KibishiiDirectory        string
+	Features                 string
 }
 
 type SnapshotCheckPoint struct {

test/e2e/upgrade/upgrade.go

@@ -149,7 +149,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, upgradeFromVelero Upgrade
 
 			By("Deploy sample workload of Kibishii", func() {
 				Expect(KibishiiPrepareBeforeBackup(oneHourTimeout, client, tmpCfg1.CloudProvider,
-					upgradeNamespace, tmpCfg1.RegistryCredentialFile, tmpCfg1.KibishiiDirectory)).To(Succeed())
+					upgradeNamespace, tmpCfg1.RegistryCredentialFile, tmpCfg1.Features, tmpCfg1.KibishiiDirectory)).To(Succeed())
 			})
 
 			By(fmt.Sprintf("Backup namespace %s", upgradeNamespace), func() {

test/e2e/util/csi/AzureVolumeSnapshotClass.yaml

@@ -0,0 +1,8 @@
+apiVersion: snapshot.storage.k8s.io/v1
+kind: VolumeSnapshotClass
+metadata:
+  name: velero
+  labels:
+    velero.io/csi-volumesnapshot-class: "true"
+driver: disk.csi.azure.com
+deletionPolicy: Retain

test/e2e/util/k8s/common.go

@@ -136,3 +136,8 @@ func AddLabelToPod(ctx context.Context, podName, namespace, label string) error
 	fmt.Println(args)
 	return exec.CommandContext(ctx, "kubectl", args...).Run()
 }
+
+func KubectlApplyByFile(ctx context.Context, file string) error {
+	args := []string{"apply", "-f", file, "--force=true"}
+	return exec.CommandContext(ctx, "kubectl", args...).Run()
+}

test/e2e/util/kibishii/kibishii_utils.go

@@ -20,38 +20,43 @@ import (
 	"fmt"
 	"os/exec"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/pkg/errors"
-	"golang.org/x/net/context"
-
 	veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec"
+	. "github.com/vmware-tanzu/velero/test/e2e"
 	. "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
 	. "github.com/vmware-tanzu/velero/test/e2e/util/velero"
+	"golang.org/x/net/context"
 )
 
 const (
-	kibishiiNamespace = "kibishii-workload"
-	jumpPadPod        = "jump-pad"
+	jumpPadPod = "jump-pad"
 )
 
 // RunKibishiiTests runs kibishii tests on the provider.
-func RunKibishiiTests(client TestClient, providerName, veleroCLI, veleroNamespace, backupName, restoreName, backupLocation string,
-	useVolumeSnapshots bool, registryCredentialFile string, kibishiiDirectory string) error {
+func RunKibishiiTests(client TestClient, veleroCfg VerleroConfig, backupName, restoreName, backupLocation, kibishiiNamespace string,
+	useVolumeSnapshots bool) error {
 	oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60)
+	veleroCLI := VeleroCfg.VeleroCLI
+	providerName := VeleroCfg.CloudProvider
+	veleroNamespace := VeleroCfg.VeleroNamespace
+	registryCredentialFile := VeleroCfg.RegistryCredentialFile
+	veleroFeatures := VeleroCfg.Features
+	kibishiiDirectory := VeleroCfg.KibishiiDirectory
+
 	if err := CreateNamespace(oneHourTimeout, client, kibishiiNamespace); err != nil {
 		return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", kibishiiNamespace)
 	}
-	defer func() {
-		if err := DeleteNamespace(context.Background(), client, kibishiiNamespace, true); err != nil {
-			fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", kibishiiNamespace))
-		}
-	}()
-	if err := KibishiiPrepareBeforeBackup(oneHourTimeout, client, providerName, kibishiiNamespace, registryCredentialFile, kibishiiDirectory); err != nil {
+
+	if err := KibishiiPrepareBeforeBackup(oneHourTimeout, client, providerName,
+		kibishiiNamespace, registryCredentialFile, veleroFeatures, kibishiiDirectory); err != nil {
 		return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", kibishiiNamespace)
 	}
 
-	if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, kibishiiNamespace, backupLocation, useVolumeSnapshots, ""); err != nil {
+	if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName,
+		kibishiiNamespace, backupLocation, useVolumeSnapshots, ""); err != nil {
 		RunDebug(context.Background(), veleroCLI, veleroNamespace, backupName, "")
 		return errors.Wrapf(err, "Failed to backup kibishii namespace %s", kibishiiNamespace)
 	}
@@ -90,10 +95,14 @@ func RunKibishiiTests(client TestClient, providerName, veleroCLI, veleroNamespac
 	return nil
 }
 
-func installKibishii(ctx context.Context, namespace string, cloudPlatform string, kibishiiDirectory string) error {
+func installKibishii(ctx context.Context, namespace string, cloudPlatform, veleroFeatures, kibishiiDirectory string) error {
+	if strings.EqualFold(cloudPlatform, "azure") && strings.EqualFold(veleroFeatures, "EnableCSI") {
+		cloudPlatform = "azure-csi"
+	}
 	// We use kustomize to generate YAML for Kibishii from the checked-in yaml directories
 	kibishiiInstallCmd := exec.CommandContext(ctx, "kubectl", "apply", "-n", namespace, "-k",
-		kibishiiDirectory+cloudPlatform)
+		//"github.com/vmware-tanzu-experiments/distributed-data-generator/kubernetes/yaml/"+cloudPlatform)
+		"github.com/danfengliu/distributed-data-generator/kubernetes/yaml/"+cloudPlatform)
 	_, stderr, err := veleroexec.RunCommand(kibishiiInstallCmd)
 	if err != nil {
 		return errors.Wrapf(err, "failed to install kibishii, stderr=%s", stderr)
@@ -149,7 +158,7 @@ func waitForKibishiiPods(ctx context.Context, client TestClient, kibishiiNamespa
 	return WaitForPods(ctx, client, kibishiiNamespace, []string{"jump-pad", "etcd0", "etcd1", "etcd2", "kibishii-deployment-0", "kibishii-deployment-1"})
 }
 
-func KibishiiPrepareBeforeBackup(oneHourTimeout context.Context, client TestClient, providerName, kibishiiNamespace, registryCredentialFile, kibishiiDirectory string) error {
+func KibishiiPrepareBeforeBackup(oneHourTimeout context.Context, client TestClient, providerName, kibishiiNamespace, registryCredentialFile, veleroFeatures, kibishiiDirectory string) error {
 	serviceAccountName := "default"
 
 	// wait until the service account is created before patch the image pull secret
@@ -161,7 +170,7 @@ func KibishiiPrepareBeforeBackup(oneHourTimeout context.Context, client TestClie
 		return errors.Wrapf(err, "failed to patch the service account %q under the namespace %q", serviceAccountName, kibishiiNamespace)
 	}
 
-	if err := installKibishii(oneHourTimeout, kibishiiNamespace, providerName, kibishiiDirectory); err != nil {
+	if err := installKibishii(oneHourTimeout, kibishiiNamespace, providerName, veleroFeatures, kibishiiDirectory); err != nil {
 		return errors.Wrap(err, "Failed to install Kibishii workload")
 	}
 

test/e2e/util/velero/install.go

@@ -24,6 +24,7 @@ import (
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"strings"
 	"time"
 
 	"github.com/pkg/errors"
@@ -199,6 +200,14 @@ func installVeleroServer(ctx context.Context, cli string, options *installOption
 	}
 	if len(options.Features) > 0 {
 		args = append(args, "--features", options.Features)
+		if strings.EqualFold(options.Features, "EnableCSI") {
+			if strings.EqualFold(options.ProviderName, "Azure") {
+				if err := KubectlApplyByFile(ctx, "util/csi/AzureVolumeSnapshotClass.yaml"); err != nil {
+					return err
+				}
+			}
+
+		}
 	}
 
 	if err := createVelereResources(ctx, cli, namespace, args, options.RegistryCredentialFile, options.ResticHelperImage); err != nil {