Set `GOBIN` so Makefile don't modify $PATH on `go install` Fix realPath resolving when cloud credentials is prefixed by `~` for home dir Use `~/.docker/config.json` if REGISTRY_CREDENTIAL_FILE not defined and skip step if does not exists since it is optional

Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> Set `GOBIN` so Makefile don't modify $PATH on `go install` Fix realPath resolving when cloud credentials is prefixed by `~` for home dir Use `~/.docker/config.json` if REGISTRY_CREDENTIAL_FILE not defined and skip step if does not exists since it is optional Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> Add kind testdata storageclass Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> Add kind testdata storageclass Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> log `Start to install Azure VolumeSnapshotClass ...` only on azure when csi is enabled Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> Add BSL_CONFIG example and notes Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> Makefile: Set `GOBIN` for `_output/...` Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> README spacing Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> StandbyClusterObjectStoreProvider typo Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> Specify velero namespace during get/delete command Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> Use object stores rather than cloudProvider for bucket queries Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> Remove debug print Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> simplify NS get changes, add velero NS to `DeleteBackupResource` Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> Skip file system backups on kind which uses hostPath volumes Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> Add StorageClass change test to PR kind e2e Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> Add more tests to pr Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> Add NS mapping to PR e2e Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> Add `SKIP_KIND` to some jobs containing volumes Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> Remove kind from kibishii tests Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> Label volume resource policies as restic, skip restic/snapshot tests, add more tests Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> TTLTest is a snapshot test Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> Remove non working tests Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> Resolve https://github.com/vmware-tanzu/velero/pull/7353#issuecomment-1925660077 Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> address https://github.com/vmware-tanzu/velero/pull/7353/files#r1477218762 Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> Address https://github.com/vmware-tanzu/velero/pull/7353#issuecomment-1923414840 Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com>
2024-01-24 14:28:58 -05:00 · 2024-01-24 14:28:58 -05:00 · b1d95cf2aa
parent 99e0c483c7
commit b1d95cf2aa
20 changed files with 161 additions and 63 deletions
--- a/.github/workflows/e2e-test-kind.yaml
+++ b/.github/workflows/e2e-test-kind.yaml
@ -67,6 +67,16 @@ jobs:
          - 1.23.6
          - 1.24.0
          - 1.25.3
+        focus:
+          # tests to focus on, use `|` to concatenate multiple regexes to run on the same job
+          # ordered according to e2e_suite_test.go order
+          - Basic\]\[ClusterResource
+          - ResourceFiltering
+          - ResourceModifier|Backups|PrivilegesMgmt\]\[SSR
+          - Schedule\]\[OrderedResources
+          - NamespaceMapping\]\[Single\]\[Restic|NamespaceMapping\]\[Multiple\]\[Restic
+          - Basic\]\[Nodeport
+          - Basic\]\[StorageClass
      fail-fast: false
    steps:
      - name: Set up Go
@ -123,7 +133,8 @@ jobs:
              CREDS_FILE=/tmp/credential BSL_BUCKET=bucket \
              ADDITIONAL_OBJECT_STORE_PROVIDER=aws ADDITIONAL_BSL_CONFIG=region=minio,s3ForcePathStyle="true",s3Url=http://$(hostname -i):9000 \
              ADDITIONAL_CREDS_FILE=/tmp/credential ADDITIONAL_BSL_BUCKET=additional-bucket \
-              GINKGO_FOCUS='Basic\]\[ClusterResource' VELERO_IMAGE=velero:pr-test \
+              GINKGO_FOCUS='${{ matrix.focus }}' VELERO_IMAGE=velero:pr-test \
+              GINKGO_SKIP='SKIP_KIND|pv-backup|Restic|Snapshot|LongTime' \
              make -C test/e2e run
        timeout-minutes: 30
      - name: Upload debug bundle
--- a/3
+++ b/3
@ -108,6 +108,7 @@ platform_temp = $(subst -, ,$(ARCH))
 GOOS = $(word 1, $(platform_temp))
 GOARCH = $(word 2, $(platform_temp))
 GOPROXY ?= https://proxy.golang.org
+GOBIN=$$(pwd)/.go/bin

 # If you want to build all binaries, see the 'all-build' rule.
 # If you want to build all containers, see the 'all-containers' rule.
@ -129,6 +130,7 @@ local: build-dirs
 # Add DEBUG=1 to enable debug locally
 	GOOS=$(GOOS) \
 	GOARCH=$(GOARCH) \
+	GOBIN=$(GOBIN) \
 	VERSION=$(VERSION) \
 	REGISTRY=$(REGISTRY) \
 	PKG=$(PKG) \
@ -145,6 +147,7 @@ _output/bin/$(GOOS)/$(GOARCH)/$(BIN): build-dirs
 	$(MAKE) shell CMD="-c '\
 		GOOS=$(GOOS) \
 		GOARCH=$(GOARCH) \
+		GOBIN=$(GOBIN) \
 		VERSION=$(VERSION) \
 		REGISTRY=$(REGISTRY) \
 		PKG=$(PKG) \
--- a/hack/build.sh
+++ b/hack/build.sh
@ -32,6 +32,10 @@ if [[ -z "${GOOS}" ]]; then
    echo "GOOS must be set"
    exit 1
 fi
+if [[ -z "${GOBIN}" ]]; then
+    echo "GOBIN must be set"
+    exit 1
+fi
 if [[ -z "${GOARCH}" ]]; then
    echo "GOARCH must be set"
    exit 1
--- a/test/e2e/Makefile
+++ b/test/e2e/Makefile
@ -41,8 +41,10 @@ help:  ## Display this help

 TOOLS_DIR := $(REPO_ROOT)/hack/tools
 BIN_DIR := bin
+# Try to not modify PATH if possible
+GOBIN := $(REPO_ROOT)/.go/bin
 TOOLS_BIN_DIR := $(TOOLS_DIR)/$(BIN_DIR)
-GINKGO := $(shell go env GOPATH)/bin/ginkgo
+GINKGO := $(GOBIN)/ginkgo
 KUSTOMIZE := $(TOOLS_BIN_DIR)/kustomize
 OUTPUT_DIR := _output/$(GOOS)/$(GOARCH)/bin
 GINKGO_FOCUS ?=
@ -113,9 +115,13 @@ STANDBY_CLUSTER_NAME ?=
 EKS_POLICY_ARN ?=


+# Make sure ginkgo is in $GOBIN
 .PHONY:ginkgo
-ginkgo: # Make sure ginkgo is in $GOPATH/bin
-	go install github.com/onsi/ginkgo/ginkgo@v1.16.5
+ginkgo: ${GOBIN}/ginkgo
+
+# This target does not run if ginkgo is already in $GOBIN
+${GOBIN}/ginkgo:
+	GOBIN=${GOBIN} go install github.com/onsi/ginkgo/ginkgo@v1.16.5

 .PHONY: run
 run: ginkgo
--- a/test/e2e/README.md
+++ b/test/e2e/README.md
@ -66,7 +66,7 @@ the object-store-provider to be specified.
 1. `-features`: Comma-separated list of features to enable for this Velero process.
 1. `-registry-credential-file`: File containing credential for the image registry, follows the same format rules as the ~/.docker/config.json file. This credential will be loaded in Velero server pod to help on Docker Hub rate limit issue.
 1. `-kibishii-directory`: The file directory or URL path to install Kibishii. It's configurable in case the default path is not accessible for your own test environment.
-1. `-debug-e2e-test`: A Switch for enable or disable test data cleaning action.
+1. `-debug-e2e-test`: <true/false> A Switch for enable or disable test data cleaning action.
 1. `-garbage-collection-frequency`: frequency of garbage collection. It is a parameter for Velero installation. Optional.
 1. `-velero-server-debug-mode`: A switch for enable or disable having debug log of Velero server.
 1. `-default-cluster-context`: Default (source) cluster's kube config context, it's for migration test.
@ -101,7 +101,7 @@ Below is a mapping between `make` variables to E2E configuration flags.
 1. `VELERO_NAMESPACE `: the `-velero-namespace`. Optional.
 1. `PLUGINS `: the `-plugins`. Optional.
 1. `BSL_PREFIX`: `-prefix`. Optional.
-1. `BSL_CONFIG`: `-bsl-config`. Optional.
+1. `BSL_CONFIG`: `-bsl-config`. Optional. Example: BSL_CONFIG="region=us-east-1". May be required for some object store provider
 1. `VSL_CONFIG`: `-vsl-config`. Optional.
 1. `UPGRADE_FROM_VELERO_CLI `: `-upgrade-from-velero-cli`. Optional.
 1. `UPGRADE_FROM_VELERO_VERSION `: `-upgrade-from-velero-version`. Optional.
@ -136,9 +136,18 @@ Below is a mapping between `make` variables to E2E configuration flags.
 Basic examples:

 1. Run Velero tests in a kind cluster with AWS (or Minio) as the storage provider:
+    
+    Start kind cluster
+    ```bash
+    kind create cluster
+    ```
    ```bash
    BSL_PREFIX=<PREFIX_UNDER_BUCKET> BSL_BUCKET=<BUCKET_FOR_E2E_TEST_BACKUP> CREDS_FILE=/path/to/aws-creds CLOUD_PROVIDER=kind OBJECT_STORE_PROVIDER=aws make test-e2e
    ```
+    Stop kind cluster
+    ```bash
+    kind delete cluster
+    ```
 1. Run Velero tests in an AWS cluster:
    ```bash
    BSL_PREFIX=<PREFIX_UNDER_BUCKET> BSL_BUCKET=<BUCKET_FOR_E2E_TEST_BACKUP> CREDS_FILE=/path/to/aws-creds CLOUD_PROVIDER=aws make test-e2e
@ -303,4 +312,8 @@ The TestFunc function concatenate all the flows in a test.
 It will reduce the frequency of velero installation by installing and checking Velero with the global Velero. It's Need to explicit reinstall Velero if the case has specicial configuration, such as API Group test case we need to enable feature EnableCSI.

 ### Tips
-Look for the ⛵ emoji printed at the end of each install and uninstall log. There should not be two install/unintall in a row, and there should be tests between an install and an uninstall. 
+Look for the ⛵ emoji printed at the end of each install and uninstall log. There should not be two install/unintall in a row, and there should be tests between an install and an uninstall. 
+
+#### Troubleshooting
+
+If velero log shows `level=error msg="Failed to get bucket region, bucket: xbucket, error: operation error S3: HeadBucket, failed to resolve service endpoint, endpoint rule error, A region must be set when sending requests to S3." backup-storage-location=velero/default cmd=/plugins/velero-plugin-for-aws controller=backup-storage-location logSource="/go/src/velero-plugin-for-aws/velero-plugin-for-aws/object_store.go:136" pluginName=velero-plugin-for-aws`, it means you need to set `BSL_CONFIG` to include `region=<region>`.
--- a/test/e2e/backup/backup.go
+++ b/test/e2e/backup/backup.go
@ -74,8 +74,12 @@ func BackupRestoreTest(backupRestoreTestConfig BackupRestoreTestConfig) {
 		veleroCfg.KibishiiDirectory = veleroCfg.KibishiiDirectory + backupRestoreTestConfig.kibishiiPatchSubDir
 		veleroCfg.UseVolumeSnapshots = useVolumeSnapshots
 		veleroCfg.UseNodeAgent = !useVolumeSnapshots
-		if useVolumeSnapshots && veleroCfg.CloudProvider == "kind" {
-			Skip("Volume snapshots not supported on kind")
+		if veleroCfg.CloudProvider == "kind" {
+			Skip("Volume snapshots plugin and File System Backups are not supported on kind")
+			// on kind cluster snapshots are not supported since there is no velero snapshot plugin for kind volumes.
+			// and PodVolumeBackups are not supported because PVB creation gets skipped for hostpath volumes, which are the only
+			// volumes created on kind clusters using the default storage class and provisioner (provisioner: rancher.io/local-path)
+			// This test suite checks for volume snapshots and PVBs generated from FileSystemBackups, so skip it on kind clusters
 		}
 		// [SKIP]: Static provisioning for vSphere CSI driver works differently from other drivers.
 		//         For vSphere CSI, after you create a PV specifying an existing volume handle, CSI
--- a/test/e2e/backups/deletion.go
+++ b/test/e2e/backups/deletion.go
@ -86,6 +86,9 @@ func backup_deletion_test(useVolumeSnapshots bool) {
 // runUpgradeTests runs upgrade test on the provider by kibishii.
 func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupName, backupLocation string,
 	useVolumeSnapshots bool, kibishiiDirectory string) error {
+	if useVolumeSnapshots && veleroCfg.CloudProvider == "kind" {
+		Skip("Volume snapshots not supported on kind")
+	}
 	oneHourTimeout, ctxCancel := context.WithTimeout(context.Background(), time.Minute*60)
 	defer ctxCancel()
 	veleroCLI := veleroCfg.VeleroCLI
@ -111,7 +114,7 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam
 		registryCredentialFile, veleroFeatures, kibishiiDirectory, useVolumeSnapshots, DefaultKibishiiData); err != nil {
 		return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", deletionTest)
 	}
-	err := ObjectsShouldNotBeInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig, backupName, BackupObjectsPrefix, 1)
+	err := ObjectsShouldNotBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig, backupName, BackupObjectsPrefix, 1)
 	if err != nil {
 		return err
 	}
@ -139,7 +142,7 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam
 			return errors.Wrapf(err, "Error waiting for uploads to complete")
 		}
 	}
-	err = ObjectsShouldBeInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix)
+	err = ObjectsShouldBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix)
 	if err != nil {
 		return err
 	}
@ -167,7 +170,7 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam
 		}
 	}

-	err = ObjectsShouldNotBeInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 5)
+	err = ObjectsShouldNotBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 5)
 	if err != nil {
 		return err
 	}
@ -194,12 +197,12 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam
 		})
 	})

-	err = DeleteObjectsInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix)
+	err = DeleteObjectsInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix)
 	if err != nil {
 		return err
 	}

-	err = ObjectsShouldNotBeInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 1)
+	err = ObjectsShouldNotBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 1)
 	if err != nil {
 		return err
 	}
--- a/test/e2e/backups/sync_backups.go
+++ b/test/e2e/backups/sync_backups.go
@ -148,13 +148,13 @@ func BackupsSyncTest() {
 		})

 		By(fmt.Sprintf("Delete %s backup files in object store", test.backupName), func() {
-			err = DeleteObjectsInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
+			err = DeleteObjectsInBucket(VeleroCfg.ObjectStoreProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
 				VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, test.backupName, BackupObjectsPrefix)
 			Expect(err).To(Succeed(), fmt.Sprintf("Failed to delete object in bucket %s with err %v", test.backupName, err))
 		})

 		By(fmt.Sprintf("Check %s backup files in object store is deleted", test.backupName), func() {
-			err = ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
+			err = ObjectsShouldNotBeInBucket(VeleroCfg.ObjectStoreProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
 				VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, test.backupName, BackupObjectsPrefix, 1)
 			Expect(err).To(Succeed(), fmt.Sprintf("Failed to delete object in bucket %s with err %v", test.backupName, err))
 		})
--- a/test/e2e/backups/ttl.go
+++ b/test/e2e/backups/ttl.go
@ -155,7 +155,7 @@ func TTLTest() {
 		})

 		By("Associated Restores should be created", func() {
-			Expect(ObjectsShouldBeInBucket(veleroCfg.CloudProvider,
+			Expect(ObjectsShouldBeInBucket(veleroCfg.ObjectStoreProvider,
 				veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,
 				veleroCfg.BSLPrefix, veleroCfg.BSLConfig, test.restoreName,
 				RestoreObjectsPrefix)).NotTo(HaveOccurred(), "Fail to get restore object")
@ -179,7 +179,7 @@ func TTLTest() {
 		})

 		By("Backup file from cloud object storage should be deleted", func() {
-			Expect(ObjectsShouldNotBeInBucket(veleroCfg.CloudProvider,
+			Expect(ObjectsShouldNotBeInBucket(veleroCfg.ObjectStoreProvider,
 				veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,
 				veleroCfg.BSLPrefix, veleroCfg.BSLConfig, test.backupName,
 				BackupObjectsPrefix, 5)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
@ -194,7 +194,7 @@ func TTLTest() {
 		})

 		By("Associated Restores should be deleted", func() {
-			Expect(ObjectsShouldNotBeInBucket(veleroCfg.CloudProvider,
+			Expect(ObjectsShouldNotBeInBucket(veleroCfg.ObjectStoreProvider,
 				veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,
 				veleroCfg.BSLPrefix, veleroCfg.BSLConfig, test.restoreName,
 				RestoreObjectsPrefix, 5)).NotTo(HaveOccurred(), "Fail to get restore object")
--- a/test/e2e/basic/namespace-mapping.go
+++ b/test/e2e/basic/namespace-mapping.go
@ -38,6 +38,9 @@ func (n *NamespaceMapping) Init() error {
 	n.VeleroCfg.UseVolumeSnapshots = n.UseVolumeSnapshots
 	n.VeleroCfg.UseNodeAgent = !n.UseVolumeSnapshots
 	n.kibishiiData = &KibishiiData{Levels: 2, DirsPerLevel: 10, FilesPerLevel: 10, FileLength: 1024, BlockSize: 1024, PassNum: 0, ExpectedNodes: 2}
+	if n.VeleroCfg.CloudProvider == "kind" {
+		n.kibishiiData = &KibishiiData{Levels: 0, DirsPerLevel: 0, FilesPerLevel: 0, FileLength: 0, BlockSize: 0, PassNum: 0, ExpectedNodes: 2}
+	}
 	backupType := "restic"
 	if n.UseVolumeSnapshots {
 		backupType = "snapshot"
@ -67,7 +70,11 @@ func (n *NamespaceMapping) Init() error {
 		"create", "--namespace", n.VeleroCfg.VeleroNamespace, "backup", n.BackupName,
 		"--include-namespaces", strings.Join(*n.NSIncluded, ","), "--wait",
 	}
-	if n.UseVolumeSnapshots {
+	if VeleroCfg.CloudProvider == "kind" {
+		// don't test volume snapshotter or file system backup on kind
+		n.BackupArgs = append(n.BackupArgs, "--snapshot-volumes=false")
+		n.UseVolumeSnapshots = false
+	} else if n.UseVolumeSnapshots {
 		n.BackupArgs = append(n.BackupArgs, "--snapshot-volumes")
 	} else {
 		n.BackupArgs = append(n.BackupArgs, "--snapshot-volumes=false")
--- a/test/e2e/bsl-mgmt/deletion.go
+++ b/test/e2e/bsl-mgmt/deletion.go
@ -276,7 +276,7 @@ func BslDeletionTest(useVolumeSnapshots bool) {
 				Expect(cmp.Diff(backupsInBsl1AndBsl2, backupsBeforeDel, cmpopts.SortSlices(less))).Should(BeEmpty())

 				By(fmt.Sprintf("Backup1 %s should exist in cloud object store before bsl deletion", backupName_1), func() {
-					Expect(ObjectsShouldBeInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile,
+					Expect(ObjectsShouldBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile,
 						veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig,
 						backupName_1, BackupObjectsPrefix)).To(Succeed())
 				})
@ -295,14 +295,14 @@ func BslDeletionTest(useVolumeSnapshots bool) {
 			})

 			By(fmt.Sprintf("Backup1 %s should still exist in cloud object store after bsl deletion", backupName_1), func() {
-				Expect(ObjectsShouldBeInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile,
+				Expect(ObjectsShouldBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile,
 					veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig,
 					backupName_1, BackupObjectsPrefix)).To(Succeed())
 			})

 			// TODO: Choose additional BSL to be deleted as an new test case
 			// By(fmt.Sprintf("Backup %s should still exist in cloud object store", backupName_2), func() {
-			// 	Expect(ObjectsShouldBeInBucket(veleroCfg.CloudProvider, veleroCfg.AdditionalBSLCredentials,
+			// 	Expect(ObjectsShouldBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.AdditionalBSLCredentials,
 			// 		veleroCfg.AdditionalBSLBucket, veleroCfg.AdditionalBSLPrefix, veleroCfg.AdditionalBSLConfig,
 			// 		backupName_2, BackupObjectsPrefix)).To(Succeed())
 			// })
--- a/test/e2e/e2e_suite_test.go
+++ b/test/e2e/e2e_suite_test.go
@ -92,7 +92,7 @@ func init() {
 	flag.StringVar(&VeleroCfg.DataMoverPlugin, "data-mover-plugin", "", "customized plugin for data mover.")
 	flag.StringVar(&VeleroCfg.StandbyClusterCloudProvider, "standby-cluster-cloud-provider", "", "cloud provider for standby cluster.")
 	flag.StringVar(&VeleroCfg.StandbyClusterPlugins, "standby-cluster-plugins", "", "plugins provider for standby cluster.")
-	flag.StringVar(&VeleroCfg.StandbyClusterOjbectStoreProvider, "standby-cluster-object-store-provider", "", "object store provider for standby cluster.")
+	flag.StringVar(&VeleroCfg.StandbyClusterObjectStoreProvider, "standby-cluster-object-store-provider", "", "object store provider for standby cluster.")
 	flag.BoolVar(&VeleroCfg.DebugVeleroPodRestart, "debug-velero-pod-restart", false, "a switch for debugging velero pod restart.")
 	flag.BoolVar(&VeleroCfg.DisableInformerCache, "disable-informer-cache", false, "a switch for disable informer cache.")
 	flag.StringVar(&VeleroCfg.DefaultClusterName, "default-cluster-name", "", "default cluster's name in kube config file, it's for EKS IRSA test.")
@ -103,8 +103,8 @@ func init() {

 }

-var _ = Describe("[APIGroup][APIVersion] Velero tests with various CRD API group versions", APIGropuVersionsTest)
-var _ = Describe("[APIGroup][APIExtensions] CRD of apiextentions v1beta1 should be B/R successfully from cluster(k8s version < 1.22) to cluster(k8s version >= 1.22)", APIExtensionsVersionsTest)
+var _ = Describe("[APIGroup][APIVersion][SKIP_KIND] Velero tests with various CRD API group versions", APIGropuVersionsTest)
+var _ = Describe("[APIGroup][APIExtensions][SKIP_KIND] CRD of apiextentions v1beta1 should be B/R successfully from cluster(k8s version < 1.22) to cluster(k8s version >= 1.22)", APIExtensionsVersionsTest)

 // Test backup and restore of Kibishi using restic
 var _ = Describe("[Basic][Restic] Velero tests on cluster using the plugin provider for object storage and Restic for volume backups", BackupRestoreWithRestic)
@ -134,18 +134,18 @@ var _ = Describe("[ResourceFiltering][IncludeNamespaces][Restore] Velero test on
 var _ = Describe("[ResourceFiltering][IncludeResources][Backup] Velero test on include resources from the cluster backup", BackupWithIncludeResources)
 var _ = Describe("[ResourceFiltering][IncludeResources][Restore] Velero test on include resources from the cluster restore", RestoreWithIncludeResources)
 var _ = Describe("[ResourceFiltering][LabelSelector] Velero test on backup include resources matching the label selector", BackupWithLabelSelector)
-var _ = Describe("[ResourceFiltering][ResourcePolicies] Velero test on skip backup of volume by resource policies", ResourcePoliciesTest)
+var _ = Describe("[ResourceFiltering][ResourcePolicies][Restic] Velero test on skip backup of volume by resource policies", ResourcePoliciesTest)

 var _ = Describe("[ResourceModifier][Restore] Velero test on resource modifiers from the cluster restore", ResourceModifiersTest)

 var _ = Describe("[Backups][Deletion][Restic] Velero tests of Restic backup deletion", BackupDeletionWithRestic)
 var _ = Describe("[Backups][Deletion][Snapshot] Velero tests of snapshot backup deletion", BackupDeletionWithSnapshots)
-var _ = Describe("[Backups][TTL][LongTime] Local backups and restic repos will be deleted once the corresponding backup storage location is deleted", TTLTest)
+var _ = Describe("[Backups][TTL][LongTime][Snapshot] Local backups and restic repos will be deleted once the corresponding backup storage location is deleted", TTLTest)
 var _ = Describe("[Backups][BackupsSync] Backups in object storage are synced to a new Velero and deleted backups in object storage are synced to be deleted in Velero", BackupsSyncTest)

 var _ = Describe("[Schedule][BR][Pause][LongTime] Backup will be created periodly by schedule defined by a Cron expression", ScheduleBackupTest)
-var _ = Describe("[Schedule][OrederedResources] Backup resources should follow the specific order in schedule", ScheduleOrderedResources)
-var _ = Describe("[Schedule][BackupCreation] Schedule controller wouldn't create a new backup when it still has pending or InProgress backup", ScheduleBackupCreationTest)
+var _ = Describe("[Schedule][OrderedResources] Backup resources should follow the specific order in schedule", ScheduleOrderedResources)
+var _ = Describe("[Schedule][BackupCreation][SKIP_KIND] Schedule controller wouldn't create a new backup when it still has pending or InProgress backup", ScheduleBackupCreationTest)

 var _ = Describe("[PrivilegesMgmt][SSR] Velero test on ssr object when controller namespace mix-ups", SSRTest)

@ -165,7 +165,7 @@ var _ = Describe("[pv-backup][Opt-Out] Backup resources should follow the specif

 var _ = Describe("[Basic][Nodeport] Service nodeport reservation during restore is configurable", NodePortTest)
 var _ = Describe("[Basic][StorageClass] Storage class of persistent volumes and persistent volume claims can be changed during restores", StorageClasssChangingTest)
-var _ = Describe("[Basic][SelectedNode] Node selectors of persistent volume claims can be changed during restores", PVCSelectedNodeChangingTest)
+var _ = Describe("[Basic][SelectedNode][SKIP_KIND] Node selectors of persistent volume claims can be changed during restores", PVCSelectedNodeChangingTest)

 func GetKubeconfigContext() error {
 	var err error
@ -206,6 +206,18 @@ func TestE2e(t *testing.T) {
 		t.Skip("Skipping E2E tests")
 	}

+	if VeleroCfg.CloudProvider != "kind" {
+		fmt.Println("For cloud platforms, object store plugin provider will be set as cloud provider")
+		// If ObjectStoreProvider is not provided, then using the value same as CloudProvider
+		if VeleroCfg.ObjectStoreProvider == "" {
+			VeleroCfg.ObjectStoreProvider = VeleroCfg.CloudProvider
+		}
+	} else {
+		if VeleroCfg.ObjectStoreProvider == "" {
+			t.Error(errors.New("No object store provider specified - must be specified when using kind as the cloud provider")) // Must have an object store provider
+		}
+	}
+
 	var err error
 	if err = GetKubeconfigContext(); err != nil {
 		fmt.Println(err)
--- a/test/e2e/migration/migration.go
+++ b/test/e2e/migration/migration.go
@ -298,7 +298,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
 					// For SnapshotMoveData pipelines, we should use standby clustr setting for Velero installation
 					// In nightly CI, StandbyClusterPlugins is set properly if pipeline is for SnapshotMoveData.
 					veleroCfg.Plugins = veleroCfg.StandbyClusterPlugins
-					veleroCfg.ObjectStoreProvider = veleroCfg.StandbyClusterOjbectStoreProvider
+					veleroCfg.ObjectStoreProvider = veleroCfg.StandbyClusterObjectStoreProvider
 				}

 				Expect(VeleroInstall(context.Background(), &veleroCfg, true)).To(Succeed())
--- a/test/perf/Makefile
+++ b/test/perf/Makefile
@ -41,8 +41,10 @@ help:  ## Display this help

 TOOLS_DIR := $(REPO_ROOT)/hack/tools
 BIN_DIR := bin
+# Try to not modify PATH if possible
+GOBIN := $(REPO_ROOT)/.go/bin
 TOOLS_BIN_DIR := $(TOOLS_DIR)/$(BIN_DIR)
-GINKGO := $(GOPATH)/bin/ginkgo
+GINKGO := $(GOBIN)/ginkgo
 KUSTOMIZE := $(TOOLS_BIN_DIR)/kustomize
 OUTPUT_DIR := _output/$(GOOS)/$(GOARCH)/bin
 GINKGO_FOCUS ?=
@ -88,9 +90,13 @@ VELERO_POD_CPU_REQUEST ?= 2
 VELERO_POD_MEM_REQUEST ?= 2Gi
 POD_VOLUME_OPERATION_TIMEOUT ?= 6h

+# Make sure ginkgo is in $GOBIN
 .PHONY:ginkgo
-ginkgo: # Make sure ginkgo is in $GOPATH/bin
-	go install github.com/onsi/ginkgo/ginkgo@v1.16.5
+ginkgo: ${GOBIN}/ginkgo
+
+# This target does not run if ginkgo is already in $GOBIN
+${GOBIN}/ginkgo:
+	GOBIN=${GOBIN} go install github.com/onsi/ginkgo/ginkgo@v1.16.5

 .PHONY: run
 run: ginkgo
--- a/test/testdata/storage-class/kind.yaml
+++ b/test/testdata/storage-class/kind.yaml
@ -0,0 +1,10 @@
+allowVolumeExpansion: true
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: e2e-storage-class
+parameters:
+  type: pd-standard
+provisioner: rancher.io/local-path
+reclaimPolicy: Delete
+volumeBindingMode: WaitForFirstConsumer
--- a/test/types.go
+++ b/test/types.go
@ -85,7 +85,7 @@ type VeleroConfig struct {
 	DataMoverPlugin                   string
 	StandbyClusterCloudProvider       string
 	StandbyClusterPlugins             string
-	StandbyClusterOjbectStoreProvider string
+	StandbyClusterObjectStoreProvider string
 	DebugVeleroPodRestart             bool
 	IsUpgradeTest                     bool
 	WithoutDisableInformerCacheParam  bool
--- a/test/util/k8s/serviceaccount.go
+++ b/test/util/k8s/serviceaccount.go
@ -47,8 +47,16 @@ func WaitUntilServiceAccountCreated(ctx context.Context, client TestClient, name
 }

 func PatchServiceAccountWithImagePullSecret(ctx context.Context, client TestClient, namespace, serviceAccount, dockerCredentialFile string) error {
+	if dockerCredentialFile == "" {
+		// use the default docker credential file in the home directory
+		dockerCredentialFile = os.Getenv("HOME") + "/.docker/config.json"
+	}
+	// if file do not exist, do not patch the service account, just return
 	credential, err := os.ReadFile(dockerCredentialFile)
 	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
 		return errors.Wrapf(err, "failed to read the docker credential file %q", dockerCredentialFile)
 	}
 	secretName := "image-pull-secret"
--- a/test/util/providers/common.go
+++ b/test/util/providers/common.go
@ -34,24 +34,24 @@ type ObjectsInStorage interface {
 	IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupName string, snapshotCheck SnapshotCheckPoint) error
 }

-func ObjectsShouldBeInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) error {
+func ObjectsShouldBeInBucket(objectStoreProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) error {
 	fmt.Printf("|| VERIFICATION || - %s should exist in storage [%s %s]\n", backupName, bslPrefix, subPrefix)
-	exist, err := IsObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix)
+	exist, err := IsObjectsInBucket(objectStoreProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix)
 	if !exist {
 		return errors.Wrap(err, fmt.Sprintf("|| UNEXPECTED ||Backup object %s is not exist in object store after backup as expected\n", backupName))
 	}
 	fmt.Printf("|| EXPECTED || - Backup %s exist in object storage bucket %s\n", backupName, bslBucket)
 	return nil
 }
-func ObjectsShouldNotBeInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string, retryTimes int) error {
+func ObjectsShouldNotBeInBucket(objectStoreProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string, retryTimes int) error {
 	var err error
 	var exist bool
 	fmt.Printf("|| VERIFICATION || - %s %s should not exist in object store %s\n", subPrefix, backupName, bslPrefix)
 	if cloudCredentialsFile == "" {
-		return errors.New(fmt.Sprintf("|| ERROR || - Please provide credential file of cloud %s \n", cloudProvider))
+		return errors.New(fmt.Sprintf("|| ERROR || - Please provide credential file of cloud %s \n", objectStoreProvider))
 	}
 	for i := 0; i < retryTimes; i++ {
-		exist, err = IsObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix)
+		exist, err = IsObjectsInBucket(objectStoreProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix)
 		if err != nil {
 			return errors.Wrapf(err, "|| UNEXPECTED || - Failed to get backup %s in object store\n", backupName)
 		}
@ -63,6 +63,10 @@ func ObjectsShouldNotBeInBucket(cloudProvider, cloudCredentialsFile, bslBucket,
 	}
 	return errors.New(fmt.Sprintf("|| UNEXPECTED ||Backup object %s still exist in object store after backup deletion\n", backupName))
 }
+
+// This function returns a storage interface based on the cloud provider for querying objects and snapshots
+// When cloudProvider is kind, pass in object storage provider instead. For example, "aws".
+// Snapshots are not supported on kind.
 func getProvider(cloudProvider string) (ObjectsInStorage, error) {
 	var s ObjectsInStorage
 	switch cloudProvider {
@ -89,24 +93,24 @@ func getFullPrefix(bslPrefix, subPrefix string) string {
 	}
 	return bslPrefix
 }
-func IsObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) (bool, error) {
+func IsObjectsInBucket(objectStoreProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) (bool, error) {
 	bslPrefix = getFullPrefix(bslPrefix, subPrefix)
-	s, err := getProvider(cloudProvider)
+	s, err := getProvider(objectStoreProvider)
 	if err != nil {
-		return false, errors.Wrapf(err, fmt.Sprintf("Cloud provider %s is not valid", cloudProvider))
+		return false, errors.Wrapf(err, fmt.Sprintf("Object store provider %s is not valid", objectStoreProvider))
 	}
 	return s.IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName)
 }

-func DeleteObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) error {
+func DeleteObjectsInBucket(objectStoreProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) error {
 	bslPrefix = getFullPrefix(bslPrefix, subPrefix)
 	fmt.Printf("|| VERIFICATION || - Delete backup %s in storage %s\n", backupName, bslPrefix)
 	if cloudCredentialsFile == "" {
-		return errors.New(fmt.Sprintf("|| ERROR || - Please provide credential file of cloud %s \n", cloudProvider))
+		return errors.New(fmt.Sprintf("|| ERROR || - Please provide credential file of cloud %s \n", objectStoreProvider))
 	}
-	s, err := getProvider(cloudProvider)
+	s, err := getProvider(objectStoreProvider)
 	if err != nil {
-		return errors.Wrapf(err, fmt.Sprintf("Cloud provider %s is not valid", cloudProvider))
+		return errors.Wrapf(err, fmt.Sprintf("Object store provider %s is not valid", objectStoreProvider))
 	}
 	err = s.DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName)
 	if err != nil {
--- a/test/util/velero/install.go
+++ b/test/util/velero/install.go
@ -75,6 +75,7 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluste
 	if isStandbyCluster {
 		veleroCfg.CloudProvider = veleroCfg.StandbyClusterCloudProvider
 	}
+
 	if veleroCfg.CloudProvider != "kind" {
 		fmt.Println("For cloud platforms, object store plugin provider will be set as cloud provider")
 		// If ObjectStoreProvider is not provided, then using the value same as CloudProvider
@ -278,14 +279,11 @@ func installVeleroServer(ctx context.Context, cli, cloudProvider string, options

 	if len(options.Features) > 0 {
 		args = append(args, "--features", options.Features)
-		if strings.EqualFold(options.Features, FeatureCSI) && options.UseVolumeSnapshots {
-			if strings.EqualFold(cloudProvider, "azure") {
-				fmt.Println("Start to install Azure VolumeSnapshotClass ...")
-				if err := KubectlApplyByFile(ctx, "../util/csi/AzureVolumeSnapshotClass.yaml"); err != nil {
-					return err
-				}
+		if strings.EqualFold(cloudProvider, "azure") && strings.EqualFold(options.Features, FeatureCSI) && options.UseVolumeSnapshots {
+			fmt.Println("Start to install Azure VolumeSnapshotClass ...")
+			if err := KubectlApplyByFile(ctx, "../util/csi/AzureVolumeSnapshotClass.yaml"); err != nil {
+				return err
 			}
-
 		}
 	}
 	if options.GarbageCollectionFrequency > 0 {
--- a/test/util/velero/velero_utils.go
+++ b/test/util/velero/velero_utils.go
@ -26,6 +26,7 @@ import (
 	"net/http"
 	"os"
 	"os/exec"
+	"os/user"
 	"path/filepath"
 	"reflect"
 	"regexp"
@ -184,6 +185,14 @@ func getProviderVeleroInstallOptions(veleroCfg *VeleroConfig,
 	io.ProviderName = veleroCfg.ObjectStoreProvider

 	if veleroCfg.CloudCredentialsFile != "" {
+		// Expand home directory if it is specified. https://stackoverflow.com/a/17617721
+		if strings.HasPrefix(veleroCfg.CloudCredentialsFile, "~/") {
+			usr, _ := user.Current()
+			dir := usr.HomeDir
+			// Use strings.HasPrefix so we don't match paths like
+			// "/something/~/something/"
+			veleroCfg.CloudCredentialsFile = filepath.Join(dir, veleroCfg.CloudCredentialsFile[2:])
+		}
 		realPath, err := filepath.Abs(veleroCfg.CloudCredentialsFile)
 		if err != nil {
 			return nil, err
@ -927,7 +936,7 @@ func getVeleroCliTarball(cliTarballUrl string) (*os.File, error) {
 }

 func DeleteBackupResource(ctx context.Context, veleroCLI string, backupName string) error {
-	args := []string{"backup", "delete", backupName, "--confirm"}
+	args := []string{"--namespace", VeleroCfg.VeleroNamespace, "backup", "delete", backupName, "--confirm"}

 	cmd := exec.CommandContext(ctx, veleroCLI, args...)
 	fmt.Println("Delete backup Command:" + cmd.String())
@ -939,7 +948,7 @@ func DeleteBackupResource(ctx context.Context, veleroCLI string, backupName stri
 	output := strings.Replace(stdout, "\n", " ", -1)
 	fmt.Println("Backup delete command output:" + output)

-	args = []string{"backup", "get", backupName}
+	args = []string{"--namespace", VeleroCfg.VeleroNamespace, "backup", "get", backupName}

 	retryTimes := 5
 	for i := 1; i < retryTimes+1; i++ {
@ -958,13 +967,13 @@ func DeleteBackupResource(ctx context.Context, veleroCLI string, backupName stri
 	return nil
 }

-func GetBackup(ctx context.Context, veleroCLI string, backupName string) (string, string, error) {
-	args := []string{"backup", "get", backupName}
+func GetBackup(ctx context.Context, veleroCLI, backupName string) (string, string, error) {
+	args := []string{"--namespace", VeleroCfg.VeleroNamespace, "backup", "get", backupName}
 	cmd := exec.CommandContext(ctx, veleroCLI, args...)
 	return veleroexec.RunCommand(cmd)
 }

-func IsBackupExist(ctx context.Context, veleroCLI string, backupName string) (bool, error) {
+func IsBackupExist(ctx context.Context, veleroCLI, backupName string) (bool, error) {
 	out, outerr, err := GetBackup(ctx, veleroCLI, backupName)
 	if err != nil {
 		if strings.Contains(outerr, "not found") {
@ -977,7 +986,7 @@ func IsBackupExist(ctx context.Context, veleroCLI string, backupName string) (bo
 	return true, nil
 }

-func WaitBackupDeleted(ctx context.Context, veleroCLI string, backupName string, timeout time.Duration) error {
+func WaitBackupDeleted(ctx context.Context, veleroCLI, backupName string, timeout time.Duration) error {
 	return wait.PollImmediate(10*time.Second, timeout, func() (bool, error) {
 		if exist, err := IsBackupExist(ctx, veleroCLI, backupName); err != nil {
 			return false, err
@ -992,7 +1001,7 @@ func WaitBackupDeleted(ctx context.Context, veleroCLI string, backupName string,
 	})
 }

-func WaitForExpectedStateOfBackup(ctx context.Context, veleroCLI string, backupName string,
+func WaitForExpectedStateOfBackup(ctx context.Context, veleroCLI, backupName string,
 	timeout time.Duration, existing bool) error {
 	return wait.PollImmediate(10*time.Second, timeout, func() (bool, error) {
 		if exist, err := IsBackupExist(ctx, veleroCLI, backupName); err != nil {
@ -1013,7 +1022,7 @@ func WaitForExpectedStateOfBackup(ctx context.Context, veleroCLI string, backupN
 	})
 }

-func WaitForBackupToBeCreated(ctx context.Context, veleroCLI string, backupName string, timeout time.Duration) error {
+func WaitForBackupToBeCreated(ctx context.Context, veleroCLI, backupName string, timeout time.Duration) error {
 	return WaitForExpectedStateOfBackup(ctx, veleroCLI, backupName, timeout, true)
 }