Set `GOBIN` so Makefile don't modify $PATH on `go install` Fix realPath resolving when cloud credentials is prefixed by `~` for home dir Use `~/.docker/config.json` if REGISTRY_CREDENTIAL_FILE not defined and skip step if does not exists since it is optional

Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> Set `GOBIN` so Makefile don't modify $PATH on `go install` Fix realPath resolving when cloud credentials is prefixed by `~` for home dir Use `~/.docker/config.json` if REGISTRY_CREDENTIAL_FILE not defined and skip step if does not exists since it is optional Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> Add kind testdata storageclass Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> Add kind testdata storageclass Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> log `Start to install Azure VolumeSnapshotClass ...` only on azure when csi is enabled Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> Add BSL_CONFIG example and notes Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> Makefile: Set `GOBIN` for `_output/...` Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> README spacing Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> StandbyClusterObjectStoreProvider typo Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> Specify velero namespace during get/delete command Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> Use object stores rather than cloudProvider for bucket queries Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> Remove debug print Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> simplify NS get changes, add velero NS to `DeleteBackupResource` Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> Skip file system backups on kind which uses hostPath volumes Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> Add StorageClass change test to PR kind e2e Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> Add more tests to pr Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> Add NS mapping to PR e2e Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> Add `SKIP_KIND` to some jobs containing volumes Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> Remove kind from kibishii tests Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> Label volume resource policies as restic, skip restic/snapshot tests, add more tests Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> TTLTest is a snapshot test Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> Remove non working tests Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> Resolve https://github.com/vmware-tanzu/velero/pull/7353#issuecomment-1925660077 Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> address https://github.com/vmware-tanzu/velero/pull/7353/files#r1477218762 Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com> Address https://github.com/vmware-tanzu/velero/pull/7353#issuecomment-1923414840 Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com>
2024-01-24 14:28:58 -05:00 · 2024-01-24 14:28:58 -05:00 · b1d95cf2aa
parent 99e0c483c7
commit b1d95cf2aa
20 changed files with 161 additions and 63 deletions
--- a/.github/workflows/e2e-test-kind.yaml
+++ b/.github/workflows/e2e-test-kind.yaml
@ -67,6 +67,16 @@ jobs:
          - 1.23.6
          - 1.24.0
          - 1.25.3
+        focus:
+          # tests to focus on, use `|` to concatenate multiple regexes to run on the same job
+          # ordered according to e2e_suite_test.go order
+          - Basic\]\[ClusterResource
+          - ResourceFiltering
+          - ResourceModifier|Backups|PrivilegesMgmt\]\[SSR
+          - Schedule\]\[OrderedResources
+          - NamespaceMapping\]\[Single\]\[Restic|NamespaceMapping\]\[Multiple\]\[Restic
+          - Basic\]\[Nodeport
+          - Basic\]\[StorageClass
      fail-fast: false
    steps:
      - name: Set up Go
@ -123,7 +133,8 @@ jobs:
              CREDS_FILE=/tmp/credential BSL_BUCKET=bucket \
              ADDITIONAL_OBJECT_STORE_PROVIDER=aws ADDITIONAL_BSL_CONFIG=region=minio,s3ForcePathStyle="true",s3Url=http://$(hostname -i):9000 \
              ADDITIONAL_CREDS_FILE=/tmp/credential ADDITIONAL_BSL_BUCKET=additional-bucket \
-              GINKGO_FOCUS='Basic\]\[ClusterResource' VELERO_IMAGE=velero:pr-test \
+              GINKGO_FOCUS='${{ matrix.focus }}' VELERO_IMAGE=velero:pr-test \
+              GINKGO_SKIP='SKIP_KIND|pv-backup|Restic|Snapshot|LongTime' \
              make -C test/e2e run
        timeout-minutes: 30
      - name: Upload debug bundle
--- a/3
+++ b/3
@ -108,6 +108,7 @@ platform_temp = $(subst -, ,$(ARCH))
 GOOS = $(word 1, $(platform_temp))
 GOARCH = $(word 2, $(platform_temp))
 GOPROXY ?= https://proxy.golang.org
+GOBIN=$$(pwd)/.go/bin

 # If you want to build all binaries, see the 'all-build' rule.
 # If you want to build all containers, see the 'all-containers' rule.
@ -129,6 +130,7 @@ local: build-dirs
 # Add DEBUG=1 to enable debug locally
 	GOOS=$(GOOS) \
 	GOARCH=$(GOARCH) \
+	GOBIN=$(GOBIN) \
 	VERSION=$(VERSION) \
 	REGISTRY=$(REGISTRY) \
 	PKG=$(PKG) \
@ -145,6 +147,7 @@ _output/bin/$(GOOS)/$(GOARCH)/$(BIN): build-dirs
 	$(MAKE) shell CMD="-c '\
 		GOOS=$(GOOS) \
 		GOARCH=$(GOARCH) \
+		GOBIN=$(GOBIN) \
 		VERSION=$(VERSION) \
 		REGISTRY=$(REGISTRY) \
 		PKG=$(PKG) \
--- a/hack/build.sh
+++ b/hack/build.sh
@ -32,6 +32,10 @@ if [[ -z "${GOOS}" ]]; then
    echo "GOOS must be set"
    exit 1
 fi
+if [[ -z "${GOBIN}" ]]; then
+    echo "GOBIN must be set"
+    exit 1
+fi
 if [[ -z "${GOARCH}" ]]; then
    echo "GOARCH must be set"
    exit 1
--- a/test/e2e/Makefile
+++ b/test/e2e/Makefile
@ -41,8 +41,10 @@ help:  ## Display this help

 TOOLS_DIR := $(REPO_ROOT)/hack/tools
 BIN_DIR := bin
+# Try to not modify PATH if possible
+GOBIN := $(REPO_ROOT)/.go/bin
 TOOLS_BIN_DIR := $(TOOLS_DIR)/$(BIN_DIR)
-GINKGO := $(shell go env GOPATH)/bin/ginkgo
+GINKGO := $(GOBIN)/ginkgo
 KUSTOMIZE := $(TOOLS_BIN_DIR)/kustomize
 OUTPUT_DIR := _output/$(GOOS)/$(GOARCH)/bin
 GINKGO_FOCUS ?=
@ -113,9 +115,13 @@ STANDBY_CLUSTER_NAME ?=
 EKS_POLICY_ARN ?=


+# Make sure ginkgo is in $GOBIN
 .PHONY:ginkgo
-ginkgo: # Make sure ginkgo is in $GOPATH/bin
-	go install github.com/onsi/ginkgo/ginkgo@v1.16.5
+ginkgo: ${GOBIN}/ginkgo
+
+# This target does not run if ginkgo is already in $GOBIN
+${GOBIN}/ginkgo:
+	GOBIN=${GOBIN} go install github.com/onsi/ginkgo/ginkgo@v1.16.5

 .PHONY: run
 run: ginkgo
--- a/test/e2e/README.md
+++ b/test/e2e/README.md
@ -66,7 +66,7 @@ the object-store-provider to be specified.
 1. `-features`: Comma-separated list of features to enable for this Velero process.
 1. `-registry-credential-file`: File containing credential for the image registry, follows the same format rules as the ~/.docker/config.json file. This credential will be loaded in Velero server pod to help on Docker Hub rate limit issue.
 1. `-kibishii-directory`: The file directory or URL path to install Kibishii. It's configurable in case the default path is not accessible for your own test environment.
-1. `-debug-e2e-test`: A Switch for enable or disable test data cleaning action.
+1. `-debug-e2e-test`: <true/false> A Switch for enable or disable test data cleaning action.
 1. `-garbage-collection-frequency`: frequency of garbage collection. It is a parameter for Velero installation. Optional.
 1. `-velero-server-debug-mode`: A switch for enable or disable having debug log of Velero server.
 1. `-default-cluster-context`: Default (source) cluster's kube config context, it's for migration test.
@ -101,7 +101,7 @@ Below is a mapping between `make` variables to E2E configuration flags.
 1. `VELERO_NAMESPACE `: the `-velero-namespace`. Optional.
 1. `PLUGINS `: the `-plugins`. Optional.
 1. `BSL_PREFIX`: `-prefix`. Optional.
-1. `BSL_CONFIG`: `-bsl-config`. Optional.
+1. `BSL_CONFIG`: `-bsl-config`. Optional. Example: BSL_CONFIG="region=us-east-1". May be required for some object store provider
 1. `VSL_CONFIG`: `-vsl-config`. Optional.
 1. `UPGRADE_FROM_VELERO_CLI `: `-upgrade-from-velero-cli`. Optional.
 1. `UPGRADE_FROM_VELERO_VERSION `: `-upgrade-from-velero-version`. Optional.
@ -136,9 +136,18 @@ Below is a mapping between `make` variables to E2E configuration flags.
 Basic examples:

 1. Run Velero tests in a kind cluster with AWS (or Minio) as the storage provider:
+    
+    Start kind cluster
+    ```bash
+    kind create cluster
+    ```
    ```bash
    BSL_PREFIX=<PREFIX_UNDER_BUCKET> BSL_BUCKET=<BUCKET_FOR_E2E_TEST_BACKUP> CREDS_FILE=/path/to/aws-creds CLOUD_PROVIDER=kind OBJECT_STORE_PROVIDER=aws make test-e2e
    ```
+    Stop kind cluster
+    ```bash
+    kind delete cluster
+    ```
 1. Run Velero tests in an AWS cluster:
    ```bash
    BSL_PREFIX=<PREFIX_UNDER_BUCKET> BSL_BUCKET=<BUCKET_FOR_E2E_TEST_BACKUP> CREDS_FILE=/path/to/aws-creds CLOUD_PROVIDER=aws make test-e2e
@ -304,3 +313,7 @@ It will reduce the frequency of velero installation by installing and checking V

 ### Tips
 Look for the ⛵ emoji printed at the end of each install and uninstall log. There should not be two install/unintall in a row, and there should be tests between an install and an uninstall. 
+
+#### Troubleshooting
+
+If velero log shows `level=error msg="Failed to get bucket region, bucket: xbucket, error: operation error S3: HeadBucket, failed to resolve service endpoint, endpoint rule error, A region must be set when sending requests to S3." backup-storage-location=velero/default cmd=/plugins/velero-plugin-for-aws controller=backup-storage-location logSource="/go/src/velero-plugin-for-aws/velero-plugin-for-aws/object_store.go:136" pluginName=velero-plugin-for-aws`, it means you need to set `BSL_CONFIG` to include `region=<region>`.
--- a/test/e2e/backup/backup.go
+++ b/test/e2e/backup/backup.go
@ -74,8 +74,12 @@ func BackupRestoreTest(backupRestoreTestConfig BackupRestoreTestConfig) {
 		veleroCfg.KibishiiDirectory = veleroCfg.KibishiiDirectory + backupRestoreTestConfig.kibishiiPatchSubDir
 		veleroCfg.UseVolumeSnapshots = useVolumeSnapshots
 		veleroCfg.UseNodeAgent = !useVolumeSnapshots
-		if useVolumeSnapshots && veleroCfg.CloudProvider == "kind" {
-			Skip("Volume snapshots not supported on kind")
+		if veleroCfg.CloudProvider == "kind" {
+			Skip("Volume snapshots plugin and File System Backups are not supported on kind")
+			// on kind cluster snapshots are not supported since there is no velero snapshot plugin for kind volumes.
+			// and PodVolumeBackups are not supported because PVB creation gets skipped for hostpath volumes, which are the only
+			// volumes created on kind clusters using the default storage class and provisioner (provisioner: rancher.io/local-path)
+			// This test suite checks for volume snapshots and PVBs generated from FileSystemBackups, so skip it on kind clusters
 		}
 		// [SKIP]: Static provisioning for vSphere CSI driver works differently from other drivers.
 		//         For vSphere CSI, after you create a PV specifying an existing volume handle, CSI
--- a/test/e2e/backups/deletion.go
+++ b/test/e2e/backups/deletion.go
@ -86,6 +86,9 @@ func backup_deletion_test(useVolumeSnapshots bool) {
 // runUpgradeTests runs upgrade test on the provider by kibishii.
 func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupName, backupLocation string,
 	useVolumeSnapshots bool, kibishiiDirectory string) error {
+	if useVolumeSnapshots && veleroCfg.CloudProvider == "kind" {
+		Skip("Volume snapshots not supported on kind")
+	}
 	oneHourTimeout, ctxCancel := context.WithTimeout(context.Background(), time.Minute*60)
 	defer ctxCancel()
 	veleroCLI := veleroCfg.VeleroCLI
@ -111,7 +114,7 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam
 		registryCredentialFile, veleroFeatures, kibishiiDirectory, useVolumeSnapshots, DefaultKibishiiData); err != nil {
 		return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", deletionTest)
 	}
-	err := ObjectsShouldNotBeInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig, backupName, BackupObjectsPrefix, 1)
+	err := ObjectsShouldNotBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig, backupName, BackupObjectsPrefix, 1)
 	if err != nil {
 		return err
 	}
@ -139,7 +142,7 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam
 			return errors.Wrapf(err, "Error waiting for uploads to complete")
 		}
 	}
-	err = ObjectsShouldBeInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix)
+	err = ObjectsShouldBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix)
 	if err != nil {
 		return err
 	}
@ -167,7 +170,7 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam
 		}
 	}

-	err = ObjectsShouldNotBeInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 5)
+	err = ObjectsShouldNotBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 5)
 	if err != nil {
 		return err
 	}
@ -194,12 +197,12 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam
 		})
 	})

-	err = DeleteObjectsInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix)
+	err = DeleteObjectsInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix)
 	if err != nil {
 		return err
 	}

-	err = ObjectsShouldNotBeInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 1)
+	err = ObjectsShouldNotBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 1)
 	if err != nil {
 		return err
 	}
--- a/test/e2e/backups/sync_backups.go
+++ b/test/e2e/backups/sync_backups.go
@ -148,13 +148,13 @@ func BackupsSyncTest() {
 		})

 		By(fmt.Sprintf("Delete %s backup files in object store", test.backupName), func() {
-			err = DeleteObjectsInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
+			err = DeleteObjectsInBucket(VeleroCfg.ObjectStoreProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
 				VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, test.backupName, BackupObjectsPrefix)
 			Expect(err).To(Succeed(), fmt.Sprintf("Failed to delete object in bucket %s with err %v", test.backupName, err))
 		})

 		By(fmt.Sprintf("Check %s backup files in object store is deleted", test.backupName), func() {
-			err = ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
+			err = ObjectsShouldNotBeInBucket(VeleroCfg.ObjectStoreProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
 				VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, test.backupName, BackupObjectsPrefix, 1)
 			Expect(err).To(Succeed(), fmt.Sprintf("Failed to delete object in bucket %s with err %v", test.backupName, err))
 		})
--- a/test/e2e/backups/ttl.go
+++ b/test/e2e/backups/ttl.go
@ -155,7 +155,7 @@ func TTLTest() {
 		})

 		By("Associated Restores should be created", func() {
-			Expect(ObjectsShouldBeInBucket(veleroCfg.CloudProvider,
+			Expect(ObjectsShouldBeInBucket(veleroCfg.ObjectStoreProvider,
 				veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,
 				veleroCfg.BSLPrefix, veleroCfg.BSLConfig, test.restoreName,
 				RestoreObjectsPrefix)).NotTo(HaveOccurred(), "Fail to get restore object")
@ -179,7 +179,7 @@ func TTLTest() {
 		})

 		By("Backup file from cloud object storage should be deleted", func() {
-			Expect(ObjectsShouldNotBeInBucket(veleroCfg.CloudProvider,
+			Expect(ObjectsShouldNotBeInBucket(veleroCfg.ObjectStoreProvider,
 				veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,
 				veleroCfg.BSLPrefix, veleroCfg.BSLConfig, test.backupName,
 				BackupObjectsPrefix, 5)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
@ -194,7 +194,7 @@ func TTLTest() {
 		})

 		By("Associated Restores should be deleted", func() {
-			Expect(ObjectsShouldNotBeInBucket(veleroCfg.CloudProvider,
+			Expect(ObjectsShouldNotBeInBucket(veleroCfg.ObjectStoreProvider,
 				veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,
 				veleroCfg.BSLPrefix, veleroCfg.BSLConfig, test.restoreName,
 				RestoreObjectsPrefix, 5)).NotTo(HaveOccurred(), "Fail to get restore object")
--- a/test/e2e/basic/namespace-mapping.go
+++ b/test/e2e/basic/namespace-mapping.go
@ -38,6 +38,9 @@ func (n *NamespaceMapping) Init() error {
 	n.VeleroCfg.UseVolumeSnapshots = n.UseVolumeSnapshots
 	n.VeleroCfg.UseNodeAgent = !n.UseVolumeSnapshots
 	n.kibishiiData = &KibishiiData{Levels: 2, DirsPerLevel: 10, FilesPerLevel: 10, FileLength: 1024, BlockSize: 1024, PassNum: 0, ExpectedNodes: 2}
+	if n.VeleroCfg.CloudProvider == "kind" {
+		n.kibishiiData = &KibishiiData{Levels: 0, DirsPerLevel: 0, FilesPerLevel: 0, FileLength: 0, BlockSize: 0, PassNum: 0, ExpectedNodes: 2}
+	}
 	backupType := "restic"
 	if n.UseVolumeSnapshots {
 		backupType = "snapshot"
@ -67,7 +70,11 @@ func (n *NamespaceMapping) Init() error {
 		"create", "--namespace", n.VeleroCfg.VeleroNamespace, "backup", n.BackupName,
 		"--include-namespaces", strings.Join(*n.NSIncluded, ","), "--wait",
 	}
-	if n.UseVolumeSnapshots {
+	if VeleroCfg.CloudProvider == "kind" {
+		// don't test volume snapshotter or file system backup on kind
+		n.BackupArgs = append(n.BackupArgs, "--snapshot-volumes=false")
+		n.UseVolumeSnapshots = false
+	} else if n.UseVolumeSnapshots {
 		n.BackupArgs = append(n.BackupArgs, "--snapshot-volumes")
 	} else {
 		n.BackupArgs = append(n.BackupArgs, "--snapshot-volumes=false")
--- a/test/e2e/bsl-mgmt/deletion.go
+++ b/test/e2e/bsl-mgmt/deletion.go
@ -276,7 +276,7 @@ func BslDeletionTest(useVolumeSnapshots bool) {
 				Expect(cmp.Diff(backupsInBsl1AndBsl2, backupsBeforeDel, cmpopts.SortSlices(less))).Should(BeEmpty())

 				By(fmt.Sprintf("Backup1 %s should exist in cloud object store before bsl deletion", backupName_1), func() {
-					Expect(ObjectsShouldBeInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile,
+					Expect(ObjectsShouldBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile,
 						veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig,
 						backupName_1, BackupObjectsPrefix)).To(Succeed())
 				})
@ -295,14 +295,14 @@ func BslDeletionTest(useVolumeSnapshots bool) {
 			})

 			By(fmt.Sprintf("Backup1 %s should still exist in cloud object store after bsl deletion", backupName_1), func() {
-				Expect(ObjectsShouldBeInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile,
+				Expect(ObjectsShouldBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile,
 					veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig,
 					backupName_1, BackupObjectsPrefix)).To(Succeed())
 			})

 			// TODO: Choose additional BSL to be deleted as an new test case
 			// By(fmt.Sprintf("Backup %s should still exist in cloud object store", backupName_2), func() {
-			// 	Expect(ObjectsShouldBeInBucket(veleroCfg.CloudProvider, veleroCfg.AdditionalBSLCredentials,
+			// 	Expect(ObjectsShouldBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.AdditionalBSLCredentials,
 			// 		veleroCfg.AdditionalBSLBucket, veleroCfg.AdditionalBSLPrefix, veleroCfg.AdditionalBSLConfig,
 			// 		backupName_2, BackupObjectsPrefix)).To(Succeed())
 			// })
--- a/test/e2e/e2e_suite_test.go
+++ b/test/e2e/e2e_suite_test.go
@ -92,7 +92,7 @@ func init() {
 	flag.StringVar(&VeleroCfg.DataMoverPlugin, "data-mover-plugin", "", "customized plugin for data mover.")
 	flag.StringVar(&VeleroCfg.StandbyClusterCloudProvider, "standby-cluster-cloud-provider", "", "cloud provider for standby cluster.")
 	flag.StringVar(&VeleroCfg.StandbyClusterPlugins, "standby-cluster-plugins", "", "plugins provider for standby cluster.")
-	flag.StringVar(&VeleroCfg.StandbyClusterOjbectStoreProvider, "standby-cluster-object-store-provider", "", "object store provider for standby cluster.")
+	flag.StringVar(&VeleroCfg.StandbyClusterObjectStoreProvider, "standby-cluster-object-store-provider", "", "object store provider for standby cluster.")
 	flag.BoolVar(&VeleroCfg.DebugVeleroPodRestart, "debug-velero-pod-restart", false, "a switch for debugging velero pod restart.")
 	flag.BoolVar(&VeleroCfg.DisableInformerCache, "disable-informer-cache", false, "a switch for disable informer cache.")
 	flag.StringVar(&VeleroCfg.DefaultClusterName, "default-cluster-name", "", "default cluster's name in kube config file, it's for EKS IRSA test.")
@ -103,8 +103,8 @@ func init() {

 }

-var _ = Describe("[APIGroup][APIVersion] Velero tests with various CRD API group versions", APIGropuVersionsTest)
-var _ = Describe("[APIGroup][APIExtensions] CRD of apiextentions v1beta1 should be B/R successfully from cluster(k8s version < 1.22) to cluster(k8s version >= 1.22)", APIExtensionsVersionsTest)
+var _ = Describe("[APIGroup][APIVersion][SKIP_KIND] Velero tests with various CRD API group versions", APIGropuVersionsTest)
+var _ = Describe("[APIGroup][APIExtensions][SKIP_KIND] CRD of apiextentions v1beta1 should be B/R successfully from cluster(k8s version < 1.22) to cluster(k8s version >= 1.22)", APIExtensionsVersionsTest)

 // Test backup and restore of Kibishi using restic
 var _ = Describe("[Basic][Restic] Velero tests on cluster using the plugin provider for object storage and Restic for volume backups", BackupRestoreWithRestic)
@ -134,18 +134,18 @@ var _ = Describe("[ResourceFiltering][IncludeNamespaces][Restore] Velero test on
 var _ = Describe("[ResourceFiltering][IncludeResources][Backup] Velero test on include resources from the cluster backup", BackupWithIncludeResources)
 var _ = Describe("[ResourceFiltering][IncludeResources][Restore] Velero test on include resources from the cluster restore", RestoreWithIncludeResources)
 var _ = Describe("[ResourceFiltering][LabelSelector] Velero test on backup include resources matching the label selector", BackupWithLabelSelector)
-var _ = Describe("[ResourceFiltering][ResourcePolicies] Velero test on skip backup of volume by resource policies", ResourcePoliciesTest)
+var _ = Describe("[ResourceFiltering][ResourcePolicies][Restic] Velero test on skip backup of volume by resource policies", ResourcePoliciesTest)

 var _ = Describe("[ResourceModifier][Restore] Velero test on resource modifiers from the cluster restore", ResourceModifiersTest)

 var _ = Describe("[Backups][Deletion][Restic] Velero tests of Restic backup deletion", BackupDeletionWithRestic)
 var _ = Describe("[Backups][Deletion][Snapshot] Velero tests of snapshot backup deletion", BackupDeletionWithSnapshots)
-var _ = Describe("[Backups][TTL][LongTime] Local backups and restic repos will be deleted once the corresponding backup storage location is deleted", TTLTest)
+var _ = Describe("[Backups][TTL][LongTime][Snapshot] Local backups and restic repos will be deleted once the corresponding backup storage location is deleted", TTLTest)
 var _ = Describe("[Backups][BackupsSync] Backups in object storage are synced to a new Velero and deleted backups in object storage are synced to be deleted in Velero", BackupsSyncTest)

 var _ = Describe("[Schedule][BR][Pause][LongTime] Backup will be created periodly by schedule defined by a Cron expression", ScheduleBackupTest)
-var _ = Describe("[Schedule][OrederedResources] Backup resources should follow the specific order in schedule", ScheduleOrderedResources)
-var _ = Describe("[Schedule][BackupCreation] Schedule controller wouldn't create a new backup when it still has pending or InProgress backup", ScheduleBackupCreationTest)
+var _ = Describe("[Schedule][OrderedResources] Backup resources should follow the specific order in schedule", ScheduleOrderedResources)
+var _ = Describe("[Schedule][BackupCreation][SKIP_KIND] Schedule controller wouldn't create a new backup when it still has pending or InProgress backup", ScheduleBackupCreationTest)

 var _ = Describe("[PrivilegesMgmt][SSR] Velero test on ssr object when controller namespace mix-ups", SSRTest)

@ -165,7 +165,7 @@ var _ = Describe("[pv-backup][Opt-Out] Backup resources should follow the specif

 var _ = Describe("[Basic][Nodeport] Service nodeport reservation during restore is configurable", NodePortTest)
 var _ = Describe("[Basic][StorageClass] Storage class of persistent volumes and persistent volume claims can be changed during restores", StorageClasssChangingTest)
-var _ = Describe("[Basic][SelectedNode] Node selectors of persistent volume claims can be changed during restores", PVCSelectedNodeChangingTest)
+var _ = Describe("[Basic][SelectedNode][SKIP_KIND] Node selectors of persistent volume claims can be changed during restores", PVCSelectedNodeChangingTest)

 func GetKubeconfigContext() error {
 	var err error
@ -206,6 +206,18 @@ func TestE2e(t *testing.T) {
 		t.Skip("Skipping E2E tests")
 	}

+	if VeleroCfg.CloudProvider != "kind" {
+		fmt.Println("For cloud platforms, object store plugin provider will be set as cloud provider")
+		// If ObjectStoreProvider is not provided, then using the value same as CloudProvider
+		if VeleroCfg.ObjectStoreProvider == "" {
+			VeleroCfg.ObjectStoreProvider = VeleroCfg.CloudProvider
+		}
+	} else {
+		if VeleroCfg.ObjectStoreProvider == "" {
+			t.Error(errors.New("No object store provider specified - must be specified when using kind as the cloud provider")) // Must have an object store provider
+		}
+	}
+
 	var err error
 	if err = GetKubeconfigContext(); err != nil {
 		fmt.Println(err)
--- a/test/e2e/migration/migration.go
+++ b/test/e2e/migration/migration.go
@ -298,7 +298,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
 					// For SnapshotMoveData pipelines, we should use standby clustr setting for Velero installation
 					// In nightly CI, StandbyClusterPlugins is set properly if pipeline is for SnapshotMoveData.
 					veleroCfg.Plugins = veleroCfg.StandbyClusterPlugins
-					veleroCfg.ObjectStoreProvider = veleroCfg.StandbyClusterOjbectStoreProvider
+					veleroCfg.ObjectStoreProvider = veleroCfg.StandbyClusterObjectStoreProvider
 				}

 				Expect(VeleroInstall(context.Background(), &veleroCfg, true)).To(Succeed())
--- a/test/perf/Makefile
+++ b/test/perf/Makefile
@ -41,8 +41,10 @@ help:  ## Display this help

 TOOLS_DIR := $(REPO_ROOT)/hack/tools
 BIN_DIR := bin
+# Try to not modify PATH if possible
+GOBIN := $(REPO_ROOT)/.go/bin
 TOOLS_BIN_DIR := $(TOOLS_DIR)/$(BIN_DIR)
-GINKGO := $(GOPATH)/bin/ginkgo
+GINKGO := $(GOBIN)/ginkgo
 KUSTOMIZE := $(TOOLS_BIN_DIR)/kustomize
 OUTPUT_DIR := _output/$(GOOS)/$(GOARCH)/bin
 GINKGO_FOCUS ?=
@ -88,9 +90,13 @@ VELERO_POD_CPU_REQUEST ?= 2
 VELERO_POD_MEM_REQUEST ?= 2Gi
 POD_VOLUME_OPERATION_TIMEOUT ?= 6h

+# Make sure ginkgo is in $GOBIN
 .PHONY:ginkgo
-ginkgo: # Make sure ginkgo is in $GOPATH/bin
-	go install github.com/onsi/ginkgo/ginkgo@v1.16.5
+ginkgo: ${GOBIN}/ginkgo
+
+# This target does not run if ginkgo is already in $GOBIN
+${GOBIN}/ginkgo:
+	GOBIN=${GOBIN} go install github.com/onsi/ginkgo/ginkgo@v1.16.5

 .PHONY: run
 run: ginkgo
--- a/test/testdata/storage-class/kind.yaml
+++ b/test/testdata/storage-class/kind.yaml
@ -0,0 +1,10 @@
+allowVolumeExpansion: true
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: e2e-storage-class
+parameters:
+  type: pd-standard
+provisioner: rancher.io/local-path
+reclaimPolicy: Delete
+volumeBindingMode: WaitForFirstConsumer
--- a/test/types.go
+++ b/test/types.go
@ -85,7 +85,7 @@ type VeleroConfig struct {
 	DataMoverPlugin                   string
 	StandbyClusterCloudProvider       string
 	StandbyClusterPlugins             string
-	StandbyClusterOjbectStoreProvider string
+	StandbyClusterObjectStoreProvider string
 	DebugVeleroPodRestart             bool
 	IsUpgradeTest                     bool
 	WithoutDisableInformerCacheParam  bool
--- a/test/util/k8s/serviceaccount.go
+++ b/test/util/k8s/serviceaccount.go
@ -47,8 +47,16 @@ func WaitUntilServiceAccountCreated(ctx context.Context, client TestClient, name
 }

 func PatchServiceAccountWithImagePullSecret(ctx context.Context, client TestClient, namespace, serviceAccount, dockerCredentialFile string) error {
+	if dockerCredentialFile == "" {
+		// use the default docker credential file in the home directory
+		dockerCredentialFile = os.Getenv("HOME") + "/.docker/config.json"
+	}
+	// if file do not exist, do not patch the service account, just return
 	credential, err := os.ReadFile(dockerCredentialFile)
 	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
 		return errors.Wrapf(err, "failed to read the docker credential file %q", dockerCredentialFile)
 	}
 	secretName := "image-pull-secret"
--- a/test/util/providers/common.go
+++ b/test/util/providers/common.go
@ -34,24 +34,24 @@ type ObjectsInStorage interface {
 	IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupName string, snapshotCheck SnapshotCheckPoint) error
 }

-func ObjectsShouldBeInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) error {
+func ObjectsShouldBeInBucket(objectStoreProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) error {
 	fmt.Printf("|| VERIFICATION || - %s should exist in storage [%s %s]\n", backupName, bslPrefix, subPrefix)
-	exist, err := IsObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix)
+	exist, err := IsObjectsInBucket(objectStoreProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix)
 	if !exist {
 		return errors.Wrap(err, fmt.Sprintf("|| UNEXPECTED ||Backup object %s is not exist in object store after backup as expected\n", backupName))
 	}
 	fmt.Printf("|| EXPECTED || - Backup %s exist in object storage bucket %s\n", backupName, bslBucket)
 	return nil
 }
-func ObjectsShouldNotBeInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string, retryTimes int) error {
+func ObjectsShouldNotBeInBucket(objectStoreProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string, retryTimes int) error {
 	var err error
 	var exist bool
 	fmt.Printf("|| VERIFICATION || - %s %s should not exist in object store %s\n", subPrefix, backupName, bslPrefix)
 	if cloudCredentialsFile == "" {
-		return errors.New(fmt.Sprintf("|| ERROR || - Please provide credential file of cloud %s \n", cloudProvider))
+		return errors.New(fmt.Sprintf("|| ERROR || - Please provide credential file of cloud %s \n", objectStoreProvider))
 	}
 	for i := 0; i < retryTimes; i++ {
-		exist, err = IsObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix)
+		exist, err = IsObjectsInBucket(objectStoreProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix)
 		if err != nil {
 			return errors.Wrapf(err, "|| UNEXPECTED || - Failed to get backup %s in object store\n", backupName)
 		}
@ -63,6 +63,10 @@ func ObjectsShouldNotBeInBucket(cloudProvider, cloudCredentialsFile, bslBucket,
 	}
 	return errors.New(fmt.Sprintf("|| UNEXPECTED ||Backup object %s still exist in object store after backup deletion\n", backupName))
 }
+
+// This function returns a storage interface based on the cloud provider for querying objects and snapshots
+// When cloudProvider is kind, pass in object storage provider instead. For example, "aws".
+// Snapshots are not supported on kind.
 func getProvider(cloudProvider string) (ObjectsInStorage, error) {
 	var s ObjectsInStorage
 	switch cloudProvider {
@ -89,24 +93,24 @@ func getFullPrefix(bslPrefix, subPrefix string) string {
 	}
 	return bslPrefix
 }
-func IsObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) (bool, error) {
+func IsObjectsInBucket(objectStoreProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) (bool, error) {
 	bslPrefix = getFullPrefix(bslPrefix, subPrefix)
-	s, err := getProvider(cloudProvider)
+	s, err := getProvider(objectStoreProvider)
 	if err != nil {
-		return false, errors.Wrapf(err, fmt.Sprintf("Cloud provider %s is not valid", cloudProvider))
+		return false, errors.Wrapf(err, fmt.Sprintf("Object store provider %s is not valid", objectStoreProvider))
 	}
 	return s.IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName)
 }

-func DeleteObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) error {
+func DeleteObjectsInBucket(objectStoreProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) error {
 	bslPrefix = getFullPrefix(bslPrefix, subPrefix)
 	fmt.Printf("|| VERIFICATION || - Delete backup %s in storage %s\n", backupName, bslPrefix)
 	if cloudCredentialsFile == "" {
-		return errors.New(fmt.Sprintf("|| ERROR || - Please provide credential file of cloud %s \n", cloudProvider))
+		return errors.New(fmt.Sprintf("|| ERROR || - Please provide credential file of cloud %s \n", objectStoreProvider))
 	}
-	s, err := getProvider(cloudProvider)
+	s, err := getProvider(objectStoreProvider)
 	if err != nil {
-		return errors.Wrapf(err, fmt.Sprintf("Cloud provider %s is not valid", cloudProvider))
+		return errors.Wrapf(err, fmt.Sprintf("Object store provider %s is not valid", objectStoreProvider))
 	}
 	err = s.DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName)
 	if err != nil {
--- a/test/util/velero/install.go
+++ b/test/util/velero/install.go
@ -75,6 +75,7 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluste
 	if isStandbyCluster {
 		veleroCfg.CloudProvider = veleroCfg.StandbyClusterCloudProvider
 	}
+
 	if veleroCfg.CloudProvider != "kind" {
 		fmt.Println("For cloud platforms, object store plugin provider will be set as cloud provider")
 		// If ObjectStoreProvider is not provided, then using the value same as CloudProvider
@ -278,15 +279,12 @@ func installVeleroServer(ctx context.Context, cli, cloudProvider string, options

 	if len(options.Features) > 0 {
 		args = append(args, "--features", options.Features)
-		if strings.EqualFold(options.Features, FeatureCSI) && options.UseVolumeSnapshots {
-			if strings.EqualFold(cloudProvider, "azure") {
+		if strings.EqualFold(cloudProvider, "azure") && strings.EqualFold(options.Features, FeatureCSI) && options.UseVolumeSnapshots {
 			fmt.Println("Start to install Azure VolumeSnapshotClass ...")
 			if err := KubectlApplyByFile(ctx, "../util/csi/AzureVolumeSnapshotClass.yaml"); err != nil {
 				return err
 			}
 		}
-
-		}
 	}
 	if options.GarbageCollectionFrequency > 0 {
 		args = append(args, fmt.Sprintf("--garbage-collection-frequency=%v", options.GarbageCollectionFrequency))
--- a/test/util/velero/velero_utils.go
+++ b/test/util/velero/velero_utils.go
@ -26,6 +26,7 @@ import (
 	"net/http"
 	"os"
 	"os/exec"
+	"os/user"
 	"path/filepath"
 	"reflect"
 	"regexp"
@ -184,6 +185,14 @@ func getProviderVeleroInstallOptions(veleroCfg *VeleroConfig,
 	io.ProviderName = veleroCfg.ObjectStoreProvider

 	if veleroCfg.CloudCredentialsFile != "" {
+		// Expand home directory if it is specified. https://stackoverflow.com/a/17617721
+		if strings.HasPrefix(veleroCfg.CloudCredentialsFile, "~/") {
+			usr, _ := user.Current()
+			dir := usr.HomeDir
+			// Use strings.HasPrefix so we don't match paths like
+			// "/something/~/something/"
+			veleroCfg.CloudCredentialsFile = filepath.Join(dir, veleroCfg.CloudCredentialsFile[2:])
+		}
 		realPath, err := filepath.Abs(veleroCfg.CloudCredentialsFile)
 		if err != nil {
 			return nil, err
@ -927,7 +936,7 @@ func getVeleroCliTarball(cliTarballUrl string) (*os.File, error) {
 }

 func DeleteBackupResource(ctx context.Context, veleroCLI string, backupName string) error {
-	args := []string{"backup", "delete", backupName, "--confirm"}
+	args := []string{"--namespace", VeleroCfg.VeleroNamespace, "backup", "delete", backupName, "--confirm"}

 	cmd := exec.CommandContext(ctx, veleroCLI, args...)
 	fmt.Println("Delete backup Command:" + cmd.String())
@ -939,7 +948,7 @@ func DeleteBackupResource(ctx context.Context, veleroCLI string, backupName stri
 	output := strings.Replace(stdout, "\n", " ", -1)
 	fmt.Println("Backup delete command output:" + output)

-	args = []string{"backup", "get", backupName}
+	args = []string{"--namespace", VeleroCfg.VeleroNamespace, "backup", "get", backupName}

 	retryTimes := 5
 	for i := 1; i < retryTimes+1; i++ {
@ -958,13 +967,13 @@ func DeleteBackupResource(ctx context.Context, veleroCLI string, backupName stri
 	return nil
 }

-func GetBackup(ctx context.Context, veleroCLI string, backupName string) (string, string, error) {
-	args := []string{"backup", "get", backupName}
+func GetBackup(ctx context.Context, veleroCLI, backupName string) (string, string, error) {
+	args := []string{"--namespace", VeleroCfg.VeleroNamespace, "backup", "get", backupName}
 	cmd := exec.CommandContext(ctx, veleroCLI, args...)
 	return veleroexec.RunCommand(cmd)
 }

-func IsBackupExist(ctx context.Context, veleroCLI string, backupName string) (bool, error) {
+func IsBackupExist(ctx context.Context, veleroCLI, backupName string) (bool, error) {
 	out, outerr, err := GetBackup(ctx, veleroCLI, backupName)
 	if err != nil {
 		if strings.Contains(outerr, "not found") {
@ -977,7 +986,7 @@ func IsBackupExist(ctx context.Context, veleroCLI string, backupName string) (bo
 	return true, nil
 }

-func WaitBackupDeleted(ctx context.Context, veleroCLI string, backupName string, timeout time.Duration) error {
+func WaitBackupDeleted(ctx context.Context, veleroCLI, backupName string, timeout time.Duration) error {
 	return wait.PollImmediate(10*time.Second, timeout, func() (bool, error) {
 		if exist, err := IsBackupExist(ctx, veleroCLI, backupName); err != nil {
 			return false, err
@ -992,7 +1001,7 @@ func WaitBackupDeleted(ctx context.Context, veleroCLI string, backupName string,
 	})
 }

-func WaitForExpectedStateOfBackup(ctx context.Context, veleroCLI string, backupName string,
+func WaitForExpectedStateOfBackup(ctx context.Context, veleroCLI, backupName string,
 	timeout time.Duration, existing bool) error {
 	return wait.PollImmediate(10*time.Second, timeout, func() (bool, error) {
 		if exist, err := IsBackupExist(ctx, veleroCLI, backupName); err != nil {
@ -1013,7 +1022,7 @@ func WaitForExpectedStateOfBackup(ctx context.Context, veleroCLI string, backupN
 	})
 }

-func WaitForBackupToBeCreated(ctx context.Context, veleroCLI string, backupName string, timeout time.Duration) error {
+func WaitForBackupToBeCreated(ctx context.Context, veleroCLI, backupName string, timeout time.Duration) error {
 	return WaitForExpectedStateOfBackup(ctx, veleroCLI, backupName, timeout, true)
 }