Merge pull request #8315 from blackpiglet/8298_fix

Modifications to support VKS environment

test/Makefile

@@ -197,7 +197,7 @@ run-e2e: ginkgo
 		--standby-cluster-name=$(STANDBY_CLUSTER_NAME) \
 		--eks-policy-arn=$(EKS_POLICY_ARN) \
 		--default-cls-service-account-name=$(DEFAULT_CLS_SERVICE_ACCOUNT_NAME) \
-		--standby-cls-service-account-name=$(STANDBY_CLS_SERVICE_ACCOUNT_NAME)
+		--standby-cls-service-account-name=$(STANDBY_CLS_SERVICE_ACCOUNT_NAME) \
 		--kibishii-directory=$(KIBISHII_DIRECTORY) \
 		--disable-informer-cache=$(DISABLE_INFORMER_CACHE)
 

test/e2e/migration/migration.go

@@ -25,6 +25,7 @@ import (
 	"github.com/google/uuid"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	"golang.org/x/mod/semver"
 
 	. "github.com/vmware-tanzu/velero/test"
 	util "github.com/vmware-tanzu/velero/test/util/csi"
@@ -146,22 +147,26 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
 						OriginVeleroCfg.RestoreHelperImage = ""
 						OriginVeleroCfg.Plugins = ""
 
-						// It is for v1.13.x migration scenario only, because since v1.14, nightly CI won't
+						versionWithoutPatch := semver.MajorMinor(veleroCLI2Version.VeleroVersion)
-						// pass velero-plugin-for-csi to E2E test anymore, and velero installation will not
+						// Read migration case needs plugins from the PluginsMatrix map.
-						// fetch velero-plugin-for-csi automatically, so add it as hardcode below.
+						migrationNeedPlugins, ok := PluginsMatrix[versionWithoutPatch]
-						// TODO: remove this section from future version like v1.15, e.g.
+						Expect(ok).To(BeTrue())
+
 						if OriginVeleroCfg.CloudProvider == Azure {
-							OriginVeleroCfg.Plugins = "velero/velero-plugin-for-microsoft-azure:v1.9.0"
+							OriginVeleroCfg.Plugins = migrationNeedPlugins[Azure][0]
 						}
 						if OriginVeleroCfg.CloudProvider == AWS {
-							OriginVeleroCfg.Plugins = "velero/velero-plugin-for-aws:v1.9.0"
+							OriginVeleroCfg.Plugins = migrationNeedPlugins[AWS][0]
 						}
-						if strings.Contains(OriginVeleroCfg.Features, FeatureCSI) {
+						// Because Velero CSI plugin is deprecated in v1.14,
-							OriginVeleroCfg.Plugins = OriginVeleroCfg.Plugins + ",velero/velero-plugin-for-csi:v0.7.0"
+						// only need to install it for version lower than v1.14.
+						if strings.Contains(OriginVeleroCfg.Features, FeatureCSI) &&
+							semver.Compare(versionWithoutPatch, "v1.14") < 0 {
+							OriginVeleroCfg.Plugins = OriginVeleroCfg.Plugins + "," + migrationNeedPlugins[CSI][0]
 						}
 						if OriginVeleroCfg.SnapshotMoveData {
 							if OriginVeleroCfg.CloudProvider == Azure {
-								OriginVeleroCfg.Plugins = OriginVeleroCfg.Plugins + ",velero/velero-plugin-for-aws:v1.9.0"
+								OriginVeleroCfg.Plugins = OriginVeleroCfg.Plugins + "," + migrationNeedPlugins[AWS][0]
 							}
 						}
 						veleroCLI2Version.VeleroCLI, err = InstallVeleroCLI(veleroCLI2Version.VeleroVersion)

test/testdata/storage-class/vsphere-csi.yaml

@@ -5,7 +5,8 @@ metadata:
   annotations:
     storageclass.kubernetes.io/is-default-class: "false"
 parameters:
-  StoragePolicyName: "vSAN Default Storage Policy"
+  # StoragePolicyName: "vSAN Default Storage Policy" # This is used for the TKGm environment.
+  svStorageClass: worker-storagepolicy # This is used for TKGs/uTKG environment.
 provisioner: csi.vsphere.vmware.com
 reclaimPolicy: Delete
 volumeBindingMode: WaitForFirstConsumer

test/testdata/storage-class/vsphere.yaml

@@ -5,7 +5,8 @@ metadata:
   annotations:
     storageclass.kubernetes.io/is-default-class: "false"
 parameters:
-  StoragePolicyName: "vSAN Default Storage Policy"
+  #StoragePolicyName: "vSAN Default Storage Policy" # This is used for TKGm environment.
+  svStorageClass: worker-storagepolicy # This is used for TKGs/uTKG environment.
 provisioner: csi.vsphere.vmware.com
 reclaimPolicy: Delete
 volumeBindingMode: WaitForFirstConsumer

test/types.go

@@ -22,7 +22,7 @@ import (
 	"github.com/google/uuid"
 
 	"github.com/vmware-tanzu/velero/pkg/cmd/cli/install"
-	. "github.com/vmware-tanzu/velero/test/util/k8s"
+	"github.com/vmware-tanzu/velero/test/util/k8s"
 )
 
 const StorageClassName = "e2e-storage-class"
@@ -35,12 +35,20 @@ const Azure = "azure"
 const AzureCSI = "azure-csi"
 const AwsCSI = "aws-csi"
 const AWS = "aws"
-const Gcp = "gcp"
+const GCP = "gcp"
 const Vsphere = "vsphere"
+const CSI = "csi"
 
 const UploaderTypeRestic = "restic"
 
-var PublicCloudProviders = []string{AWS, Azure, Gcp, Vsphere}
+const (
+	KubeSystemNamespace           = "kube-system"
+	VSphereCSIControllerNamespace = "vmware-system-csi"
+	VeleroVSphereSecretName       = "velero-vsphere-config-secret"
+	VeleroVSphereConfigMapName    = "velero-vsphere-plugin-config"
+)
+
+var PublicCloudProviders = []string{AWS, Azure, GCP, Vsphere}
 var LocalCloudProviders = []string{Kind, VanillaZFS}
 var CloudProviders = append(PublicCloudProviders, LocalCloudProviders...)
 
@@ -87,9 +95,9 @@ type VeleroConfig struct {
 	GCFrequency                       string
 	DefaultClusterContext             string
 	StandbyClusterContext             string
-	ClientToInstallVelero             *TestClient
+	ClientToInstallVelero             *k8s.TestClient
-	DefaultClient                     *TestClient
+	DefaultClient                     *k8s.TestClient
-	StandbyClient                     *TestClient
+	StandbyClient                     *k8s.TestClient
 	ClusterToInstallVelero            string
 	DefaultClusterName                string
 	StandbyClusterName                string

test/util/providers/common.go

@@ -78,7 +78,7 @@ func getProvider(cloudProvider string) (ObjectsInStorage, error) {
 	case AWS, Vsphere:
 		aws := AWSStorage("")
 		s = &aws
-	case Gcp:
+	case GCP:
 		gcs := GCSStorage("")
 		s = &gcs
 	case Azure:

test/util/velero/install.go

@@ -39,14 +39,9 @@ import (
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	"github.com/vmware-tanzu/velero/pkg/cmd/cli/install"
 	velerexec "github.com/vmware-tanzu/velero/pkg/util/exec"
-	. "github.com/vmware-tanzu/velero/test"
+	"github.com/vmware-tanzu/velero/test"
-	. "github.com/vmware-tanzu/velero/test/util/eks"
+	eksutil "github.com/vmware-tanzu/velero/test/util/eks"
-	. "github.com/vmware-tanzu/velero/test/util/k8s"
+	"github.com/vmware-tanzu/velero/test/util/k8s"
-)
-
-const (
-	KubeSystemNamespace           = "kube-system"
-	VSphereCSIControllerNamespace = "vmware-system-csi"
 )
 
 // we provide more install options other than the standard install.InstallOptions in E2E test
@@ -58,7 +53,7 @@ type installOptions struct {
 	WithoutDisableInformerCacheParam bool
 }
 
-func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluster bool) error {
+func VeleroInstall(ctx context.Context, veleroCfg *test.VeleroConfig, isStandbyCluster bool) error {
 	fmt.Printf("Velero install %s\n", time.Now().Format("2006-01-02 15:04:05"))
 
 	// veleroCfg struct including a set of BSL params and a set of additional BSL params,
@@ -77,7 +72,7 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluste
 		veleroCfg.CloudProvider = veleroCfg.StandbyClusterCloudProvider
 	}
 
-	if slices.Contains(PublicCloudProviders, veleroCfg.CloudProvider) {
+	if slices.Contains(test.PublicCloudProviders, veleroCfg.CloudProvider) {
 		fmt.Println("For public cloud platforms, object store plugin provider will be set as cloud provider")
 		// If ObjectStoreProvider is not provided, then using the value same as CloudProvider
 		if veleroCfg.ObjectStoreProvider == "" {
@@ -93,20 +88,29 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluste
 	if err != nil {
 		return errors.WithMessage(err, "Failed to get provider plugins")
 	}
-	err = EnsureClusterExists(ctx)
+	err = k8s.EnsureClusterExists(ctx)
 	if err != nil {
 		return errors.WithMessage(err, "Failed to ensure Kubernetes cluster exists")
 	}
 
 	// TODO - handle this better
-	if veleroCfg.CloudProvider == Vsphere {
+	if veleroCfg.CloudProvider == test.Vsphere {
 		// We overrider the ObjectStoreProvider here for vSphere because we want to use the aws plugin for the
 		// backup, but needed to pick up the provider plugins earlier.  vSphere plugin no longer needs a Volume
 		// Snapshot location specified
 		if veleroCfg.ObjectStoreProvider == "" {
-			veleroCfg.ObjectStoreProvider = AWS
+			veleroCfg.ObjectStoreProvider = test.AWS
 		}
-		if err := configvSpherePlugin(veleroCfg); err != nil {
+
+		if err := cleanVSpherePluginConfig(
+			veleroCfg.ClientToInstallVelero.ClientGo,
+			veleroCfg.VeleroNamespace,
+			test.VeleroVSphereSecretName,
+			test.VeleroVSphereConfigMapName,
+		); err != nil {
+			return errors.WithMessagef(err, "Failed to clear up vsphere plugin config %s namespace", veleroCfg.VeleroNamespace)
+		}
+		if err := generateVSpherePlugin(veleroCfg); err != nil {
 			return errors.WithMessagef(err, "Failed to config vsphere plugin")
 		}
 	}
@@ -118,11 +122,11 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluste
 
 	// For AWS IRSA credential test, AWS IAM service account is required, so if ServiceAccountName and EKSPolicyARN
 	// are both provided, we assume IRSA test is running, otherwise skip this IAM service account creation part.
-	if veleroCfg.CloudProvider == AWS && veleroInstallOptions.ServiceAccountName != "" {
+	if veleroCfg.CloudProvider == test.AWS && veleroInstallOptions.ServiceAccountName != "" {
 		if veleroCfg.EKSPolicyARN == "" {
 			return errors.New("Please provide EKSPolicyARN for IRSA test.")
 		}
-		_, err = GetNamespace(ctx, *veleroCfg.ClientToInstallVelero, veleroCfg.VeleroNamespace)
+		_, err = k8s.GetNamespace(ctx, *veleroCfg.ClientToInstallVelero, veleroCfg.VeleroNamespace)
 		// We should uninstall Velero for a new service account creation.
 		if !apierrors.IsNotFound(err) {
 			if err := VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace); err != nil {
@@ -130,19 +134,19 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluste
 			}
 		}
 		// If velero namespace does not exist, we should create it for service account creation
-		if err := KubectlCreateNamespace(ctx, veleroCfg.VeleroNamespace); err != nil {
+		if err := k8s.KubectlCreateNamespace(ctx, veleroCfg.VeleroNamespace); err != nil {
 			return errors.Wrapf(err, "Failed to create namespace %s to install Velero", veleroCfg.VeleroNamespace)
 		}
-		if err := KubectlDeleteClusterRoleBinding(ctx, "velero-cluster-role"); err != nil {
+		if err := k8s.KubectlDeleteClusterRoleBinding(ctx, "velero-cluster-role"); err != nil {
 			return errors.Wrapf(err, "Failed to delete clusterrolebinding %s to %s namespace", "velero-cluster-role", veleroCfg.VeleroNamespace)
 		}
-		if err := KubectlCreateClusterRoleBinding(ctx, "velero-cluster-role", "cluster-admin", veleroCfg.VeleroNamespace, veleroInstallOptions.ServiceAccountName); err != nil {
+		if err := k8s.KubectlCreateClusterRoleBinding(ctx, "velero-cluster-role", "cluster-admin", veleroCfg.VeleroNamespace, veleroInstallOptions.ServiceAccountName); err != nil {
 			return errors.Wrapf(err, "Failed to create clusterrolebinding %s to %s namespace", "velero-cluster-role", veleroCfg.VeleroNamespace)
 		}
-		if err := KubectlDeleteIAMServiceAcount(ctx, veleroInstallOptions.ServiceAccountName, veleroCfg.VeleroNamespace, veleroCfg.ClusterToInstallVelero); err != nil {
+		if err := eksutil.KubectlDeleteIAMServiceAcount(ctx, veleroInstallOptions.ServiceAccountName, veleroCfg.VeleroNamespace, veleroCfg.ClusterToInstallVelero); err != nil {
 			return errors.Wrapf(err, "Failed to delete service account %s to %s namespace", veleroInstallOptions.ServiceAccountName, veleroCfg.VeleroNamespace)
 		}
-		if err := EksctlCreateIAMServiceAcount(ctx, veleroInstallOptions.ServiceAccountName, veleroCfg.VeleroNamespace, veleroCfg.EKSPolicyARN, veleroCfg.ClusterToInstallVelero); err != nil {
+		if err := eksutil.EksctlCreateIAMServiceAcount(ctx, veleroInstallOptions.ServiceAccountName, veleroCfg.VeleroNamespace, veleroCfg.EKSPolicyARN, veleroCfg.ClusterToInstallVelero); err != nil {
 			return errors.Wrapf(err, "Failed to create service account %s to %s namespace", veleroInstallOptions.ServiceAccountName, veleroCfg.VeleroNamespace)
 		}
 	}
@@ -163,45 +167,79 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluste
 	return nil
 }
 
-// configvSpherePlugin refers to https://github.com/vmware-tanzu/velero-plugin-for-vsphere/blob/v1.3.0/docs/vanilla.md
+// generateVSpherePlugin refers to
-func configvSpherePlugin(veleroCfg *VeleroConfig) error {
+// https://github.com/vmware-tanzu/velero-plugin-for-vsphere/blob/v1.3.0/docs/vanilla.md
+func generateVSpherePlugin(veleroCfg *test.VeleroConfig) error {
 	cli := veleroCfg.ClientToInstallVelero
-	var err error
+
-	vsphereSecret := "velero-vsphere-config-secret"
+	if err := k8s.CreateNamespace(
-	configmaptName := "velero-vsphere-plugin-config"
+		context.Background(),
-	if err := clearupvSpherePluginConfig(cli.ClientGo, veleroCfg.VeleroNamespace, vsphereSecret, configmaptName); err != nil {
+		*cli,
-		return errors.WithMessagef(err, "Failed to clear up vsphere plugin config %s namespace", veleroCfg.VeleroNamespace)
+		veleroCfg.VeleroNamespace,
-	}
+	); err != nil {
-	if err := CreateNamespace(context.Background(), *cli, veleroCfg.VeleroNamespace); err != nil {
+		return errors.WithMessagef(
-		return errors.WithMessagef(err, "Failed to create Velero %s namespace", veleroCfg.VeleroNamespace)
+			err,
-	}
+			"Failed to create Velero %s namespace",
-	if err := createVCCredentialSecret(cli.ClientGo, veleroCfg.VeleroNamespace); err != nil {
+			veleroCfg.VeleroNamespace,
-		return errors.WithMessagef(err, "Failed to create virtual center credential secret in %s namespace", veleroCfg.VeleroNamespace)
+		)
-	}
-	if err := WaitForSecretsComplete(cli.ClientGo, veleroCfg.VeleroNamespace, vsphereSecret); err != nil {
-		return errors.Wrap(err, "Failed to ensure velero-vsphere-config-secret secret completion in namespace kube-system")
-	}
-	_, err = CreateConfigMap(cli.ClientGo, veleroCfg.VeleroNamespace, configmaptName, map[string]string{
-		"cluster_flavor":           "VANILLA",
-		"vsphere_secret_name":      vsphereSecret,
-		"vsphere_secret_namespace": veleroCfg.VeleroNamespace,
-	}, nil)
-	if err != nil {
-		return errors.WithMessagef(err, "Failed to create velero-vsphere-plugin-config configmap in %s namespace", veleroCfg.VeleroNamespace)
 	}
 
-	err = WaitForConfigMapComplete(cli.ClientGo, veleroCfg.VeleroNamespace, configmaptName)
+	clusterFlavor := "VANILLA"
-	if err != nil {
+
-		return errors.Wrap(err, fmt.Sprintf("Failed to ensure configmap %s completion in namespace: %s", configmaptName, veleroCfg.VeleroNamespace))
+	if err := createVCCredentialSecret(cli.ClientGo, veleroCfg.VeleroNamespace); err != nil {
+		// For TKGs/uTKG the VC secret is not supposed to exist.
+		if apierrors.IsNotFound(err) {
+			clusterFlavor = "GUEST"
+		} else {
+			return errors.WithMessagef(
+				err,
+				"Failed to create virtual center credential secret in %s namespace",
+				veleroCfg.VeleroNamespace,
+			)
+		}
 	}
+
+	_, err := k8s.CreateConfigMap(
+		cli.ClientGo,
+		veleroCfg.VeleroNamespace,
+		test.VeleroVSphereConfigMapName,
+		nil,
+		map[string]string{
+			"cluster_flavor":           clusterFlavor,
+			"vsphere_secret_name":      test.VeleroVSphereSecretName,
+			"vsphere_secret_namespace": veleroCfg.VeleroNamespace,
+		},
+	)
+	if err != nil {
+		return errors.WithMessagef(
+			err,
+			"Failed to create velero-vsphere-plugin-config ConfigMap in %s namespace",
+			veleroCfg.VeleroNamespace,
+		)
+	}
+
+	if err := k8s.WaitForConfigMapComplete(
+		cli.ClientGo,
+		veleroCfg.VeleroNamespace,
+		test.VeleroVSphereConfigMapName,
+	); err != nil {
+		return errors.Wrap(
+			err,
+			fmt.Sprintf("Failed to ensure ConfigMap %s completion in namespace: %s",
+				test.VeleroVSphereConfigMapName,
+				veleroCfg.VeleroNamespace,
+			),
+		)
+	}
+
 	return nil
 }
 
-func clearupvSpherePluginConfig(c clientset.Interface, ns, secretName, configMapName string) error {
+func cleanVSpherePluginConfig(c clientset.Interface, ns, secretName, configMapName string) error {
 	//clear secret
-	_, err := GetSecret(c, ns, secretName)
+	_, err := k8s.GetSecret(c, ns, secretName)
 	if err == nil { //exist
-		if err := WaitForSecretDelete(c, ns, secretName); err != nil {
+		if err := k8s.WaitForSecretDelete(c, ns, secretName); err != nil {
 			return errors.WithMessagef(err, "Failed to clear up vsphere plugin secret in %s namespace", ns)
 		}
 	} else if !apierrors.IsNotFound(err) {
@@ -209,9 +247,9 @@ func clearupvSpherePluginConfig(c clientset.Interface, ns, secretName, configMap
 	}
 
 	//clear configmap
-	_, err = GetConfigmap(c, ns, configMapName)
+	_, err = k8s.GetConfigmap(c, ns, configMapName)
 	if err == nil {
-		if err := WaitForConfigmapDelete(c, ns, configMapName); err != nil {
+		if err := k8s.WaitForConfigmapDelete(c, ns, configMapName); err != nil {
 			return errors.WithMessagef(err, "Failed to clear up vsphere plugin configmap in %s namespace", ns)
 		}
 	} else if !apierrors.IsNotFound(err) {
@@ -282,10 +320,10 @@ func installVeleroServer(ctx context.Context, cli, cloudProvider string, options
 
 	if len(options.Features) > 0 {
 		args = append(args, "--features", options.Features)
-		if !strings.EqualFold(cloudProvider, Vsphere) && strings.EqualFold(options.Features, FeatureCSI) && options.UseVolumeSnapshots {
+		if !strings.EqualFold(cloudProvider, test.Vsphere) && strings.EqualFold(options.Features, test.FeatureCSI) && options.UseVolumeSnapshots {
 			// https://github.com/openebs/zfs-localpv/blob/develop/docs/snapshot.md
 			fmt.Printf("Start to install %s VolumeSnapshotClass ... \n", cloudProvider)
-			if err := KubectlApplyByFile(ctx, fmt.Sprintf("../testdata/volume-snapshot-class/%s.yaml", cloudProvider)); err != nil {
+			if err := k8s.KubectlApplyByFile(ctx, fmt.Sprintf("../testdata/volume-snapshot-class/%s.yaml", cloudProvider)); err != nil {
 				fmt.Println("Fail to install VolumeSnapshotClass when CSI feature is enabled: ", err)
 				return err
 			}
@@ -454,11 +492,11 @@ func patchResources(resources *unstructured.UnstructuredList, namespace string,
 			i++
 		} else if options.VeleroServerDebugMode && resource.GetKind() == "Deployment" &&
 			resource.GetName() == "velero" {
-			deployJsonStr, err := json.Marshal(resource.Object)
+			deployJSONStr, err := json.Marshal(resource.Object)
 			if err != nil {
 				return errors.Wrapf(err, "failed to marshal velero deployment")
 			}
-			if err := json.Unmarshal(deployJsonStr, &deploy); err != nil {
+			if err := json.Unmarshal(deployJSONStr, &deploy); err != nil {
 				return errors.Wrapf(err, "failed to unmarshal velero deployment")
 			}
 			veleroDeployIndex := -1
@@ -565,7 +603,7 @@ func waitVeleroReady(ctx context.Context, namespace string, useNodeAgent bool) e
 	return nil
 }
 
-func IsVeleroReady(ctx context.Context, veleroCfg *VeleroConfig) (bool, error) {
+func IsVeleroReady(ctx context.Context, veleroCfg *test.VeleroConfig) (bool, error) {
 	namespace := veleroCfg.VeleroNamespace
 	useNodeAgent := veleroCfg.UseNodeAgent
 	if useNodeAgent {
@@ -599,7 +637,7 @@ func IsVeleroReady(ctx context.Context, veleroCfg *VeleroConfig) (bool, error) {
 	}
 
 	// Check BSL with poll
-	err = wait.PollUntilContextTimeout(ctx, PollInterval, time.Minute, true, func(ctx context.Context) (bool, error) {
+	err = wait.PollUntilContextTimeout(ctx, k8s.PollInterval, time.Minute, true, func(ctx context.Context) (bool, error) {
 		return checkBSL(ctx, veleroCfg) == nil, nil
 	})
 	if err != nil {
@@ -608,7 +646,7 @@ func IsVeleroReady(ctx context.Context, veleroCfg *VeleroConfig) (bool, error) {
 	return true, nil
 }
 
-func checkBSL(ctx context.Context, veleroCfg *VeleroConfig) error {
+func checkBSL(ctx context.Context, veleroCfg *test.VeleroConfig) error {
 	namespace := veleroCfg.VeleroNamespace
 	stdout, stderr, err := velerexec.RunCommand(exec.CommandContext(ctx, "kubectl", "get", "bsl", "default",
 		"-o", "json", "-n", namespace))
@@ -626,7 +664,7 @@ func checkBSL(ctx context.Context, veleroCfg *VeleroConfig) error {
 	return nil
 }
 
-func PrepareVelero(ctx context.Context, caseName string, veleroCfg VeleroConfig) error {
+func PrepareVelero(ctx context.Context, caseName string, veleroCfg test.VeleroConfig) error {
 	ready, err := IsVeleroReady(context.Background(), &veleroCfg)
 	if err != nil {
 		fmt.Printf("error in checking velero status with %v", err)
@@ -653,17 +691,20 @@ func VeleroUninstall(ctx context.Context, cli, namespace string) error {
 	return nil
 }
 
-// createVCCredentialSecret refer to https://github.com/vmware-tanzu/velero-plugin-for-vsphere/blob/v1.3.0/docs/vanilla.md
+// createVCCredentialSecret refer to
+// https://github.com/vmware-tanzu/velero-plugin-for-vsphere/blob/v1.3.0/docs/vanilla.md
 func createVCCredentialSecret(c clientset.Interface, veleroNamespace string) error {
 	secret, err := getVCCredentialSecret(c)
 	if err != nil {
 		return err
 	}
+
 	vsphereCfg, exist := secret.Data["csi-vsphere.conf"]
 	if !exist {
 		return errors.New("failed to retrieve csi-vsphere config")
 	}
-	se := &corev1.Secret{
+
+	vsphereSecret := &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "velero-vsphere-config-secret",
 			Namespace: veleroNamespace,
@@ -671,17 +712,36 @@ func createVCCredentialSecret(c clientset.Interface, veleroNamespace string) err
 		Type: corev1.SecretTypeOpaque,
 		Data: map[string][]byte{"csi-vsphere.conf": vsphereCfg},
 	}
-	_, err = c.CoreV1().Secrets(veleroNamespace).Create(context.TODO(), se, metav1.CreateOptions{})
+	_, err = c.CoreV1().Secrets(veleroNamespace).Create(
-	return err
+		context.TODO(),
+		vsphereSecret,
+		metav1.CreateOptions{},
+	)
+	if err != nil {
+		return err
+	}
+
+	if err := k8s.WaitForSecretsComplete(
+		c,
+		veleroNamespace,
+		test.VeleroVSphereSecretName,
+	); err != nil {
+		return errors.Wrap(
+			err,
+			"Failed to ensure velero-vsphere-config-secret secret completion in namespace kube-system",
+		)
+	}
+
+	return nil
 }
 
 // Reference https://github.com/vmware-tanzu/velero-plugin-for-vsphere/blob/main/docs/vanilla.md#create-vc-credential-secret
 // Read secret from kube-system namespace first, if not found, try with vmware-system-csi.
 func getVCCredentialSecret(c clientset.Interface) (secret *corev1.Secret, err error) {
-	secret, err = GetSecret(c, KubeSystemNamespace, "vsphere-config-secret")
+	secret, err = k8s.GetSecret(c, test.KubeSystemNamespace, "vsphere-config-secret")
 	if err != nil {
 		if apierrors.IsNotFound(err) {
-			secret, err = GetSecret(c, VSphereCSIControllerNamespace, "vsphere-config-secret")
+			secret, err = k8s.GetSecret(c, test.VSphereCSIControllerNamespace, "vsphere-config-secret")
 		}
 	}
 	return

test/util/velero/velero_utils.go

@@ -54,77 +54,56 @@ const BackupObjectsPrefix = "backups"
 const RestoreObjectsPrefix = "restores"
 const PluginsObjectsPrefix = "plugins"
 
-var pluginsMatrix = map[string]map[string][]string{
+var PluginsMatrix = map[string]map[string][]string{
-	"v1.7": {
-		"aws":     {"velero/velero-plugin-for-aws:v1.3.0"},
-		"azure":   {"velero/velero-plugin-for-microsoft-azure:v1.3.0"},
-		"vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.3.0"},
-		"gcp":     {"velero/velero-plugin-for-gcp:v1.3.0"},
-		"csi":     {"velero/velero-plugin-for-csi:v0.2.0"},
-	},
-	"v1.8": {
-		"aws":     {"velero/velero-plugin-for-aws:v1.4.0"},
-		"azure":   {"velero/velero-plugin-for-microsoft-azure:v1.4.0"},
-		"vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.3.1"},
-		"gcp":     {"velero/velero-plugin-for-gcp:v1.4.0"},
-		"csi":     {"velero/velero-plugin-for-csi:v0.2.0"},
-	},
-	"v1.9": {
-		"aws":     {"velero/velero-plugin-for-aws:v1.5.0"},
-		"azure":   {"velero/velero-plugin-for-microsoft-azure:v1.5.0"},
-		"vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.4.0"},
-		"gcp":     {"velero/velero-plugin-for-gcp:v1.5.0"},
-		"csi":     {"velero/velero-plugin-for-csi:v0.3.0"},
-	},
 	"v1.10": {
-		"aws":     {"velero/velero-plugin-for-aws:v1.6.0"},
+		"aws":     {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.6.0"},
-		"azure":   {"velero/velero-plugin-for-microsoft-azure:v1.6.0"},
+		"azure":   {"gcr.io/velero-gcp/velero-plugin-for-microsoft-azure:v1.6.0"},
-		"vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.1"},
+		"vsphere": {"gcr.io/velero-gcp/velero-plugin-for-vsphere:v1.5.1"},
-		"gcp":     {"velero/velero-plugin-for-gcp:v1.6.0"},
+		"gcp":     {"gcr.io/velero-gcp/velero-plugin-for-gcp:v1.6.0"},
-		"csi":     {"velero/velero-plugin-for-csi:v0.4.0"},
+		"csi":     {"gcr.io/velero-gcp/velero-plugin-for-csi:v0.4.0"},
 	},
 	"v1.11": {
-		"aws":     {"velero/velero-plugin-for-aws:v1.7.0"},
+		"aws":     {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.7.0"},
-		"azure":   {"velero/velero-plugin-for-microsoft-azure:v1.7.0"},
+		"azure":   {"gcr.io/velero-gcp/velero-plugin-for-microsoft-azure:v1.7.0"},
-		"vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.1"},
+		"vsphere": {"gcr.io/velero-gcp/velero-plugin-for-vsphere:v1.5.1"},
-		"gcp":     {"velero/velero-plugin-for-gcp:v1.7.0"},
+		"gcp":     {"gcr.io/velero-gcp/velero-plugin-for-gcp:v1.7.0"},
-		"csi":     {"velero/velero-plugin-for-csi:v0.5.0"},
+		"csi":     {"gcr.io/velero-gcp/velero-plugin-for-csi:v0.5.0"},
 	},
 	"v1.12": {
-		"aws":     {"velero/velero-plugin-for-aws:v1.8.0"},
+		"aws":     {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.8.0"},
-		"azure":   {"velero/velero-plugin-for-microsoft-azure:v1.8.0"},
+		"azure":   {"gcr.io/velero-gcp/velero-plugin-for-microsoft-azure:v1.8.0"},
-		"vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.1"},
+		"vsphere": {"gcr.io/velero-gcp/velero-plugin-for-vsphere:v1.5.1"},
-		"gcp":     {"velero/velero-plugin-for-gcp:v1.8.0"},
+		"gcp":     {"gcr.io/velero-gcp/velero-plugin-for-gcp:v1.8.0"},
-		"csi":     {"velero/velero-plugin-for-csi:v0.6.0"},
+		"csi":     {"gcr.io/velero-gcp/velero-plugin-for-csi:v0.6.0"},
 	},
 	"v1.13": {
-		"aws":       {"velero/velero-plugin-for-aws:v1.9.2"},
+		"aws":       {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.9.2"},
-		"azure":     {"velero/velero-plugin-for-microsoft-azure:v1.9.2"},
+		"azure":     {"gcr.io/velero-gcp/velero-plugin-for-microsoft-azure:v1.9.2"},
-		"vsphere":   {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.2"},
+		"vsphere":   {"gcr.io/velero-gcp/velero-plugin-for-vsphere:v1.5.2"},
-		"gcp":       {"velero/velero-plugin-for-gcp:v1.9.2"},
+		"gcp":       {"gcr.io/velero-gcp/velero-plugin-for-gcp:v1.9.2"},
-		"csi":       {"velero/velero-plugin-for-csi:v0.7.1"},
+		"csi":       {"gcr.io/velero-gcp/velero-plugin-for-csi:v0.7.1"},
-		"datamover": {"velero/velero-plugin-for-aws:v1.9.2"},
+		"datamover": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.9.2"},
 	},
 	"v1.14": {
-		"aws":       {"velero/velero-plugin-for-aws:v1.10.1"},
+		"aws":       {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.10.1"},
-		"azure":     {"velero/velero-plugin-for-microsoft-azure:v1.10.1"},
+		"azure":     {"gcr.io/velero-gcp/velero-plugin-for-microsoft-azure:v1.10.1"},
-		"vsphere":   {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.2"},
+		"vsphere":   {"gcr.io/velero-gcp/velero-plugin-for-vsphere:v1.5.2"},
-		"gcp":       {"velero/velero-plugin-for-gcp:v1.10.1"},
+		"gcp":       {"gcr.io/velero-gcp/velero-plugin-for-gcp:v1.10.1"},
-		"datamover": {"velero/velero-plugin-for-aws:v1.10.1"},
+		"datamover": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.10.1"},
 	},
 	"v1.15": {
-		"aws":       {"velero/velero-plugin-for-aws:v1.11.0"},
+		"aws":       {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.11.0"},
-		"azure":     {"velero/velero-plugin-for-microsoft-azure:v1.11.0"},
+		"azure":     {"gcr.io/velero-gcp/velero-plugin-for-microsoft-azure:v1.11.0"},
-		"vsphere":   {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.2"},
+		"vsphere":   {"gcr.io/velero-gcp/velero-plugin-for-vsphere:v1.5.2"},
-		"gcp":       {"velero/velero-plugin-for-gcp:v1.11.0"},
+		"gcp":       {"gcr.io/velero-gcp/velero-plugin-for-gcp:v1.11.0"},
-		"datamover": {"velero/velero-plugin-for-aws:v1.11.0"},
+		"datamover": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.11.0"},
 	},
 	"main": {
-		"aws":       {"velero/velero-plugin-for-aws:main"},
+		"aws":       {"gcr.io/velero-gcp/velero-plugin-for-aws:main"},
-		"azure":     {"velero/velero-plugin-for-microsoft-azure:main"},
+		"azure":     {"gcr.io/velero-gcp/velero-plugin-for-microsoft-azure:main"},
-		"vsphere":   {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.2"},
+		"vsphere":   {"gcr.io/velero-gcp/velero-plugin-for-vsphere:v1.5.2"},
-		"gcp":       {"velero/velero-plugin-for-gcp:main"},
+		"gcp":       {"gcr.io/velero-gcp/velero-plugin-for-gcp:main"},
-		"datamover": {"velero/velero-plugin-for-aws:main"},
+		"datamover": {"gcr.io/velero-gcp/velero-plugin-for-aws:main"},
 	},
 }
 
@@ -132,10 +111,10 @@ func getPluginsByVersion(version, cloudProvider, objectStoreProvider string, nee
 	var cloudMap map[string][]string
 	arr := strings.Split(version, ".")
 	if len(arr) >= 3 {
-		cloudMap = pluginsMatrix[arr[0]+"."+arr[1]]
+		cloudMap = PluginsMatrix[arr[0]+"."+arr[1]]
 	}
 	if len(cloudMap) == 0 {
-		cloudMap = pluginsMatrix["main"]
+		cloudMap = PluginsMatrix["main"]
 		if len(cloudMap) == 0 {
 			return nil, errors.Errorf("fail to get plugins by version: main")
 		}