vmware-tanzu
/
velero
mirror of https://github.com/vmware-tanzu/velero.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #4440 from qiuming-best/upgrade-e2e-vsphere-plugin 

Upgrade e2e test vsphere plugin
pull/4455/head
qiuming 2021-12-16 09:45:06 +08:00 committed by GitHub
parent 5aaeb3ebbe d8b1ed7dba
commit e350ce5bb4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 150 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
changelogs/unreleased/4440-mqiu Normal file
View File

@ -0,0 +1 @@
Upgrade e2e test vsphere plugin

28
test/e2e/util/k8s/configmap.go
View File

@ -17,10 +17,17 @@ limitations under the License.
package k8s
import (
"fmt"
"time"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"golang.org/x/net/context"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
waitutil "k8s.io/apimachinery/pkg/util/wait"
clientset "k8s.io/client-go/kubernetes"
)
@ -45,3 +52,24 @@ func WaitForConfigMapComplete(c clientset.Interface, ns, configmapName string) e
return true, nil
})
}
func GetConfigmap(c clientset.Interface, ns, secretName string) (*v1.ConfigMap, error) {
return c.CoreV1().ConfigMaps(ns).Get(context.TODO(), secretName, metav1.GetOptions{})
}
func WaitForConfigmapDelete(c clientset.Interface, ns, name string) error {
if err := c.CoreV1().ConfigMaps(ns).Delete(context.TODO(), name, metav1.DeleteOptions{}); err != nil {
return errors.Wrap(err, fmt.Sprintf("failed to delete configmap in namespace %q", ns))
}
return waitutil.PollImmediateInfinite(5*time.Second,
func() (bool, error) {
if _, err := c.CoreV1().ConfigMaps(ns).Get(context.TODO(), ns, metav1.GetOptions{}); err != nil {
if apierrors.IsNotFound(err) {
return true, nil
}
return false, err
}
logrus.Debugf("configmap %q in namespace %q is still being deleted...", name, ns)
return false, nil
})
}

51
test/e2e/util/k8s/secret.go
View File

@ -17,10 +17,18 @@ limitations under the License.
package k8s
import (
"fmt"
"time"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"golang.org/x/net/context"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
waitutil "k8s.io/apimachinery/pkg/util/wait"
clientset "k8s.io/client-go/kubernetes"
)
@ -34,6 +42,23 @@ func CreateSecret(c clientset.Interface, ns, name string, labels map[string]stri
return c.CoreV1().Secrets(ns).Create(context.TODO(), secret, metav1.CreateOptions{})
}
func WaitForSecretDelete(c clientset.Interface, ns, name string) error {
if err := c.CoreV1().Secrets(ns).Delete(context.TODO(), name, metav1.DeleteOptions{}); err != nil {
return errors.Wrap(err, fmt.Sprintf("failed to delete secret in namespace %q", ns))
}
return waitutil.PollImmediateInfinite(5*time.Second,
func() (bool, error) {
if _, err := c.CoreV1().Secrets(ns).Get(context.TODO(), ns, metav1.GetOptions{}); err != nil {
if apierrors.IsNotFound(err) {
return true, nil
}
return false, err
}
logrus.Debugf("secret %q in namespace %q is still being deleted...", name, ns)
return false, nil
})
}
// WaitForSecretsComplete uses c to wait for completions to complete for the Job jobName in namespace ns.
func WaitForSecretsComplete(c clientset.Interface, ns, secretName string) error {
return wait.Poll(PollInterval, PollTimeout, func() (bool, error) {
@ -44,3 +69,29 @@ func WaitForSecretsComplete(c clientset.Interface, ns, secretName string) error
return true, nil
})
}
func GetSecret(c clientset.Interface, ns, secretName string) (*v1.Secret, error) {
return c.CoreV1().Secrets(ns).Get(context.TODO(), secretName, metav1.GetOptions{})
}
//CreateVCCredentialSecret refer to https://github.com/vmware-tanzu/velero-plugin-for-vsphere/blob/v1.3.0/docs/vanilla.md
func CreateVCCredentialSecret(c clientset.Interface, veleroNamespace string) error {
secret, err := GetSecret(c, "kube-system", "vsphere-config-secret")
if err != nil {
return err
}
vsphereCfg, exist := secret.Data["csi-vsphere.conf"]
if !exist {
return errors.New("failed to retrieve csi-vsphere config")
}
se := &v1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "velero-vsphere-config-secret",
Namespace: veleroNamespace,
},
Type: v1.SecretTypeOpaque,
Data: map[string][]byte{"csi-vsphere.conf": vsphereCfg},
}
_, err = c.CoreV1().Secrets(veleroNamespace).Create(context.TODO(), se, metav1.CreateOptions{})
return err
}

72
test/e2e/util/velero/install.go
View File

@ -29,9 +29,11 @@ import (
"github.com/pkg/errors"
apps "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/util/wait"
clientset "k8s.io/client-go/kubernetes"
"github.com/vmware-tanzu/velero/pkg/cmd/cli/install"
velerexec "github.com/vmware-tanzu/velero/pkg/util/exec"
@ -62,16 +64,20 @@ func VeleroInstall(ctx context.Context, veleroCfg *VerleroConfig, features strin
if err != nil {
return errors.WithMessage(err, "Failed to get provider plugins")
}
err = EnsureClusterExists(ctx)
if err != nil {
return errors.WithMessage(err, "Failed to ensure Kubernetes cluster exists")
}
// TODO - handle this better
if veleroCfg.CloudProvider == "vsphere" {
// We overrider the ObjectStoreProvider here for vSphere because we want to use the aws plugin for the
// backup, but needed to pick up the provider plugins earlier. vSphere plugin no longer needs a Volume
// Snapshot location specified
veleroCfg.ObjectStoreProvider = "aws"
}
err = EnsureClusterExists(ctx)
if err != nil {
return errors.WithMessage(err, "Failed to ensure Kubernetes cluster exists")
if err := configvSpherePlugin(); err != nil {
return errors.WithMessagef(err, "Failed to config vsphere plugin")
}
}
veleroInstallOptions, err := getProviderVeleroInstallOptions(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,
@ -96,6 +102,64 @@ func VeleroInstall(ctx context.Context, veleroCfg *VerleroConfig, features strin
return nil
}
//configvSpherePlugin refers to https://github.com/vmware-tanzu/velero-plugin-for-vsphere/blob/v1.3.0/docs/vanilla.md
func configvSpherePlugin() error {
cli, err := NewTestClient()
if err != nil {
return errors.WithMessagef(err, "Failed to instantiate cluster client to config vsphere plugin")
}
vsphereSecret := "velero-vsphere-config-secret"
configmaptName := "velero-vsphere-plugin-config"
if err := clearupvSpherePluginConfig(cli.ClientGo, VeleroCfg.VeleroNamespace, vsphereSecret, configmaptName); err != nil {
return errors.WithMessagef(err, "Failed to clear up vsphere plugin config %s namespace", VeleroCfg.VeleroNamespace)
}
if err := CreateNamespace(context.Background(), cli, VeleroCfg.VeleroNamespace); err != nil {
return errors.WithMessagef(err, "Failed to create Velero %s namespace", VeleroCfg.VeleroNamespace)
}
if err := CreateVCCredentialSecret(cli.ClientGo, VeleroCfg.VeleroNamespace); err != nil {
return errors.WithMessagef(err, "Failed to create virtual center credential secret in %s namespace", VeleroCfg.VeleroNamespace)
}
if err := WaitForSecretsComplete(cli.ClientGo, VeleroCfg.VeleroNamespace, vsphereSecret); err != nil {
return errors.Wrap(err, "Failed to ensure velero-vsphere-config-secret secret completion in namespace kube-system")
}
_, err = CreateConfigMap(cli.ClientGo, VeleroCfg.VeleroNamespace, configmaptName, map[string]string{
"cluster_flavor": "VANILLA",
"vsphere_secret_name": vsphereSecret,
"vsphere_secret_namespace": VeleroCfg.VeleroNamespace,
})
if err != nil {
return errors.WithMessagef(err, "Failed to create velero-vsphere-plugin-config configmap in %s namespace", VeleroCfg.VeleroNamespace)
}
err = WaitForConfigMapComplete(cli.ClientGo, VeleroCfg.VeleroNamespace, configmaptName)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("Failed to ensure configmap %s completion in namespace: %s", configmaptName, VeleroCfg.VeleroNamespace))
}
return nil
}
func clearupvSpherePluginConfig(c clientset.Interface, ns, secretName, configMapName string) error {
//clear secret
_, err := GetSecret(c, ns, secretName)
if err == nil { //exist
if err := WaitForSecretDelete(c, ns, secretName); err != nil {
return errors.WithMessagef(err, "Failed to clear up vsphere plugin secret in %s namespace", ns)
}
} else if !apierrors.IsNotFound(err) {
return errors.WithMessagef(err, "Failed to retrieve vsphere plugin secret in %s namespace", ns)
}
//clear configmap
_, err = GetConfigmap(c, ns, configMapName)
if err == nil {
if err := WaitForConfigmapDelete(c, ns, configMapName); err != nil {
return errors.WithMessagef(err, "Failed to clear up vsphere plugin configmap in %s namespace", ns)
}
} else if !apierrors.IsNotFound(err) {
return errors.WithMessagef(err, "Failed to retrieve vsphere plugin configmap in %s namespace", ns)
}
return nil
}
func installVeleroServer(ctx context.Context, cli string, options *installOptions) error {
args := []string{"install"}
namespace := "velero"

4
test/e2e/util/velero/velero_utils.go
View File

@ -63,13 +63,13 @@ var pluginsMatrix = map[string]map[string][]string{
"v1.7": {
"aws": {"velero/velero-plugin-for-aws:v1.3.0"},
"azure": {"velero/velero-plugin-for-microsoft-azure:v1.3.0"},
"vsphere": {"velero/velero-plugin-for-aws:v1.3.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.1.1"},
"vsphere": {"velero/velero-plugin-for-aws:v1.3.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.3.0"},
"gcp": {"velero/velero-plugin-for-gcp:v1.3.0"},
},
"main": {
"aws": {"velero/velero-plugin-for-aws:main"},
"azure": {"velero/velero-plugin-for-microsoft-azure:main"},
"vsphere": {"velero/velero-plugin-for-aws:main", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.1.1"},
"vsphere": {"velero/velero-plugin-for-aws:main", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.3.0"},
"gcp": {"velero/velero-plugin-for-gcp:main"},
},
}