E2E tests now run in multiple clouds in addition to KIND (#3286)
Split plug-in provider into cloud provider/object provider Moved velero install/uninstall for tests into velero_utils Added remove of CRDs to test v elero uninstall Added remove of cluster role binding to test velero uninstall Added dump of velero describe and logs on error Added velero namespace argument to velero_utils functions Modified api group versions e2e tests to use VeleroInstall Added velero logs dumps for api group versions e2e testing Added DeleteNamespace to test/e2e/common.go Fixed VeleroInstall to use the image specified Changed enable_api_group_versions_test to use veleroNamespace instead of hardcoded "velero" Signed-off-by: Dave Smith-Uchida <dsmithuchida@vmware.com>pull/3477/head
David L. Smith-Uchida
GitHub
parent
52504b548d
commit
45d53178ae
|
|
@ -48,12 +48,14 @@ OUTPUT_DIR := _output/$(GOOS)/$(GOARCH)/bin
|
|||
GINKGO_FOCUS ?=
|
||||
VELERO_CLI ?=$$(pwd)/../../_output/bin/$(GOOS)/$(GOARCH)/velero
|
||||
VELERO_IMAGE ?= velero/velero:main
|
||||
VELERO_NAMESPACE ?=
|
||||
CREDS_FILE ?=
|
||||
BSL_BUCKET ?=
|
||||
BSL_PREFIX ?=
|
||||
BSL_CONFIG ?=
|
||||
VSL_CONFIG ?=
|
||||
PLUGIN_PROVIDER ?=
|
||||
CLOUD_PROVIDER ?=
|
||||
OBJECT_STORE_PROVIDER ?=
|
||||
|
||||
.PHONY:ginkgo
|
||||
ginkgo: # Make sure ginkgo is in $GOPATH/bin
|
||||
|
|
@ -65,16 +67,18 @@ run: ginkgo
|
|||
( echo "A credentials file is required to run E2E tests, please re-run the make target with CREDS_FILE=<PathToCredentialsFile>"; exit 1 )
|
||||
@[ "${BSL_BUCKET}" ] && echo "Using bucket ${BSL_BUCKET} to store backups from E2E tests" || \
|
||||
(echo "Bucket to store the backups from E2E tests is required, please re-run with BSL_BUCKET=<BucketName>"; exit 1 )
|
||||
@[ "${PLUGIN_PROVIDER}" ] && echo "Using plugin provider ${PLUGIN_PROVIDER} for object storage and volume snaphotting" || \
|
||||
(echo "Plugin provider to store the backups from E2E tests is required, please re-run with PLUGIN_PROVIDER=<pluginProviderName>"; exit 1 )
|
||||
@[ "${CLOUD_PROVIDER}" ] && echo "Using cloud provider ${CLOUD_PROVIDER}" || \
|
||||
(echo "Cloud provider for target cloud/plug-in provider is required, please rerun with CLOUD_PROVIDER=<aws,azure,kind,vsphere>"; exit 1)
|
||||
@$(GINKGO) -v -focus="$(GINKGO_FOCUS)" . -- -velerocli=$(VELERO_CLI) \
|
||||
-velero-image=$(VELERO_IMAGE) \
|
||||
-velero-namespace=$(VELERO_NAMESPACE) \
|
||||
-credentials-file=$(CREDS_FILE) \
|
||||
-bucket=$(BSL_BUCKET) \
|
||||
-prefix=$(BSL_PREFIX) \
|
||||
-bsl-config=$(BSL_CONFIG) \
|
||||
-vsl-config=$(VSL_CONFIG) \
|
||||
-plugin-provider=$(PLUGIN_PROVIDER)
|
||||
-cloud-provider=$(CLOUD_PROVIDER) \
|
||||
-object-store-provider="$(OBJECT_STORE_PROVIDER)"
|
||||
|
||||
build: ginkgo
|
||||
mkdir -p $(OUTPUT_DIR)
|
||||
|
|
|
|||
|
|
@ -31,15 +31,21 @@ These configuration parameters are expected as values to the following command l
|
|||
|
||||
1. `-credentials-file`: File containing credentials for backup and volume provider. Required.
|
||||
1. `-bucket`: Name of the object storage bucket where backups from e2e tests should be stored. Required.
|
||||
1. `-plugin-provider`: Provider of object store and volume snapshotter plugins. Required.
|
||||
1. `-cloud-provider`: The cloud the tests will be run in. Appropriate plug-ins will be installed except for kind which requires
|
||||
the object-store-provider to be specified.
|
||||
1. `-object-store-provider`: Object store provider to use. Required when kind is the cloud provider.
|
||||
1. `-velerocli`: Path to the velero application to use. Optional, by default uses `velero` in the `$PATH`
|
||||
1. `-velero-image`: Image for the velero server to be tested. Optional, by default uses `velero/velero:main`
|
||||
1. `-bsl-config`: Configuration to use for the backup storage location. Format is key1=value1,key2=value2. Optional.
|
||||
1. `-prefix`: Prefix in the `bucket`, under which all Velero data should be stored within the bucket. Optional.
|
||||
1. `-vsl-config`: Configuration to use for the volume snapshot location. Format is key1=value1,key2=value2. Optional.
|
||||
1. `-velero-namespace`: Namespace to install velero in. Optional, defaults to "velero".
|
||||
1. `-install-velero`: Specifies whether to install/uninstall velero for the tests. Optional, defaults to "true".
|
||||
|
||||
These configurations or parameters are used to generate install options for Velero for each test suite.
|
||||
|
||||
Tests can be run with the Kubernetes cluster hosted in various cloud providers or in a _kind_ cluster with storage in
|
||||
a specified object store type. Currently supported cloud provider types are _aws_, _azure_, _vsphere_ and _kind_.
|
||||
## Running tests locally
|
||||
|
||||
### Running using `make`
|
||||
|
|
@ -49,22 +55,27 @@ E2E tests can be run from the Velero repository root by running `make test-e2e`.
|
|||
Below is a mapping between `make` variables to E2E configuration flags.
|
||||
1. `CREDS_FILE`: `-credentials-file`. Required.
|
||||
1. `BSL_BUCKET`: `-bucket`. Required.
|
||||
1. `PLUGIN_PROVIDER`: `-plugin-provider`. Required.
|
||||
1. `CLOUD_PROVIDER`: `-cloud-provider`. Required
|
||||
1. `OBJECT_STORE_PROVIDER`: `-object-store-provider`. Required when kind is the cloud provider.
|
||||
1. `VELERO_CLI`: the `-velerocli`. Optional.
|
||||
1. `VELERO_IMAGE`: the `-velero-image`. Optional.
|
||||
1. `BSL_PREFIX`: `-prefix`. Optional.
|
||||
1. `BSL_CONFIG`: `-bsl-config`. Optional.
|
||||
1. `VSL_CONFIG`: `-vsl-config`. Optional.
|
||||
|
||||
For example, E2E tests can be run from Velero repository roots using the below command:
|
||||
For example, E2E tests can be run from Velero repository roots using the commands below:
|
||||
|
||||
1. Run Velero tests using AWS as the storage provider:
|
||||
1. Run Velero tests in a kind cluster with AWS (or Minio) as the storage provider:
|
||||
```bash
|
||||
BSL_PREFIX=<PREFIX_UNDER_BUCKET> BSL_BUCKET=<BUCKET_FOR_E2E_TEST_BACKUP> CREDS_FILE=/path/to/aws-creds PLUGIN_PROVIDER=aws make test-e2e
|
||||
BSL_PREFIX=<PREFIX_UNDER_BUCKET> BSL_BUCKET=<BUCKET_FOR_E2E_TEST_BACKUP> CREDS_FILE=/path/to/aws-creds CLOUD_PROVIDER=kind OBJECT_STORE_PROVIDER=aws make test-e2e
|
||||
```
|
||||
1. Run Velero tests using Microsoft Azure as the storage provider:
|
||||
1. Run Velero tests in an AWS cluster:
|
||||
```bash
|
||||
BSL_CONFIG="resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,storageAccount=$AZURE_STORAGE_ACCOUNT_ID,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID" BSL_BUCKET=velero CREDS_FILE=~/bin/velero-dev/aks-creds PLUGIN_PROVIDER=azure make test-e2e
|
||||
BSL_PREFIX=<PREFIX_UNDER_BUCKET> BSL_BUCKET=<BUCKET_FOR_E2E_TEST_BACKUP> CREDS_FILE=/path/to/aws-creds CLOUD_PROVIDER=aws make test-e2e
|
||||
```
|
||||
1. Run Velero tests in a Microsoft Azure cluster:
|
||||
```bash
|
||||
BSL_CONFIG="resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,storageAccount=$AZURE_STORAGE_ACCOUNT_ID,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID" BSL_BUCKET=<BUCKET_FOR_E2E_TEST_BACKUP> CREDS_FILE=/path/to/azure-creds CLOUD_PROVIDER=azure make test-e2e
|
||||
```
|
||||
Please refer to `velero-plugin-for-microsoft-azure` documentation for instruction to [set up permissions for Velero](https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure#set-permissions-for-velero) and to [set up azure storage account and blob container](https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure#setup-azure-storage-account-and-blob-container)
|
||||
1. Run Ginko-focused Restore Multi-API Groups tests using an image built for PR #3133 and Minio as the backup storage location:
|
||||
|
|
|
|||
|
|
@ -1,63 +1,60 @@
|
|||
package e2e
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
"golang.org/x/net/context"
|
||||
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
|
||||
"github.com/vmware-tanzu/velero/pkg/cmd/cli/install"
|
||||
)
|
||||
|
||||
var (
|
||||
uuidgen uuid.UUID
|
||||
veleroInstallOptions *install.InstallOptions
|
||||
uuidgen uuid.UUID
|
||||
)
|
||||
|
||||
func veleroInstall(pluginProvider string, useRestic bool) {
|
||||
var err error
|
||||
flag.Parse()
|
||||
Expect(EnsureClusterExists(context.TODO())).To(Succeed(), "Failed to ensure kubernetes cluster exists")
|
||||
uuidgen, err = uuid.NewRandom()
|
||||
Expect(err).To(Succeed())
|
||||
veleroInstallOptions, err = GetProviderVeleroInstallOptions(pluginProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, getProviderPlugins(pluginProvider))
|
||||
Expect(err).To(Succeed(), fmt.Sprintf("Failed to get Velero InstallOptions for plugin provider %s", pluginProvider))
|
||||
veleroInstallOptions.UseRestic = useRestic
|
||||
Expect(InstallVeleroServer(veleroInstallOptions)).To(Succeed(), "Failed to install Velero on KinD cluster")
|
||||
}
|
||||
|
||||
// Test backup and restore of Kibishi using restic
|
||||
var _ = Describe("[Restic] [KinD] Velero tests on KinD cluster using the plugin provider for object storage and Restic for volume backups", func() {
|
||||
var _ = Describe("[Restic] Velero tests on cluster using the plugin provider for object storage and Restic for volume backups", func() {
|
||||
var (
|
||||
client *kubernetes.Clientset
|
||||
backupName string
|
||||
restoreName string
|
||||
client *kubernetes.Clientset
|
||||
extensionsClient *apiextensionsclientset.Clientset
|
||||
backupName string
|
||||
restoreName string
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
var err error
|
||||
veleroInstall(pluginProvider, true)
|
||||
client, err = GetClusterClient()
|
||||
flag.Parse()
|
||||
uuidgen, err = uuid.NewRandom()
|
||||
Expect(err).To(Succeed())
|
||||
if installVelero {
|
||||
VeleroInstall(context.Background(), veleroImage, veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots,
|
||||
cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, "")
|
||||
}
|
||||
client, extensionsClient, err = GetClusterClient()
|
||||
Expect(err).To(Succeed(), "Failed to instantiate cluster client")
|
||||
})
|
||||
|
||||
AfterEach(func() {
|
||||
timeoutCTX, _ := context.WithTimeout(context.Background(), time.Minute)
|
||||
err := client.CoreV1().Namespaces().Delete(timeoutCTX, veleroInstallOptions.Namespace, metav1.DeleteOptions{})
|
||||
Expect(err).To(Succeed())
|
||||
if installVelero {
|
||||
timeoutCTX, _ := context.WithTimeout(context.Background(), time.Minute)
|
||||
err := VeleroUninstall(timeoutCTX, client, extensionsClient, veleroNamespace)
|
||||
Expect(err).To(Succeed())
|
||||
}
|
||||
|
||||
})
|
||||
|
||||
Context("When kibishii is the sample workload", func() {
|
||||
It("should be successfully backed up and restored", func() {
|
||||
backupName = "backup-" + uuidgen.String()
|
||||
restoreName = "restore-" + uuidgen.String()
|
||||
// Even though we are using Velero's CloudProvider plugin for object storage, the kubernetes cluster is running on
|
||||
// KinD. So use the kind installation for Kibishii.
|
||||
Expect(RunKibishiiTests(client, "kind", veleroCLI, backupName, restoreName)).To(Succeed(), "Failed to successfully backup and restore Kibishii namespace")
|
||||
Expect(RunKibishiiTests(client, cloudProvider, veleroCLI, veleroNamespace, backupName, restoreName)).To(Succeed(),
|
||||
"Failed to successfully backup and restore Kibishii namespace")
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
|
|||
|
|
@ -1,10 +1,15 @@
|
|||
package e2e
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"time"
|
||||
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"golang.org/x/net/context"
|
||||
apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
|
|
@ -19,20 +24,26 @@ func EnsureClusterExists(ctx context.Context) error {
|
|||
}
|
||||
|
||||
// GetClusterClient instantiates and returns a client for the cluster.
|
||||
func GetClusterClient() (*kubernetes.Clientset, error) {
|
||||
func GetClusterClient() (*kubernetes.Clientset, *apiextensionsclientset.Clientset, error) {
|
||||
loadingRules := clientcmd.NewDefaultClientConfigLoadingRules()
|
||||
configOverrides := &clientcmd.ConfigOverrides{}
|
||||
kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides)
|
||||
clientConfig, err := kubeConfig.ClientConfig()
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
return nil, nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
client, err := kubernetes.NewForConfig(clientConfig)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
return nil, nil, errors.WithStack(err)
|
||||
}
|
||||
return client, nil
|
||||
|
||||
extensionClientSet, err := apiextensionsclientset.NewForConfig(clientConfig)
|
||||
if err != nil {
|
||||
return nil, nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return client, extensionClientSet, nil
|
||||
}
|
||||
|
||||
// CreateNamespace creates a kubernetes namespace
|
||||
|
|
@ -44,3 +55,18 @@ func CreateNamespace(ctx context.Context, client *kubernetes.Clientset, namespac
|
|||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func DeleteNamespace(ctx context.Context, client *kubernetes.Clientset, namespace string) error {
|
||||
err := client.CoreV1().Namespaces().Delete(ctx, namespace, metav1.DeleteOptions{})
|
||||
if err != nil {
|
||||
return errors.WithMessagef(err, "Delete namespace failed removing namespace %s", namespace)
|
||||
}
|
||||
return wait.Poll(1*time.Second, 3*time.Minute, func() (bool, error) {
|
||||
_, err := client.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
fmt.Printf("Namespaces.Get after delete return err %v\n", err)
|
||||
return true, nil // Assume any error means the delete was successful
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,11 +9,13 @@ import (
|
|||
)
|
||||
|
||||
var (
|
||||
veleroCLI, veleroImage, cloudCredentialsFile, bslConfig, bslBucket, bslPrefix, vslConfig, pluginProvider string
|
||||
veleroCLI, veleroImage, cloudCredentialsFile, bslConfig, bslBucket, bslPrefix, vslConfig, cloudProvider, objectStoreProvider, veleroNamespace string
|
||||
installVelero, useVolumeSnapshots bool
|
||||
)
|
||||
|
||||
func init() {
|
||||
flag.StringVar(&pluginProvider, "plugin-provider", "", "Provider of object store and volume snapshotter plugins. Required.")
|
||||
flag.StringVar(&cloudProvider, "cloud-provider", "", "Cloud that Velero will be installed into. Required.")
|
||||
flag.StringVar(&objectStoreProvider, "object-store-provider", "", "Provider of object store plugin. Required if cloud-provider is kind, otherwise ignored.")
|
||||
flag.StringVar(&bslBucket, "bucket", "", "name of the object storage bucket where backups from e2e tests should be stored. Required.")
|
||||
flag.StringVar(&cloudCredentialsFile, "credentials-file", "", "file containing credentials for backup and volume provider. Required.")
|
||||
flag.StringVar(&veleroCLI, "velerocli", "velero", "path to the velero application to use.")
|
||||
|
|
@ -21,6 +23,9 @@ func init() {
|
|||
flag.StringVar(&bslConfig, "bsl-config", "", "configuration to use for the backup storage location. Format is key1=value1,key2=value2")
|
||||
flag.StringVar(&bslPrefix, "prefix", "", "prefix under which all Velero data should be stored within the bucket. Optional.")
|
||||
flag.StringVar(&vslConfig, "vsl-config", "", "configuration to use for the volume snapshot location. Format is key1=value1,key2=value2")
|
||||
flag.StringVar(&veleroNamespace, "velero-namespace", "velero", "Namespace to install Velero into")
|
||||
flag.BoolVar(&installVelero, "install-velero", true, "Install/uninstall velero during the test. Optional.")
|
||||
flag.BoolVar(&useVolumeSnapshots, "use-volume-snapshots", false, "Use volume-snapshotter plugin for volume backup. Optional")
|
||||
}
|
||||
|
||||
func TestE2e(t *testing.T) {
|
||||
|
|
|
|||
|
|
@ -10,6 +10,8 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
|
||||
|
||||
"github.com/google/uuid"
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
|
|
@ -23,13 +25,14 @@ import (
|
|||
veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec"
|
||||
)
|
||||
|
||||
var _ = Describe("[KinD] Velero tests on KinD clusters with various CRD API group versions", func() {
|
||||
var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", func() {
|
||||
var (
|
||||
resource, group string
|
||||
certMgrCRD map[string]string
|
||||
client *kubernetes.Clientset
|
||||
err error
|
||||
ctx = context.Background()
|
||||
resource, group string
|
||||
certMgrCRD map[string]string
|
||||
client *kubernetes.Clientset
|
||||
extensionsClient *apiextensionsclient.Clientset
|
||||
err error
|
||||
ctx = context.Background()
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
|
|
@ -40,7 +43,7 @@ var _ = Describe("[KinD] Velero tests on KinD clusters with various CRD API grou
|
|||
"namespace": "cert-manager",
|
||||
}
|
||||
|
||||
client, err = GetClusterClient()
|
||||
client, extensionsClient, err = GetClusterClient() // Currently we ignore the API extensions client
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
err = InstallCRD(ctx, certMgrCRD["url"], certMgrCRD["namespace"])
|
||||
|
|
@ -58,15 +61,6 @@ var _ = Describe("[KinD] Velero tests on KinD clusters with various CRD API grou
|
|||
_, _, _ = veleroexec.RunCommand(cmd)
|
||||
|
||||
_ = DeleteCRD(ctx, certMgrCRD["url"], certMgrCRD["namespace"])
|
||||
|
||||
// Uninstall Velero.
|
||||
if client != nil {
|
||||
_ = client.CoreV1().Namespaces().Delete(
|
||||
context.Background(),
|
||||
"velero",
|
||||
metav1.DeleteOptions{},
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
Context("When EnableAPIGroupVersions flag is set", func() {
|
||||
|
|
@ -76,12 +70,14 @@ var _ = Describe("[KinD] Velero tests on KinD clusters with various CRD API grou
|
|||
resource,
|
||||
group,
|
||||
client,
|
||||
extensionsClient,
|
||||
)).To(Succeed(), "Failed to successfully backup and restore multiple API Groups")
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, client *kubernetes.Clientset) error {
|
||||
func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, client *kubernetes.Clientset,
|
||||
extensionsClient *apiextensionsclient.Clientset) error {
|
||||
tests := []struct {
|
||||
name string
|
||||
namespaces []string
|
||||
|
|
@ -181,7 +177,7 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string,
|
|||
"namespace": "music-system",
|
||||
},
|
||||
tgtVer: "v2beta1",
|
||||
cm: builder.ForConfigMap("velero", "enableapigroupversions").Data(
|
||||
cm: builder.ForConfigMap(veleroNamespace, "enableapigroupversions").Data(
|
||||
"restoreResourcesVersionPriority",
|
||||
`rockbands.music.example.io=v2beta1,v2beta2,v2`,
|
||||
).Result(),
|
||||
|
|
@ -220,17 +216,22 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string,
|
|||
tc.namespaces = append(tc.namespaces, ns)
|
||||
}
|
||||
|
||||
if err := installVeleroForAPIGroups(ctx); err != nil {
|
||||
return errors.Wrap(err, "install velero")
|
||||
// TODO - Velero needs to be installed AFTER CRDs are installed because of https://github.com/vmware-tanzu/velero/issues/3471
|
||||
// Once that issue is fixed, we should install Velero once for the test suite
|
||||
if installVelero {
|
||||
VeleroInstall(context.Background(), veleroImage, veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots,
|
||||
cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig,
|
||||
"EnableAPIGroupVersions" /* TODO - remove this when the feature flag is removed */)
|
||||
fmt.Println("Sleep 20s to wait for Velero to stabilize after install.")
|
||||
time.Sleep(time.Second * 20)
|
||||
}
|
||||
fmt.Println("Sleep 20s to wait for Velero to stabilize after install.")
|
||||
time.Sleep(time.Second * 20)
|
||||
|
||||
backup := "backup-rockbands-" + uuidgen.String() + "-" + strconv.Itoa(i)
|
||||
namespacesStr := strings.Join(tc.namespaces, ",")
|
||||
|
||||
err = VeleroBackupNamespace(ctx, veleroCLI, backup, namespacesStr)
|
||||
err = VeleroBackupNamespace(ctx, veleroCLI, veleroNamespace, backup, namespacesStr)
|
||||
if err != nil {
|
||||
VeleroBackupLogs(ctx, veleroCLI, veleroNamespace, backup)
|
||||
return errors.Wrapf(err, "backing up %s namespaces on source cluster", namespacesStr)
|
||||
}
|
||||
|
||||
|
|
@ -256,14 +257,14 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string,
|
|||
|
||||
// Apply config map if there is one.
|
||||
if tc.cm != nil {
|
||||
_, err := client.CoreV1().ConfigMaps("velero").Create(ctx, tc.cm, metav1.CreateOptions{})
|
||||
_, err := client.CoreV1().ConfigMaps(veleroNamespace).Create(ctx, tc.cm, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "creating config map with user version priorities")
|
||||
}
|
||||
}
|
||||
|
||||
// Reset Velero to recognize music-system CRD.
|
||||
if err := RestartPods(ctx, "velero"); err != nil {
|
||||
if err := RestartPods(ctx, veleroNamespace); err != nil {
|
||||
return errors.Wrapf(err, "restarting Velero pods")
|
||||
}
|
||||
fmt.Println("Sleep 20s to wait for Velero to stabilize after restart.")
|
||||
|
|
@ -273,7 +274,8 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string,
|
|||
restore := "restore-rockbands-" + uuidgen.String() + "-" + strconv.Itoa(i)
|
||||
|
||||
if tc.want != nil {
|
||||
if err := VeleroRestore(ctx, veleroCLI, restore, backup); err != nil {
|
||||
if err := VeleroRestore(ctx, veleroCLI, veleroNamespace, restore, backup); err != nil {
|
||||
VeleroRestoreLogs(ctx, veleroCLI, veleroNamespace, restore)
|
||||
return errors.Wrapf(err, "restoring %s namespaces on target cluster", namespacesStr)
|
||||
}
|
||||
|
||||
|
|
@ -310,7 +312,7 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string,
|
|||
} else {
|
||||
// No custom resource should have been restored. Expect "no resource found"
|
||||
// error during restore.
|
||||
err := VeleroRestore(ctx, veleroCLI, restore, backup)
|
||||
err := VeleroRestore(ctx, veleroCLI, veleroNamespace, restore, backup)
|
||||
|
||||
if err.Error() != "Unexpected restore phase got PartiallyFailed, expecting Completed" {
|
||||
return errors.New("expected error but not none")
|
||||
|
|
@ -338,9 +340,13 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string,
|
|||
tc.srcCRD["namespace"],
|
||||
)
|
||||
|
||||
// Delete Velero namespace
|
||||
_ = client.CoreV1().Namespaces().Delete(ctx, "velero", metav1.DeleteOptions{})
|
||||
_ = WaitNamespaceDelete(ctx, "velero")
|
||||
// Uninstall Velero
|
||||
if installVelero {
|
||||
err = VeleroUninstall(ctx, client, extensionsClient, veleroNamespace)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
|
|
@ -353,13 +359,14 @@ func installVeleroForAPIGroups(ctx context.Context) error {
|
|||
|
||||
// Pass global variables to option parameters.
|
||||
options, err := GetProviderVeleroInstallOptions(
|
||||
pluginProvider,
|
||||
cloudProvider,
|
||||
cloudCredentialsFile,
|
||||
bslBucket,
|
||||
bslPrefix,
|
||||
bslConfig,
|
||||
vslConfig,
|
||||
getProviderPlugins(pluginProvider),
|
||||
getProviderPlugins(cloudProvider),
|
||||
"EnableAPIGroupVersions",
|
||||
)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "get velero install options")
|
||||
|
|
@ -428,7 +435,7 @@ func WaitForPodContainers(ctx context.Context, ns string) error {
|
|||
|
||||
func DeleteCRD(ctx context.Context, crdFile, ns string) error {
|
||||
fmt.Println("Delete CRD", crdFile)
|
||||
cmd := exec.CommandContext(ctx, "kubectl", "delete", "-f", crdFile)
|
||||
cmd := exec.CommandContext(ctx, "kubectl", "delete", "-f", crdFile, "--wait")
|
||||
|
||||
_, stderr, err := veleroexec.RunCommand(cmd)
|
||||
if strings.Contains(stderr, "not found") {
|
||||
|
|
@ -454,7 +461,6 @@ func DeleteCRD(ctx context.Context, crdFile, ns string) error {
|
|||
re := regexp.MustCompile(ns)
|
||||
return re.MatchString(stdout), nil
|
||||
})
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -37,9 +37,15 @@ func installKibishii(ctx context.Context, namespace string, cloudPlatform string
|
|||
return err
|
||||
}
|
||||
|
||||
fmt.Printf("Waiting for kibishii jump-pad ready\n")
|
||||
jumpPadWaitCmd := exec.CommandContext(ctx, "kubectl", "wait", "--for=condition=ready", "-n", namespace, "pod/jump-pad")
|
||||
_, _, err = veleroexec.RunCommand(jumpPadWaitCmd)
|
||||
|
||||
// TODO - Fix kibishii so we can check that it is ready to go
|
||||
if err == nil {
|
||||
fmt.Printf("Sleeping for Kibishii startup\n")
|
||||
time.Sleep(time.Minute * 1)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
|
|
@ -73,23 +79,24 @@ func verifyData(ctx context.Context, namespace string, levels int, filesPerLevel
|
|||
}
|
||||
|
||||
// RunKibishiiTests runs kibishii tests on the provider.
|
||||
func RunKibishiiTests(client *kubernetes.Clientset, providerName, veleroCLI, backupName, restoreName string) error {
|
||||
func RunKibishiiTests(client *kubernetes.Clientset, providerName, veleroCLI, veleroNamespace, backupName, restoreName string) error {
|
||||
fiveMinTimeout, _ := context.WithTimeout(context.Background(), 5*time.Minute)
|
||||
oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60)
|
||||
|
||||
if err := CreateNamespace(fiveMinTimeout, client, kibishiiNamespace); err != nil {
|
||||
errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", kibishiiNamespace)
|
||||
return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", kibishiiNamespace)
|
||||
}
|
||||
|
||||
if err := installKibishii(fiveMinTimeout, kibishiiNamespace, providerName); err != nil {
|
||||
errors.Wrap(err, "Failed to install Kibishii workload")
|
||||
return errors.Wrap(err, "Failed to install Kibishii workload")
|
||||
}
|
||||
|
||||
if err := generateData(oneHourTimeout, kibishiiNamespace, 2, 10, 10, 1024, 1024, 0, 2); err != nil {
|
||||
return errors.Wrap(err, "Failed to generate data")
|
||||
}
|
||||
|
||||
if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, backupName, kibishiiNamespace); err != nil {
|
||||
if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, kibishiiNamespace); err != nil {
|
||||
VeleroBackupLogs(fiveMinTimeout, veleroCLI, veleroNamespace, backupName)
|
||||
return errors.Wrapf(err, "Failed to backup kibishii namespace %s", kibishiiNamespace)
|
||||
}
|
||||
|
||||
|
|
@ -98,7 +105,8 @@ func RunKibishiiTests(client *kubernetes.Clientset, providerName, veleroCLI, bac
|
|||
return errors.Wrap(err, "Failed to simulate a disaster")
|
||||
}
|
||||
|
||||
if err := VeleroRestore(oneHourTimeout, veleroCLI, restoreName, backupName); err != nil {
|
||||
if err := VeleroRestore(oneHourTimeout, veleroCLI, veleroNamespace, restoreName, backupName); err != nil {
|
||||
VeleroRestoreLogs(fiveMinTimeout, veleroCLI, veleroNamespace, restoreName)
|
||||
return errors.Wrapf(err, "Restore %s failed from backup %s", restoreName, backupName)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,24 +0,0 @@
|
|||
package e2e
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
var _ = Describe("Testing Velero on a kind cluster", func() {
|
||||
BeforeEach(func() {
|
||||
flag.Parse()
|
||||
ctx := context.TODO()
|
||||
err := EnsureClusterExists(ctx)
|
||||
Expect(err).To(Succeed())
|
||||
})
|
||||
Describe("Dummy test", func() {
|
||||
Context("Dummy test", func() {
|
||||
It("is a dummy test", func() {
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
@ -9,7 +9,13 @@ import (
|
|||
"os/exec"
|
||||
"path/filepath"
|
||||
|
||||
"k8s.io/apimachinery/pkg/labels"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
|
||||
apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
|
||||
|
||||
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
|
||||
"github.com/vmware-tanzu/velero/pkg/client"
|
||||
|
|
@ -34,13 +40,14 @@ func getProviderPlugins(providerName string) []string {
|
|||
|
||||
// GetProviderVeleroInstallOptions returns Velero InstallOptions for the provider.
|
||||
func GetProviderVeleroInstallOptions(
|
||||
providerName,
|
||||
pluginProvider,
|
||||
credentialsFile,
|
||||
objectStoreBucket,
|
||||
objectStorePrefix string,
|
||||
bslConfig,
|
||||
vslConfig string,
|
||||
plugins []string,
|
||||
features string,
|
||||
) (*cliinstall.InstallOptions, error) {
|
||||
|
||||
if credentialsFile == "" {
|
||||
|
|
@ -55,7 +62,7 @@ func GetProviderVeleroInstallOptions(
|
|||
io := cliinstall.NewInstallOptions()
|
||||
// always wait for velero and restic pods to be running.
|
||||
io.Wait = true
|
||||
io.ProviderName = providerName
|
||||
io.ProviderName = pluginProvider
|
||||
io.SecretFile = credentialsFile
|
||||
|
||||
io.BucketName = objectStoreBucket
|
||||
|
|
@ -68,6 +75,7 @@ func GetProviderVeleroInstallOptions(
|
|||
|
||||
io.SecretFile = realPath
|
||||
io.Plugins = flag.NewStringArray(plugins...)
|
||||
io.Features = features
|
||||
return io, nil
|
||||
}
|
||||
|
||||
|
|
@ -101,7 +109,7 @@ func InstallVeleroServer(io *cliinstall.InstallOptions) error {
|
|||
}
|
||||
|
||||
fmt.Println("Waiting for Velero deployment to be ready.")
|
||||
if _, err = install.DeploymentIsReady(factory, "velero"); err != nil {
|
||||
if _, err = install.DeploymentIsReady(factory, io.Namespace); err != nil {
|
||||
return errors.Wrap(err, errorMsg)
|
||||
}
|
||||
|
||||
|
|
@ -116,8 +124,11 @@ func InstallVeleroServer(io *cliinstall.InstallOptions) error {
|
|||
}
|
||||
|
||||
// CheckBackupPhase uses veleroCLI to inspect the phase of a Velero backup.
|
||||
func CheckBackupPhase(ctx context.Context, veleroCLI string, backupName string, expectedPhase velerov1api.BackupPhase) error {
|
||||
checkCMD := exec.CommandContext(ctx, veleroCLI, "backup", "get", "-o", "json", backupName)
|
||||
func CheckBackupPhase(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string,
|
||||
expectedPhase velerov1api.BackupPhase) error {
|
||||
checkCMD := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "backup", "get", "-o", "json",
|
||||
backupName)
|
||||
|
||||
fmt.Printf("get backup cmd =%v\n", checkCMD)
|
||||
stdoutPipe, err := checkCMD.StdoutPipe()
|
||||
if err != nil {
|
||||
|
|
@ -157,8 +168,11 @@ func CheckBackupPhase(ctx context.Context, veleroCLI string, backupName string,
|
|||
}
|
||||
|
||||
// CheckRestorePhase uses veleroCLI to inspect the phase of a Velero restore.
|
||||
func CheckRestorePhase(ctx context.Context, veleroCLI string, restoreName string, expectedPhase velerov1api.RestorePhase) error {
|
||||
checkCMD := exec.CommandContext(ctx, veleroCLI, "restore", "get", "-o", "json", restoreName)
|
||||
func CheckRestorePhase(ctx context.Context, veleroCLI string, veleroNamespace string, restoreName string,
|
||||
expectedPhase velerov1api.RestorePhase) error {
|
||||
checkCMD := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "restore", "get", "-o", "json",
|
||||
restoreName)
|
||||
|
||||
fmt.Printf("get restore cmd =%v\n", checkCMD)
|
||||
stdoutPipe, err := checkCMD.StdoutPipe()
|
||||
if err != nil {
|
||||
|
|
@ -198,23 +212,135 @@ func CheckRestorePhase(ctx context.Context, veleroCLI string, restoreName string
|
|||
}
|
||||
|
||||
// VeleroBackupNamespace uses the veleroCLI to backup a namespace.
|
||||
func VeleroBackupNamespace(ctx context.Context, veleroCLI string, backupName string, namespace string) error {
|
||||
backupCmd := exec.CommandContext(ctx, veleroCLI, "create", "backup", backupName, "--include-namespaces", namespace,
|
||||
func VeleroBackupNamespace(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, namespace string) error {
|
||||
backupCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "create", "backup", backupName,
|
||||
"--include-namespaces", namespace,
|
||||
"--default-volumes-to-restic", "--wait")
|
||||
|
||||
backupCmd.Stdout = os.Stdout
|
||||
backupCmd.Stderr = os.Stderr
|
||||
fmt.Printf("backup cmd =%v\n", backupCmd)
|
||||
err := backupCmd.Run()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return CheckBackupPhase(ctx, veleroCLI, backupName, velerov1api.BackupPhaseCompleted)
|
||||
err = CheckBackupPhase(ctx, veleroCLI, veleroNamespace, backupName, velerov1api.BackupPhaseCompleted)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// VeleroRestore uses the veleroCLI to restore from a Velero backup.
|
||||
func VeleroRestore(ctx context.Context, veleroCLI string, restoreName string, backupName string) error {
|
||||
restoreCmd := exec.CommandContext(ctx, veleroCLI, "create", "restore", restoreName, "--from-backup", backupName, "--wait")
|
||||
func VeleroRestore(ctx context.Context, veleroCLI string, veleroNamespace string, restoreName string, backupName string) error {
|
||||
restoreCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "create", "restore", restoreName,
|
||||
"--from-backup", backupName, "--wait")
|
||||
|
||||
restoreCmd.Stdout = os.Stdout
|
||||
restoreCmd.Stderr = os.Stderr
|
||||
fmt.Printf("restore cmd =%v\n", restoreCmd)
|
||||
err := restoreCmd.Run()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return CheckRestorePhase(ctx, veleroCLI, restoreName, velerov1api.RestorePhaseCompleted)
|
||||
return CheckRestorePhase(ctx, veleroCLI, veleroNamespace, restoreName, velerov1api.RestorePhaseCompleted)
|
||||
}
|
||||
|
||||
func VeleroInstall(ctx context.Context, veleroImage string, veleroNamespace string, cloudProvider string, objectStoreProvider string, useVolumeSnapshots bool,
|
||||
cloudCredentialsFile string, bslBucket string, bslPrefix string, bslConfig string, vslConfig string,
|
||||
features string) error {
|
||||
|
||||
if cloudProvider != "kind" {
|
||||
if objectStoreProvider != "" {
|
||||
return errors.New("For cloud platforms, object store plugin cannot be overridden") // Can't set an object store provider that is different than your cloud
|
||||
}
|
||||
objectStoreProvider = cloudProvider
|
||||
} else {
|
||||
if objectStoreProvider == "" {
|
||||
return errors.New("No object store provider specified - must be specified when using kind as the cloud provider") // Gotta have an object store provider
|
||||
}
|
||||
}
|
||||
err := EnsureClusterExists(ctx)
|
||||
if err != nil {
|
||||
return errors.WithMessage(err, "Failed to ensure kubernetes cluster exists")
|
||||
}
|
||||
veleroInstallOptions, err := GetProviderVeleroInstallOptions(objectStoreProvider, cloudCredentialsFile, bslBucket,
|
||||
bslPrefix, bslConfig, vslConfig, getProviderPlugins(objectStoreProvider), features)
|
||||
if err != nil {
|
||||
return errors.WithMessagef(err, "Failed to get Velero InstallOptions for plugin provider %s", objectStoreProvider)
|
||||
}
|
||||
veleroInstallOptions.UseRestic = !useVolumeSnapshots
|
||||
veleroInstallOptions.Image = veleroImage
|
||||
veleroInstallOptions.Namespace = veleroNamespace
|
||||
err = InstallVeleroServer(veleroInstallOptions)
|
||||
if err != nil {
|
||||
return errors.WithMessagef(err, "Failed to install Velero in cluster")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func VeleroUninstall(ctx context.Context, client *kubernetes.Clientset, extensionsClient *apiextensionsclient.Clientset,
|
||||
veleroNamespace string) error {
|
||||
// TODO - replace with invocation of "velero uninstall" when that becomes available
|
||||
err := DeleteNamespace(ctx, client, veleroNamespace)
|
||||
if err != nil {
|
||||
return errors.WithMessagef(err, "Uninstall failed removing Velero namespace %s", veleroNamespace)
|
||||
}
|
||||
|
||||
return err
|
||||
rolebinding := install.ClusterRoleBinding(veleroNamespace)
|
||||
|
||||
err = client.RbacV1().ClusterRoleBindings().Delete(ctx, rolebinding.Name, metav1.DeleteOptions{})
|
||||
if err != nil {
|
||||
return errors.WithMessagef(err, "Uninstall failed removing Velero cluster role binding %s", rolebinding)
|
||||
}
|
||||
veleroLabels := labels.FormatLabels(install.Labels())
|
||||
|
||||
crds, err := extensionsClient.ApiextensionsV1().CustomResourceDefinitions().List(ctx, metav1.ListOptions{
|
||||
LabelSelector: veleroLabels,
|
||||
})
|
||||
if err != nil {
|
||||
return errors.WithMessagef(err, "Uninstall failed listing Velero crds")
|
||||
}
|
||||
for _, removeCRD := range crds.Items {
|
||||
err = extensionsClient.ApiextensionsV1().CustomResourceDefinitions().Delete(ctx, removeCRD.ObjectMeta.Name, metav1.DeleteOptions{})
|
||||
if err != nil {
|
||||
return errors.WithMessagef(err, "Uninstall failed removing CRD %s", removeCRD.ObjectMeta.Name)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func VeleroBackupLogs(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string) error {
|
||||
describeCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "backup", "describe", backupName)
|
||||
describeCmd.Stdout = os.Stdout
|
||||
describeCmd.Stderr = os.Stderr
|
||||
err := describeCmd.Run()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
logCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "backup", "logs", backupName)
|
||||
logCmd.Stdout = os.Stdout
|
||||
logCmd.Stderr = os.Stderr
|
||||
err = logCmd.Run()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func VeleroRestoreLogs(ctx context.Context, veleroCLI string, veleroNamespace string, restoreName string) error {
|
||||
describeCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "restore", "describe", restoreName)
|
||||
describeCmd.Stdout = os.Stdout
|
||||
describeCmd.Stderr = os.Stderr
|
||||
err := describeCmd.Run()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
logCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "restore", "logs", restoreName)
|
||||
logCmd.Stdout = os.Stdout
|
||||
logCmd.Stderr = os.Stderr
|
||||
err = logCmd.Run()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue