Merge pull request #4913 from qiuming-best/order-resources-test
Add schedule ordered resources E2E testpull/4927/head
Wenkai Yin(尹文开)
GitHub
commit
51de990a8a
|
|
@ -0,0 +1 @@
|
|||
Add schedule ordered resources E2E test
|
||||
|
|
@ -30,6 +30,7 @@ import (
|
|||
. "github.com/vmware-tanzu/velero/test/e2e/basic"
|
||||
. "github.com/vmware-tanzu/velero/test/e2e/basic/resources-check"
|
||||
. "github.com/vmware-tanzu/velero/test/e2e/bsl-mgmt"
|
||||
. "github.com/vmware-tanzu/velero/test/e2e/orderedresources"
|
||||
. "github.com/vmware-tanzu/velero/test/e2e/privilegesmgmt"
|
||||
. "github.com/vmware-tanzu/velero/test/e2e/resource-filtering"
|
||||
. "github.com/vmware-tanzu/velero/test/e2e/scale"
|
||||
|
|
@ -105,6 +106,8 @@ var _ = Describe("[PrivilegesMgmt][SSR] Velero test on ssr object when controlle
|
|||
var _ = Describe("[BSL][Deletion][Snapshot] Local backups will be deleted once the corresponding backup storage location is deleted", BslDeletionWithSnapshots)
|
||||
var _ = Describe("[BSL][Deletion][Restic] Local backups and restic repos will be deleted once the corresponding backup storage location is deleted", BslDeletionWithRestic)
|
||||
|
||||
var _ = Describe("[Schedule][OrederedResources] Backup resources should follow the specific order in schedule", ScheduleOrderedResources)
|
||||
|
||||
func TestE2e(t *testing.T) {
|
||||
// Skip running E2E tests when running only "short" tests because:
|
||||
// 1. E2E tests are long running tests involving installation of Velero and performing backup and restore operations.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,191 @@
|
|||
package orderedresources
|
||||
|
||||
/*
|
||||
Copyright the Velero contributors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
//the ordered resources test related to https://github.com/vmware-tanzu/velero/issues/4561
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/pkg/errors"
|
||||
waitutil "k8s.io/apimachinery/pkg/util/wait"
|
||||
kbclient "sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
|
||||
. "github.com/vmware-tanzu/velero/test/e2e"
|
||||
. "github.com/vmware-tanzu/velero/test/e2e/test"
|
||||
. "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
|
||||
. "github.com/vmware-tanzu/velero/test/e2e/util/velero"
|
||||
)
|
||||
|
||||
type OrderedResources struct {
|
||||
Namespace string
|
||||
ScheduleName string
|
||||
OrderMap map[string]string
|
||||
ScheduleArgs []string
|
||||
TestCase
|
||||
}
|
||||
|
||||
func ScheduleOrderedResources() {
|
||||
BeforeEach(func() {
|
||||
flag.Parse()
|
||||
if VeleroCfg.InstallVelero {
|
||||
Expect(VeleroInstall(context.Background(), &VeleroCfg, false)).To(Succeed())
|
||||
}
|
||||
})
|
||||
|
||||
AfterEach(func() {
|
||||
if VeleroCfg.InstallVelero && !VeleroCfg.Debug {
|
||||
Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To(Succeed())
|
||||
}
|
||||
})
|
||||
|
||||
It("Create a schedule to backup resources in a specific order should be successful", func() {
|
||||
test := &OrderedResources{}
|
||||
err := test.Init()
|
||||
Expect(err).To(Succeed(), err)
|
||||
defer func() {
|
||||
Expect(DeleteNamespace(test.Ctx, test.Client, test.Namespace, false)).To(Succeed(), fmt.Sprintf("Failed to delete the namespace %s", test.Namespace))
|
||||
err = VeleroScheduleDelete(test.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.ScheduleName)
|
||||
Expect(err).To(Succeed(), fmt.Sprintf("Failed to delete schedule with err %v", err))
|
||||
}()
|
||||
|
||||
By(fmt.Sprintf("Prepare workload as target to backup in base namespace %s", test.Namespace), func() {
|
||||
err = test.CreateResources()
|
||||
Expect(err).To(Succeed(), fmt.Sprintf("Failed to create resources to backup with err %v", err))
|
||||
})
|
||||
|
||||
By(fmt.Sprintf("Create schedule the workload in %s namespace", test.Namespace), func() {
|
||||
err = VeleroScheduleCreate(test.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.ScheduleName, test.ScheduleArgs)
|
||||
Expect(err).To(Succeed(), fmt.Sprintf("Failed to create schedule %s with err %v", test.ScheduleName, err))
|
||||
})
|
||||
|
||||
By(fmt.Sprintf("Checking resource order in %s schedule cr", test.ScheduleName), func() {
|
||||
err = CheckScheduleWithResourceOrder(test.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.ScheduleName, test.OrderMap)
|
||||
Expect(err).To(Succeed(), fmt.Sprintf("Failed to check schedule %s with err %v", test.ScheduleName, err))
|
||||
})
|
||||
|
||||
By("Checking resource order in backup cr", func() {
|
||||
backupList := new(velerov1api.BackupList)
|
||||
err = waitutil.PollImmediate(10*time.Second, time.Minute*5, func() (bool, error) {
|
||||
if err = test.Client.Kubebuilder.List(test.Ctx, backupList, &kbclient.ListOptions{Namespace: VeleroCfg.VeleroNamespace}); err != nil {
|
||||
return false, fmt.Errorf("failed to list backup object in %s namespace with err %v", VeleroCfg.VeleroNamespace, err)
|
||||
}
|
||||
|
||||
for _, backup := range backupList.Items {
|
||||
if err = CheckBackupWithResourceOrder(test.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backup.Name, test.OrderMap); err == nil {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
fmt.Printf("still finding backup created by schedule %s ...\n", test.ScheduleName)
|
||||
return false, nil
|
||||
})
|
||||
Expect(err).To(Succeed(), fmt.Sprintf("Failed to check schedule %s created backup with err %v", test.ScheduleName, err))
|
||||
})
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
func (o *OrderedResources) Init() error {
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
UUIDgen, _ = uuid.NewRandom()
|
||||
client, err := NewTestClient()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to init ordered resources test with err %v", err)
|
||||
}
|
||||
o.Client = client
|
||||
o.ScheduleName = "schedule-ordered-resources-" + UUIDgen.String()
|
||||
o.NSBaseName = "schedule-ordered-resources"
|
||||
o.Namespace = o.NSBaseName + "-" + UUIDgen.String()
|
||||
o.OrderMap = map[string]string{
|
||||
"deployments": fmt.Sprintf("deploy-%s", o.NSBaseName),
|
||||
"secrets": fmt.Sprintf("secret-%s", o.NSBaseName),
|
||||
"configmaps": fmt.Sprintf("configmap-%s", o.NSBaseName),
|
||||
}
|
||||
|
||||
o.ScheduleArgs = []string{"--schedule", "@every 1m",
|
||||
"--include-namespaces", o.Namespace, "--default-volumes-to-restic", "--ordered-resources"}
|
||||
var orderStr string
|
||||
for kind, resource := range o.OrderMap {
|
||||
orderStr += fmt.Sprintf("%s=%s;", kind, resource)
|
||||
}
|
||||
o.ScheduleArgs = append(o.ScheduleArgs, strings.TrimRight(orderStr, ";"))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (o *OrderedResources) CreateResources() error {
|
||||
o.Ctx, _ = context.WithTimeout(context.Background(), 5*time.Minute)
|
||||
label := map[string]string{
|
||||
"orderedresources": "true",
|
||||
}
|
||||
fmt.Printf("Creating resources in %s namespace ...\n", o.Namespace)
|
||||
if err := CreateNamespace(o.Ctx, o.Client, o.Namespace); err != nil {
|
||||
return errors.Wrapf(err, "failed to create namespace %s", o.Namespace)
|
||||
}
|
||||
serviceAccountName := "default"
|
||||
// wait until the service account is created before patch the image pull secret
|
||||
if err := WaitUntilServiceAccountCreated(o.Ctx, o.Client, o.Namespace, serviceAccountName, 10*time.Minute); err != nil {
|
||||
return errors.Wrapf(err, "failed to wait the service account %q created under the namespace %q", serviceAccountName, o.Namespace)
|
||||
}
|
||||
// add the image pull secret to avoid the image pull limit issue of Docker Hub
|
||||
if err := PatchServiceAccountWithImagePullSecret(o.Ctx, o.Client, o.Namespace, serviceAccountName, VeleroCfg.RegistryCredentialFile); err != nil {
|
||||
return errors.Wrapf(err, "failed to patch the service account %q under the namespace %q", serviceAccountName, o.Namespace)
|
||||
}
|
||||
//Create deployment
|
||||
deploymentName := fmt.Sprintf("deploy-%s", o.NSBaseName)
|
||||
fmt.Printf("Creating deployment %s in %s namespaces ...\n", deploymentName, o.Namespace)
|
||||
deployment := NewDeployment(deploymentName, o.Namespace, 1, label)
|
||||
deployment, err := CreateDeployment(o.Client.ClientGo, o.Namespace, deployment)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, fmt.Sprintf("failed to create namespace %q with err %v", o.Namespace, err))
|
||||
}
|
||||
err = WaitForReadyDeployment(o.Client.ClientGo, o.Namespace, deployment.Name)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, fmt.Sprintf("failed to ensure job completion in namespace: %q", o.Namespace))
|
||||
}
|
||||
//Create Secret
|
||||
secretName := fmt.Sprintf("secret-%s", o.NSBaseName)
|
||||
fmt.Printf("Creating secret %s in %s namespaces ...\n", secretName, o.Namespace)
|
||||
_, err = CreateSecret(o.Client.ClientGo, o.Namespace, secretName, label)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, fmt.Sprintf("failed to create secret in the namespace %q", o.Namespace))
|
||||
}
|
||||
err = WaitForSecretsComplete(o.Client.ClientGo, o.Namespace, secretName)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, fmt.Sprintf("failed to ensure secret completion in namespace: %q", o.Namespace))
|
||||
}
|
||||
//Create Configmap
|
||||
configmapName := fmt.Sprintf("configmap-%s", o.NSBaseName)
|
||||
fmt.Printf("Creating configmap %s in %s namespaces ...\n", configmapName, o.Namespace)
|
||||
_, err = CreateConfigMap(o.Client.ClientGo, o.Namespace, configmapName, label)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, fmt.Sprintf("failed to create configmap in the namespace %q", o.Namespace))
|
||||
}
|
||||
err = WaitForConfigMapComplete(o.Client.ClientGo, o.Namespace, configmapName)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, fmt.Sprintf("failed to ensure secret completion in namespace: %q", o.Namespace))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -28,6 +28,7 @@ import (
|
|||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"strings"
|
||||
|
|
@ -158,6 +159,36 @@ func getProviderVeleroInstallOptions(
|
|||
return io, nil
|
||||
}
|
||||
|
||||
func CMDExecWithOutput(checkCMD *exec.Cmd) (*[]byte, error) {
|
||||
stdoutPipe, err := checkCMD.StdoutPipe()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
jsonBuf := make([]byte, 16*1024) // If the YAML is bigger than 16K, there's probably something bad happening
|
||||
|
||||
err = checkCMD.Start()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
bytesRead, err := io.ReadFull(stdoutPipe, jsonBuf)
|
||||
|
||||
if err != nil && err != io.ErrUnexpectedEOF {
|
||||
return nil, err
|
||||
}
|
||||
if bytesRead == len(jsonBuf) {
|
||||
return nil, errors.New("yaml returned bigger than max allowed")
|
||||
}
|
||||
|
||||
jsonBuf = jsonBuf[0:bytesRead]
|
||||
err = checkCMD.Wait()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &jsonBuf, err
|
||||
}
|
||||
|
||||
// checkBackupPhase uses VeleroCLI to inspect the phase of a Velero backup.
|
||||
func checkBackupPhase(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string,
|
||||
expectedPhase velerov1api.BackupPhase) error {
|
||||
|
|
@ -165,34 +196,12 @@ func checkBackupPhase(ctx context.Context, veleroCLI string, veleroNamespace str
|
|||
backupName)
|
||||
|
||||
fmt.Printf("get backup cmd =%v\n", checkCMD)
|
||||
stdoutPipe, err := checkCMD.StdoutPipe()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
jsonBuf := make([]byte, 16*1024) // If the YAML is bigger than 16K, there's probably something bad happening
|
||||
|
||||
err = checkCMD.Start()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
bytesRead, err := io.ReadFull(stdoutPipe, jsonBuf)
|
||||
|
||||
if err != nil && err != io.ErrUnexpectedEOF {
|
||||
return err
|
||||
}
|
||||
if bytesRead == len(jsonBuf) {
|
||||
return errors.New("yaml returned bigger than max allowed")
|
||||
}
|
||||
|
||||
jsonBuf = jsonBuf[0:bytesRead]
|
||||
err = checkCMD.Wait()
|
||||
jsonBuf, err := CMDExecWithOutput(checkCMD)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
backup := velerov1api.Backup{}
|
||||
err = json.Unmarshal(jsonBuf, &backup)
|
||||
err = json.Unmarshal(*jsonBuf, &backup)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -209,34 +218,12 @@ func checkRestorePhase(ctx context.Context, veleroCLI string, veleroNamespace st
|
|||
restoreName)
|
||||
|
||||
fmt.Printf("get restore cmd =%v\n", checkCMD)
|
||||
stdoutPipe, err := checkCMD.StdoutPipe()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
jsonBuf := make([]byte, 16*1024) // If the YAML is bigger than 16K, there's probably something bad happening
|
||||
|
||||
err = checkCMD.Start()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
bytesRead, err := io.ReadFull(stdoutPipe, jsonBuf)
|
||||
|
||||
if err != nil && err != io.ErrUnexpectedEOF {
|
||||
return err
|
||||
}
|
||||
if bytesRead == len(jsonBuf) {
|
||||
return errors.New("yaml returned bigger than max allowed")
|
||||
}
|
||||
|
||||
jsonBuf = jsonBuf[0:bytesRead]
|
||||
err = checkCMD.Wait()
|
||||
jsonBuf, err := CMDExecWithOutput(checkCMD)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
restore := velerov1api.Restore{}
|
||||
err = json.Unmarshal(jsonBuf, &restore)
|
||||
err = json.Unmarshal(*jsonBuf, &restore)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -246,6 +233,67 @@ func checkRestorePhase(ctx context.Context, veleroCLI string, veleroNamespace st
|
|||
return nil
|
||||
}
|
||||
|
||||
func checkSchedulePhase(ctx context.Context, veleroCLI, veleroNamespace, scheduleName string) error {
|
||||
checkCMD := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "schedule", "get", scheduleName, "-ojson")
|
||||
jsonBuf, err := CMDExecWithOutput(checkCMD)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
schedule := velerov1api.Schedule{}
|
||||
err = json.Unmarshal(*jsonBuf, &schedule)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if schedule.Status.Phase != velerov1api.SchedulePhaseEnabled {
|
||||
return errors.Errorf("Unexpected restore phase got %s, expecting %s", schedule.Status.Phase, velerov1api.SchedulePhaseEnabled)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func CheckScheduleWithResourceOrder(ctx context.Context, veleroCLI, veleroNamespace, scheduleName string, order map[string]string) error {
|
||||
checkCMD := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "schedule", "get", scheduleName, "-ojson")
|
||||
jsonBuf, err := CMDExecWithOutput(checkCMD)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
schedule := velerov1api.Schedule{}
|
||||
err = json.Unmarshal(*jsonBuf, &schedule)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if schedule.Status.Phase != velerov1api.SchedulePhaseEnabled {
|
||||
return errors.Errorf("Unexpected restore phase got %s, expecting %s", schedule.Status.Phase, velerov1api.SchedulePhaseEnabled)
|
||||
}
|
||||
if reflect.DeepEqual(schedule.Spec.Template.OrderedResources, order) {
|
||||
return nil
|
||||
} else {
|
||||
return fmt.Errorf("resource order %v set in schedule command is not equal with order %v stored in schedule cr", order, schedule.Spec.Template.OrderedResources)
|
||||
}
|
||||
}
|
||||
|
||||
func CheckBackupWithResourceOrder(ctx context.Context, veleroCLI, veleroNamespace, backupName string, order map[string]string) error {
|
||||
checkCMD := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "get", "backup", backupName, "-ojson")
|
||||
jsonBuf, err := CMDExecWithOutput(checkCMD)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
backup := velerov1api.Backup{}
|
||||
err = json.Unmarshal(*jsonBuf, &backup)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if backup.Status.Phase != velerov1api.BackupPhaseCompleted {
|
||||
return errors.Errorf("Unexpected restore phase got %s, expecting %s", backup.Status.Phase, velerov1api.BackupPhaseCompleted)
|
||||
}
|
||||
if reflect.DeepEqual(backup.Spec.OrderedResources, order) {
|
||||
return nil
|
||||
} else {
|
||||
return fmt.Errorf("resource order %v set in backup command is not equal with order %v stored in backup cr", order, backup.Spec.OrderedResources)
|
||||
}
|
||||
}
|
||||
|
||||
// VeleroBackupNamespace uses the veleroCLI to backup a namespace.
|
||||
func VeleroBackupNamespace(ctx context.Context, veleroCLI, veleroNamespace string, backupCfg BackupConfig) error {
|
||||
args := []string{
|
||||
|
|
@ -322,6 +370,21 @@ func VeleroBackupExec(ctx context.Context, veleroCLI string, veleroNamespace str
|
|||
return checkBackupPhase(ctx, veleroCLI, veleroNamespace, backupName, velerov1api.BackupPhaseCompleted)
|
||||
}
|
||||
|
||||
func VeleroScheduleDelete(ctx context.Context, veleroCLI string, veleroNamespace string, scheduleName string) error {
|
||||
args := []string{"--namespace", veleroNamespace, "delete", "schedule", scheduleName, "--confirm"}
|
||||
return VeleroCmdExec(ctx, veleroCLI, args)
|
||||
}
|
||||
|
||||
func VeleroScheduleCreate(ctx context.Context, veleroCLI string, veleroNamespace string, scheduleName string, args []string) error {
|
||||
args = append([]string{
|
||||
"--namespace", veleroNamespace, "create", "schedule", scheduleName,
|
||||
}, args...)
|
||||
if err := VeleroCmdExec(ctx, veleroCLI, args); err != nil {
|
||||
return err
|
||||
}
|
||||
return checkSchedulePhase(ctx, veleroCLI, veleroNamespace, scheduleName)
|
||||
}
|
||||
|
||||
func VeleroCmdExec(ctx context.Context, veleroCLI string, args []string) error {
|
||||
cmd := exec.CommandContext(ctx, veleroCLI, args...)
|
||||
cmd.Stdout = os.Stdout
|
||||
|
|
|
|||
Loading…
Reference in New Issue