From 80430542df8760ff49074fd0685f102ba775758e Mon Sep 17 00:00:00 2001 From: danfengl Date: Sun, 18 Sep 2022 12:22:47 +0000 Subject: [PATCH] Add schedule backup timing E2E test Signed-off-by: danfengl --- changelogs/unreleased/5355-danfengliu | 1 + test/e2e/backups/schedule.go | 166 ++++++++++++++++++++++++++ test/e2e/e2e_suite_test.go | 1 + test/e2e/util/common/common.go | 2 +- test/e2e/util/k8s/common.go | 2 +- test/e2e/util/k8s/namespace.go | 2 +- test/e2e/util/velero/velero_utils.go | 72 ++++++++--- 7 files changed, 229 insertions(+), 17 deletions(-) create mode 100644 changelogs/unreleased/5355-danfengliu create mode 100644 test/e2e/backups/schedule.go diff --git a/changelogs/unreleased/5355-danfengliu b/changelogs/unreleased/5355-danfengliu new file mode 100644 index 000000000..e365367f3 --- /dev/null +++ b/changelogs/unreleased/5355-danfengliu @@ -0,0 +1 @@ +Add E2E test for schedule backup \ No newline at end of file diff --git a/test/e2e/backups/schedule.go b/test/e2e/backups/schedule.go new file mode 100644 index 000000000..8cabe529c --- /dev/null +++ b/test/e2e/backups/schedule.go @@ -0,0 +1,166 @@ +package backups + +import ( + "context" + "fmt" + "math/rand" + "strings" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + . "github.com/vmware-tanzu/velero/test/e2e" + . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" + . "github.com/vmware-tanzu/velero/test/e2e/util/velero" +) + +type ScheduleBackup struct { + TestCase + ScheduleName string + ScheduleArgs []string + Period int //Limitation: The unit is minitue only and 60 is divisible by it + randBackupName string + verifyTimes int +} + +var ScheduleBackupTest func() = TestFunc(&ScheduleBackup{TestCase: TestCase{NSBaseName: "ns", NSIncluded: &[]string{"ns1"}}}) + +func (n *ScheduleBackup) Init() error { + n.Client = TestClientInstance + n.Period = 3 + n.verifyTimes = 5 // More verify times more confidence + n.TestMsg = &TestMSG{ + Desc: "Set up a scheduled backup defined by a Cron expression", + FailedMSG: "Failed to schedule a backup", + Text: "should backup periodly according to the schedule", + } + return nil +} + +func (n *ScheduleBackup) StartRun() error { + + n.ScheduleName = n.ScheduleName + "schedule-" + UUIDgen.String() + n.RestoreName = n.RestoreName + "restore-ns-mapping-" + UUIDgen.String() + + n.ScheduleArgs = []string{ + "schedule", "create", "--namespace", VeleroCfg.VeleroNamespace, n.ScheduleName, + "--include-namespaces", strings.Join(*n.NSIncluded, ","), + "--schedule=*/" + fmt.Sprintf("%v", n.Period) + " * * * *", + } + + return nil +} +func (n *ScheduleBackup) CreateResources() error { + n.Ctx, _ = context.WithTimeout(context.Background(), 60*time.Minute) + for _, ns := range *n.NSIncluded { + By(fmt.Sprintf("Creating namespaces %s ......\n", ns), func() { + Expect(CreateNamespace(n.Ctx, n.Client, ns)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", ns)) + }) + configmaptName := n.NSBaseName + fmt.Printf("Creating configmap %s in namespaces ...%s\n", configmaptName, ns) + _, err := CreateConfigMap(n.Client.ClientGo, ns, configmaptName, nil) + Expect(err).To(Succeed(), fmt.Sprintf("failed to create configmap in the namespace %q", ns)) + Expect(WaitForConfigMapComplete(n.Client.ClientGo, ns, configmaptName)).To(Succeed(), + fmt.Sprintf("ailed to ensure secret completion in namespace: %q", ns)) + } + return nil +} + +func (n *ScheduleBackup) Backup() error { + // Wait until the beginning of the given period to create schedule, it will give us + // a predictable period to wait for the first scheduled backup, and verify no immediate + // scheduled backup was created between schedule creation and first scheduled backup. + By(fmt.Sprintf("Creating schedule %s ......\n", n.ScheduleName), func() { + for i := 0; i < n.Period*60/30; i++ { + time.Sleep(30 * time.Second) + now := time.Now().Minute() + triggerNow := now % n.Period + if triggerNow == 0 { + Expect(VeleroCmdExec(n.Ctx, VeleroCfg.VeleroCLI, n.ScheduleArgs)).To(Succeed()) + break + } + } + }) + return nil +} +func (n *ScheduleBackup) Destroy() error { + By(fmt.Sprintf("Schedule %s is created without any delay\n", n.ScheduleName), func() { + creationTimestamp, err := GetSchedule(context.Background(), VeleroCfg.VeleroNamespace, n.ScheduleName) + Expect(err).To(Succeed()) + + creationTime, err := time.Parse(time.RFC3339, strings.Replace(creationTimestamp, "'", "", -1)) + Expect(err).To(Succeed()) + fmt.Printf("Schedule %s created at %s\n", n.ScheduleName, creationTime) + now := time.Now() + diff := creationTime.Sub(now) + Expect(diff.Minutes() < 1).To(Equal(true)) + }) + + By(fmt.Sprintf("No immediate backup is created by schedule %s\n", n.ScheduleName), func() { + for i := 0; i < n.Period; i++ { + time.Sleep(1 * time.Minute) + now := time.Now() + fmt.Printf("Get backup for #%d time at %v\n", i, now) + //Ignore the last minute in the period avoiding met the 1st backup by schedule + if i != n.Period-1 { + backupsInfo, err := GetScheduledBackupsCreationTime(context.Background(), VeleroCfg.VeleroCLI, "default", n.ScheduleName) + Expect(err).To(Succeed()) + Expect(len(backupsInfo) == 0).To(Equal(true)) + } + } + }) + + By("Delay one more minute to make sure the new backup was created in the given period", func() { + time.Sleep(1 * time.Minute) + }) + + By(fmt.Sprintf("Get backups every %d minute, and backups count should increase 1 more step in the same pace\n", n.Period), func() { + for i := 0; i < n.verifyTimes; i++ { + fmt.Printf("Start to sleep %d minute #%d time...\n", n.Period, i+1) + time.Sleep(time.Duration(n.Period) * time.Minute) + bMap := make(map[string]string) + backupsInfo, err := GetScheduledBackupsCreationTime(context.Background(), VeleroCfg.VeleroCLI, "default", n.ScheduleName) + Expect(err).To(Succeed()) + Expect(len(backupsInfo) == i+2).To(Equal(true)) + for index, bi := range backupsInfo { + bList := strings.Split(bi, ",") + fmt.Printf("Backup %d: %v\n", index, bList) + bMap[bList[0]] = bList[1] + _, err := time.Parse("2006-01-02 15:04:05 -0700 MST", bList[1]) + Expect(err).To(Succeed()) + } + if i == n.verifyTimes-1 { + backupInfo := backupsInfo[rand.Intn(len(backupsInfo))] + n.randBackupName = strings.Split(backupInfo, ",")[0] + } + } + }) + + n.BackupName = strings.Replace(n.randBackupName, " ", "", -1) + + By("Delete all namespaces", func() { + Expect(CleanupNamespacesWithPoll(n.Ctx, n.Client, n.NSBaseName)).To(Succeed(), "Could cleanup retrieve namespaces") + }) + + n.RestoreArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "restore", n.RestoreName, + "--from-backup", n.BackupName, + "--wait", + } + + return nil +} + +func (n *ScheduleBackup) Verify() error { + By("Namespaces were restored", func() { + for _, ns := range *n.NSIncluded { + configmap, err := GetConfigmap(n.Client.ClientGo, ns, n.NSBaseName) + fmt.Printf("Restored configmap is %v\n", configmap) + Expect(err).ShouldNot(HaveOccurred(), fmt.Sprintf("failed to list configmap in namespace: %q\n", ns)) + } + + }) + return nil +} diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 9e509b302..0f06f6a19 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -111,6 +111,7 @@ var _ = Describe("[Backups][Deletion][Restic] Velero tests of Restic backup dele var _ = Describe("[Backups][Deletion][Snapshot] Velero tests of snapshot backup deletion", BackupDeletionWithSnapshots) var _ = Describe("[Backups][TTL] Local backups and restic repos will be deleted once the corresponding backup storage location is deleted", TTLTest) var _ = Describe("[Backups][BackupsSync] Backups in object storage are synced to a new Velero and deleted backups in object storage are synced to be deleted in Velero", BackupsSyncTest) +var _ = Describe("[Backups][Schedule] Backup will be created periodly by schedule defined by a Cron expression", ScheduleBackupTest) var _ = Describe("[PrivilegesMgmt][SSR] Velero test on ssr object when controller namespace mix-ups", SSRTest) diff --git a/test/e2e/util/common/common.go b/test/e2e/util/common/common.go index 17e942cab..f7a43fc71 100644 --- a/test/e2e/util/common/common.go +++ b/test/e2e/util/common/common.go @@ -38,7 +38,7 @@ func GetListBy2Pipes(ctx context.Context, cmdline1, cmdline2, cmdline3 OsCommand _ = c2.Wait() _ = c3.Wait() - fmt.Println(&b2) + //fmt.Println(&b2) scanner := bufio.NewScanner(&b2) var ret []string for scanner.Scan() { diff --git a/test/e2e/util/k8s/common.go b/test/e2e/util/k8s/common.go index 394a8d506..9032d36f2 100644 --- a/test/e2e/util/k8s/common.go +++ b/test/e2e/util/k8s/common.go @@ -63,7 +63,7 @@ func WaitForPods(ctx context.Context, client TestClient, namespace string, pods checkPod, err := client.ClientGo.CoreV1().Pods(namespace).Get(context.TODO(), podName, metav1.GetOptions{}) if err != nil { //Should ignore "etcdserver: request timed out" kind of errors, try to get pod status again before timeout. - fmt.Println(errors.Wrap(err, fmt.Sprintf("Failed to verify pod %s/%s is %s, try again...", namespace, podName, corev1api.PodRunning))) + fmt.Println(errors.Wrap(err, fmt.Sprintf("Failed to verify pod %s/%s is %s, try again...\n", namespace, podName, corev1api.PodRunning))) return false, nil } // If any pod is still waiting we don't need to check any more so return and wait for next poll interval diff --git a/test/e2e/util/k8s/namespace.go b/test/e2e/util/k8s/namespace.go index cbdc644eb..767aba7ba 100644 --- a/test/e2e/util/k8s/namespace.go +++ b/test/e2e/util/k8s/namespace.go @@ -100,7 +100,7 @@ func CleanupNamespacesWithPoll(ctx context.Context, client TestClient, nsBaseNam if err != nil { return errors.Wrapf(err, "Could not delete namespace %s", checkNamespace.Name) } - fmt.Printf("Delete namespace %s", checkNamespace.Name) + fmt.Printf("Delete namespace %s\n", checkNamespace.Name) } } return nil diff --git a/test/e2e/util/velero/velero_utils.go b/test/e2e/util/velero/velero_utils.go index e7bc0404e..3e3fc67f0 100644 --- a/test/e2e/util/velero/velero_utils.go +++ b/test/e2e/util/velero/velero_utils.go @@ -431,17 +431,19 @@ func VeleroScheduleCreate(ctx context.Context, veleroCLI string, veleroNamespace func VeleroCmdExec(ctx context.Context, veleroCLI string, args []string) error { cmd := exec.CommandContext(ctx, veleroCLI, args...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr + var errBuf, outBuf bytes.Buffer + cmd.Stderr = io.MultiWriter(os.Stderr, &errBuf) + cmd.Stdout = io.MultiWriter(os.Stdout, &outBuf) fmt.Printf("velero cmd =%v\n", cmd) err := cmd.Run() - if strings.Contains(fmt.Sprint(cmd.Stdout), "Failed") { + retAll := outBuf.String() + " " + errBuf.String() + if strings.Contains(strings.ToLower(retAll), "failed") { return errors.New(fmt.Sprintf("velero cmd =%v return with failure\n", cmd)) } if err != nil { return err } - return err + return nil } func VeleroBackupLogs(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string) error { @@ -871,6 +873,43 @@ func GetBackupsFromBsl(ctx context.Context, veleroCLI, bslName string) ([]string return common.GetListBy2Pipes(ctx, *CmdLine1, *CmdLine2, *CmdLine3) } +func GetScheduledBackupsCreationTime(ctx context.Context, veleroCLI, bslName, scheduleName string) ([]string, error) { + var creationTimes []string + backups, err := GetBackupsCreationTime(ctx, veleroCLI, bslName) + if err != nil { + return nil, err + } + for _, b := range backups { + if strings.Contains(b, scheduleName) { + creationTimes = append(creationTimes, b) + } + } + return creationTimes, nil +} +func GetBackupsCreationTime(ctx context.Context, veleroCLI, bslName string) ([]string, error) { + args1 := []string{"get", "backups"} + createdTime := "$1,\",\" $5,$6,$7,$8" + if strings.TrimSpace(bslName) != "" { + args1 = append(args1, "-l", "velero.io/storage-location="+bslName) + } + CmdLine1 := &common.OsCommandLine{ + Cmd: veleroCLI, + Args: args1, + } + + CmdLine2 := &common.OsCommandLine{ + Cmd: "awk", + Args: []string{"{print " + createdTime + "}"}, + } + + CmdLine3 := &common.OsCommandLine{ + Cmd: "tail", + Args: []string{"-n", "+2"}, + } + + return common.GetListBy2Pipes(ctx, *CmdLine1, *CmdLine2, *CmdLine3) +} + func GetAllBackups(ctx context.Context, veleroCLI string) ([]string, error) { return GetBackupsFromBsl(ctx, veleroCLI, "") } @@ -976,7 +1015,6 @@ func GetSnapshotCheckPoint(client TestClient, VeleroCfg VerleroConfig, expectCou } func GetBackupTTL(ctx context.Context, veleroNamespace, backupName string) (string, error) { - checkSnapshotCmd := exec.CommandContext(ctx, "kubectl", "get", "backup", "-n", veleroNamespace, backupName, "-o=jsonpath='{.spec.ttl}'") fmt.Printf("checkSnapshotCmd cmd =%v\n", checkSnapshotCmd) @@ -984,15 +1022,8 @@ func GetBackupTTL(ctx context.Context, veleroNamespace, backupName string) (stri if err != nil { fmt.Print(stdout) fmt.Print(stderr) - return "", errors.Wrap(err, "failed to verify") + return "", errors.Wrap(err, fmt.Sprintf("failed to run command %s", checkSnapshotCmd)) } - // lines := strings.Split(stdout, "\n") - // complete := true - // for _, curLine := range lines { - // fmt.Println(curLine) - - // } - // return complete, nil return stdout, err } @@ -1019,10 +1050,23 @@ func DeleteBackups(ctx context.Context, client TestClient) error { return fmt.Errorf("failed to list backup object in %s namespace with err %v", VeleroCfg.VeleroNamespace, err) } for _, backup := range backupList.Items { - fmt.Printf("Backup %s is going to be deleted...", backup.Name) + fmt.Printf("Backup %s is going to be deleted...\n", backup.Name) if err := VeleroBackupDelete(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backup.Name); err != nil { return err } } return nil } + +func GetSchedule(ctx context.Context, veleroNamespace, scheduleName string) (string, error) { + checkSnapshotCmd := exec.CommandContext(ctx, "kubectl", + "get", "schedule", "-n", veleroNamespace, scheduleName, "-o=jsonpath='{.metadata.creationTimestamp}'") + fmt.Printf("Cmd =%v\n", checkSnapshotCmd) + stdout, stderr, err := veleroexec.RunCommand(checkSnapshotCmd) + if err != nil { + fmt.Print(stdout) + fmt.Print(stderr) + return "", errors.Wrap(err, fmt.Sprintf("failed to run command %s", checkSnapshotCmd)) + } + return stdout, err +}