From 56f72797db51d1d07659956e0bc8a8cae04796d2 Mon Sep 17 00:00:00 2001 From: Ming Date: Sat, 4 Jun 2022 05:33:48 +0000 Subject: [PATCH] Fix nightly running failure Signed-off-by: Ming --- test/e2e/backups/deletion.go | 19 +++------ test/e2e/backups/ttl.go | 17 ++++---- test/e2e/basic/resources-check/namespaces.go | 2 +- test/e2e/bsl-mgmt/deletion.go | 8 ++-- .../e2e/orderedresources/ordered_resources.go | 15 +++++++ test/e2e/upgrade/upgrade.go | 2 +- test/e2e/util/kibishii/kibishii_utils.go | 2 +- test/e2e/util/providers/common.go | 31 -------------- test/e2e/util/velero/velero_utils.go | 41 ++++++++++++------- 9 files changed, 62 insertions(+), 75 deletions(-) diff --git a/test/e2e/backups/deletion.go b/test/e2e/backups/deletion.go index a531865f1..89d118631 100644 --- a/test/e2e/backups/deletion.go +++ b/test/e2e/backups/deletion.go @@ -25,7 +25,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/pkg/errors" - waitutil "k8s.io/apimachinery/pkg/util/wait" . "github.com/vmware-tanzu/velero/test/e2e" . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" @@ -147,7 +146,7 @@ func runBackupDeletionTests(client TestClient, veleroCfg VerleroConfig, backupNa if useVolumeSnapshots { snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 2, deletionTest, backupName, KibishiiPodNameList) Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") - err = WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider, + err = SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslConfig, backupName, snapshotCheckPoint) if err != nil { @@ -159,7 +158,7 @@ func runBackupDeletionTests(client TestClient, veleroCfg VerleroConfig, backupNa return err } if useVolumeSnapshots { - err = WaitUntilSnapshotsNotExistInCloud(VeleroCfg.CloudProvider, + err = SnapshotsShouldNotExistInCloud(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, veleroCfg.BSLConfig, backupName, snapshotCheckPoint) if err != nil { @@ -172,17 +171,9 @@ func runBackupDeletionTests(client TestClient, veleroCfg VerleroConfig, backupNa return err } if useVolumeSnapshots { - err = waitutil.PollImmediate(30*time.Second, 3*time.Minute, - func() (bool, error) { - err := SnapshotsShouldNotExistInCloud(VeleroCfg.CloudProvider, - VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, - bslConfig, backupName, snapshotCheckPoint) - if err == nil { - return true, nil - } - return false, err - }) - if err != nil { + if err := SnapshotsShouldNotExistInCloud(VeleroCfg.CloudProvider, + VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, + bslConfig, backupName, snapshotCheckPoint); err != nil { return errors.Wrap(err, "exceed waiting for snapshot created in cloud") } } diff --git a/test/e2e/backups/ttl.go b/test/e2e/backups/ttl.go index 6b024808e..439ada693 100644 --- a/test/e2e/backups/ttl.go +++ b/test/e2e/backups/ttl.go @@ -52,8 +52,8 @@ func (b *TTL) Init() { b.testNS = "backup-ttl-test-" + UUIDgen.String() b.backupName = "backup-ttl-test-" + UUIDgen.String() b.restoreName = "restore-ttl-test-" + UUIDgen.String() - b.ctx, _ = context.WithTimeout(context.Background(), time.Duration(time.Minute*30)) - b.ttl = time.Duration(10 * time.Minute) + b.ctx, _ = context.WithTimeout(context.Background(), time.Duration(time.Hour)) + b.ttl = time.Duration(20 * time.Minute) } @@ -87,14 +87,13 @@ func TTLTest() { It("Backups in object storage should be synced to a new Velero successfully", func() { test.Init() - oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60) By(fmt.Sprintf("Prepare workload as target to backup by creating namespace %s namespace", test.testNS), func() { - Expect(CreateNamespace(oneHourTimeout, client, test.testNS)).To(Succeed(), + Expect(CreateNamespace(test.ctx, client, test.testNS)).To(Succeed(), fmt.Sprintf("Failed to create %s namespace", test.testNS)) }) By("Deploy sample workload of Kibishii", func() { - Expect(KibishiiPrepareBeforeBackup(oneHourTimeout, client, VeleroCfg.CloudProvider, + Expect(KibishiiPrepareBeforeBackup(test.ctx, client, VeleroCfg.CloudProvider, test.testNS, VeleroCfg.RegistryCredentialFile, VeleroCfg.Features, VeleroCfg.KibishiiDirectory, useVolumeSnapshots)).To(Succeed()) }) @@ -108,7 +107,7 @@ func TTLTest() { BackupCfg.TTL = test.ttl By(fmt.Sprintf("Backup the workload in %s namespace", test.testNS), func() { - Expect(VeleroBackupNamespace(oneHourTimeout, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string { + Expect(VeleroBackupNamespace(test.ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string { RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.backupName, "") return "Fail to backup workload" }) @@ -119,14 +118,14 @@ func TTLTest() { if VeleroCfg.CloudProvider == "vsphere" { // TODO - remove after upload progress monitoring is implemented By("Waiting for vSphere uploads to complete", func() { - Expect(WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, + Expect(WaitForVSphereUploadCompletion(test.ctx, time.Hour, test.testNS)).To(Succeed()) }) } snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 2, test.testNS, test.backupName, KibishiiPodNameList) Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") - Expect(WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider, + Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, VeleroCfg.BSLConfig, test.backupName, snapshotCheckPoint)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") } @@ -183,7 +182,7 @@ func TTLTest() { By("PersistentVolume snapshots should be deleted", func() { if useVolumeSnapshots { - Expect(WaitUntilSnapshotsNotExistInCloud(VeleroCfg.CloudProvider, + Expect(SnapshotsShouldNotExistInCloud(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, VeleroCfg.BSLConfig, test.backupName, snapshotCheckPoint)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") } diff --git a/test/e2e/basic/resources-check/namespaces.go b/test/e2e/basic/resources-check/namespaces.go index 366e3adfb..ae6e15455 100644 --- a/test/e2e/basic/resources-check/namespaces.go +++ b/test/e2e/basic/resources-check/namespaces.go @@ -57,7 +57,7 @@ func (m *MultiNSBackup) Init() error { } } else { m.NamespacesTotal = 2 - m.TimeoutDuration = time.Minute * 5 + m.TimeoutDuration = time.Minute * 10 m.TestMsg = &TestMSG{ Text: "When I create 2 namespaces should be successfully backed up and restored", FailedMSG: "Failed to successfully backup and restore multiple namespaces", diff --git a/test/e2e/bsl-mgmt/deletion.go b/test/e2e/bsl-mgmt/deletion.go index 91307f19e..fd75ba180 100644 --- a/test/e2e/bsl-mgmt/deletion.go +++ b/test/e2e/bsl-mgmt/deletion.go @@ -220,7 +220,7 @@ func BslDeletionTest(useVolumeSnapshots bool) { By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation_1), func() { snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 1, bslDeletionTestNs, backupName_1, []string{podName_1}) Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") - Expect(WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider, + Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.AdditionalBSLBucket, VeleroCfg.BSLConfig, backupName_1, snapshotCheckPoint)).To(Succeed()) }) @@ -236,7 +236,7 @@ func BslDeletionTest(useVolumeSnapshots bool) { BSLConfig = VeleroCfg.BSLConfig } - Expect(WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider, + Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider, BSLCredentials, VeleroCfg.AdditionalBSLBucket, BSLConfig, backupName_2, snapshotCheckPoint)).To(Succeed()) }) @@ -318,7 +318,7 @@ func BslDeletionTest(useVolumeSnapshots bool) { snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 1, bslDeletionTestNs, backupName_1, []string{podName_1}) Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") - Expect(WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider, + Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, VeleroCfg.BSLConfig, backupName_1, snapshotCheckPoint)).To(Succeed()) }) @@ -333,7 +333,7 @@ func BslDeletionTest(useVolumeSnapshots bool) { } snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 1, bslDeletionTestNs, backupName_2, []string{podName_2}) Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") - Expect(WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider, + Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider, BSLCredentials, VeleroCfg.AdditionalBSLBucket, BSLConfig, backupName_2, snapshotCheckPoint)).To(Succeed()) }) diff --git a/test/e2e/orderedresources/ordered_resources.go b/test/e2e/orderedresources/ordered_resources.go index ff141470d..2a9fe4c97 100644 --- a/test/e2e/orderedresources/ordered_resources.go +++ b/test/e2e/orderedresources/ordered_resources.go @@ -69,6 +69,8 @@ func ScheduleOrderedResources() { Expect(DeleteNamespace(test.Ctx, test.Client, test.Namespace, false)).To(Succeed(), fmt.Sprintf("Failed to delete the namespace %s", test.Namespace)) err = VeleroScheduleDelete(test.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.ScheduleName) Expect(err).To(Succeed(), fmt.Sprintf("Failed to delete schedule with err %v", err)) + err = test.DeleteBackups() + Expect(err).To(Succeed(), fmt.Sprintf("Failed to delete backups with err %v", err)) }() By(fmt.Sprintf("Prepare workload as target to backup in base namespace %s", test.Namespace), func() { @@ -189,3 +191,16 @@ func (o *OrderedResources) CreateResources() error { } return nil } + +func (o *OrderedResources) DeleteBackups() error { + backupList := new(velerov1api.BackupList) + if err := o.Client.Kubebuilder.List(o.Ctx, backupList, &kbclient.ListOptions{Namespace: VeleroCfg.VeleroNamespace}); err != nil { + return fmt.Errorf("failed to list backup object in %s namespace with err %v", VeleroCfg.VeleroNamespace, err) + } + for _, backup := range backupList.Items { + if err := VeleroBackupDelete(o.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backup.Name); err != nil { + return err + } + } + return nil +} diff --git a/test/e2e/upgrade/upgrade.go b/test/e2e/upgrade/upgrade.go index 794feba25..7461697a2 100644 --- a/test/e2e/upgrade/upgrade.go +++ b/test/e2e/upgrade/upgrade.go @@ -190,7 +190,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, upgradeFromVelero Upgrade snapshotCheckPoint, err := GetSnapshotCheckPoint(client, VeleroCfg, 2, upgradeNamespace, backupName, KibishiiPodNameList) Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint") - Expect(WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider, + Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, VeleroCfg.BSLConfig, backupName, snapshotCheckPoint)).To(Succeed()) }) diff --git a/test/e2e/util/kibishii/kibishii_utils.go b/test/e2e/util/kibishii/kibishii_utils.go index 45d4d8f8e..8406cda55 100644 --- a/test/e2e/util/kibishii/kibishii_utils.go +++ b/test/e2e/util/kibishii/kibishii_utils.go @@ -96,7 +96,7 @@ func RunKibishiiTests(client TestClient, veleroCfg VerleroConfig, backupName, re if err != nil { return errors.Wrap(err, "Fail to get snapshot checkpoint") } - err = WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider, + err = SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, veleroCfg.BSLConfig, backupName, snapshotCheckPoint) if err != nil { diff --git a/test/e2e/util/providers/common.go b/test/e2e/util/providers/common.go index ee5dd47da..4ca8ac143 100644 --- a/test/e2e/util/providers/common.go +++ b/test/e2e/util/providers/common.go @@ -24,8 +24,6 @@ import ( "github.com/pkg/errors" - waitutil "k8s.io/apimachinery/pkg/util/wait" - . "github.com/vmware-tanzu/velero/test/e2e" velero "github.com/vmware-tanzu/velero/test/e2e/util/velero" ) @@ -111,35 +109,6 @@ func DeleteObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPr return nil } -func WaitUntilSnapshotsExistInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheckPoint SnapshotCheckPoint) error { - err := waitutil.PollImmediate(30*time.Second, 3*time.Minute, - func() (bool, error) { - err := SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider, - VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, - bslConfig, backupName, snapshotCheckPoint) - if err == nil { - return true, nil - } - return false, err - }) - return err -} - -func WaitUntilSnapshotsNotExistInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheckPoint SnapshotCheckPoint) error { - err := waitutil.PollImmediate(30*time.Second, 3*time.Minute, - func() (bool, error) { - snapshotCheckPoint.ExpectCount = 0 - err := SnapshotsShouldNotExistInCloud(VeleroCfg.CloudProvider, - VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, - bslConfig, backupName, snapshotCheckPoint) - if err == nil { - return true, nil - } - return false, err - }) - return err -} - func SnapshotsShouldNotExistInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheckPoint SnapshotCheckPoint) error { fmt.Printf("|| VERIFICATION || - Snapshots should not exist in cloud, backup %s\n", backupName) snapshotCheckPoint.ExpectCount = 0 diff --git a/test/e2e/util/velero/velero_utils.go b/test/e2e/util/velero/velero_utils.go index 0a4a014b3..30126d219 100644 --- a/test/e2e/util/velero/velero_utils.go +++ b/test/e2e/util/velero/velero_utils.go @@ -235,20 +235,23 @@ func checkRestorePhase(ctx context.Context, veleroCLI string, veleroNamespace st func checkSchedulePhase(ctx context.Context, veleroCLI, veleroNamespace, scheduleName string) error { checkCMD := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "schedule", "get", scheduleName, "-ojson") - jsonBuf, err := CMDExecWithOutput(checkCMD) - if err != nil { - return err - } - schedule := velerov1api.Schedule{} - err = json.Unmarshal(*jsonBuf, &schedule) - if err != nil { - return err - } + return wait.PollImmediate(time.Second*5, time.Minute*2, func() (bool, error) { + jsonBuf, err := CMDExecWithOutput(checkCMD) + if err != nil { + return false, err + } + schedule := velerov1api.Schedule{} + err = json.Unmarshal(*jsonBuf, &schedule) + if err != nil { + return false, err + } - if schedule.Status.Phase != velerov1api.SchedulePhaseEnabled { - return errors.Errorf("Unexpected restore phase got %s, expecting %s", schedule.Status.Phase, velerov1api.SchedulePhaseEnabled) - } - return nil + if schedule.Status.Phase != velerov1api.SchedulePhaseEnabled { + fmt.Printf("Unexpected schedule phase got %s, expecting %s, still waiting...", schedule.Status.Phase, velerov1api.SchedulePhaseEnabled) + return false, nil + } + return true, nil + }) } func CheckScheduleWithResourceOrder(ctx context.Context, veleroCLI, veleroNamespace, scheduleName string, order map[string]string) error { @@ -370,6 +373,16 @@ func VeleroBackupExec(ctx context.Context, veleroCLI string, veleroNamespace str return checkBackupPhase(ctx, veleroCLI, veleroNamespace, backupName, velerov1api.BackupPhaseCompleted) } +func VeleroBackupDelete(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string) error { + args := []string{"--namespace", veleroNamespace, "delete", "backup", backupName, "--confirm"} + return VeleroCmdExec(ctx, veleroCLI, args) +} + +func VeleroRestoreDelete(ctx context.Context, veleroCLI string, veleroNamespace string, restoreName string) error { + args := []string{"--namespace", veleroNamespace, "delete", "restore", restoreName, "--confirm"} + return VeleroCmdExec(ctx, veleroCLI, args) +} + func VeleroScheduleDelete(ctx context.Context, veleroCLI string, veleroNamespace string, scheduleName string) error { args := []string{"--namespace", veleroNamespace, "delete", "schedule", scheduleName, "--confirm"} return VeleroCmdExec(ctx, veleroCLI, args) @@ -514,7 +527,7 @@ func VeleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNa // WaitForVSphereUploadCompletion waits for uploads started by the Velero Plug-in for vSphere to complete // TODO - remove after upload progress monitoring is implemented func WaitForVSphereUploadCompletion(ctx context.Context, timeout time.Duration, namespace string) error { - err := wait.PollImmediate(time.Minute, timeout, func() (bool, error) { + err := wait.PollImmediate(time.Second*5, timeout, func() (bool, error) { checkSnapshotCmd := exec.CommandContext(ctx, "kubectl", "get", "-n", namespace, "snapshots.backupdriver.cnsdp.vmware.com", "-o=jsonpath='{range .items[*]}{.spec.resourceHandle.name}{\"=\"}{.status.phase}{\"\\n\"}{end}'") fmt.Printf("checkSnapshotCmd cmd =%v\n", checkSnapshotCmd)