vmware-tanzu
/
velero
mirror of https://github.com/vmware-tanzu/velero.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Fix nightly running failure 

Signed-off-by: Ming <mqiu@vmware.com>
pull/4965/head
Ming 2022-06-04 05:33:48 +00:00
parent 9af031b84a
commit 56f72797db
9 changed files with 62 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
test/e2e/backups/deletion.go
View File

@ -25,7 +25,6 @@ import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/pkg/errors"
waitutil "k8s.io/apimachinery/pkg/util/wait"
. "github.com/vmware-tanzu/velero/test/e2e"
. "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
@ -147,7 +146,7 @@ func runBackupDeletionTests(client TestClient, veleroCfg VerleroConfig, backupNa
if useVolumeSnapshots {
snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 2, deletionTest, backupName, KibishiiPodNameList)
Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
err = WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider,
err = SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider,
VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslConfig,
backupName, snapshotCheckPoint)
if err != nil {
@ -159,7 +158,7 @@ func runBackupDeletionTests(client TestClient, veleroCfg VerleroConfig, backupNa
return err
}
if useVolumeSnapshots {
err = WaitUntilSnapshotsNotExistInCloud(VeleroCfg.CloudProvider,
err = SnapshotsShouldNotExistInCloud(VeleroCfg.CloudProvider,
VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, veleroCfg.BSLConfig,
backupName, snapshotCheckPoint)
if err != nil {
@ -172,17 +171,9 @@ func runBackupDeletionTests(client TestClient, veleroCfg VerleroConfig, backupNa
return err
}
if useVolumeSnapshots {
err = waitutil.PollImmediate(30*time.Second, 3*time.Minute,
func() (bool, error) {
err := SnapshotsShouldNotExistInCloud(VeleroCfg.CloudProvider,
VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
bslConfig, backupName, snapshotCheckPoint)
if err == nil {
return true, nil
}
return false, err
})
if err != nil {
if err := SnapshotsShouldNotExistInCloud(VeleroCfg.CloudProvider,
VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
bslConfig, backupName, snapshotCheckPoint); err != nil {
return errors.Wrap(err, "exceed waiting for snapshot created in cloud")
}
}

17
test/e2e/backups/ttl.go
View File

@ -52,8 +52,8 @@ func (b *TTL) Init() {
b.testNS = "backup-ttl-test-" + UUIDgen.String()
b.backupName = "backup-ttl-test-" + UUIDgen.String()
b.restoreName = "restore-ttl-test-" + UUIDgen.String()
b.ctx, _ = context.WithTimeout(context.Background(), time.Duration(time.Minute*30))
b.ttl = time.Duration(10 * time.Minute)
b.ctx, _ = context.WithTimeout(context.Background(), time.Duration(time.Hour))
b.ttl = time.Duration(20 * time.Minute)
}
@ -87,14 +87,13 @@ func TTLTest() {
It("Backups in object storage should be synced to a new Velero successfully", func() {
test.Init()
oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60)
By(fmt.Sprintf("Prepare workload as target to backup by creating namespace %s namespace", test.testNS), func() {
Expect(CreateNamespace(oneHourTimeout, client, test.testNS)).To(Succeed(),
Expect(CreateNamespace(test.ctx, client, test.testNS)).To(Succeed(),
fmt.Sprintf("Failed to create %s namespace", test.testNS))
})
By("Deploy sample workload of Kibishii", func() {
Expect(KibishiiPrepareBeforeBackup(oneHourTimeout, client, VeleroCfg.CloudProvider,
Expect(KibishiiPrepareBeforeBackup(test.ctx, client, VeleroCfg.CloudProvider,
test.testNS, VeleroCfg.RegistryCredentialFile, VeleroCfg.Features,
VeleroCfg.KibishiiDirectory, useVolumeSnapshots)).To(Succeed())
})
@ -108,7 +107,7 @@ func TTLTest() {
BackupCfg.TTL = test.ttl
By(fmt.Sprintf("Backup the workload in %s namespace", test.testNS), func() {
Expect(VeleroBackupNamespace(oneHourTimeout, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string {
Expect(VeleroBackupNamespace(test.ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string {
RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.backupName, "")
return "Fail to backup workload"
})
@ -119,14 +118,14 @@ func TTLTest() {
if VeleroCfg.CloudProvider == "vsphere" {
// TODO - remove after upload progress monitoring is implemented
By("Waiting for vSphere uploads to complete", func() {
Expect(WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour,
Expect(WaitForVSphereUploadCompletion(test.ctx, time.Hour,
test.testNS)).To(Succeed())
})
}
snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 2, test.testNS, test.backupName, KibishiiPodNameList)
Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
Expect(WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider,
Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider,
VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, VeleroCfg.BSLConfig,
test.backupName, snapshotCheckPoint)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
}
@ -183,7 +182,7 @@ func TTLTest() {
By("PersistentVolume snapshots should be deleted", func() {
if useVolumeSnapshots {
Expect(WaitUntilSnapshotsNotExistInCloud(VeleroCfg.CloudProvider,
Expect(SnapshotsShouldNotExistInCloud(VeleroCfg.CloudProvider,
VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, VeleroCfg.BSLConfig,
test.backupName, snapshotCheckPoint)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
}

2
test/e2e/basic/resources-check/namespaces.go
View File

@ -57,7 +57,7 @@ func (m *MultiNSBackup) Init() error {
}
} else {
m.NamespacesTotal = 2
m.TimeoutDuration = time.Minute * 5
m.TimeoutDuration = time.Minute * 10
m.TestMsg = &TestMSG{
Text: "When I create 2 namespaces should be successfully backed up and restored",
FailedMSG: "Failed to successfully backup and restore multiple namespaces",

8
test/e2e/bsl-mgmt/deletion.go
View File

@ -220,7 +220,7 @@ func BslDeletionTest(useVolumeSnapshots bool) {
By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation_1), func() {
snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 1, bslDeletionTestNs, backupName_1, []string{podName_1})
Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
Expect(WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider,
Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider,
VeleroCfg.CloudCredentialsFile, VeleroCfg.AdditionalBSLBucket,
VeleroCfg.BSLConfig, backupName_1, snapshotCheckPoint)).To(Succeed())
})
@ -236,7 +236,7 @@ func BslDeletionTest(useVolumeSnapshots bool) {
BSLConfig = VeleroCfg.BSLConfig
}
Expect(WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider,
Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider,
BSLCredentials, VeleroCfg.AdditionalBSLBucket,
BSLConfig, backupName_2, snapshotCheckPoint)).To(Succeed())
})
@ -318,7 +318,7 @@ func BslDeletionTest(useVolumeSnapshots bool) {
snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 1, bslDeletionTestNs, backupName_1, []string{podName_1})
Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
Expect(WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider,
Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider,
VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
VeleroCfg.BSLConfig, backupName_1, snapshotCheckPoint)).To(Succeed())
})
@ -333,7 +333,7 @@ func BslDeletionTest(useVolumeSnapshots bool) {
}
snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 1, bslDeletionTestNs, backupName_2, []string{podName_2})
Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
Expect(WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider,
Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider,
BSLCredentials, VeleroCfg.AdditionalBSLBucket,
BSLConfig, backupName_2, snapshotCheckPoint)).To(Succeed())
})

15
test/e2e/orderedresources/ordered_resources.go
View File

@ -69,6 +69,8 @@ func ScheduleOrderedResources() {
Expect(DeleteNamespace(test.Ctx, test.Client, test.Namespace, false)).To(Succeed(), fmt.Sprintf("Failed to delete the namespace %s", test.Namespace))
err = VeleroScheduleDelete(test.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.ScheduleName)
Expect(err).To(Succeed(), fmt.Sprintf("Failed to delete schedule with err %v", err))
err = test.DeleteBackups()
Expect(err).To(Succeed(), fmt.Sprintf("Failed to delete backups with err %v", err))
}()
By(fmt.Sprintf("Prepare workload as target to backup in base namespace %s", test.Namespace), func() {
@ -189,3 +191,16 @@ func (o *OrderedResources) CreateResources() error {
}
return nil
}
func (o *OrderedResources) DeleteBackups() error {
backupList := new(velerov1api.BackupList)
if err := o.Client.Kubebuilder.List(o.Ctx, backupList, &kbclient.ListOptions{Namespace: VeleroCfg.VeleroNamespace}); err != nil {
return fmt.Errorf("failed to list backup object in %s namespace with err %v", VeleroCfg.VeleroNamespace, err)
}
for _, backup := range backupList.Items {
if err := VeleroBackupDelete(o.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backup.Name); err != nil {
return err
}
}
return nil
}

2
test/e2e/upgrade/upgrade.go
View File

@ -190,7 +190,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, upgradeFromVelero Upgrade
snapshotCheckPoint, err := GetSnapshotCheckPoint(client, VeleroCfg, 2,
upgradeNamespace, backupName, KibishiiPodNameList)
Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint")
Expect(WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider,
Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider,
VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
VeleroCfg.BSLConfig, backupName, snapshotCheckPoint)).To(Succeed())
})

2
test/e2e/util/kibishii/kibishii_utils.go
View File

@ -96,7 +96,7 @@ func RunKibishiiTests(client TestClient, veleroCfg VerleroConfig, backupName, re
if err != nil {
return errors.Wrap(err, "Fail to get snapshot checkpoint")
}
err = WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider,
err = SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider,
VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, veleroCfg.BSLConfig,
backupName, snapshotCheckPoint)
if err != nil {

31
test/e2e/util/providers/common.go
View File

@ -24,8 +24,6 @@ import (
"github.com/pkg/errors"
waitutil "k8s.io/apimachinery/pkg/util/wait"
. "github.com/vmware-tanzu/velero/test/e2e"
velero "github.com/vmware-tanzu/velero/test/e2e/util/velero"
)
@ -111,35 +109,6 @@ func DeleteObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPr
return nil
}
func WaitUntilSnapshotsExistInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheckPoint SnapshotCheckPoint) error {
err := waitutil.PollImmediate(30*time.Second, 3*time.Minute,
func() (bool, error) {
err := SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider,
VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
bslConfig, backupName, snapshotCheckPoint)
if err == nil {
return true, nil
}
return false, err
})
return err
}
func WaitUntilSnapshotsNotExistInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheckPoint SnapshotCheckPoint) error {
err := waitutil.PollImmediate(30*time.Second, 3*time.Minute,
func() (bool, error) {
snapshotCheckPoint.ExpectCount = 0
err := SnapshotsShouldNotExistInCloud(VeleroCfg.CloudProvider,
VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
bslConfig, backupName, snapshotCheckPoint)
if err == nil {
return true, nil
}
return false, err
})
return err
}
func SnapshotsShouldNotExistInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheckPoint SnapshotCheckPoint) error {
fmt.Printf("|| VERIFICATION || - Snapshots should not exist in cloud, backup %s\n", backupName)
snapshotCheckPoint.ExpectCount = 0

41
test/e2e/util/velero/velero_utils.go
View File

@ -235,20 +235,23 @@ func checkRestorePhase(ctx context.Context, veleroCLI string, veleroNamespace st
func checkSchedulePhase(ctx context.Context, veleroCLI, veleroNamespace, scheduleName string) error {
checkCMD := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "schedule", "get", scheduleName, "-ojson")
jsonBuf, err := CMDExecWithOutput(checkCMD)
if err != nil {
return err
}
schedule := velerov1api.Schedule{}
err = json.Unmarshal(*jsonBuf, &schedule)
if err != nil {
return err
}
return wait.PollImmediate(time.Second*5, time.Minute*2, func() (bool, error) {
jsonBuf, err := CMDExecWithOutput(checkCMD)
if err != nil {
return false, err
}
schedule := velerov1api.Schedule{}
err = json.Unmarshal(*jsonBuf, &schedule)
if err != nil {
return false, err
}
if schedule.Status.Phase != velerov1api.SchedulePhaseEnabled {
return errors.Errorf("Unexpected restore phase got %s, expecting %s", schedule.Status.Phase, velerov1api.SchedulePhaseEnabled)
}
return nil
if schedule.Status.Phase != velerov1api.SchedulePhaseEnabled {
fmt.Printf("Unexpected schedule phase got %s, expecting %s, still waiting...", schedule.Status.Phase, velerov1api.SchedulePhaseEnabled)
return false, nil
}
return true, nil
})
}
func CheckScheduleWithResourceOrder(ctx context.Context, veleroCLI, veleroNamespace, scheduleName string, order map[string]string) error {
@ -370,6 +373,16 @@ func VeleroBackupExec(ctx context.Context, veleroCLI string, veleroNamespace str
return checkBackupPhase(ctx, veleroCLI, veleroNamespace, backupName, velerov1api.BackupPhaseCompleted)
}
func VeleroBackupDelete(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string) error {
args := []string{"--namespace", veleroNamespace, "delete", "backup", backupName, "--confirm"}
return VeleroCmdExec(ctx, veleroCLI, args)
}
func VeleroRestoreDelete(ctx context.Context, veleroCLI string, veleroNamespace string, restoreName string) error {
args := []string{"--namespace", veleroNamespace, "delete", "restore", restoreName, "--confirm"}
return VeleroCmdExec(ctx, veleroCLI, args)
}
func VeleroScheduleDelete(ctx context.Context, veleroCLI string, veleroNamespace string, scheduleName string) error {
args := []string{"--namespace", veleroNamespace, "delete", "schedule", scheduleName, "--confirm"}
return VeleroCmdExec(ctx, veleroCLI, args)
@ -514,7 +527,7 @@ func VeleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNa
// WaitForVSphereUploadCompletion waits for uploads started by the Velero Plug-in for vSphere to complete
// TODO - remove after upload progress monitoring is implemented
func WaitForVSphereUploadCompletion(ctx context.Context, timeout time.Duration, namespace string) error {
err := wait.PollImmediate(time.Minute, timeout, func() (bool, error) {
err := wait.PollImmediate(time.Second*5, timeout, func() (bool, error) {
checkSnapshotCmd := exec.CommandContext(ctx, "kubectl",
"get", "-n", namespace, "snapshots.backupdriver.cnsdp.vmware.com", "-o=jsonpath='{range .items[*]}{.spec.resourceHandle.name}{\"=\"}{.status.phase}{\"\\n\"}{end}'")
fmt.Printf("checkSnapshotCmd cmd =%v\n", checkSnapshotCmd)