Modify other cases to support VKS environment.
Signed-off-by: Xun Jiang <xun.jiang@broadcom.com>pull/8371/head
Xun Jiang
parent
b02fc1da96
commit
bcb60ed783
|
|
@ -99,8 +99,6 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupLoc
|
|||
providerName := veleroCfg.CloudProvider
|
||||
veleroNamespace := veleroCfg.VeleroNamespace
|
||||
registryCredentialFile := veleroCfg.RegistryCredentialFile
|
||||
bslPrefix := veleroCfg.BSLPrefix
|
||||
bslConfig := veleroCfg.BSLConfig
|
||||
veleroFeatures := veleroCfg.Features
|
||||
for _, ns := range workloadNamespaceList {
|
||||
if err := CreateNamespace(oneHourTimeout, client, ns); err != nil {
|
||||
|
|
@ -153,7 +151,7 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupLoc
|
|||
}
|
||||
}
|
||||
}
|
||||
err = ObjectsShouldBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix)
|
||||
err = ObjectsShouldBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig, backupName, BackupObjectsPrefix)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -165,9 +163,12 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupLoc
|
|||
for _, ns := range workloadNamespaceList {
|
||||
snapshotCheckPoint, err = GetSnapshotCheckPoint(client, veleroCfg, DefaultKibishiiWorkerCounts, ns, backupName, KibishiiPVCNameList)
|
||||
Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
|
||||
err = SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider,
|
||||
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslConfig,
|
||||
backupName, snapshotCheckPoint)
|
||||
err = CheckSnapshotsInProvider(
|
||||
veleroCfg,
|
||||
backupName,
|
||||
snapshotCheckPoint,
|
||||
false,
|
||||
)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "exceed waiting for snapshot created in cloud")
|
||||
}
|
||||
|
|
@ -179,9 +180,12 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupLoc
|
|||
Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
|
||||
|
||||
// Get all snapshots base on backup name, regardless of namespaces
|
||||
err = SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider,
|
||||
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslConfig,
|
||||
backupName, snapshotCheckPoint)
|
||||
err = CheckSnapshotsInProvider(
|
||||
veleroCfg,
|
||||
backupName,
|
||||
snapshotCheckPoint,
|
||||
false,
|
||||
)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "exceed waiting for snapshot created in cloud")
|
||||
}
|
||||
|
|
@ -207,26 +211,34 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupLoc
|
|||
return err
|
||||
}
|
||||
|
||||
// Verify snapshots are deleted after backup deletion.
|
||||
if useVolumeSnapshots {
|
||||
err = SnapshotsShouldNotExistInCloud(veleroCfg.CloudProvider,
|
||||
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLConfig,
|
||||
backupName, snapshotCheckPoint)
|
||||
snapshotCheckPoint.ExpectCount = 0
|
||||
err = CheckSnapshotsInProvider(
|
||||
veleroCfg,
|
||||
backupName,
|
||||
snapshotCheckPoint,
|
||||
false,
|
||||
)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "exceed waiting for snapshot created in cloud")
|
||||
return errors.Wrap(err, "fail to verify snapshots are deleted in provider.")
|
||||
}
|
||||
}
|
||||
|
||||
err = ObjectsShouldNotBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 5)
|
||||
// Verify backup metadata files are deleted in OSS after backup deletion.
|
||||
err = ObjectsShouldNotBeInBucket(
|
||||
veleroCfg.ObjectStoreProvider,
|
||||
veleroCfg.CloudCredentialsFile,
|
||||
veleroCfg.BSLBucket,
|
||||
veleroCfg.BSLPrefix,
|
||||
veleroCfg.BSLConfig,
|
||||
backupName,
|
||||
BackupObjectsPrefix,
|
||||
5,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if useVolumeSnapshots {
|
||||
if err := SnapshotsShouldNotExistInCloud(veleroCfg.CloudProvider,
|
||||
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,
|
||||
bslConfig, backupName, snapshotCheckPoint); err != nil {
|
||||
return errors.Wrap(err, "exceed waiting for snapshot created in cloud")
|
||||
}
|
||||
}
|
||||
|
||||
// Hit issue: https://docs.aws.amazon.com/AWSEC2/latest/APIReference/errors-overview.html#:~:text=SnapshotCreationPerVolumeRateExceeded
|
||||
// Sleep for more than 15 seconds to avoid this issue.
|
||||
|
|
@ -243,13 +255,28 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupLoc
|
|||
})
|
||||
})
|
||||
|
||||
err = DeleteObjectsInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix)
|
||||
if err != nil {
|
||||
if err := DeleteObjectsInBucket(
|
||||
veleroCfg.ObjectStoreProvider,
|
||||
veleroCfg.CloudCredentialsFile,
|
||||
veleroCfg.BSLBucket,
|
||||
veleroCfg.BSLPrefix,
|
||||
veleroCfg.BSLConfig,
|
||||
backupName,
|
||||
BackupObjectsPrefix,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = ObjectsShouldNotBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 1)
|
||||
if err != nil {
|
||||
if err := ObjectsShouldNotBeInBucket(
|
||||
veleroCfg.ObjectStoreProvider,
|
||||
veleroCfg.CloudCredentialsFile,
|
||||
veleroCfg.BSLBucket,
|
||||
veleroCfg.BSLPrefix,
|
||||
veleroCfg.BSLConfig,
|
||||
backupName,
|
||||
BackupObjectsPrefix,
|
||||
1,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -140,13 +140,11 @@ func TTLTest() {
|
|||
Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint")
|
||||
|
||||
Expect(
|
||||
SnapshotsShouldBeCreatedInCloud(
|
||||
veleroCfg.CloudProvider,
|
||||
veleroCfg.CloudCredentialsFile,
|
||||
veleroCfg.BSLBucket,
|
||||
veleroCfg.BSLConfig,
|
||||
CheckSnapshotsInProvider(
|
||||
veleroCfg,
|
||||
test.backupName,
|
||||
snapshotCheckPoint,
|
||||
false,
|
||||
),
|
||||
).NotTo(HaveOccurred(), "Fail to verify the created snapshots")
|
||||
}
|
||||
|
|
@ -202,9 +200,13 @@ func TTLTest() {
|
|||
|
||||
By("PersistentVolume snapshots should be deleted", func() {
|
||||
if useVolumeSnapshots {
|
||||
Expect(SnapshotsShouldNotExistInCloud(veleroCfg.CloudProvider,
|
||||
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLConfig,
|
||||
test.backupName, snapshotCheckPoint)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
|
||||
snapshotCheckPoint.ExpectCount = 0
|
||||
Expect(CheckSnapshotsInProvider(
|
||||
veleroCfg,
|
||||
test.backupName,
|
||||
snapshotCheckPoint,
|
||||
false,
|
||||
)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
|
||||
}
|
||||
})
|
||||
|
||||
|
|
|
|||
|
|
@ -36,8 +36,8 @@ import (
|
|||
|
||||
const (
|
||||
// Please make sure length of this namespace should be shorter,
|
||||
// otherwise ResticRepositories name verification will be wrong
|
||||
// when making combination of ResticRepositories name(max length is 63)
|
||||
// otherwise BackupRepositories name verification will be wrong
|
||||
// when making combination of BackupRepositories name(max length is 63)
|
||||
bslDeletionTestNs = "bsl-deletion"
|
||||
)
|
||||
|
||||
|
|
@ -134,12 +134,12 @@ func BslDeletionTest(useVolumeSnapshots bool) {
|
|||
)).To(Succeed())
|
||||
})
|
||||
|
||||
backupName_1 := "backup1-" + UUIDgen.String()
|
||||
backupName_2 := "backup2-" + UUIDgen.String()
|
||||
backupLocation_1 := "default"
|
||||
backupLocation_2 := additionalBsl
|
||||
podName_1 := "kibishii-deployment-0"
|
||||
podName_2 := "kibishii-deployment-1"
|
||||
backupName1 := "backup1-" + UUIDgen.String()
|
||||
backupName2 := "backup2-" + UUIDgen.String()
|
||||
backupLocation1 := "default"
|
||||
backupLocation2 := additionalBsl
|
||||
podName1 := "kibishii-deployment-0"
|
||||
podName2 := "kibishii-deployment-1"
|
||||
|
||||
label_1 := "for=1"
|
||||
// TODO remove when issue https://github.com/vmware-tanzu/velero/issues/4724 is fixed
|
||||
|
|
@ -157,17 +157,17 @@ func BslDeletionTest(useVolumeSnapshots bool) {
|
|||
|
||||
// Restic can not backup PV only, so pod need to be labeled also
|
||||
By("Label all 2 worker-pods of Kibishii", func() {
|
||||
Expect(AddLabelToPod(context.Background(), podName_1, bslDeletionTestNs, label_1)).To(Succeed())
|
||||
Expect(AddLabelToPod(context.Background(), podName1, bslDeletionTestNs, label_1)).To(Succeed())
|
||||
Expect(AddLabelToPod(context.Background(), "kibishii-deployment-1", bslDeletionTestNs, label_2)).To(Succeed())
|
||||
})
|
||||
|
||||
By("Get all 2 PVCs of Kibishii and label them separately ", func() {
|
||||
pvc, err := GetPvcByPVCName(context.Background(), bslDeletionTestNs, podName_1)
|
||||
pvc, err := GetPvcByPVCName(context.Background(), bslDeletionTestNs, podName1)
|
||||
Expect(err).To(Succeed())
|
||||
fmt.Println(pvc)
|
||||
Expect(pvc).To(HaveLen(1))
|
||||
pvc1 := pvc[0]
|
||||
pvc, err = GetPvcByPVCName(context.Background(), bslDeletionTestNs, podName_2)
|
||||
pvc, err = GetPvcByPVCName(context.Background(), bslDeletionTestNs, podName2)
|
||||
Expect(err).To(Succeed())
|
||||
fmt.Println(pvc)
|
||||
Expect(pvc).To(HaveLen(1))
|
||||
|
|
@ -177,13 +177,13 @@ func BslDeletionTest(useVolumeSnapshots bool) {
|
|||
})
|
||||
|
||||
var BackupCfg BackupConfig
|
||||
BackupCfg.BackupName = backupName_1
|
||||
BackupCfg.BackupName = backupName1
|
||||
BackupCfg.Namespace = bslDeletionTestNs
|
||||
BackupCfg.BackupLocation = backupLocation_1
|
||||
BackupCfg.BackupLocation = backupLocation1
|
||||
BackupCfg.UseVolumeSnapshots = useVolumeSnapshots
|
||||
BackupCfg.DefaultVolumesToFsBackup = !useVolumeSnapshots
|
||||
BackupCfg.Selector = label_1
|
||||
By(fmt.Sprintf("Backup one of PV of sample workload by label-1 - Kibishii by the first BSL %s", backupLocation_1), func() {
|
||||
By(fmt.Sprintf("Backup one of PV of sample workload by label-1 - Kibishii by the first BSL %s", backupLocation1), func() {
|
||||
// TODO currently, the upgrade case covers the upgrade path from 1.6 to main and the velero v1.6 doesn't support "debug" command
|
||||
// TODO move to "runDebug" after we bump up to 1.7 in the upgrade case
|
||||
Expect(VeleroBackupNamespace(oneHourTimeout, veleroCfg.VeleroCLI,
|
||||
|
|
@ -193,10 +193,10 @@ func BslDeletionTest(useVolumeSnapshots bool) {
|
|||
})
|
||||
})
|
||||
|
||||
BackupCfg.BackupName = backupName_2
|
||||
BackupCfg.BackupLocation = backupLocation_2
|
||||
BackupCfg.BackupName = backupName2
|
||||
BackupCfg.BackupLocation = backupLocation2
|
||||
BackupCfg.Selector = label_2
|
||||
By(fmt.Sprintf("Back up the other one PV of sample workload with label-2 into the additional BSL %s", backupLocation_2), func() {
|
||||
By(fmt.Sprintf("Back up the other one PV of sample workload with label-2 into the additional BSL %s", backupLocation2), func() {
|
||||
Expect(VeleroBackupNamespace(oneHourTimeout, veleroCfg.VeleroCLI,
|
||||
veleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string {
|
||||
RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, BackupCfg.BackupName, "")
|
||||
|
|
@ -211,82 +211,94 @@ func BslDeletionTest(useVolumeSnapshots bool) {
|
|||
Expect(WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour,
|
||||
bslDeletionTestNs, 2)).To(Succeed())
|
||||
})
|
||||
By(fmt.Sprintf("Snapshot CR in backup %s should be created", backupName_1), func() {
|
||||
By(fmt.Sprintf("Snapshot CR in backup %s should be created", backupName1), func() {
|
||||
Expect(SnapshotCRsCountShouldBe(context.Background(), bslDeletionTestNs,
|
||||
backupName_1, 1)).To(Succeed())
|
||||
backupName1, 1)).To(Succeed())
|
||||
})
|
||||
By(fmt.Sprintf("Snapshot CR in backup %s should be created", backupName_2), func() {
|
||||
By(fmt.Sprintf("Snapshot CR in backup %s should be created", backupName2), func() {
|
||||
Expect(SnapshotCRsCountShouldBe(context.Background(), bslDeletionTestNs,
|
||||
backupName_2, 1)).To(Succeed())
|
||||
backupName2, 1)).To(Succeed())
|
||||
})
|
||||
}
|
||||
if veleroCfg.CloudProvider != VanillaZFS {
|
||||
var snapshotCheckPoint SnapshotCheckPoint
|
||||
snapshotCheckPoint.NamespaceBackedUp = bslDeletionTestNs
|
||||
By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation_1), func() {
|
||||
snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName_1, []string{podName_1})
|
||||
By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation1), func() {
|
||||
snapshotCheckPoint, err = GetSnapshotCheckPoint(
|
||||
*veleroCfg.ClientToInstallVelero,
|
||||
veleroCfg,
|
||||
1,
|
||||
bslDeletionTestNs,
|
||||
backupName1,
|
||||
[]string{podName1},
|
||||
)
|
||||
Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
|
||||
Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider,
|
||||
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,
|
||||
veleroCfg.BSLConfig, backupName_1, snapshotCheckPoint)).To(Succeed())
|
||||
Expect(CheckSnapshotsInProvider(
|
||||
veleroCfg,
|
||||
backupName1,
|
||||
snapshotCheckPoint,
|
||||
false,
|
||||
)).To(Succeed())
|
||||
})
|
||||
By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation_2), func() {
|
||||
snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName_2, []string{podName_2})
|
||||
By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation2), func() {
|
||||
snapshotCheckPoint, err = GetSnapshotCheckPoint(
|
||||
*veleroCfg.ClientToInstallVelero,
|
||||
veleroCfg,
|
||||
1,
|
||||
bslDeletionTestNs,
|
||||
backupName2,
|
||||
[]string{podName2},
|
||||
)
|
||||
Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint")
|
||||
var BSLCredentials, BSLConfig string
|
||||
if veleroCfg.CloudProvider == Vsphere {
|
||||
BSLCredentials = veleroCfg.AdditionalBSLCredentials
|
||||
BSLConfig = veleroCfg.AdditionalBSLConfig
|
||||
} else { // Snapshotting with non-vSphere provider has nothing to do with BSL
|
||||
BSLCredentials = veleroCfg.CloudCredentialsFile
|
||||
BSLConfig = veleroCfg.BSLConfig
|
||||
}
|
||||
|
||||
Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider,
|
||||
BSLCredentials, veleroCfg.AdditionalBSLBucket,
|
||||
BSLConfig, backupName_2, snapshotCheckPoint)).To(Succeed())
|
||||
Expect(CheckSnapshotsInProvider(
|
||||
veleroCfg,
|
||||
backupName2,
|
||||
snapshotCheckPoint,
|
||||
true,
|
||||
)).To(Succeed())
|
||||
})
|
||||
}
|
||||
} else { // For Restics
|
||||
By(fmt.Sprintf("Resticrepositories for BSL %s should be created in Velero namespace", backupLocation_1), func() {
|
||||
} else {
|
||||
By(fmt.Sprintf("BackupRepositories for BSL %s should be created in Velero namespace", backupLocation1), func() {
|
||||
Expect(BackupRepositoriesCountShouldBe(context.Background(),
|
||||
veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation_1, 1)).To(Succeed())
|
||||
veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation1, 1)).To(Succeed())
|
||||
})
|
||||
By(fmt.Sprintf("Resticrepositories for BSL %s should be created in Velero namespace", backupLocation_2), func() {
|
||||
By(fmt.Sprintf("BackupRepositories for BSL %s should be created in Velero namespace", backupLocation2), func() {
|
||||
Expect(BackupRepositoriesCountShouldBe(context.Background(),
|
||||
veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation_2, 1)).To(Succeed())
|
||||
veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation2, 1)).To(Succeed())
|
||||
})
|
||||
}
|
||||
|
||||
By(fmt.Sprintf("Backup 1 %s should be created.", backupName_1), func() {
|
||||
By(fmt.Sprintf("Backup 1 %s should be created.", backupName1), func() {
|
||||
Expect(WaitForBackupToBeCreated(context.Background(),
|
||||
backupName_1, 10*time.Minute, &veleroCfg)).To(Succeed())
|
||||
backupName1, 10*time.Minute, &veleroCfg)).To(Succeed())
|
||||
})
|
||||
|
||||
By(fmt.Sprintf("Backup 2 %s should be created.", backupName_2), func() {
|
||||
By(fmt.Sprintf("Backup 2 %s should be created.", backupName2), func() {
|
||||
Expect(WaitForBackupToBeCreated(context.Background(),
|
||||
backupName_2, 10*time.Minute, &veleroCfg)).To(Succeed())
|
||||
backupName2, 10*time.Minute, &veleroCfg)).To(Succeed())
|
||||
})
|
||||
|
||||
backupsInBSL1, err := GetBackupsFromBsl(context.Background(), veleroCfg.VeleroCLI, backupLocation_1)
|
||||
backupsInBSL1, err := GetBackupsFromBsl(context.Background(), veleroCfg.VeleroCLI, backupLocation1)
|
||||
Expect(err).To(Succeed())
|
||||
backupsInBSL2, err := GetBackupsFromBsl(context.Background(), veleroCfg.VeleroCLI, backupLocation_2)
|
||||
backupsInBSL2, err := GetBackupsFromBsl(context.Background(), veleroCfg.VeleroCLI, backupLocation2)
|
||||
Expect(err).To(Succeed())
|
||||
backupsInBsl1AndBsl2 := append(backupsInBSL1, backupsInBSL2...)
|
||||
|
||||
By(fmt.Sprintf("Get all backups from 2 BSLs %s before deleting one of them", backupLocation_1), func() {
|
||||
By(fmt.Sprintf("Get all backups from 2 BSLs %s before deleting one of them", backupLocation1), func() {
|
||||
backupsBeforeDel, err := GetAllBackups(context.Background(), veleroCfg.VeleroCLI)
|
||||
Expect(err).To(Succeed())
|
||||
Expect(cmp.Diff(backupsInBsl1AndBsl2, backupsBeforeDel, cmpopts.SortSlices(less))).Should(BeEmpty())
|
||||
|
||||
By(fmt.Sprintf("Backup1 %s should exist in cloud object store before bsl deletion", backupName_1), func() {
|
||||
By(fmt.Sprintf("Backup1 %s should exist in cloud object store before bsl deletion", backupName1), func() {
|
||||
Expect(ObjectsShouldBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile,
|
||||
veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig,
|
||||
backupName_1, BackupObjectsPrefix)).To(Succeed())
|
||||
backupName1, BackupObjectsPrefix)).To(Succeed())
|
||||
})
|
||||
|
||||
By(fmt.Sprintf("Delete one of backup locations - %s", backupLocation_1), func() {
|
||||
Expect(DeleteBslResource(context.Background(), veleroCfg.VeleroCLI, backupLocation_1)).To(Succeed())
|
||||
By(fmt.Sprintf("Delete one of backup locations - %s", backupLocation1), func() {
|
||||
Expect(DeleteBslResource(context.Background(), veleroCfg.VeleroCLI, backupLocation1)).To(Succeed())
|
||||
Expect(WaitForBackupsToBeDeleted(context.Background(), backupsInBSL1, 10*time.Minute, &veleroCfg)).To(Succeed())
|
||||
})
|
||||
|
||||
|
|
@ -298,10 +310,10 @@ func BslDeletionTest(useVolumeSnapshots bool) {
|
|||
})
|
||||
})
|
||||
|
||||
By(fmt.Sprintf("Backup1 %s should still exist in cloud object store after bsl deletion", backupName_1), func() {
|
||||
By(fmt.Sprintf("Backup1 %s should still exist in cloud object store after bsl deletion", backupName1), func() {
|
||||
Expect(ObjectsShouldBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile,
|
||||
veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig,
|
||||
backupName_1, BackupObjectsPrefix)).To(Succeed())
|
||||
backupName1, BackupObjectsPrefix)).To(Succeed())
|
||||
})
|
||||
|
||||
// TODO: Choose additional BSL to be deleted as an new test case
|
||||
|
|
@ -313,45 +325,51 @@ func BslDeletionTest(useVolumeSnapshots bool) {
|
|||
|
||||
if useVolumeSnapshots {
|
||||
if veleroCfg.CloudProvider == Vsphere {
|
||||
By(fmt.Sprintf("Snapshot in backup %s should still exist, because snapshot CR will be deleted 24 hours later if the status is a success", backupName_2), func() {
|
||||
By(fmt.Sprintf("Snapshot in backup %s should still exist, because snapshot CR will be deleted 24 hours later if the status is a success", backupName2), func() {
|
||||
Expect(SnapshotCRsCountShouldBe(context.Background(), bslDeletionTestNs,
|
||||
backupName_1, 1)).To(Succeed())
|
||||
backupName1, 1)).To(Succeed())
|
||||
Expect(SnapshotCRsCountShouldBe(context.Background(), bslDeletionTestNs,
|
||||
backupName_2, 1)).To(Succeed())
|
||||
backupName2, 1)).To(Succeed())
|
||||
})
|
||||
}
|
||||
var snapshotCheckPoint SnapshotCheckPoint
|
||||
snapshotCheckPoint.NamespaceBackedUp = bslDeletionTestNs
|
||||
By(fmt.Sprintf("Snapshot should not be deleted in cloud object store after deleting bsl %s", backupLocation_1), func() {
|
||||
snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName_1, []string{podName_1})
|
||||
By(fmt.Sprintf("Snapshot should not be deleted in cloud object store after deleting bsl %s", backupLocation1), func() {
|
||||
snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName1, []string{podName1})
|
||||
Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
|
||||
Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider,
|
||||
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,
|
||||
veleroCfg.BSLConfig, backupName_1, snapshotCheckPoint)).To(Succeed())
|
||||
Expect(CheckSnapshotsInProvider(
|
||||
veleroCfg,
|
||||
backupName1,
|
||||
snapshotCheckPoint,
|
||||
false,
|
||||
)).To(Succeed())
|
||||
})
|
||||
By(fmt.Sprintf("Snapshot should not be deleted in cloud object store after deleting bsl %s", backupLocation_2), func() {
|
||||
var BSLCredentials, BSLConfig string
|
||||
if veleroCfg.CloudProvider == Vsphere {
|
||||
BSLCredentials = veleroCfg.AdditionalBSLCredentials
|
||||
BSLConfig = veleroCfg.AdditionalBSLConfig
|
||||
} else {
|
||||
BSLCredentials = veleroCfg.CloudCredentialsFile
|
||||
BSLConfig = veleroCfg.BSLConfig
|
||||
}
|
||||
snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName_2, []string{podName_2})
|
||||
By(fmt.Sprintf("Snapshot should not be deleted in cloud object store after deleting bsl %s", backupLocation2), func() {
|
||||
snapshotCheckPoint, err = GetSnapshotCheckPoint(
|
||||
*veleroCfg.ClientToInstallVelero,
|
||||
veleroCfg,
|
||||
1,
|
||||
bslDeletionTestNs,
|
||||
backupName2,
|
||||
[]string{podName2},
|
||||
)
|
||||
Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
|
||||
Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider,
|
||||
BSLCredentials, veleroCfg.AdditionalBSLBucket,
|
||||
BSLConfig, backupName_2, snapshotCheckPoint)).To(Succeed())
|
||||
|
||||
Expect(CheckSnapshotsInProvider(
|
||||
veleroCfg,
|
||||
backupName2,
|
||||
snapshotCheckPoint,
|
||||
true,
|
||||
)).To(Succeed())
|
||||
})
|
||||
} else { // For Restic
|
||||
By(fmt.Sprintf("Resticrepositories for BSL %s should be deleted in Velero namespace", backupLocation_1), func() {
|
||||
} else {
|
||||
By(fmt.Sprintf("BackupRepositories for BSL %s should be deleted in Velero namespace", backupLocation1), func() {
|
||||
Expect(BackupRepositoriesCountShouldBe(context.Background(),
|
||||
veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation_1, 0)).To(Succeed())
|
||||
veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation1, 0)).To(Succeed())
|
||||
})
|
||||
By(fmt.Sprintf("Resticrepositories for BSL %s should still exist in Velero namespace", backupLocation_2), func() {
|
||||
By(fmt.Sprintf("BackupRepositories for BSL %s should still exist in Velero namespace", backupLocation2), func() {
|
||||
Expect(BackupRepositoriesCountShouldBe(context.Background(),
|
||||
veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation_2, 1)).To(Succeed())
|
||||
veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation2, 1)).To(Succeed())
|
||||
})
|
||||
}
|
||||
fmt.Printf("|| EXPECTED || - Backup deletion test completed successfully\n")
|
||||
|
|
|
|||
|
|
@ -288,9 +288,12 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
|
|||
snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.DefaultClient, veleroCfg, kibishiiWorkerCount,
|
||||
migrationNamespace, backupName, GetKibishiiPVCNameList(kibishiiWorkerCount))
|
||||
Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint")
|
||||
Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider,
|
||||
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,
|
||||
veleroCfg.BSLConfig, backupName, snapshotCheckPoint)).To(Succeed())
|
||||
Expect(CheckSnapshotsInProvider(
|
||||
veleroCfg,
|
||||
backupName,
|
||||
snapshotCheckPoint,
|
||||
false,
|
||||
)).To(Succeed())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -202,9 +202,12 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
|
|||
snapshotCheckPoint, err := GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 2,
|
||||
upgradeNamespace, backupName, KibishiiPVCNameList)
|
||||
Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint")
|
||||
Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider,
|
||||
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,
|
||||
veleroCfg.BSLConfig, backupName, snapshotCheckPoint)).To(Succeed())
|
||||
Expect(CheckSnapshotsInProvider(
|
||||
veleroCfg,
|
||||
backupName,
|
||||
snapshotCheckPoint,
|
||||
false,
|
||||
)).To(Succeed())
|
||||
})
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -131,9 +131,12 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc
|
|||
if err != nil {
|
||||
return errors.Wrap(err, "Fail to get snapshot checkpoint")
|
||||
}
|
||||
err = SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider,
|
||||
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLConfig,
|
||||
backupName, snapshotCheckPoint)
|
||||
err = CheckSnapshotsInProvider(
|
||||
veleroCfg,
|
||||
backupName,
|
||||
snapshotCheckPoint,
|
||||
false,
|
||||
)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "exceed waiting for snapshot created in cloud")
|
||||
}
|
||||
|
|
@ -166,9 +169,12 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc
|
|||
return errors.Wrap(err, "failed to get snapshot checkPoint")
|
||||
}
|
||||
} else {
|
||||
err = SnapshotsShouldNotExistInCloud(veleroCfg.CloudProvider,
|
||||
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLConfig,
|
||||
backupName, SnapshotCheckPoint{})
|
||||
err = CheckSnapshotsInProvider(
|
||||
veleroCfg,
|
||||
backupName,
|
||||
SnapshotCheckPoint{},
|
||||
false,
|
||||
)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "exceed waiting for snapshot created in cloud")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,14 +28,14 @@ import (
|
|||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/vmware-tanzu/velero/internal/volume"
|
||||
. "github.com/vmware-tanzu/velero/test"
|
||||
velerotest "github.com/vmware-tanzu/velero/test"
|
||||
velero "github.com/vmware-tanzu/velero/test/util/velero"
|
||||
)
|
||||
|
||||
type ObjectsInStorage interface {
|
||||
IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) (bool, error)
|
||||
DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) error
|
||||
IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupName string, snapshotCheck SnapshotCheckPoint) error
|
||||
IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupName string, snapshotCheck velerotest.SnapshotCheckPoint) error
|
||||
GetObject(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, objectKey string) (io.ReadCloser, error)
|
||||
}
|
||||
|
||||
|
|
@ -75,13 +75,13 @@ func ObjectsShouldNotBeInBucket(objectStoreProvider, cloudCredentialsFile, bslBu
|
|||
func getProvider(cloudProvider string) (ObjectsInStorage, error) {
|
||||
var s ObjectsInStorage
|
||||
switch cloudProvider {
|
||||
case AWS, Vsphere:
|
||||
case velerotest.AWS, velerotest.Vsphere:
|
||||
aws := AWSStorage("")
|
||||
s = &aws
|
||||
case GCP:
|
||||
case velerotest.GCP:
|
||||
gcs := GCSStorage("")
|
||||
s = &gcs
|
||||
case Azure:
|
||||
case velerotest.Azure:
|
||||
az := AzureStorage("")
|
||||
s = &az
|
||||
default:
|
||||
|
|
@ -102,7 +102,7 @@ func IsObjectsInBucket(objectStoreProvider, cloudCredentialsFile, bslBucket, bsl
|
|||
bslPrefix = getFullPrefix(bslPrefix, subPrefix)
|
||||
s, err := getProvider(objectStoreProvider)
|
||||
if err != nil {
|
||||
return false, errors.Wrapf(err, fmt.Sprintf("Object store provider %s is not valid", objectStoreProvider))
|
||||
return false, errors.Wrapf(err, "Object store provider %s is not valid", objectStoreProvider)
|
||||
}
|
||||
return s.IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName)
|
||||
}
|
||||
|
|
@ -116,66 +116,77 @@ func DeleteObjectsInBucket(objectStoreProvider, cloudCredentialsFile, bslBucket,
|
|||
}
|
||||
s, err := getProvider(objectStoreProvider)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, fmt.Sprintf("Object store provider %s is not valid", objectStoreProvider))
|
||||
return errors.Wrapf(err, "Object store provider %s is not valid", objectStoreProvider)
|
||||
}
|
||||
err = s.DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, fmt.Sprintf("Fail to delete %s", bslPrefix))
|
||||
return errors.Wrapf(err, "Fail to delete %s", bslPrefix)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func SnapshotsShouldNotExistInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheckPoint SnapshotCheckPoint) error {
|
||||
fmt.Printf("|| VERIFICATION || - Snapshots should not exist in cloud, backup %s\n", backupName)
|
||||
func CheckSnapshotsInProvider(
|
||||
veleroCfg velerotest.VeleroConfig,
|
||||
backupName string,
|
||||
snapshotCheckPoint velerotest.SnapshotCheckPoint,
|
||||
checkStandbyCluster bool,
|
||||
) error {
|
||||
fmt.Printf("|| VERIFICATION || - there should %d snapshots in provider, backup %s\n",
|
||||
snapshotCheckPoint.ExpectCount, backupName)
|
||||
|
||||
if cloudProvider == VanillaZFS {
|
||||
fmt.Printf("Skip snapshot check for cloud provider %s", cloudProvider)
|
||||
if veleroCfg.CloudProvider == velerotest.VanillaZFS {
|
||||
fmt.Printf("Skip snapshot check for cloud provider %s", veleroCfg.CloudProvider)
|
||||
return nil
|
||||
}
|
||||
if cloudCredentialsFile == "" {
|
||||
return errors.New(fmt.Sprintf("|| ERROR || - Please provide credential file of cloud %s \n", cloudProvider))
|
||||
}
|
||||
|
||||
snapshotCheckPoint.ExpectCount = 0
|
||||
err := IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName, snapshotCheckPoint)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, fmt.Sprintf("|| UNEXPECTED ||Snapshots %s exist in cloud after backup as expected", backupName))
|
||||
}
|
||||
fmt.Printf("|| EXPECTED || - Snapshots do not exist in cloud, backup %s\n", backupName)
|
||||
return nil
|
||||
}
|
||||
|
||||
func SnapshotsShouldBeCreatedInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheckPoint SnapshotCheckPoint) error {
|
||||
fmt.Printf("|| VERIFICATION || - Snapshots should exist in cloud, backup %s\n", backupName)
|
||||
|
||||
if cloudProvider == VanillaZFS {
|
||||
fmt.Printf("Skip snapshot check for cloud provider %s", cloudProvider)
|
||||
if veleroCfg.CloudProvider == velerotest.Vsphere && !veleroCfg.HasVspherePlugin {
|
||||
fmt.Printf("Skip snapshot check for vSphere environment that doesn't have Velero vSphere plugin.")
|
||||
return nil
|
||||
}
|
||||
if cloudCredentialsFile == "" {
|
||||
return errors.New(fmt.Sprintf("|| ERROR || - Please provide credential file of cloud %s \n", cloudProvider))
|
||||
if veleroCfg.CloudCredentialsFile == "" {
|
||||
return errors.New(fmt.Sprintf("|| ERROR || - Please provide credential file of cloud %s \n", veleroCfg.CloudProvider))
|
||||
}
|
||||
|
||||
err := IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName, snapshotCheckPoint)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, fmt.Sprintf("|| UNEXPECTED || - Snapshots %s do not exist in cloud after backup as expected", backupName))
|
||||
cloudCredentialsFile := veleroCfg.CloudCredentialsFile
|
||||
bslBucket := veleroCfg.BSLBucket
|
||||
bslConfig := veleroCfg.BSLConfig
|
||||
if checkStandbyCluster {
|
||||
bslBucket = veleroCfg.AdditionalBSLBucket
|
||||
|
||||
// Only vSphere environment's snapshot on standby cluster is related to BSL.
|
||||
if veleroCfg.CloudProvider == velerotest.Vsphere {
|
||||
cloudCredentialsFile = veleroCfg.AdditionalBSLCredentials
|
||||
bslConfig = veleroCfg.AdditionalBSLConfig
|
||||
}
|
||||
}
|
||||
fmt.Printf("|| EXPECTED || - Snapshots of backup %s exist in cloud %s\n", backupName, cloudProvider)
|
||||
|
||||
err := isSnapshotExisted(
|
||||
veleroCfg.CloudProvider,
|
||||
cloudCredentialsFile,
|
||||
bslBucket,
|
||||
bslConfig,
|
||||
backupName,
|
||||
snapshotCheckPoint,
|
||||
)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "|| UNEXPECTED || - Snapshots count is not as expected after backup %s", backupName)
|
||||
}
|
||||
|
||||
fmt.Printf("|| EXPECTED || - Snapshots of backup %s exist in provider %s\n", backupName, veleroCfg.CloudProvider)
|
||||
return nil
|
||||
}
|
||||
|
||||
func IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheck SnapshotCheckPoint) error {
|
||||
func isSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheck velerotest.SnapshotCheckPoint) error {
|
||||
s, err := getProvider(cloudProvider)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, fmt.Sprintf("Cloud provider %s is not valid", cloudProvider))
|
||||
return errors.Wrapf(err, "Cloud provider %s is not valid", cloudProvider)
|
||||
}
|
||||
if cloudProvider == Vsphere {
|
||||
if cloudProvider == velerotest.Vsphere {
|
||||
var retSnapshotIDs []string
|
||||
ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*2)
|
||||
defer ctxCancel()
|
||||
retSnapshotIDs, err = velero.GetVsphereSnapshotIDs(ctx, time.Hour, snapshotCheck.NamespaceBackedUp, snapshotCheck.PodName)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, fmt.Sprintf("Fail to get snapshot CRs of backup%s", backupName))
|
||||
return errors.Wrapf(err, "Fail to get snapshot CRs of backup%s", backupName)
|
||||
}
|
||||
|
||||
bslPrefix := "plugins"
|
||||
|
|
@ -201,7 +212,7 @@ func IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig
|
|||
} else {
|
||||
err = s.IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupName, snapshotCheck)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, fmt.Sprintf("Fail to get snapshot of backup %s", backupName))
|
||||
return errors.Wrapf(err, "Fail to get snapshot of backup %s", backupName)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
|
|
@ -221,7 +232,7 @@ func GetVolumeInfoMetadataContent(
|
|||
fmt.Printf("|| VERIFICATION || - Get backup %s volumeinfo file in storage %s\n", backupName, bslPrefix)
|
||||
s, err := getProvider(objectStoreProvider)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, fmt.Sprintf("Cloud provider %s is not valid", objectStoreProvider))
|
||||
return nil, errors.Wrapf(err, "Cloud provider %s is not valid", objectStoreProvider)
|
||||
}
|
||||
|
||||
return s.GetObject(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, volumeFileName)
|
||||
|
|
|
|||
Loading…
Reference in New Issue