vmware-tanzu
/
velero
mirror of https://github.com/vmware-tanzu/velero.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add snapshot checkpoint for CSI E2E test and fix CSI snapshot name issue 

1. Add checkpoint in snapshot E2E test to verify snapshot CR should be created and snapshot should be created in cloud side after backup completion;
2. Fix snapshot name issue that CSI snapshot name in cloud side is not the same with other non-CSI cloud snapshots;

Signed-off-by: danfengl <danfengl@vmware.com>
pull/4836/head
danfengl 2022-04-10 10:20:16 +00:00
parent 001229a8b3
commit 33bf59a910
17 changed files with 474 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
test/e2e/backup/backup.go
View File

@ -39,13 +39,17 @@ func BackupRestoreWithRestic() {
}
func BackupRestoreTest(useVolumeSnapshots bool) {
kibishiiNamespace := "kibishii-workload"
var (
backupName, restoreName string
client TestClient
err error
)
kibishiiNamespace := "kibishii-workload"
client, err := NewTestClient()
Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests")
By("Create test client instance", func() {
client, err = NewTestClient()
Expect(err).NotTo(HaveOccurred(), "Failed to instantiate cluster client for backup tests")
})
BeforeEach(func() {
if useVolumeSnapshots && VeleroCfg.CloudProvider == "kind" {
Skip("Volume snapshots not supported on kind")

58
test/e2e/backups/deletion.go
View File

@ -25,6 +25,7 @@ import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/pkg/errors"
waitutil "k8s.io/apimachinery/pkg/util/wait"
. "github.com/vmware-tanzu/velero/test/e2e"
. "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
@ -47,11 +48,14 @@ func BackupDeletionWithRestic() {
func backup_deletion_test(useVolumeSnapshots bool) {
var (
backupName string
client TestClient
err error
)
client, err := NewTestClient()
Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup deletion tests")
By("Create test client instance", func() {
client, err = NewTestClient()
Expect(err).NotTo(HaveOccurred(), "Failed to instantiate cluster client for backup tests")
})
BeforeEach(func() {
if useVolumeSnapshots && VeleroCfg.CloudProvider == "kind" {
Skip("Volume snapshots not supported on kind")
@ -98,11 +102,13 @@ func runBackupDeletionTests(client TestClient, veleroCfg VerleroConfig, backupNa
if err := CreateNamespace(oneHourTimeout, client, deletionTest); err != nil {
return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", deletionTest)
}
defer func() {
if err := DeleteNamespace(context.Background(), client, deletionTest, true); err != nil {
fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", deletionTest))
}
}()
if !VeleroCfg.Debug {
defer func() {
if err := DeleteNamespace(context.Background(), client, deletionTest, true); err != nil {
fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", deletionTest))
}
}()
}
if err := KibishiiPrepareBeforeBackup(oneHourTimeout, client, providerName, deletionTest,
registryCredentialFile, veleroFeatures, kibishiiDirectory, useVolumeSnapshots); err != nil {
@ -131,28 +137,50 @@ func runBackupDeletionTests(client TestClient, veleroCfg VerleroConfig, backupNa
if err != nil {
return err
}
var snapshotCheckPoint SnapshotCheckPoint
if useVolumeSnapshots {
var snapshotCheckPoint SnapshotCheckPoint
snapshotCheckPoint.ExpectCount = 2
snapshotCheckPoint.NamespaceBackedUp = deletionTest
err = SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslConfig, backupName, snapshotCheckPoint)
snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 2, deletionTest, backupName, KibishiiPodNameList)
Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
if err != nil {
return err
return errors.Wrap(err, "exceed waiting for snapshot created in cloud")
}
err = WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider,
VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslConfig,
backupName, snapshotCheckPoint)
if err != nil {
return errors.Wrap(err, "exceed waiting for snapshot created in cloud")
}
}
err = DeleteBackupResource(context.Background(), veleroCLI, backupName)
if err != nil {
return err
}
if useVolumeSnapshots {
err = WaitUntilSnapshotsNotExistInCloud(VeleroCfg.CloudProvider,
VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, veleroCfg.BSLConfig,
backupName, snapshotCheckPoint)
if err != nil {
return errors.Wrap(err, "exceed waiting for snapshot created in cloud")
}
}
err = ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 5)
if err != nil {
return err
}
if useVolumeSnapshots {
err = SnapshotsShouldNotExistInCloud(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslConfig, backupName)
err = waitutil.PollImmediate(30*time.Second, 3*time.Minute,
func() (bool, error) {
err := SnapshotsShouldNotExistInCloud(VeleroCfg.CloudProvider,
VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
bslConfig, backupName, snapshotCheckPoint)
if err == nil {
return true, nil
}
return false, err
})
if err != nil {
return err
return errors.Wrap(err, "exceed waiting for snapshot created in cloud")
}
}

14
test/e2e/backups/sync_backups.go
View File

@ -52,10 +52,16 @@ func (b *SyncBackups) Init() {
func BackupsSyncTest() {
test := new(SyncBackups)
client, err := NewTestClient()
if err != nil {
println(err.Error())
}
var (
client TestClient
err error
)
By("Create test client instance", func() {
client, err = NewTestClient()
Expect(err).NotTo(HaveOccurred(), "Failed to instantiate cluster client for backup tests")
})
Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests")
BeforeEach(func() {

7
test/e2e/basic/enable_api_group_versions.go
View File

@ -45,10 +45,13 @@ func APIGropuVersionsTest() {
resource, group string
err error
ctx = context.Background()
client TestClient
)
client, err := NewTestClient()
Expect(err).To(Succeed(), "Failed to instantiate cluster client for group version tests")
By("Create test client instance", func() {
client, err = NewTestClient()
Expect(err).NotTo(HaveOccurred(), "Failed to instantiate cluster client for backup tests")
})
BeforeEach(func() {
resource = "rockbands"

55
test/e2e/bsl-mgmt/deletion.go
View File

@ -19,6 +19,7 @@ import (
"context"
"flag"
"fmt"
"strings"
"time"
"github.com/google/go-cmp/cmp"
@ -28,6 +29,7 @@ import (
. "github.com/onsi/gomega"
. "github.com/vmware-tanzu/velero/test/e2e"
util "github.com/vmware-tanzu/velero/test/e2e/util/csi"
. "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
. "github.com/vmware-tanzu/velero/test/e2e/util/kibishii"
@ -52,8 +54,16 @@ func BslDeletionWithRestic() {
BslDeletionTest(false)
}
func BslDeletionTest(useVolumeSnapshots bool) {
client, err := NewTestClient()
Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup deletion tests")
var (
client TestClient
err error
)
By("Create test client instance", func() {
client, err = NewTestClient()
Expect(err).NotTo(HaveOccurred(), "Failed to instantiate cluster client for backup tests")
})
less := func(a, b string) bool { return a < b }
BeforeEach(func() {
if useVolumeSnapshots && VeleroCfg.CloudProvider == "kind" {
@ -198,19 +208,28 @@ func BslDeletionTest(useVolumeSnapshots bool) {
Expect(SnapshotCRsCountShouldBe(context.Background(), bslDeletionTestNs,
backupName_2, 1)).To(Succeed())
})
} else if VeleroCfg.CloudProvider == "azure" && strings.EqualFold(VeleroCfg.Features, "EnableCSI") {
By(fmt.Sprintf("CSI VolumeSnapshotContent CR in backup %s should be created", backupName_1), func() {
Expect(util.CheckVolumeSnapshotCR(client, []string{podName_1}, bslDeletionTestNs, backupName_1)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot content CR.")
})
By(fmt.Sprintf("CSI VolumeSnapshotContent CR in backup %s should be created", backupName_2), func() {
Expect(util.CheckVolumeSnapshotCR(client, []string{podName_2}, bslDeletionTestNs, backupName_2)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot content CR.")
})
}
var snapshotCheckPoint SnapshotCheckPoint
snapshotCheckPoint.NamespaceBackedUp = bslDeletionTestNs
By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation_1), func() {
snapshotCheckPoint.ExpectCount = 1
snapshotCheckPoint.PodName = podName_1
Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider,
VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 1, bslDeletionTestNs, backupName_1, []string{podName_1})
Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
Expect(WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider,
VeleroCfg.CloudCredentialsFile, VeleroCfg.AdditionalBSLBucket,
VeleroCfg.BSLConfig, backupName_1, snapshotCheckPoint)).To(Succeed())
})
By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation_2), func() {
snapshotCheckPoint.ExpectCount = 1
snapshotCheckPoint.PodName = podName_2
snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 1, bslDeletionTestNs, backupName_2, []string{podName_2})
Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
var BSLCredentials, BSLConfig string
if VeleroCfg.CloudProvider == "vsphere" {
BSLCredentials = VeleroCfg.AdditionalBSLCredentials
@ -219,7 +238,8 @@ func BslDeletionTest(useVolumeSnapshots bool) {
BSLCredentials = VeleroCfg.CloudCredentialsFile
BSLConfig = VeleroCfg.BSLConfig
}
Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider,
Expect(WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider,
BSLCredentials, VeleroCfg.AdditionalBSLBucket,
BSLConfig, backupName_2, snapshotCheckPoint)).To(Succeed())
})
@ -234,12 +254,12 @@ func BslDeletionTest(useVolumeSnapshots bool) {
})
}
By(fmt.Sprintf("Verify if backup %s is created or not", backupName_1), func() {
By(fmt.Sprintf("Backup 1 %s should be created.", backupName_1), func() {
Expect(WaitForBackupCreated(context.Background(), VeleroCfg.VeleroCLI,
backupName_1, 10*time.Minute)).To(Succeed())
})
By(fmt.Sprintf("Verify if backup %s is created or not", backupName_2), func() {
By(fmt.Sprintf("Backup 2 %s should be created.", backupName_2), func() {
Expect(WaitForBackupCreated(context.Background(), VeleroCfg.VeleroCLI,
backupName_2, 10*time.Minute)).To(Succeed())
})
@ -298,15 +318,14 @@ func BslDeletionTest(useVolumeSnapshots bool) {
var snapshotCheckPoint SnapshotCheckPoint
snapshotCheckPoint.NamespaceBackedUp = bslDeletionTestNs
By(fmt.Sprintf("Snapshot should not be deleted in cloud object store after deleting bsl %s", backupLocation_1), func() {
snapshotCheckPoint.ExpectCount = 1
snapshotCheckPoint.PodName = podName_1
Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider,
snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 1, bslDeletionTestNs, backupName_1, []string{podName_1})
Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
Expect(WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider,
VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
VeleroCfg.BSLConfig, backupName_1, snapshotCheckPoint)).To(Succeed())
})
By(fmt.Sprintf("Snapshot should not be deleted in cloud object store after deleting bsl %s", backupLocation_2), func() {
snapshotCheckPoint.ExpectCount = 1
snapshotCheckPoint.PodName = podName_2
var BSLCredentials, BSLConfig string
if VeleroCfg.CloudProvider == "vsphere" {
BSLCredentials = VeleroCfg.AdditionalBSLCredentials
@ -315,7 +334,9 @@ func BslDeletionTest(useVolumeSnapshots bool) {
BSLCredentials = VeleroCfg.CloudCredentialsFile
BSLConfig = VeleroCfg.BSLConfig
}
Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider,
snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 1, bslDeletionTestNs, backupName_2, []string{podName_2})
Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
Expect(WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider,
BSLCredentials, VeleroCfg.AdditionalBSLBucket,
BSLConfig, backupName_2, snapshotCheckPoint)).To(Succeed())
})

2
test/e2e/e2e_suite_test.go
View File

@ -94,9 +94,11 @@ var _ = Describe("[ResourceFiltering][IncludeResources][Backup] Velero test on i
var _ = Describe("[ResourceFiltering][IncludeResources][Restore] Velero test on include resources from the cluster restore", RestoreWithIncludeResources)
var _ = Describe("[ResourceFiltering][LabelSelector] Velero test on backup include resources matching the label selector", BackupWithLabelSelector)
var _ = Describe("[Backups][Deletion][Restic] Velero tests of Restic backup deletion", BackupDeletionWithRestic)
var _ = Describe("[Backups][Deletion][Snapshot] Velero tests of snapshot backup deletion", BackupDeletionWithSnapshots)
var _ = Describe("[PrivilegesMgmt][SSR] Velero test on ssr object when controller namespace mix-ups", SSRTest)
var _ = Describe("[Backups][BackupsSync] Backups in object storage are synced to a new Velero and deleted backups in object storage are synced to be deleted in Velero", BackupsSyncTest)
var _ = Describe("[BSL][Deletion][Snapshot] Local backups will be deleted once the corresponding backup storage location is deleted", BslDeletionWithSnapshots)
var _ = Describe("[BSL][Deletion][Restic] Local backups and restic repos will be deleted once the corresponding backup storage location is deleted", BslDeletionWithRestic)

14
test/e2e/privilegesmgmt/ssr.go
View File

@ -36,11 +36,15 @@ import (
func SSRTest() {
testNS := "ssr-test"
client, err := NewTestClient()
if err != nil {
println(err.Error())
}
Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests")
var (
client TestClient
err error
)
By("Create test client instance", func() {
client, err = NewTestClient()
Expect(err).NotTo(HaveOccurred(), "Failed to instantiate cluster client for backup tests")
})
BeforeEach(func() {
flag.Parse()

15
test/e2e/test/test.go
View File

@ -72,9 +72,11 @@ var TestClientInstance TestClient
func TestFunc(test VeleroBackupRestoreTest) func() {
return func() {
var err error
TestClientInstance, err = NewTestClient()
Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests")
By("Create test client instance", func() {
var err error
TestClientInstance, err = NewTestClient()
Expect(err).NotTo(HaveOccurred(), "Failed to instantiate cluster client for backup tests")
})
Expect(test.Init()).To(Succeed(), "Failed to instantiate test cases")
BeforeEach(func() {
flag.Parse()
@ -99,8 +101,11 @@ func TestFuncWithMultiIt(tests []VeleroBackupRestoreTest) func() {
return func() {
var err error
var countIt int
TestClientInstance, err = NewTestClient()
Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests")
By("Create test client instance", func() {
TestClientInstance, err = NewTestClient()
Expect(err).NotTo(HaveOccurred(), "Failed to instantiate cluster client for backup tests")
})
//Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests")
for k := range tests {
Expect(tests[k].Init()).To(Succeed(), fmt.Sprintf("Failed to instantiate test %s case", tests[k].GetTestMsg().Desc))
}

9
test/e2e/types.go
View File

@ -55,7 +55,10 @@ type VerleroConfig struct {
type SnapshotCheckPoint struct {
NamespaceBackedUp string
SnapshotIDList []string
ExpectCount int
PodName string
// SnapshotIDList is for Azure CSI Verification
// we can get SnapshotID from VolumeSnapshotContent from a certain backup
SnapshotIDList []string
ExpectCount int
PodName []string
EnableCSI bool
}

51
test/e2e/upgrade/upgrade.go
View File

@ -27,8 +27,10 @@ import (
. "github.com/onsi/gomega"
. "github.com/vmware-tanzu/velero/test/e2e"
util "github.com/vmware-tanzu/velero/test/e2e/util/csi"
. "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
. "github.com/vmware-tanzu/velero/test/e2e/util/kibishii"
. "github.com/vmware-tanzu/velero/test/e2e/util/providers"
. "github.com/vmware-tanzu/velero/test/e2e/util/velero"
)
@ -74,10 +76,13 @@ func BackupUpgradeRestoreWithRestic() {
func BackupUpgradeRestoreTest(useVolumeSnapshots bool, upgradeFromVelero UpgradeFromVelero) {
var (
backupName, restoreName string
client TestClient
err error
)
client, err := NewTestClient()
Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests")
By("Create test client instance", func() {
client, err = NewTestClient()
Expect(err).NotTo(HaveOccurred(), "Failed to instantiate cluster client for backup tests")
})
BeforeEach(func() {
if !VeleroCfg.InstallVelero {
Skip("Upgrade test should not be triggered if VeleroCfg.InstallVelero is set to false")
@ -166,11 +171,27 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, upgradeFromVelero Upgrade
})
})
if useVolumeSnapshots && VeleroCfg.CloudProvider == "vsphere" {
// TODO - remove after upload progress monitoring is implemented
By("Waiting for vSphere uploads to complete", func() {
Expect(WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour,
upgradeNamespace)).To(Succeed())
if useVolumeSnapshots {
if VeleroCfg.CloudProvider == "vsphere" {
// TODO - remove after upload progress monitoring is implemented
By("Waiting for vSphere uploads to complete", func() {
Expect(WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour,
upgradeNamespace)).To(Succeed())
})
} else if VeleroCfg.CloudProvider == "azure" && strings.EqualFold(VeleroCfg.Features, "EnableCSI") {
By("CSI VolumeSnapshotContent CR should be created", func() {
Expect(util.CheckVolumeSnapshotCR(client, KibishiiPodNameList, upgradeNamespace, backupName)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot content")
})
}
var snapshotCheckPoint SnapshotCheckPoint
snapshotCheckPoint.NamespaceBackedUp = upgradeNamespace
By("Snapshot should be created in cloud object store", func() {
snapshotCheckPoint, err := GetSnapshotCheckPoint(client, VeleroCfg, 2,
upgradeNamespace, backupName, KibishiiPodNameList)
Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint")
Expect(WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider,
VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
VeleroCfg.BSLConfig, backupName, snapshotCheckPoint)).To(Succeed())
})
}
@ -179,6 +200,20 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, upgradeFromVelero Upgrade
fmt.Sprintf("failed to delete namespace %s", upgradeNamespace))
})
if useVolumeSnapshots && VeleroCfg.CloudProvider == "azure" && strings.EqualFold(VeleroCfg.Features, "EnableCSI") {
// Upgrade test is not running daily since no CSI plugin v1.0 released, because builds before
// v1.0 have issues to fail upgrade case.
By("Sleep 5 minutes to avoid snapshot recreated by unknown reason ", func() {
time.Sleep(5 * time.Minute)
})
// TODO: add WaitUntilSnapshotsNotExistInCloud verification when Upgrade test is enabled in nightly
// err = WaitUntilSnapshotsNotExistInCloud(VeleroCfg.CloudProvider,
// VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, veleroCfg.BSLConfig,
// backupName, snapshotCheckPoint)
// if err != nil {
// return errors.Wrap(err, "exceed waiting for snapshot created in cloud")
// }
}
// the snapshots of AWS may be still in pending status when do the restore, wait for a while
// to avoid this https://github.com/vmware-tanzu/velero/issues/1799
// TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed

132
test/e2e/util/csi/common.go Normal file
View File

@ -0,0 +1,132 @@
/*
Copyright 2020 the Velero contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
"context"
"fmt"
"strings"
"github.com/pkg/errors"
snapshotterClientSet "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/clientcmd"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
. "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
)
func GetClients() (*kubernetes.Clientset, *snapshotterClientSet.Clientset, error) {
loadingRules := clientcmd.NewDefaultClientConfigLoadingRules()
configOverrides := &clientcmd.ConfigOverrides{}
kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides)
clientConfig, err := kubeConfig.ClientConfig()
if err != nil {
return nil, nil, errors.WithStack(err)
}
client, err := kubernetes.NewForConfig(clientConfig)
if err != nil {
return nil, nil, errors.WithStack(err)
}
snapshotterClient, err := snapshotterClientSet.NewForConfig(clientConfig)
if err != nil {
return nil, nil, errors.WithStack(err)
}
return client, snapshotterClient, nil
}
func GetCsiSnapshotHandle(client TestClient, backupName string) ([]string, error) {
_, snapshotClient, err := GetClients()
if err != nil {
return nil, err
}
vscList, err1 := snapshotClient.SnapshotV1beta1().VolumeSnapshotContents().List(context.TODO(), metav1.ListOptions{})
if err1 != nil {
return nil, err
}
var snapshotHandleList []string
for _, i := range vscList.Items {
if i.Status.SnapshotHandle == nil {
fmt.Println("SnapshotHandle is nil")
continue
}
fmt.Println(*i.Status.SnapshotHandle)
if i.Labels["velero.io/backup-name"] == backupName {
tmp := strings.Split(*i.Status.SnapshotHandle, "/")
snapshotHandleList = append(snapshotHandleList, tmp[len(tmp)-1])
}
}
if err != nil {
return nil, err
}
if len(snapshotHandleList) == 0 {
return snapshotHandleList, errors.New(fmt.Sprintf("No VolumeSnapshotContent from backup %s", backupName))
}
return snapshotHandleList, nil
}
func GetVolumeSnapshotContentNameByPod(client TestClient, podName, namespace, backupName string) (string, error) {
pvcList, err := GetPvcByPodName(context.Background(), namespace, podName)
if err != nil {
return "", err
}
if len(pvcList) != 1 {
return "", errors.New(fmt.Sprintf("Only 1 PVC of pod %s should be found under namespace %s", podName, namespace))
}
pvList, err := GetPvByPvc(context.Background(), namespace, pvcList[0])
if err != nil {
return "", err
}
if len(pvList) != 1 {
return "", errors.New(fmt.Sprintf("Only 1 PV of PVC %s pod %s should be found under namespace %s", pvcList[0], podName, namespace))
}
pv_value, err := GetPersistentVolume(context.Background(), client, "", pvList[0])
fmt.Println(pv_value.Annotations["pv.kubernetes.io/provisioned-by"])
if err != nil {
return "", err
}
_, snapshotClient, err := GetClients()
if err != nil {
return "", err
}
vsList, err := snapshotClient.SnapshotV1beta1().VolumeSnapshots(namespace).List(context.TODO(), metav1.ListOptions{})
if err != nil {
return "", err
}
for _, i := range vsList.Items {
if pvcList[0] == *i.Spec.Source.PersistentVolumeClaimName &&
i.Labels["velero.io/backup-name"] == backupName {
return *i.Status.BoundVolumeSnapshotContentName, nil
}
}
return "", errors.New(fmt.Sprintf("Fail to get VolumeSnapshotContentName for pod %s under namespace %s", podName, namespace))
}
func CheckVolumeSnapshotCR(client TestClient, pods []string, nampespace, backupName string) error {
for _, podName := range pods {
if snapshotContentName, err := GetVolumeSnapshotContentNameByPod(client, podName, nampespace, backupName); err != nil || snapshotContentName == "" {
return errors.Wrap(err, "Fail to get Azure CSI snapshot content")
} else {
fmt.Println("Found volumesnapshotcontent: " + snapshotContentName)
}
}
return nil
}

4
test/e2e/util/k8s/common.go
View File

@ -99,7 +99,7 @@ func GetPvcByPodName(ctx context.Context, namespace, podName string) ([]string,
return common.GetListBy2Pipes(ctx, *CmdLine1, *CmdLine2, *CmdLine3)
}
func GetPvByPvc(ctx context.Context, pvc string) ([]string, error) {
func GetPvByPvc(ctx context.Context, namespace, pvc string) ([]string, error) {
// Example:
// NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE
// pvc-3f784366-58db-40b2-8fec-77307807e74b 1Gi RWO Delete Bound bsl-deletion/kibishii-data-kibishii-deployment-0 kibishii-storage-class 6h41m
@ -110,7 +110,7 @@ func GetPvByPvc(ctx context.Context, pvc string) ([]string, error) {
CmdLine2 := &common.OsCommandLine{
Cmd: "grep",
Args: []string{pvc},
Args: []string{namespace + "/" + pvc},
}
CmdLine3 := &common.OsCommandLine{

28
test/e2e/util/k8s/persistentvolumes.go Normal file
View File

@ -0,0 +1,28 @@
/*
Copyright the Velero contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package k8s
import (
"context"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func GetPersistentVolume(ctx context.Context, client TestClient, namespace string, persistentVolume string) (*corev1.PersistentVolume, error) {
return client.ClientGo.CoreV1().PersistentVolumes().Get(ctx, persistentVolume, metav1.GetOptions{})
}

47
test/e2e/util/kibishii/kibishii_utils.go
View File

@ -28,7 +28,9 @@ import (
veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec"
. "github.com/vmware-tanzu/velero/test/e2e"
util "github.com/vmware-tanzu/velero/test/e2e/util/csi"
. "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
. "github.com/vmware-tanzu/velero/test/e2e/util/providers"
. "github.com/vmware-tanzu/velero/test/e2e/util/velero"
)
@ -36,6 +38,8 @@ const (
jumpPadPod = "jump-pad"
)
var KibishiiPodNameList = []string{"kibishii-deployment-0", "kibishii-deployment-1"}
// RunKibishiiTests runs kibishii tests on the provider.
func RunKibishiiTests(client TestClient, veleroCfg VerleroConfig, backupName, restoreName, backupLocation, kibishiiNamespace string,
useVolumeSnapshots bool) error {
@ -47,7 +51,7 @@ func RunKibishiiTests(client TestClient, veleroCfg VerleroConfig, backupName, re
veleroFeatures := VeleroCfg.Features
kibishiiDirectory := VeleroCfg.KibishiiDirectory
if _, err := GetNamespace(context.Background(), client, kibishiiNamespace); err == nil {
fmt.Printf("Workload namespace %s exists, delete it first.", kibishiiNamespace)
fmt.Printf("Workload namespace %s exists, delete it first.\n", kibishiiNamespace)
if err = DeleteNamespace(context.Background(), client, kibishiiNamespace, true); err != nil {
fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", kibishiiNamespace))
}
@ -73,19 +77,46 @@ func RunKibishiiTests(client TestClient, veleroCfg VerleroConfig, backupName, re
RunDebug(context.Background(), veleroCLI, veleroNamespace, backupName, "")
return errors.Wrapf(err, "Failed to backup kibishii namespace %s", kibishiiNamespace)
}
if providerName == "vsphere" && useVolumeSnapshots {
// Wait for uploads started by the Velero Plug-in for vSphere to complete
// TODO - remove after upload progress monitoring is implemented
fmt.Println("Waiting for vSphere uploads to complete")
if err := WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, kibishiiNamespace); err != nil {
return errors.Wrapf(err, "Error waiting for uploads to complete")
var snapshotCheckPoint SnapshotCheckPoint
var err error
if useVolumeSnapshots {
if providerName == "vsphere" {
// Wait for uploads started by the Velero Plug-in for vSphere to complete
// TODO - remove after upload progress monitoring is implemented
fmt.Println("Waiting for vSphere uploads to complete")
if err := WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, kibishiiNamespace); err != nil {
return errors.Wrapf(err, "Error waiting for uploads to complete")
}
} else if providerName == "azure" && strings.EqualFold(veleroFeatures, "EnableCSI") {
if err := util.CheckVolumeSnapshotCR(client, KibishiiPodNameList, kibishiiNamespace, backupName); err != nil {
return errors.Wrapf(err, "Fail to get Azure CSI snapshot content")
}
}
snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 2, kibishiiNamespace, backupName, KibishiiPodNameList)
if err != nil {
return errors.Wrap(err, "Fail to get snapshot checkpoint")
}
err = WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider,
VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, veleroCfg.BSLConfig,
backupName, snapshotCheckPoint)
if err != nil {
return errors.Wrap(err, "exceed waiting for snapshot created in cloud")
}
}
fmt.Printf("Simulating a disaster by removing namespace %s\n", kibishiiNamespace)
if err := DeleteNamespace(oneHourTimeout, client, kibishiiNamespace, true); err != nil {
return errors.Wrapf(err, "failed to delete namespace %s", kibishiiNamespace)
}
if useVolumeSnapshots && providerName == "azure" && strings.EqualFold(veleroFeatures, "EnableCSI") {
err = WaitUntilSnapshotsNotExistInCloud(VeleroCfg.CloudProvider,
VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, veleroCfg.BSLConfig,
backupName, snapshotCheckPoint)
if err != nil {
return errors.Wrap(err, "exceed waiting for snapshot created in cloud")
}
}
time.Sleep(5 * time.Minute)
// the snapshots of AWS may be still in pending status when do the restore, wait for a while
// to avoid this https://github.com/vmware-tanzu/velero/issues/1799

39
test/e2e/util/providers/azure_utils.go
View File

@ -226,12 +226,11 @@ func deleteBlob(p pipeline.Pipeline, accountName, containerName, blobName string
_, err = blobURL.Delete(ctx, azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{})
return err
}
func (s AzureStorage) IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) (bool, error) {
func (s AzureStorage) IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName string) (bool, error) {
accountName, accountKey, err := getStorageCredential(cloudCredentialsFile, bslConfig)
if err != nil {
log.Fatal("Fail to get : accountName and accountKey, " + err.Error())
}
credential, err := azblob.NewSharedKeyCredential(accountName, accountKey)
if err != nil {
log.Fatal("Invalid credentials with error: " + err.Error())
@ -250,7 +249,7 @@ func (s AzureStorage) IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPref
_, err = containerURL.Create(ctx, azblob.Metadata{}, azblob.PublicAccessNone)
handleErrors(err)
fmt.Printf("Finding backup %s blobs in Azure container/bucket %s\n", backupObject, containerName)
fmt.Printf("Finding backup %s blobs in Azure container/bucket %s\n", backupName, containerName)
for marker := (azblob.Marker{}); marker.NotDone(); {
listBlob, err := containerURL.ListBlobsFlatSegment(ctx, marker, azblob.ListBlobsSegmentOptions{})
if err != nil {
@ -259,8 +258,8 @@ func (s AzureStorage) IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPref
marker = listBlob.NextMarker
for _, blobInfo := range listBlob.Segment.BlobItems {
if strings.Contains(blobInfo.Name, backupObject) {
fmt.Printf("Blob name: %s exist in %s\n", backupObject, blobInfo.Name)
if strings.Contains(blobInfo.Name, backupName) {
fmt.Printf("Blob name: %s exist in %s\n", backupName, blobInfo.Name)
return true, nil
}
}
@ -317,7 +316,7 @@ func mapLookup(data map[string]string) func(string) string {
return data[key]
}
}
func (s AzureStorage) IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupObject string, snapshotCheck SnapshotCheckPoint) error {
func (s AzureStorage) IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupName string, snapshotCheck SnapshotCheckPoint) error {
ctx := context.Background()
@ -351,7 +350,7 @@ func (s AzureStorage) IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupO
// // if config["snapsIncrementalConfigKey"] is empty, default to nil; otherwise, parse i
snapsClient.Authorizer = authorizer
snaps := &snapsClient
//return ListByResourceGroup(ctx, snaps, envVars[resourceGroupEnvVar], backupObject, snapshotCount)
//return ListByResourceGroup(ctx, snaps, envVars[resourceGroupEnvVar], backupName, snapshotCount)
req, err := snaps.ListByResourceGroupPreparer(ctx, envVars[resourceGroupEnvVar])
if err != nil {
return autorest.NewErrorWithError(err, "compute.SnapshotsClient", "ListByResourceGroup", nil, "Failure preparing request")
@ -364,11 +363,29 @@ func (s AzureStorage) IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupO
result, err := snaps.ListByResourceGroupResponder(resp)
snapshotCountFound := 0
backupNameInSnapshot := ""
if err != nil {
errors.Wrap(err, fmt.Sprintf("Fail to list snapshots %s\n", envVars[resourceGroupEnvVar]))
}
if result.Value == nil {
errors.New(fmt.Sprintf("No snapshots in Azure resource group %s\n", envVars[resourceGroupEnvVar]))
}
for _, v := range *result.Value {
backupNameInSnapshot = *v.Tags["velero.io-backup"]
fmt.Println(backupNameInSnapshot)
if backupObject == backupNameInSnapshot {
snapshotCountFound++
if snapshotCheck.EnableCSI {
for _, s := range snapshotCheck.SnapshotIDList {
fmt.Println("Azure CSI local snapshot CR: " + s)
fmt.Println("Azure provider snapshot name: " + *v.Name)
if s == *v.Name {
fmt.Printf("Azure snapshot %s is created.\n", s)
snapshotCountFound++
}
}
} else {
fmt.Println(v.Tags)
backupNameInSnapshot = *v.Tags["velero.io-backup"]
fmt.Println(backupNameInSnapshot)
if backupName == backupNameInSnapshot {
snapshotCountFound++
}
}
}
if err != nil {

38
test/e2e/util/providers/common.go
View File

@ -24,6 +24,8 @@ import (
"github.com/pkg/errors"
waitutil "k8s.io/apimachinery/pkg/util/wait"
. "github.com/vmware-tanzu/velero/test/e2e"
velero "github.com/vmware-tanzu/velero/test/e2e/util/velero"
)
@ -31,11 +33,11 @@ import (
type ObjectsInStorage interface {
IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) (bool, error)
DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) error
IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupObject string, snapshotCheck SnapshotCheckPoint) error
IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupName string, snapshotCheck SnapshotCheckPoint) error
}
func ObjectsShouldBeInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) error {
fmt.Printf("|| VERIFICATION || - Backup %s should exist in storage %s", backupName, bslPrefix)
fmt.Printf("|| VERIFICATION || - Backup %s should exist in storage [%s]", backupName, bslPrefix)
exist, _ := IsObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix)
if !exist {
return errors.New(fmt.Sprintf("|| UNEXPECTED ||Backup object %s is not exist in object store after backup as expected", backupName))
@ -109,9 +111,37 @@ func DeleteObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPr
return nil
}
func SnapshotsShouldNotExistInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string) error {
func WaitUntilSnapshotsExistInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheckPoint SnapshotCheckPoint) error {
err := waitutil.PollImmediate(30*time.Second, 3*time.Minute,
func() (bool, error) {
err := SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider,
VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
bslConfig, backupName, snapshotCheckPoint)
if err == nil {
return true, nil
}
return false, err
})
return err
}
func WaitUntilSnapshotsNotExistInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheckPoint SnapshotCheckPoint) error {
err := waitutil.PollImmediate(30*time.Second, 3*time.Minute,
func() (bool, error) {
snapshotCheckPoint.ExpectCount = 0
err := SnapshotsShouldNotExistInCloud(VeleroCfg.CloudProvider,
VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
bslConfig, backupName, snapshotCheckPoint)
if err == nil {
return true, nil
}
return false, err
})
return err
}
func SnapshotsShouldNotExistInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheckPoint SnapshotCheckPoint) error {
fmt.Printf("|| VERIFICATION || - Snapshots should not exist in cloud, backup %s\n", backupName)
var snapshotCheckPoint SnapshotCheckPoint
snapshotCheckPoint.ExpectCount = 0
err := IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName, snapshotCheckPoint)
if err != nil {

41
test/e2e/util/velero/velero_utils.go
View File

@ -40,7 +40,10 @@ import (
cliinstall "github.com/vmware-tanzu/velero/pkg/cmd/cli/install"
"github.com/vmware-tanzu/velero/pkg/cmd/util/flag"
veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec"
. "github.com/vmware-tanzu/velero/test/e2e"
common "github.com/vmware-tanzu/velero/test/e2e/util/common"
util "github.com/vmware-tanzu/velero/test/e2e/util/csi"
. "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
)
const BackupObjectsPrefix = "backups"
@ -491,7 +494,7 @@ func WaitForVSphereUploadCompletion(ctx context.Context, timeout time.Duration,
return err
}
func GetVsphereSnapshotIDs(ctx context.Context, timeout time.Duration, namespace, podName string) ([]string, error) {
func GetVsphereSnapshotIDs(ctx context.Context, timeout time.Duration, namespace string, podNameList []string) ([]string, error) {
checkSnapshotCmd := exec.CommandContext(ctx, "kubectl",
"get", "-n", namespace, "snapshots.backupdriver.cnsdp.vmware.com", "-o=jsonpath='{range .items[*]}{.spec.resourceHandle.name}{\"=\"}{.status.snapshotID}{\"\\n\"}{end}'")
fmt.Printf("checkSnapshotCmd cmd =%v\n", checkSnapshotCmd)
@ -504,14 +507,20 @@ func GetVsphereSnapshotIDs(ctx context.Context, timeout time.Duration, namespace
stdout = strings.Replace(stdout, "'", "", -1)
lines := strings.Split(stdout, "\n")
var result []string
for _, curLine := range lines {
fmt.Println("curLine:" + curLine)
curLine = strings.Replace(curLine, "\n", "", -1)
if len(curLine) == 0 {
continue
}
if podName != "" && !strings.Contains(curLine, podName) {
var Exist bool
for _, podName := range podNameList {
if podName != "" && strings.Contains(curLine, podName) {
Exist = true
break
}
}
if !Exist {
continue
}
snapshotID := curLine[strings.LastIndex(curLine, ":")+1:]
@ -519,6 +528,7 @@ func GetVsphereSnapshotIDs(ctx context.Context, timeout time.Duration, namespace
snapshotIDDec, _ := b64.StdEncoding.DecodeString(snapshotID)
fmt.Println("snapshotIDDec:" + string(snapshotIDDec))
result = append(result, string(snapshotIDDec))
fmt.Println(result)
}
fmt.Println(result)
return result, nil
@ -797,3 +807,28 @@ func GetResticRepositories(ctx context.Context, veleroNamespace, targetNamespace
return common.GetListBy2Pipes(ctx, *CmdLine1, *CmdLine2, *CmdLine3)
}
func GetSnapshotCheckPoint(client TestClient, VeleroCfg VerleroConfig, expectCount int, namespaceBackedUp, backupName string, kibishiiPodNameList []string) (SnapshotCheckPoint, error) {
var snapshotCheckPoint SnapshotCheckPoint
snapshotCheckPoint.ExpectCount = expectCount
snapshotCheckPoint.NamespaceBackedUp = namespaceBackedUp
snapshotCheckPoint.PodName = kibishiiPodNameList
if strings.EqualFold(VeleroCfg.Features, "EnableCSI") {
snapshotCheckPoint.EnableCSI = true
if err := util.CheckVolumeSnapshotCR(client, kibishiiPodNameList, namespaceBackedUp, backupName); err != nil {
return snapshotCheckPoint, errors.Wrapf(err, "Fail to get Azure CSI snapshot content")
}
var err error
snapshotCheckPoint.SnapshotIDList, err = util.GetCsiSnapshotHandle(client, backupName)
if err != nil {
return snapshotCheckPoint, errors.New(fmt.Sprintf("Fail to get CSI SnapshotHandle for backup %s", backupName))
}
fmt.Println(snapshotCheckPoint)
if len(snapshotCheckPoint.SnapshotIDList) != expectCount {
return snapshotCheckPoint, errors.New(fmt.Sprintf("Length of SnapshotIDList is not as expected %d", expectCount))
}
}
fmt.Println(snapshotCheckPoint)
return snapshotCheckPoint, nil
}