Add E2E test for parallel files upload and download
Signed-off-by: Ming Qiu <ming.qiu@broadcom.com>pull/7691/head
Ming Qiu
parent
a798182d61
commit
a628cb525f
|
|
@ -39,6 +39,8 @@ import (
|
|||
. "github.com/vmware-tanzu/velero/test/e2e/basic/resources-check"
|
||||
. "github.com/vmware-tanzu/velero/test/e2e/bsl-mgmt"
|
||||
. "github.com/vmware-tanzu/velero/test/e2e/migration"
|
||||
. "github.com/vmware-tanzu/velero/test/e2e/parallelfilesdownload"
|
||||
. "github.com/vmware-tanzu/velero/test/e2e/parallelfilesupload"
|
||||
. "github.com/vmware-tanzu/velero/test/e2e/privilegesmgmt"
|
||||
. "github.com/vmware-tanzu/velero/test/e2e/pv-backup"
|
||||
. "github.com/vmware-tanzu/velero/test/e2e/resource-filtering"
|
||||
|
|
@ -182,6 +184,9 @@ var _ = Describe("[Basic][Nodeport] Service nodeport reservation during restore
|
|||
var _ = Describe("[Basic][StorageClass] Storage class of persistent volumes and persistent volume claims can be changed during restores", StorageClasssChangingTest)
|
||||
var _ = Describe("[Basic][SelectedNode][SKIP_KIND] Node selectors of persistent volume claims can be changed during restores", PVCSelectedNodeChangingTest)
|
||||
|
||||
var _ = Describe("[UploaderConfig][ParallelFilesUpload] Velero test on parallel files upload", ParallelFilesUploadTest)
|
||||
var _ = Describe("[UploaderConfig][ParallelFilesDownload] Velero test on parallel files download", ParallelFilesDownloadTest)
|
||||
|
||||
func GetKubeconfigContext() error {
|
||||
var err error
|
||||
var tcDefault, tcStandby TestClient
|
||||
|
|
|
|||
|
|
@ -0,0 +1,147 @@
|
|||
/*
|
||||
Copyright the Velero contributors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package parallelfilesdownload
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
. "github.com/vmware-tanzu/velero/test"
|
||||
. "github.com/vmware-tanzu/velero/test/e2e/test"
|
||||
. "github.com/vmware-tanzu/velero/test/util/k8s"
|
||||
)
|
||||
|
||||
type ParallelFilesDownload struct {
|
||||
TestCase
|
||||
parallel string
|
||||
namespace string
|
||||
pod string
|
||||
pvc string
|
||||
volume string
|
||||
fileName string
|
||||
fileNum int
|
||||
fileSize int64
|
||||
hash []string
|
||||
}
|
||||
|
||||
var ParallelFilesDownloadTest func() = TestFunc(&ParallelFilesDownload{})
|
||||
|
||||
func (p *ParallelFilesDownload) Init() error {
|
||||
// generate random number as UUIDgen and set one default timeout duration
|
||||
p.TestCase.Init()
|
||||
|
||||
// generate variable names based on CaseBaseName + UUIDgen
|
||||
p.CaseBaseName = "parallel-files-download" + p.UUIDgen
|
||||
p.BackupName = p.CaseBaseName + "-backup"
|
||||
p.RestoreName = p.CaseBaseName + "-restore"
|
||||
p.pod = p.CaseBaseName + "-pod"
|
||||
p.pvc = p.CaseBaseName + "-pvc"
|
||||
p.fileName = p.CaseBaseName + "-file"
|
||||
p.parallel = "3"
|
||||
p.fileNum = 10
|
||||
p.fileSize = 1 * 1024 * 1024 // 1MB
|
||||
p.volume = p.CaseBaseName + "-vol"
|
||||
|
||||
// generate namespace
|
||||
p.VeleroCfg.UseVolumeSnapshots = false
|
||||
p.VeleroCfg.UseNodeAgent = true
|
||||
p.namespace = p.CaseBaseName + "-ns"
|
||||
|
||||
p.BackupArgs = []string{
|
||||
"create", "--namespace", p.VeleroCfg.VeleroNamespace,
|
||||
"backup", p.BackupName,
|
||||
"--include-namespaces", p.namespace,
|
||||
"--default-volumes-to-fs-backup",
|
||||
"--snapshot-volumes=false",
|
||||
"--wait",
|
||||
}
|
||||
|
||||
p.RestoreArgs = []string{
|
||||
"create", "--namespace", p.VeleroCfg.VeleroNamespace,
|
||||
"restore", p.RestoreName,
|
||||
"--parallel-files-download", p.parallel,
|
||||
"--from-backup", p.BackupName, "--wait",
|
||||
}
|
||||
|
||||
// Message output by ginkgo
|
||||
p.TestMsg = &TestMSG{
|
||||
Desc: "Test parallel files download",
|
||||
FailedMSG: "Failed to test parallel files download",
|
||||
Text: "Test parallel files download with parallel download " + p.parallel + " files",
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *ParallelFilesDownload) CreateResources() error {
|
||||
err := InstallStorageClass(p.Ctx, fmt.Sprintf("../testdata/storage-class/%s.yaml", p.VeleroCfg.CloudProvider))
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to install storage class for pv backup filtering test")
|
||||
}
|
||||
|
||||
By(fmt.Sprintf("Create namespace %s", p.namespace), func() {
|
||||
Expect(CreateNamespace(p.Ctx, p.Client, p.namespace)).To(Succeed(),
|
||||
fmt.Sprintf("Failed to create namespace %s", p.namespace))
|
||||
})
|
||||
|
||||
By(fmt.Sprintf("Create pod %s in namespace %s", p.pod, p.namespace), func() {
|
||||
_, err := CreatePod(p.Client, p.namespace, p.pod, StorageClassName, p.pvc, []string{p.volume}, nil, nil)
|
||||
Expect(err).To(Succeed())
|
||||
err = WaitForPods(p.Ctx, p.Client, p.namespace, []string{p.pod})
|
||||
Expect(err).To(Succeed())
|
||||
})
|
||||
|
||||
podList, err := ListPods(p.Ctx, p.Client, p.namespace)
|
||||
Expect(err).To(Succeed(), fmt.Sprintf("failed to list pods in namespace: %q with error %v", p.namespace, err))
|
||||
|
||||
for _, pod := range podList.Items {
|
||||
for i := 0; i < p.fileNum; i++ {
|
||||
fileName := fmt.Sprintf("%s-%d", p.fileName, i)
|
||||
// Write random data to file in pod
|
||||
Expect(WriteRandomDataToFileInPod(p.Ctx, p.namespace, pod.Name, pod.Name, p.volume,
|
||||
fileName, p.fileSize)).To(Succeed())
|
||||
// Calculate hash of the file
|
||||
hash, err := CalFileHashInPod(p.Ctx, p.namespace, pod.Name, pod.Name, fmt.Sprintf("%s/%s", p.volume, fileName))
|
||||
Expect(err).To(Succeed())
|
||||
p.hash = append(p.hash, hash)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *ParallelFilesDownload) Verify() error {
|
||||
podList, err := ListPods(p.Ctx, p.Client, p.namespace)
|
||||
Expect(err).To(Succeed(), fmt.Sprintf("failed to list pods in namespace: %q with error %v", p.namespace, err))
|
||||
|
||||
for _, pod := range podList.Items {
|
||||
err = WaitForPods(p.Ctx, p.Client, p.namespace, []string{pod.Name})
|
||||
Expect(err).To(Succeed())
|
||||
|
||||
for i := 0; i < p.fileNum; i++ {
|
||||
fileName := fmt.Sprintf("%s-%d", p.fileName, i)
|
||||
// Calculate hash of the file
|
||||
hash, err := CalFileHashInPod(p.Ctx, p.namespace, pod.Name, pod.Name, fmt.Sprintf("%s/%s", p.volume, fileName))
|
||||
Expect(err).To(Succeed())
|
||||
Expect(hash).To(Equal(p.hash[i]))
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,112 @@
|
|||
/*
|
||||
Copyright the Velero contributors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package parallelfilesupload
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
. "github.com/vmware-tanzu/velero/test"
|
||||
. "github.com/vmware-tanzu/velero/test/e2e/test"
|
||||
. "github.com/vmware-tanzu/velero/test/util/k8s"
|
||||
)
|
||||
|
||||
type ParallelFilesUpload struct {
|
||||
TestCase
|
||||
parallel string
|
||||
namespace string
|
||||
pod string
|
||||
pvc string
|
||||
volume string
|
||||
fileName string
|
||||
fileNum int
|
||||
fileSize int64
|
||||
}
|
||||
|
||||
var ParallelFilesUploadTest func() = TestFunc(&ParallelFilesUpload{})
|
||||
|
||||
func (p *ParallelFilesUpload) Init() error {
|
||||
// generate random number as UUIDgen and set one default timeout duration
|
||||
p.TestCase.Init()
|
||||
|
||||
// generate variable names based on CaseBaseName + UUIDgen
|
||||
p.CaseBaseName = "parallel-files-upload" + p.UUIDgen
|
||||
p.BackupName = p.CaseBaseName + "-backup"
|
||||
p.pod = p.CaseBaseName + "-pod"
|
||||
p.pvc = p.CaseBaseName + "-pvc"
|
||||
p.fileName = p.CaseBaseName + "-file"
|
||||
p.parallel = "3"
|
||||
p.fileNum = 10
|
||||
p.fileSize = 1 * 1024 * 1024 // 1MB
|
||||
p.volume = p.CaseBaseName + "-vol"
|
||||
// generate namespace
|
||||
p.VeleroCfg.UseVolumeSnapshots = false
|
||||
p.VeleroCfg.UseNodeAgent = true
|
||||
p.namespace = p.CaseBaseName + "-ns"
|
||||
|
||||
p.BackupArgs = []string{
|
||||
"create", "--namespace", p.VeleroCfg.VeleroNamespace,
|
||||
"backup", p.BackupName,
|
||||
"--include-namespaces", p.namespace,
|
||||
"--parallel-files-upload", p.parallel,
|
||||
"--default-volumes-to-fs-backup",
|
||||
"--snapshot-volumes=false",
|
||||
"--wait",
|
||||
}
|
||||
|
||||
// Message output by ginkgo
|
||||
p.TestMsg = &TestMSG{
|
||||
Desc: "Test parallel files upload",
|
||||
FailedMSG: "Failed to test parallel files upload",
|
||||
Text: "Test parallel files upload with parallel upload " + p.parallel + " files",
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *ParallelFilesUpload) CreateResources() error {
|
||||
err := InstallStorageClass(p.Ctx, fmt.Sprintf("../testdata/storage-class/%s.yaml", p.VeleroCfg.CloudProvider))
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to install storage class for pv backup filtering test")
|
||||
}
|
||||
|
||||
By(fmt.Sprintf("Create namespace %s", p.namespace), func() {
|
||||
Expect(CreateNamespace(p.Ctx, p.Client, p.namespace)).To(Succeed(),
|
||||
fmt.Sprintf("Failed to create namespace %s", p.namespace))
|
||||
})
|
||||
|
||||
By(fmt.Sprintf("Create pod %s in namespace %s", p.pod, p.namespace), func() {
|
||||
_, err := CreatePod(p.Client, p.namespace, p.pod, StorageClassName, p.pvc, []string{p.volume}, nil, nil)
|
||||
Expect(err).To(Succeed())
|
||||
err = WaitForPods(p.Ctx, p.Client, p.namespace, []string{p.pod})
|
||||
Expect(err).To(Succeed())
|
||||
})
|
||||
|
||||
podList, err := ListPods(p.Ctx, p.Client, p.namespace)
|
||||
Expect(err).To(Succeed(), fmt.Sprintf("failed to list pods in namespace: %q with error %v", p.namespace, err))
|
||||
|
||||
for _, pod := range podList.Items {
|
||||
for i := 0; i < p.fileNum; i++ {
|
||||
Expect(WriteRandomDataToFileInPod(p.Ctx, p.namespace, pod.Name, pod.Name, p.volume,
|
||||
fmt.Sprintf("%s-%d", p.fileName, i), p.fileSize)).To(Succeed())
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
@ -299,6 +299,29 @@ func PrepareVolumeList(volumeNameList []string) (vols []*corev1.Volume) {
|
|||
return
|
||||
}
|
||||
|
||||
func CalFileHashInPod(ctx context.Context, namespace, podName, containerName, filePath string) (string, error) {
|
||||
arg := []string{"exec", "-n", namespace, "-c", containerName, podName,
|
||||
"--", "/bin/sh", "-c", fmt.Sprintf("sha256sum %s | awk '{ print $1 }'", filePath)}
|
||||
cmd := exec.CommandContext(ctx, "kubectl", arg...)
|
||||
output, err := cmd.Output()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// Trim any leading or trailing whitespace characters from the output
|
||||
hash := string(output)
|
||||
hash = strings.TrimSpace(hash)
|
||||
|
||||
return hash, nil
|
||||
}
|
||||
|
||||
func WriteRandomDataToFileInPod(ctx context.Context, namespace, podName, containerName, volume, filename string, fileSize int64) error {
|
||||
arg := []string{"exec", "-n", namespace, "-c", containerName, podName,
|
||||
"--", "/bin/sh", "-c", fmt.Sprintf("dd if=/dev/urandom of=/%s/%s bs=%d count=1", volume, filename, fileSize)}
|
||||
cmd := exec.CommandContext(ctx, "kubectl", arg...)
|
||||
fmt.Printf("Kubectl exec cmd =%v\n", cmd)
|
||||
return cmd.Run()
|
||||
}
|
||||
|
||||
func CreateFileToPod(ctx context.Context, namespace, podName, containerName, volume, filename, content string) error {
|
||||
arg := []string{"exec", "-n", namespace, "-c", containerName, podName,
|
||||
"--", "/bin/sh", "-c", fmt.Sprintf("echo ns-%s pod-%s volume-%s > /%s/%s", namespace, podName, volume, volume, filename)}
|
||||
|
|
@ -306,6 +329,7 @@ func CreateFileToPod(ctx context.Context, namespace, podName, containerName, vol
|
|||
fmt.Printf("Kubectl exec cmd =%v\n", cmd)
|
||||
return cmd.Run()
|
||||
}
|
||||
|
||||
func FileExistInPV(ctx context.Context, namespace, podName, containerName, volume, filename string) (bool, error) {
|
||||
stdout, stderr, err := ReadFileFromPodVolume(ctx, namespace, podName, containerName, volume, filename)
|
||||
|
||||
|
|
|
|||
|
|
@ -598,22 +598,32 @@ func IsVeleroReady(ctx context.Context, veleroCfg *VeleroConfig) (bool, error) {
|
|||
}
|
||||
}
|
||||
|
||||
// Check BSL
|
||||
stdout, stderr, err = velerexec.RunCommand(exec.CommandContext(ctx, "kubectl", "get", "bsl", "default",
|
||||
// Check BSL with poll
|
||||
err = wait.PollUntilContextTimeout(ctx, PollInterval, time.Minute, true, func(ctx context.Context) (bool, error) {
|
||||
return checkBSL(ctx, veleroCfg) == nil, nil
|
||||
})
|
||||
if err != nil {
|
||||
return false, errors.Wrap(err, "failed to check the bsl")
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func checkBSL(ctx context.Context, veleroCfg *VeleroConfig) error {
|
||||
namespace := veleroCfg.VeleroNamespace
|
||||
stdout, stderr, err := velerexec.RunCommand(exec.CommandContext(ctx, "kubectl", "get", "bsl", "default",
|
||||
"-o", "json", "-n", namespace))
|
||||
if err != nil {
|
||||
return false, errors.Wrapf(err, "failed to get bsl %s stdout=%s, stderr=%s", veleroCfg.BSLBucket, stdout, stderr)
|
||||
return errors.Wrapf(err, "failed to get bsl %s stdout=%s, stderr=%s", veleroCfg.BSLBucket, stdout, stderr)
|
||||
} else {
|
||||
bsl := &velerov1api.BackupStorageLocation{}
|
||||
if err = json.Unmarshal([]byte(stdout), bsl); err != nil {
|
||||
return false, errors.Wrapf(err, "failed to unmarshal the velero bsl")
|
||||
return errors.Wrapf(err, "failed to unmarshal the velero bsl")
|
||||
}
|
||||
if bsl.Status.Phase != velerov1api.BackupStorageLocationPhaseAvailable {
|
||||
return false, fmt.Errorf("current bsl %s is not available", veleroCfg.BSLBucket)
|
||||
return fmt.Errorf("current bsl %s is not available", veleroCfg.BSLBucket)
|
||||
}
|
||||
}
|
||||
|
||||
return true, nil
|
||||
return nil
|
||||
}
|
||||
|
||||
func PrepareVelero(ctx context.Context, caseName string, veleroCfg VeleroConfig) error {
|
||||
|
|
|
|||
Loading…
Reference in New Issue