vmware-tanzu
/
velero
mirror of https://github.com/vmware-tanzu/velero.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #5425 from danfengliu/fix-ctx-issue 

Fix E2E test and Git action CI issues
pull/5449/head^2
Xun Jiang/Bruce Jiang 2022-10-17 09:56:23 +08:00 committed by GitHub
parent c0430b8964 cf32cabddd
commit 7a535ea047
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 164 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.github/workflows/e2e-test-kind.yaml vendored
View File

@ -117,12 +117,17 @@ jobs:
aws_access_key_id=minio
aws_secret_access_key=minio123
EOF
# Match kubectl version to k8s server version
curl -LO https://dl.k8s.io/release/v${{ matrix.k8s }}/bin/linux/amd64/kubectl
sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
GOPATH=~/go CLOUD_PROVIDER=kind \
OBJECT_STORE_PROVIDER=aws BSL_CONFIG=region=minio,s3ForcePathStyle="true",s3Url=http://$(hostname -i):9000 \
CREDS_FILE=/tmp/credential BSL_BUCKET=bucket \
ADDITIONAL_OBJECT_STORE_PROVIDER=aws ADDITIONAL_BSL_CONFIG=region=minio,s3ForcePathStyle="true",s3Url=http://$(hostname -i):9000 \
ADDITIONAL_CREDS_FILE=/tmp/credential ADDITIONAL_BSL_BUCKET=additional-bucket \
GINKGO_FOCUS='Basic\].+\[ClusterResource' VELERO_IMAGE=velero:pr-test \
GINKGO_FOCUS='Basic\]\[ClusterResource' VELERO_IMAGE=velero:pr-test \
make -C test/e2e run
timeout-minutes: 30
- name: Upload debug bundle

48
pkg/cmd/cli/install/install.go
View File

@ -43,29 +43,31 @@ import (
// InstallOptions collects all the options for installing Velero into a Kubernetes cluster.
type InstallOptions struct {
Namespace string
Image string
BucketName string
Prefix string
ProviderName string
PodAnnotations flag.Map
PodLabels flag.Map
ServiceAccountAnnotations flag.Map
VeleroPodCPURequest string
VeleroPodMemRequest string
VeleroPodCPULimit string
VeleroPodMemLimit string
NodeAgentPodCPURequest string
NodeAgentPodMemRequest string
NodeAgentPodCPULimit string
NodeAgentPodMemLimit string
RestoreOnly bool
SecretFile string
NoSecret bool
DryRun bool
BackupStorageConfig flag.Map
VolumeSnapshotConfig flag.Map
UseNodeAgent bool
Namespace string
Image string
BucketName string
Prefix string
ProviderName string
PodAnnotations flag.Map
PodLabels flag.Map
ServiceAccountAnnotations flag.Map
VeleroPodCPURequest string
VeleroPodMemRequest string
VeleroPodCPULimit string
VeleroPodMemLimit string
NodeAgentPodCPURequest string
NodeAgentPodMemRequest string
NodeAgentPodCPULimit string
NodeAgentPodMemLimit string
RestoreOnly bool
SecretFile string
NoSecret bool
DryRun bool
BackupStorageConfig flag.Map
VolumeSnapshotConfig flag.Map
UseNodeAgent bool
//TODO remove UseRestic when migration test out of using it
UseRestic bool
Wait bool
UseVolumeSnapshots bool
DefaultRepoMaintenanceFrequency time.Duration

3
test/e2e/Makefile
View File

@ -48,6 +48,7 @@ OUTPUT_DIR := _output/$(GOOS)/$(GOARCH)/bin
GINKGO_FOCUS ?=
GINKGO_SKIP ?=
SKIP_STR := $(foreach var, $(subst ., ,$(GINKGO_SKIP)),-skip "$(var)")
FOCUS_STR := $(foreach var, $(subst ., ,$(GINKGO_FOCUS)),-focus "$(var)")
VELERO_CLI ?=$$(pwd)/../../_output/bin/$(GOOS)/$(GOARCH)/velero
VELERO_IMAGE ?= velero/velero:main
VELERO_VERSION ?= $(VERSION)
@ -110,7 +111,7 @@ run: ginkgo
(echo "Bucket to store the backups from E2E tests is required, please re-run with BSL_BUCKET=<BucketName>"; exit 1 )
@[ "${CLOUD_PROVIDER}" ] && echo "Using cloud provider ${CLOUD_PROVIDER}" || \
(echo "Cloud provider for target cloud/plug-in provider is required, please rerun with CLOUD_PROVIDER=<aws,azure,kind,vsphere>"; exit 1)
@$(GINKGO) -v -focus="$(GINKGO_FOCUS)" $(SKIP_STR) . -- -velerocli=$(VELERO_CLI) \
@$(GINKGO) -v $(FOCUS_STR) $(SKIP_STR) . -- -velerocli=$(VELERO_CLI) \
-velero-image=$(VELERO_IMAGE) \
-plugins=$(PLUGINS) \
-velero-version=$(VELERO_VERSION) \

8
test/e2e/backups/schedule.go
View File

@ -25,7 +25,7 @@ type ScheduleBackup struct {
verifyTimes int
}
var ScheduleBackupTest func() = TestFunc(&ScheduleBackup{TestCase: TestCase{NSBaseName: "ns", NSIncluded: &[]string{"ns1"}}})
var ScheduleBackupTest func() = TestFunc(&ScheduleBackup{TestCase: TestCase{NSBaseName: "schedule-test-ns", NSIncluded: &[]string{"ns1"}}})
func (n *ScheduleBackup) Init() error {
n.Client = TestClientInstance
@ -45,7 +45,6 @@ func (n *ScheduleBackup) StartRun() error {
n.RestoreName = n.RestoreName + "restore-ns-mapping-" + UUIDgen.String()
n.ScheduleArgs = []string{
"schedule", "create", "--namespace", VeleroCfg.VeleroNamespace, n.ScheduleName,
"--include-namespaces", strings.Join(*n.NSIncluded, ","),
"--schedule=*/" + fmt.Sprintf("%v", n.Period) + " * * * *",
}
@ -78,7 +77,10 @@ func (n *ScheduleBackup) Backup() error {
now := time.Now().Minute()
triggerNow := now % n.Period
if triggerNow == 0 {
Expect(VeleroCmdExec(n.Ctx, VeleroCfg.VeleroCLI, n.ScheduleArgs)).To(Succeed())
Expect(VeleroScheduleCreate(n.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, n.ScheduleName, n.ScheduleArgs)).To(Succeed(), func() string {
RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, "", "")
return "Fail to restore workload"
})
break
}
}

32
test/e2e/basic/namespace-mapping.go
View File

@ -21,17 +21,24 @@ type NamespaceMapping struct {
kibishiiData *KibishiiData
}
var OneNamespaceMappingTest func() = TestFunc(&NamespaceMapping{TestCase: TestCase{NSBaseName: "ns", NSIncluded: &[]string{"ns1"}}})
var MultiNamespacesMappingTest func() = TestFunc(&NamespaceMapping{TestCase: TestCase{NSBaseName: "ns", NSIncluded: &[]string{"ns1", "ns2"}}})
const NamespaceBaseName string = "ns-mp-"
var OneNamespaceMappingResticTest func() = TestFunc(&NamespaceMapping{TestCase: TestCase{NSBaseName: NamespaceBaseName, NSIncluded: &[]string{NamespaceBaseName + "1"}, UseVolumeSnapshots: false}})
var MultiNamespacesMappingResticTest func() = TestFunc(&NamespaceMapping{TestCase: TestCase{NSBaseName: NamespaceBaseName, NSIncluded: &[]string{NamespaceBaseName + "1", NamespaceBaseName + "2"}, UseVolumeSnapshots: false}})
var OneNamespaceMappingSnapshotTest func() = TestFunc(&NamespaceMapping{TestCase: TestCase{NSBaseName: NamespaceBaseName, NSIncluded: &[]string{NamespaceBaseName + "1"}, UseVolumeSnapshots: true}})
var MultiNamespacesMappingSnapshotTest func() = TestFunc(&NamespaceMapping{TestCase: TestCase{NSBaseName: NamespaceBaseName, NSIncluded: &[]string{NamespaceBaseName + "1", NamespaceBaseName + "2"}, UseVolumeSnapshots: true}})
func (n *NamespaceMapping) Init() error {
n.Client = TestClientInstance
n.kibishiiData = &KibishiiData{2, 10, 10, 1024, 1024, 0, 2}
backupType := "restic"
if n.UseVolumeSnapshots {
backupType = "snapshot"
}
n.TestMsg = &TestMSG{
Desc: "Backup resources with include namespace test",
FailedMSG: "Failed to backup with namespace include",
Text: fmt.Sprintf("should backup namespaces %s", *n.NSIncluded),
Desc: fmt.Sprintf("Restore namespace %s with namespace mapping by %s test", *n.NSIncluded, backupType),
FailedMSG: "Failed to restore with namespace mapping",
Text: fmt.Sprintf("should restore namespace %s with namespace mapping by %s", *n.NSIncluded, backupType),
}
return nil
}
@ -49,15 +56,20 @@ func (n *NamespaceMapping) StartRun() error {
n.BackupName = n.BackupName + ns
n.RestoreName = n.RestoreName + ns
}
n.BackupName = n.BackupName + "backup-ns-mapping-" + UUIDgen.String()
n.RestoreName = n.RestoreName + "restore-ns-mapping-" + UUIDgen.String()
n.BackupName = n.BackupName + UUIDgen.String()
n.RestoreName = n.RestoreName + UUIDgen.String()
n.MappedNamespaceList = mappedNSList
fmt.Println(mappedNSList)
n.BackupArgs = []string{
"create", "--namespace", VeleroCfg.VeleroNamespace, "backup", n.BackupName,
"--include-namespaces", strings.Join(*n.NSIncluded, ","),
"--default-volumes-to-fs-backup", "--wait",
"--include-namespaces", strings.Join(*n.NSIncluded, ","), "--wait",
}
if n.UseVolumeSnapshots {
n.BackupArgs = append(n.BackupArgs, "--snapshot-volumes")
} else {
n.BackupArgs = append(n.BackupArgs, "--snapshot-volumes=false")
n.BackupArgs = append(n.BackupArgs, "--default-volumes-to-fs-backup")
}
n.RestoreArgs = []string{
"create", "--namespace", VeleroCfg.VeleroNamespace, "restore", n.RestoreName,

1
test/e2e/basic/resources-check/namespaces.go
View File

@ -121,6 +121,7 @@ func (m *MultiNSBackup) Verify() error {
}
func (m *MultiNSBackup) Destroy() error {
m.Ctx, _ = context.WithTimeout(context.Background(), 60*time.Minute)
err := CleanupNamespaces(m.Ctx, m.Client, m.NSBaseName)
if err != nil {
return errors.Wrap(err, "Could cleanup retrieve namespaces")

7
test/e2e/e2e_suite_test.go
View File

@ -37,6 +37,7 @@ import (
. "github.com/vmware-tanzu/velero/test/e2e/privilegesmgmt"
. "github.com/vmware-tanzu/velero/test/e2e/pv-backup"
. "github.com/vmware-tanzu/velero/test/e2e/resource-filtering"
. "github.com/vmware-tanzu/velero/test/e2e/scale"
. "github.com/vmware-tanzu/velero/test/e2e/upgrade"
@ -126,8 +127,10 @@ var _ = Describe("[Migration][Snapshot]", MigrationWithSnapshots)
var _ = Describe("[Schedule][OrederedResources] Backup resources should follow the specific order in schedule", ScheduleOrderedResources)
var _ = Describe("[NamespaceMapping][Single] Backup resources should follow the specific order in schedule", OneNamespaceMappingTest)
var _ = Describe("[NamespaceMapping][Multiple] Backup resources should follow the specific order in schedule", MultiNamespacesMappingTest)
var _ = Describe("[NamespaceMapping][Single][Restic] Backup resources should follow the specific order in schedule", OneNamespaceMappingResticTest)
var _ = Describe("[NamespaceMapping][Multiple][Restic] Backup resources should follow the specific order in schedule", MultiNamespacesMappingResticTest)
var _ = Describe("[NamespaceMapping][Single][Snapshot] Backup resources should follow the specific order in schedule", OneNamespaceMappingSnapshotTest)
var _ = Describe("[NamespaceMapping][Multiple][Snapshot] Backup resources should follow the specific order in schedule", MultiNamespacesMappingSnapshotTest)
var _ = Describe("[pv-backup][Opt-In] Backup resources should follow the specific order in schedule", OptInPVBackupTest)
var _ = Describe("[pv-backup][Opt-Out] Backup resources should follow the specific order in schedule", OptOutPVBackupTest)

12
test/e2e/migration/migration.go
View File

@ -126,6 +126,8 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
OriginVeleroCfg.Plugins = ""
//TODO: Remove this once origin Velero version is 1.10 and upper
OriginVeleroCfg.UploaderType = ""
OriginVeleroCfg.UseNodeAgent = false
OriginVeleroCfg.UseRestic = !useVolumeSnapshots
}
fmt.Println(OriginVeleroCfg)
Expect(VeleroInstall(context.Background(), &OriginVeleroCfg, useVolumeSnapshots)).To(Succeed())
@ -157,9 +159,9 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
BackupStorageClassCfg.IncludeResources = "StorageClass"
BackupStorageClassCfg.IncludeClusterResources = true
//TODO Remove UseRestic parameter once minor version is 1.10 or upper
BackupStorageClassCfg.UseRestic = true
BackupStorageClassCfg.UseResticIfFSBackup = true
if veleroCLI2Version.VeleroVersion == "self" {
BackupStorageClassCfg.UseRestic = false
BackupStorageClassCfg.UseResticIfFSBackup = false
}
Expect(VeleroBackupNamespace(context.Background(), OriginVeleroCfg.VeleroCLI,
@ -175,9 +177,9 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
BackupCfg.BackupLocation = ""
BackupCfg.Selector = ""
//TODO Remove UseRestic parameter once minor version is 1.10 or upper
BackupCfg.UseRestic = true
BackupCfg.UseResticIfFSBackup = true
if veleroCLI2Version.VeleroVersion == "self" {
BackupCfg.UseRestic = false
BackupCfg.UseResticIfFSBackup = false
}
Expect(VeleroBackupNamespace(context.Background(), OriginVeleroCfg.VeleroCLI,
OriginVeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string {
@ -235,6 +237,8 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
VeleroCfg.ObjectStoreProvider = ""
VeleroCfg.ClientToInstallVelero = VeleroCfg.StandbyClient
VeleroCfg.UseNodeAgent = !useVolumeSnapshots
VeleroCfg.UseRestic = false
Expect(VeleroInstall(context.Background(), &VeleroCfg, useVolumeSnapshots)).To(Succeed())
})

18
test/e2e/privilegesmgmt/ssr.go
View File

@ -62,16 +62,14 @@ func SSRTest() {
Expect(CreateNamespace(ctx, *VeleroCfg.ClientToInstallVelero, testNS)).To(Succeed(),
fmt.Sprintf("Failed to create %s namespace", testNS))
By(fmt.Sprintf("Get version in %s namespace", testNS))
cmd := []string{"version", "-n", testNS}
Expect(VeleroCmdExec(context.Background(), VeleroCfg.VeleroCLI, cmd)).To(Succeed(),
fmt.Sprintf("Failed to create an ssr object in the %s namespace", testNS))
By(fmt.Sprintf("Get version in %s namespace", VeleroCfg.VeleroNamespace))
cmd = []string{"version", "-n", VeleroCfg.VeleroNamespace}
Expect(VeleroCmdExec(context.Background(), VeleroCfg.VeleroCLI, cmd)).To(Succeed(),
fmt.Sprintf("Failed to create an ssr object in %s namespace", VeleroCfg.VeleroNamespace))
By(fmt.Sprintf("Get version in %s namespace", testNS), func() {
Expect(VeleroVersion(context.Background(), VeleroCfg.VeleroCLI, testNS)).To(Succeed(),
fmt.Sprintf("Failed to create an ssr object in the %s namespace", testNS))
})
By(fmt.Sprintf("Get version in %s namespace", VeleroCfg.VeleroNamespace), func() {
Expect(VeleroVersion(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To(Succeed(),
fmt.Sprintf("Failed to create an ssr object in %s namespace", VeleroCfg.VeleroNamespace))
})
ssrListResp := new(v1.ServerStatusRequestList)
By(fmt.Sprintf("Check ssr object in %s namespace", VeleroCfg.VeleroNamespace))
err = waitutil.PollImmediate(5*time.Second, time.Minute,

6
test/e2e/pv-backup/pv-backup-filter.go
View File

@ -49,8 +49,8 @@ func (p *PVBackupFiltering) StartRun() error {
if err != nil {
return err
}
p.BackupName = p.BackupName + "backup-opt-in-" + UUIDgen.String()
p.RestoreName = p.RestoreName + "restore-opt-in-" + UUIDgen.String()
p.BackupName = p.BackupName + "backup-" + p.id + "-" + UUIDgen.String()
p.RestoreName = p.RestoreName + "restore-" + p.id + "-" + UUIDgen.String()
p.BackupArgs = []string{
"create", "--namespace", VeleroCfg.VeleroNamespace, "backup", p.BackupName,
"--include-namespaces", strings.Join(*p.NSIncluded, ","),
@ -84,7 +84,7 @@ func (p *PVBackupFiltering) CreateResources() error {
for j := 0; j <= VOLUME_COUNT_PER_POD-1; j++ {
volume := fmt.Sprintf("volume-%s-%d-%d", p.id, i, j)
volumes = append(volumes, volume)
//Volumes cherry pick policy for opt-in/out annotation to pods
//Volumes cherry-pick policy for opt-in/out annotation to apply
if j%2 == 0 {
volumeToAnnotationList = append(volumeToAnnotationList, volume)
}

47
test/e2e/test/test.go
View File

@ -20,6 +20,7 @@ import (
"context"
"flag"
"fmt"
"time"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
@ -47,6 +48,7 @@ type VeleroBackupRestoreTest interface {
Verify() error
Clean() error
GetTestMsg() *TestMSG
GetTestCase() *TestCase
}
type TestMSG struct {
@ -56,16 +58,17 @@ type TestMSG struct {
}
type TestCase struct {
BackupName string
RestoreName string
NSBaseName string
BackupArgs []string
RestoreArgs []string
NamespacesTotal int
TestMsg *TestMSG
Client TestClient
Ctx context.Context
NSIncluded *[]string
BackupName string
RestoreName string
NSBaseName string
BackupArgs []string
RestoreArgs []string
NamespacesTotal int
TestMsg *TestMSG
Client TestClient
Ctx context.Context
NSIncluded *[]string
UseVolumeSnapshots bool
}
var TestClientInstance TestClient
@ -79,7 +82,7 @@ func TestFunc(test VeleroBackupRestoreTest) func() {
BeforeEach(func() {
flag.Parse()
if VeleroCfg.InstallVelero {
Expect(VeleroInstall(context.Background(), &VeleroCfg, false)).To(Succeed())
Expect(VeleroInstall(context.Background(), &VeleroCfg, test.GetTestCase().UseVolumeSnapshots)).To(Succeed())
}
})
AfterEach(func() {
@ -101,16 +104,17 @@ func TestFuncWithMultiIt(tests []VeleroBackupRestoreTest) func() {
By("Create test client instance", func() {
TestClientInstance = *VeleroCfg.ClientToInstallVelero
})
var useVolumeSnapshots bool
for k := range tests {
Expect(tests[k].Init()).To(Succeed(), fmt.Sprintf("Failed to instantiate test %s case", tests[k].GetTestMsg().Desc))
useVolumeSnapshots = tests[k].GetTestCase().UseVolumeSnapshots
}
BeforeEach(func() {
flag.Parse()
if VeleroCfg.InstallVelero {
if countIt == 0 {
Expect(VeleroInstall(context.Background(), &VeleroCfg, false)).To(Succeed())
Expect(VeleroInstall(context.Background(), &VeleroCfg, useVolumeSnapshots)).To(Succeed())
}
countIt++
}
@ -148,7 +152,7 @@ func (t *TestCase) StartRun() error {
}
func (t *TestCase) Backup() error {
if err := VeleroCmdExec(t.Ctx, VeleroCfg.VeleroCLI, t.BackupArgs); err != nil {
if err := VeleroBackupExec(t.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, t.BackupName, t.BackupArgs); err != nil {
RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, t.BackupName, "")
return errors.Wrapf(err, "Failed to backup resources")
}
@ -164,9 +168,17 @@ func (t *TestCase) Destroy() error {
}
func (t *TestCase) Restore() error {
// the snapshots of AWS may be still in pending status when do the restore, wait for a while
// to avoid this https://github.com/vmware-tanzu/velero/issues/1799
// TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed
if t.UseVolumeSnapshots {
fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...")
time.Sleep(5 * time.Minute)
}
By("Start to restore ......", func() {
Expect(VeleroCmdExec(t.Ctx, VeleroCfg.VeleroCLI, t.RestoreArgs)).To(Succeed(), func() string {
RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, t.BackupName, "")
Expect(VeleroRestoreExec(t.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, t.RestoreName, t.RestoreArgs)).To(Succeed(), func() string {
RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, "", t.RestoreName)
return "Fail to restore workload"
})
})
@ -193,6 +205,9 @@ func (t *TestCase) GetTestMsg() *TestMSG {
return t.TestMsg
}
func (t *TestCase) GetTestCase() *TestCase {
return t
}
func RunTestCase(test VeleroBackupRestoreTest) error {
fmt.Printf("Running test case %s\n", test.GetTestMsg().Desc)
if test == nil {

4
test/e2e/types.go
View File

@ -64,6 +64,8 @@ type VerleroConfig struct {
DefaultClient *TestClient
StandbyClient *TestClient
UploaderType string
UseNodeAgent bool
UseRestic bool
}
type SnapshotCheckPoint struct {
@ -87,7 +89,7 @@ type BackupConfig struct {
ExcludeResources string
IncludeClusterResources bool
OrderedResources string
UseRestic bool
UseResticIfFSBackup bool
}
type VeleroCLI2Version struct {

8
test/e2e/upgrade/upgrade.go
View File

@ -113,6 +113,8 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
tmpCfgForOldVeleroInstall.ResticHelperImage = ""
tmpCfgForOldVeleroInstall.Plugins = ""
tmpCfgForOldVeleroInstall.UploaderType = ""
tmpCfgForOldVeleroInstall.UseNodeAgent = false
tmpCfgForOldVeleroInstall.UseRestic = !useVolumeSnapshots
Expect(VeleroInstall(context.Background(), &tmpCfgForOldVeleroInstall,
useVolumeSnapshots)).To(Succeed())
@ -144,8 +146,8 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
BackupCfg.BackupLocation = ""
BackupCfg.UseVolumeSnapshots = useVolumeSnapshots
BackupCfg.Selector = ""
//TODO: pay attention to this param
BackupCfg.UseRestic = true
//TODO: pay attention to this param, remove it when restic is not the default backup tool any more.
BackupCfg.UseResticIfFSBackup = true
Expect(VeleroBackupNamespace(oneHourTimeout, tmpCfg.UpgradeFromVeleroCLI,
tmpCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string {
RunDebug(context.Background(), tmpCfg.UpgradeFromVeleroCLI, tmpCfg.VeleroNamespace,
@ -196,6 +198,8 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
By(fmt.Sprintf("Upgrade Velero by CLI %s", tmpCfg.VeleroCLI), func() {
tmpCfg.GCFrequency = ""
tmpCfg.UseNodeAgent = !useVolumeSnapshots
tmpCfg.UseRestic = false
Expect(VeleroInstall(context.Background(), &tmpCfg, useVolumeSnapshots)).To(Succeed())
Expect(CheckVeleroVersion(context.Background(), tmpCfg.VeleroCLI,
tmpCfg.VeleroVersion)).To(Succeed())

7
test/e2e/util/kibishii/kibishii_utils.go
View File

@ -120,12 +120,11 @@ func RunKibishiiTests(client TestClient, veleroCfg VerleroConfig, backupName, re
if err := DeleteNamespace(oneHourTimeout, client, kibishiiNamespace, true); err != nil {
return errors.Wrapf(err, "failed to delete namespace %s", kibishiiNamespace)
}
time.Sleep(5 * time.Minute)
// the snapshots of AWS may be still in pending status when do the restore, wait for a while
// to avoid this https://github.com/vmware-tanzu/velero/issues/1799
// TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed
if providerName == "aws" && useVolumeSnapshots {
if useVolumeSnapshots {
fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...")
time.Sleep(5 * time.Minute)
}
@ -138,7 +137,6 @@ func RunKibishiiTests(client TestClient, veleroCfg VerleroConfig, backupName, re
if err := KibishiiVerifyAfterRestore(client, kibishiiNamespace, oneHourTimeout, DefaultKibishiiData); err != nil {
return errors.Wrapf(err, "Error verifying kibishii after restore")
}
fmt.Printf("kibishii test completed successfully\n")
return nil
}
@ -151,8 +149,7 @@ func installKibishii(ctx context.Context, namespace string, cloudPlatform, veler
}
// We use kustomize to generate YAML for Kibishii from the checked-in yaml directories
kibishiiInstallCmd := exec.CommandContext(ctx, "kubectl", "apply", "-n", namespace, "-k",
kibishiiDirectory+cloudPlatform)
kibishiiDirectory+cloudPlatform, "--timeout=90s")
_, stderr, err := veleroexec.RunCommand(kibishiiInstallCmd)
fmt.Printf("Install Kibishii cmd: %s\n", kibishiiInstallCmd)
if err != nil {

4
test/e2e/util/providers/azure_utils.go
View File

@ -364,10 +364,10 @@ func (s AzureStorage) IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupN
snapshotCountFound := 0
backupNameInSnapshot := ""
if err != nil {
errors.Wrap(err, fmt.Sprintf("Fail to list snapshots %s\n", envVars[resourceGroupEnvVar]))
return errors.Wrap(err, fmt.Sprintf("Fail to list snapshots %s\n", envVars[resourceGroupEnvVar]))
}
if result.Value == nil {
errors.New(fmt.Sprintf("No snapshots in Azure resource group %s\n", envVars[resourceGroupEnvVar]))
return errors.New(fmt.Sprintf("No snapshots in Azure resource group %s\n", envVars[resourceGroupEnvVar]))
}
for _, v := range *result.Value {
if snapshotCheck.EnableCSI {

10
test/e2e/util/providers/common.go
View File

@ -46,7 +46,7 @@ func ObjectsShouldBeInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bsl
func ObjectsShouldNotBeInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string, retryTimes int) error {
var err error
var exist bool
fmt.Printf("|| VERIFICATION || - %s %s should not exist in storage %s\n", subPrefix, backupName, bslPrefix)
fmt.Printf("|| VERIFICATION || - %s %s should not exist in object store %s\n", subPrefix, backupName, bslPrefix)
for i := 0; i < retryTimes; i++ {
exist, err = IsObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix)
if err != nil {
@ -114,9 +114,9 @@ func SnapshotsShouldNotExistInCloud(cloudProvider, cloudCredentialsFile, bslBuck
snapshotCheckPoint.ExpectCount = 0
err := IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName, snapshotCheckPoint)
if err != nil {
return errors.Wrapf(err, fmt.Sprintf("|| UNEXPECTED ||Snapshots %s is existed in cloud after backup as expected", backupName))
return errors.Wrapf(err, fmt.Sprintf("|| UNEXPECTED ||Snapshots %s exist in cloud after backup as expected", backupName))
}
fmt.Printf("|| EXPECTED || - Snapshots are not existed in cloud, backup %s\n", backupName)
fmt.Printf("|| EXPECTED || - Snapshots do not exist in cloud, backup %s\n", backupName)
return nil
}
@ -124,9 +124,9 @@ func SnapshotsShouldBeCreatedInCloud(cloudProvider, cloudCredentialsFile, bslBuc
fmt.Printf("|| VERIFICATION || - Snapshots should exist in cloud, backup %s\n", backupName)
err := IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName, snapshotCheckPoint)
if err != nil {
return errors.Wrapf(err, fmt.Sprintf("|| UNEXPECTED ||Snapshots %s are not existed in cloud after backup as expected", backupName))
return errors.Wrapf(err, fmt.Sprintf("|| UNEXPECTED ||Snapshots %s do not exist in cloud after backup as expected", backupName))
}
fmt.Printf("|| EXPECTED || - Snapshots are existed in cloud, backup %s\n", backupName)
fmt.Printf("|| EXPECTED || - Snapshots exist in cloud, backup %s\n", backupName)
return nil
}

10
test/e2e/util/velero/install.go
View File

@ -87,7 +87,10 @@ func VeleroInstall(ctx context.Context, veleroCfg *VerleroConfig, useVolumeSnaps
return errors.WithMessagef(err, "Failed to get Velero InstallOptions for plugin provider %s", veleroCfg.ObjectStoreProvider)
}
veleroInstallOptions.UseVolumeSnapshots = useVolumeSnapshots
veleroInstallOptions.UseNodeAgent = !useVolumeSnapshots
if !veleroCfg.UseRestic {
veleroInstallOptions.UseNodeAgent = !useVolumeSnapshots
}
veleroInstallOptions.UseRestic = veleroCfg.UseRestic
veleroInstallOptions.Image = veleroCfg.VeleroImage
veleroInstallOptions.Namespace = veleroCfg.VeleroNamespace
veleroInstallOptions.UploaderType = veleroCfg.UploaderType
@ -175,6 +178,9 @@ func installVeleroServer(ctx context.Context, cli string, options *installOption
if options.UseNodeAgent {
args = append(args, "--use-node-agent")
}
if options.UseRestic {
args = append(args, "--use-restic")
}
if options.UseVolumeSnapshots {
args = append(args, "--use-volume-snapshots")
}
@ -285,7 +291,7 @@ func createVelereResources(ctx context.Context, cli, namespace string, args []st
cmd.Stderr = os.Stderr
fmt.Printf("Running cmd %q \n", cmd.String())
if err = cmd.Run(); err != nil {
return errors.Wrapf(err, "failed to apply velere resources")
return errors.Wrapf(err, "failed to apply Velero resources")
}
return nil

38
test/e2e/util/velero/velero_utils.go
View File

@ -323,7 +323,7 @@ func VeleroBackupNamespace(ctx context.Context, veleroCLI, veleroNamespace strin
if backupCfg.UseVolumeSnapshots {
args = append(args, "--snapshot-volumes")
} else {
if backupCfg.UseRestic {
if backupCfg.UseResticIfFSBackup {
args = append(args, "--default-volumes-to-restic")
} else {
args = append(args, "--default-volumes-to-fs-backup")
@ -399,6 +399,7 @@ func VeleroRestoreExec(ctx context.Context, veleroCLI, veleroNamespace, restoreN
if err := VeleroCmdExec(ctx, veleroCLI, args); err != nil {
return err
}
return checkRestorePhase(ctx, veleroCLI, veleroNamespace, restoreName, velerov1api.RestorePhaseCompleted)
}
@ -436,19 +437,14 @@ func VeleroScheduleCreate(ctx context.Context, veleroCLI string, veleroNamespace
func VeleroCmdExec(ctx context.Context, veleroCLI string, args []string) error {
cmd := exec.CommandContext(ctx, veleroCLI, args...)
var errBuf, outBuf bytes.Buffer
cmd.Stderr = io.MultiWriter(os.Stderr, &errBuf)
cmd.Stdout = io.MultiWriter(os.Stdout, &outBuf)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
fmt.Printf("velero cmd =%v\n", cmd)
err := cmd.Run()
retAll := outBuf.String() + " " + errBuf.String()
if strings.Contains(strings.ToLower(retAll), "failed") {
return errors.Wrap(err, fmt.Sprintf("velero cmd =%v return with failure\n", cmd))
if strings.Contains(fmt.Sprint(cmd.Stdout), "Failed") {
return errors.New(fmt.Sprintf("velero cmd =%v return with failure\n", cmd))
}
if err != nil {
return err
}
return nil
return err
}
func VeleroBackupLogs(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string) error {
@ -511,6 +507,16 @@ func VeleroCreateBackupLocation(ctx context.Context,
return VeleroCmdExec(ctx, veleroCLI, args)
}
func VeleroVersion(ctx context.Context, veleroCLI, veleroNamespace string) error {
args := []string{
"version", "--namespace", veleroNamespace,
}
if err := VeleroCmdExec(ctx, veleroCLI, args); err != nil {
return err
}
return nil
}
func getProviderPlugins(ctx context.Context, veleroCLI, objectStoreProvider, providerPlugins, feature string) ([]string, error) {
// Fetch the plugins for the provider before checking for the object store provider below.
var plugins []string
@ -788,12 +794,11 @@ func GetBackup(ctx context.Context, veleroCLI string, backupName string) (string
func IsBackupExist(ctx context.Context, veleroCLI string, backupName string) (bool, error) {
out, outerr, err := GetBackup(ctx, veleroCLI, backupName)
if err != nil {
if err != nil {
if strings.Contains(outerr, "not found") {
return false, nil
}
return false, err
if strings.Contains(outerr, "not found") {
fmt.Printf("Backup CR %s was not found\n", backupName)
return false, nil
}
return false, err
}
fmt.Printf("Backup <%s> exist locally according to output \n[%s]\n", backupName, out)
return true, nil
@ -807,6 +812,7 @@ func WaitBackupDeleted(ctx context.Context, veleroCLI string, backupName string,
if exist {
return false, nil
} else {
fmt.Printf("Backup %s does not exist\n", backupName)
return true, nil
}
}