vmware-tanzu
/
velero
mirror of https://github.com/vmware-tanzu/velero.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #5425 from danfengliu/fix-ctx-issue 

Fix E2E test and Git action CI issues
pull/5449/head^2
Xun Jiang/Bruce Jiang 2022-10-17 09:56:23 +08:00 committed by GitHub
parent c0430b8964 cf32cabddd
commit 7a535ea047
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 164 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.github/workflows/e2e-test-kind.yaml vendored
View File

@ -117,12 +117,17 @@ jobs:
aws_access_key_id=minio
aws_secret_access_key=minio123
EOF
# Match kubectl version to k8s server version
curl -LO https://dl.k8s.io/release/v${{ matrix.k8s }}/bin/linux/amd64/kubectl
sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
GOPATH=~/go CLOUD_PROVIDER=kind \
OBJECT_STORE_PROVIDER=aws BSL_CONFIG=region=minio,s3ForcePathStyle="true",s3Url=http://$(hostname -i):9000 \
CREDS_FILE=/tmp/credential BSL_BUCKET=bucket \
ADDITIONAL_OBJECT_STORE_PROVIDER=aws ADDITIONAL_BSL_CONFIG=region=minio,s3ForcePathStyle="true",s3Url=http://$(hostname -i):9000 \
ADDITIONAL_CREDS_FILE=/tmp/credential ADDITIONAL_BSL_BUCKET=additional-bucket \
GINKGO_FOCUS='Basic\].+\[ClusterResource' VELERO_IMAGE=velero:pr-test \
GINKGO_FOCUS='Basic\]\[ClusterResource' VELERO_IMAGE=velero:pr-test \
make -C test/e2e run
timeout-minutes: 30
- name: Upload debug bundle

2
pkg/cmd/cli/install/install.go
View File

@ -66,6 +66,8 @@ type InstallOptions struct {
BackupStorageConfig flag.Map
VolumeSnapshotConfig flag.Map
UseNodeAgent bool
//TODO remove UseRestic when migration test out of using it
UseRestic bool
Wait bool
UseVolumeSnapshots bool
DefaultRepoMaintenanceFrequency time.Duration

3
test/e2e/Makefile
View File

@ -48,6 +48,7 @@ OUTPUT_DIR := _output/$(GOOS)/$(GOARCH)/bin
GINKGO_FOCUS ?=
GINKGO_SKIP ?=
SKIP_STR := $(foreach var, $(subst ., ,$(GINKGO_SKIP)),-skip "$(var)")
FOCUS_STR := $(foreach var, $(subst ., ,$(GINKGO_FOCUS)),-focus "$(var)")
VELERO_CLI ?=$$(pwd)/../../_output/bin/$(GOOS)/$(GOARCH)/velero
VELERO_IMAGE ?= velero/velero:main
VELERO_VERSION ?= $(VERSION)
@ -110,7 +111,7 @@ run: ginkgo
(echo "Bucket to store the backups from E2E tests is required, please re-run with BSL_BUCKET=<BucketName>"; exit 1 )
@[ "${CLOUD_PROVIDER}" ] && echo "Using cloud provider ${CLOUD_PROVIDER}" || \
(echo "Cloud provider for target cloud/plug-in provider is required, please rerun with CLOUD_PROVIDER=<aws,azure,kind,vsphere>"; exit 1)
@$(GINKGO) -v -focus="$(GINKGO_FOCUS)" $(SKIP_STR) . -- -velerocli=$(VELERO_CLI) \
@$(GINKGO) -v $(FOCUS_STR) $(SKIP_STR) . -- -velerocli=$(VELERO_CLI) \
-velero-image=$(VELERO_IMAGE) \
-plugins=$(PLUGINS) \
-velero-version=$(VELERO_VERSION) \

8
test/e2e/backups/schedule.go
View File

@ -25,7 +25,7 @@ type ScheduleBackup struct {
verifyTimes int
}
var ScheduleBackupTest func() = TestFunc(&ScheduleBackup{TestCase: TestCase{NSBaseName: "ns", NSIncluded: &[]string{"ns1"}}})
var ScheduleBackupTest func() = TestFunc(&ScheduleBackup{TestCase: TestCase{NSBaseName: "schedule-test-ns", NSIncluded: &[]string{"ns1"}}})
func (n *ScheduleBackup) Init() error {
n.Client = TestClientInstance
@ -45,7 +45,6 @@ func (n *ScheduleBackup) StartRun() error {
n.RestoreName = n.RestoreName + "restore-ns-mapping-" + UUIDgen.String()
n.ScheduleArgs = []string{
"schedule", "create", "--namespace", VeleroCfg.VeleroNamespace, n.ScheduleName,
"--include-namespaces", strings.Join(*n.NSIncluded, ","),
"--schedule=*/" + fmt.Sprintf("%v", n.Period) + " * * * *",
}
@ -78,7 +77,10 @@ func (n *ScheduleBackup) Backup() error {
now := time.Now().Minute()
triggerNow := now % n.Period
if triggerNow == 0 {
Expect(VeleroCmdExec(n.Ctx, VeleroCfg.VeleroCLI, n.ScheduleArgs)).To(Succeed())
Expect(VeleroScheduleCreate(n.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, n.ScheduleName, n.ScheduleArgs)).To(Succeed(), func() string {
RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, "", "")
return "Fail to restore workload"
})
break
}
}

32
test/e2e/basic/namespace-mapping.go
View File

@ -21,17 +21,24 @@ type NamespaceMapping struct {
kibishiiData *KibishiiData
}
var OneNamespaceMappingTest func() = TestFunc(&NamespaceMapping{TestCase: TestCase{NSBaseName: "ns", NSIncluded: &[]string{"ns1"}}})
var MultiNamespacesMappingTest func() = TestFunc(&NamespaceMapping{TestCase: TestCase{NSBaseName: "ns", NSIncluded: &[]string{"ns1", "ns2"}}})
const NamespaceBaseName string = "ns-mp-"
var OneNamespaceMappingResticTest func() = TestFunc(&NamespaceMapping{TestCase: TestCase{NSBaseName: NamespaceBaseName, NSIncluded: &[]string{NamespaceBaseName + "1"}, UseVolumeSnapshots: false}})
var MultiNamespacesMappingResticTest func() = TestFunc(&NamespaceMapping{TestCase: TestCase{NSBaseName: NamespaceBaseName, NSIncluded: &[]string{NamespaceBaseName + "1", NamespaceBaseName + "2"}, UseVolumeSnapshots: false}})
var OneNamespaceMappingSnapshotTest func() = TestFunc(&NamespaceMapping{TestCase: TestCase{NSBaseName: NamespaceBaseName, NSIncluded: &[]string{NamespaceBaseName + "1"}, UseVolumeSnapshots: true}})
var MultiNamespacesMappingSnapshotTest func() = TestFunc(&NamespaceMapping{TestCase: TestCase{NSBaseName: NamespaceBaseName, NSIncluded: &[]string{NamespaceBaseName + "1", NamespaceBaseName + "2"}, UseVolumeSnapshots: true}})
func (n *NamespaceMapping) Init() error {
n.Client = TestClientInstance
n.kibishiiData = &KibishiiData{2, 10, 10, 1024, 1024, 0, 2}
backupType := "restic"
if n.UseVolumeSnapshots {
backupType = "snapshot"
}
n.TestMsg = &TestMSG{
Desc: "Backup resources with include namespace test",
FailedMSG: "Failed to backup with namespace include",
Text: fmt.Sprintf("should backup namespaces %s", *n.NSIncluded),
Desc: fmt.Sprintf("Restore namespace %s with namespace mapping by %s test", *n.NSIncluded, backupType),
FailedMSG: "Failed to restore with namespace mapping",
Text: fmt.Sprintf("should restore namespace %s with namespace mapping by %s", *n.NSIncluded, backupType),
}
return nil
}
@ -49,15 +56,20 @@ func (n *NamespaceMapping) StartRun() error {
n.BackupName = n.BackupName + ns
n.RestoreName = n.RestoreName + ns
}
n.BackupName = n.BackupName + "backup-ns-mapping-" + UUIDgen.String()
n.RestoreName = n.RestoreName + "restore-ns-mapping-" + UUIDgen.String()
n.BackupName = n.BackupName + UUIDgen.String()
n.RestoreName = n.RestoreName + UUIDgen.String()
n.MappedNamespaceList = mappedNSList
fmt.Println(mappedNSList)
n.BackupArgs = []string{
"create", "--namespace", VeleroCfg.VeleroNamespace, "backup", n.BackupName,
"--include-namespaces", strings.Join(*n.NSIncluded, ","),
"--default-volumes-to-fs-backup", "--wait",
"--include-namespaces", strings.Join(*n.NSIncluded, ","), "--wait",
}
if n.UseVolumeSnapshots {
n.BackupArgs = append(n.BackupArgs, "--snapshot-volumes")
} else {
n.BackupArgs = append(n.BackupArgs, "--snapshot-volumes=false")
n.BackupArgs = append(n.BackupArgs, "--default-volumes-to-fs-backup")
}
n.RestoreArgs = []string{
"create", "--namespace", VeleroCfg.VeleroNamespace, "restore", n.RestoreName,

1
test/e2e/basic/resources-check/namespaces.go
View File

@ -121,6 +121,7 @@ func (m *MultiNSBackup) Verify() error {
}
func (m *MultiNSBackup) Destroy() error {
m.Ctx, _ = context.WithTimeout(context.Background(), 60*time.Minute)
err := CleanupNamespaces(m.Ctx, m.Client, m.NSBaseName)
if err != nil {
return errors.Wrap(err, "Could cleanup retrieve namespaces")

7
test/e2e/e2e_suite_test.go
View File

@ -37,6 +37,7 @@ import (
. "github.com/vmware-tanzu/velero/test/e2e/privilegesmgmt"
. "github.com/vmware-tanzu/velero/test/e2e/pv-backup"
. "github.com/vmware-tanzu/velero/test/e2e/resource-filtering"
. "github.com/vmware-tanzu/velero/test/e2e/scale"
. "github.com/vmware-tanzu/velero/test/e2e/upgrade"
@ -126,8 +127,10 @@ var _ = Describe("[Migration][Snapshot]", MigrationWithSnapshots)
var _ = Describe("[Schedule][OrederedResources] Backup resources should follow the specific order in schedule", ScheduleOrderedResources)
var _ = Describe("[NamespaceMapping][Single] Backup resources should follow the specific order in schedule", OneNamespaceMappingTest)
var _ = Describe("[NamespaceMapping][Multiple] Backup resources should follow the specific order in schedule", MultiNamespacesMappingTest)
var _ = Describe("[NamespaceMapping][Single][Restic] Backup resources should follow the specific order in schedule", OneNamespaceMappingResticTest)
var _ = Describe("[NamespaceMapping][Multiple][Restic] Backup resources should follow the specific order in schedule", MultiNamespacesMappingResticTest)
var _ = Describe("[NamespaceMapping][Single][Snapshot] Backup resources should follow the specific order in schedule", OneNamespaceMappingSnapshotTest)
var _ = Describe("[NamespaceMapping][Multiple][Snapshot] Backup resources should follow the specific order in schedule", MultiNamespacesMappingSnapshotTest)
var _ = Describe("[pv-backup][Opt-In] Backup resources should follow the specific order in schedule", OptInPVBackupTest)
var _ = Describe("[pv-backup][Opt-Out] Backup resources should follow the specific order in schedule", OptOutPVBackupTest)

12
test/e2e/migration/migration.go
View File

@ -126,6 +126,8 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
OriginVeleroCfg.Plugins = ""
//TODO: Remove this once origin Velero version is 1.10 and upper
OriginVeleroCfg.UploaderType = ""
OriginVeleroCfg.UseNodeAgent = false
OriginVeleroCfg.UseRestic = !useVolumeSnapshots
}
fmt.Println(OriginVeleroCfg)
Expect(VeleroInstall(context.Background(), &OriginVeleroCfg, useVolumeSnapshots)).To(Succeed())
@ -157,9 +159,9 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
BackupStorageClassCfg.IncludeResources = "StorageClass"
BackupStorageClassCfg.IncludeClusterResources = true
//TODO Remove UseRestic parameter once minor version is 1.10 or upper
BackupStorageClassCfg.UseRestic = true
BackupStorageClassCfg.UseResticIfFSBackup = true
if veleroCLI2Version.VeleroVersion == "self" {
BackupStorageClassCfg.UseRestic = false
BackupStorageClassCfg.UseResticIfFSBackup = false
}
Expect(VeleroBackupNamespace(context.Background(), OriginVeleroCfg.VeleroCLI,
@ -175,9 +177,9 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
BackupCfg.BackupLocation = ""
BackupCfg.Selector = ""
//TODO Remove UseRestic parameter once minor version is 1.10 or upper
BackupCfg.UseRestic = true
BackupCfg.UseResticIfFSBackup = true
if veleroCLI2Version.VeleroVersion == "self" {
BackupCfg.UseRestic = false
BackupCfg.UseResticIfFSBackup = false
}
Expect(VeleroBackupNamespace(context.Background(), OriginVeleroCfg.VeleroCLI,
OriginVeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string {
@ -235,6 +237,8 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
VeleroCfg.ObjectStoreProvider = ""
VeleroCfg.ClientToInstallVelero = VeleroCfg.StandbyClient
VeleroCfg.UseNodeAgent = !useVolumeSnapshots
VeleroCfg.UseRestic = false
Expect(VeleroInstall(context.Background(), &VeleroCfg, useVolumeSnapshots)).To(Succeed())
})

14
test/e2e/privilegesmgmt/ssr.go
View File

@ -62,16 +62,14 @@ func SSRTest() {
Expect(CreateNamespace(ctx, *VeleroCfg.ClientToInstallVelero, testNS)).To(Succeed(),
fmt.Sprintf("Failed to create %s namespace", testNS))
By(fmt.Sprintf("Get version in %s namespace", testNS))
cmd := []string{"version", "-n", testNS}
Expect(VeleroCmdExec(context.Background(), VeleroCfg.VeleroCLI, cmd)).To(Succeed(),
By(fmt.Sprintf("Get version in %s namespace", testNS), func() {
Expect(VeleroVersion(context.Background(), VeleroCfg.VeleroCLI, testNS)).To(Succeed(),
fmt.Sprintf("Failed to create an ssr object in the %s namespace", testNS))
By(fmt.Sprintf("Get version in %s namespace", VeleroCfg.VeleroNamespace))
cmd = []string{"version", "-n", VeleroCfg.VeleroNamespace}
Expect(VeleroCmdExec(context.Background(), VeleroCfg.VeleroCLI, cmd)).To(Succeed(),
})
By(fmt.Sprintf("Get version in %s namespace", VeleroCfg.VeleroNamespace), func() {
Expect(VeleroVersion(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To(Succeed(),
fmt.Sprintf("Failed to create an ssr object in %s namespace", VeleroCfg.VeleroNamespace))
})
ssrListResp := new(v1.ServerStatusRequestList)
By(fmt.Sprintf("Check ssr object in %s namespace", VeleroCfg.VeleroNamespace))
err = waitutil.PollImmediate(5*time.Second, time.Minute,

6
test/e2e/pv-backup/pv-backup-filter.go
View File

@ -49,8 +49,8 @@ func (p *PVBackupFiltering) StartRun() error {
if err != nil {
return err
}
p.BackupName = p.BackupName + "backup-opt-in-" + UUIDgen.String()
p.RestoreName = p.RestoreName + "restore-opt-in-" + UUIDgen.String()
p.BackupName = p.BackupName + "backup-" + p.id + "-" + UUIDgen.String()
p.RestoreName = p.RestoreName + "restore-" + p.id + "-" + UUIDgen.String()
p.BackupArgs = []string{
"create", "--namespace", VeleroCfg.VeleroNamespace, "backup", p.BackupName,
"--include-namespaces", strings.Join(*p.NSIncluded, ","),
@ -84,7 +84,7 @@ func (p *PVBackupFiltering) CreateResources() error {
for j := 0; j <= VOLUME_COUNT_PER_POD-1; j++ {
volume := fmt.Sprintf("volume-%s-%d-%d", p.id, i, j)
volumes = append(volumes, volume)
//Volumes cherry pick policy for opt-in/out annotation to pods
//Volumes cherry-pick policy for opt-in/out annotation to apply
if j%2 == 0 {
volumeToAnnotationList = append(volumeToAnnotationList, volume)
}

27
test/e2e/test/test.go
View File

@ -20,6 +20,7 @@ import (
"context"
"flag"
"fmt"
"time"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
@ -47,6 +48,7 @@ type VeleroBackupRestoreTest interface {
Verify() error
Clean() error
GetTestMsg() *TestMSG
GetTestCase() *TestCase
}
type TestMSG struct {
@ -66,6 +68,7 @@ type TestCase struct {
Client TestClient
Ctx context.Context
NSIncluded *[]string
UseVolumeSnapshots bool
}
var TestClientInstance TestClient
@ -79,7 +82,7 @@ func TestFunc(test VeleroBackupRestoreTest) func() {
BeforeEach(func() {
flag.Parse()
if VeleroCfg.InstallVelero {
Expect(VeleroInstall(context.Background(), &VeleroCfg, false)).To(Succeed())
Expect(VeleroInstall(context.Background(), &VeleroCfg, test.GetTestCase().UseVolumeSnapshots)).To(Succeed())
}
})
AfterEach(func() {
@ -101,16 +104,17 @@ func TestFuncWithMultiIt(tests []VeleroBackupRestoreTest) func() {
By("Create test client instance", func() {
TestClientInstance = *VeleroCfg.ClientToInstallVelero
})
var useVolumeSnapshots bool
for k := range tests {
Expect(tests[k].Init()).To(Succeed(), fmt.Sprintf("Failed to instantiate test %s case", tests[k].GetTestMsg().Desc))
useVolumeSnapshots = tests[k].GetTestCase().UseVolumeSnapshots
}
BeforeEach(func() {
flag.Parse()
if VeleroCfg.InstallVelero {
if countIt == 0 {
Expect(VeleroInstall(context.Background(), &VeleroCfg, false)).To(Succeed())
Expect(VeleroInstall(context.Background(), &VeleroCfg, useVolumeSnapshots)).To(Succeed())
}
countIt++
}
@ -148,7 +152,7 @@ func (t *TestCase) StartRun() error {
}
func (t *TestCase) Backup() error {
if err := VeleroCmdExec(t.Ctx, VeleroCfg.VeleroCLI, t.BackupArgs); err != nil {
if err := VeleroBackupExec(t.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, t.BackupName, t.BackupArgs); err != nil {
RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, t.BackupName, "")
return errors.Wrapf(err, "Failed to backup resources")
}
@ -164,9 +168,17 @@ func (t *TestCase) Destroy() error {
}
func (t *TestCase) Restore() error {
// the snapshots of AWS may be still in pending status when do the restore, wait for a while
// to avoid this https://github.com/vmware-tanzu/velero/issues/1799
// TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed
if t.UseVolumeSnapshots {
fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...")
time.Sleep(5 * time.Minute)
}
By("Start to restore ......", func() {
Expect(VeleroCmdExec(t.Ctx, VeleroCfg.VeleroCLI, t.RestoreArgs)).To(Succeed(), func() string {
RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, t.BackupName, "")
Expect(VeleroRestoreExec(t.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, t.RestoreName, t.RestoreArgs)).To(Succeed(), func() string {
RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, "", t.RestoreName)
return "Fail to restore workload"
})
})
@ -193,6 +205,9 @@ func (t *TestCase) GetTestMsg() *TestMSG {
return t.TestMsg
}
func (t *TestCase) GetTestCase() *TestCase {
return t
}
func RunTestCase(test VeleroBackupRestoreTest) error {
fmt.Printf("Running test case %s\n", test.GetTestMsg().Desc)
if test == nil {

4
test/e2e/types.go
View File

@ -64,6 +64,8 @@ type VerleroConfig struct {
DefaultClient *TestClient
StandbyClient *TestClient
UploaderType string
UseNodeAgent bool
UseRestic bool
}
type SnapshotCheckPoint struct {
@ -87,7 +89,7 @@ type BackupConfig struct {
ExcludeResources string
IncludeClusterResources bool
OrderedResources string
UseRestic bool
UseResticIfFSBackup bool
}
type VeleroCLI2Version struct {

8
test/e2e/upgrade/upgrade.go
View File

@ -113,6 +113,8 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
tmpCfgForOldVeleroInstall.ResticHelperImage = ""
tmpCfgForOldVeleroInstall.Plugins = ""
tmpCfgForOldVeleroInstall.UploaderType = ""
tmpCfgForOldVeleroInstall.UseNodeAgent = false
tmpCfgForOldVeleroInstall.UseRestic = !useVolumeSnapshots
Expect(VeleroInstall(context.Background(), &tmpCfgForOldVeleroInstall,
useVolumeSnapshots)).To(Succeed())
@ -144,8 +146,8 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
BackupCfg.BackupLocation = ""
BackupCfg.UseVolumeSnapshots = useVolumeSnapshots
BackupCfg.Selector = ""
//TODO: pay attention to this param
BackupCfg.UseRestic = true
//TODO: pay attention to this param, remove it when restic is not the default backup tool any more.
BackupCfg.UseResticIfFSBackup = true
Expect(VeleroBackupNamespace(oneHourTimeout, tmpCfg.UpgradeFromVeleroCLI,
tmpCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string {
RunDebug(context.Background(), tmpCfg.UpgradeFromVeleroCLI, tmpCfg.VeleroNamespace,
@ -196,6 +198,8 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
By(fmt.Sprintf("Upgrade Velero by CLI %s", tmpCfg.VeleroCLI), func() {
tmpCfg.GCFrequency = ""
tmpCfg.UseNodeAgent = !useVolumeSnapshots
tmpCfg.UseRestic = false
Expect(VeleroInstall(context.Background(), &tmpCfg, useVolumeSnapshots)).To(Succeed())
Expect(CheckVeleroVersion(context.Background(), tmpCfg.VeleroCLI,
tmpCfg.VeleroVersion)).To(Succeed())

7
test/e2e/util/kibishii/kibishii_utils.go
View File

@ -120,12 +120,11 @@ func RunKibishiiTests(client TestClient, veleroCfg VerleroConfig, backupName, re
if err := DeleteNamespace(oneHourTimeout, client, kibishiiNamespace, true); err != nil {
return errors.Wrapf(err, "failed to delete namespace %s", kibishiiNamespace)
}
time.Sleep(5 * time.Minute)
// the snapshots of AWS may be still in pending status when do the restore, wait for a while
// to avoid this https://github.com/vmware-tanzu/velero/issues/1799
// TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed
if providerName == "aws" && useVolumeSnapshots {
if useVolumeSnapshots {
fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...")
time.Sleep(5 * time.Minute)
}
@ -138,7 +137,6 @@ func RunKibishiiTests(client TestClient, veleroCfg VerleroConfig, backupName, re
if err := KibishiiVerifyAfterRestore(client, kibishiiNamespace, oneHourTimeout, DefaultKibishiiData); err != nil {
return errors.Wrapf(err, "Error verifying kibishii after restore")
}
fmt.Printf("kibishii test completed successfully\n")
return nil
}
@ -151,8 +149,7 @@ func installKibishii(ctx context.Context, namespace string, cloudPlatform, veler
}
// We use kustomize to generate YAML for Kibishii from the checked-in yaml directories
kibishiiInstallCmd := exec.CommandContext(ctx, "kubectl", "apply", "-n", namespace, "-k",
kibishiiDirectory+cloudPlatform)
kibishiiDirectory+cloudPlatform, "--timeout=90s")
_, stderr, err := veleroexec.RunCommand(kibishiiInstallCmd)
fmt.Printf("Install Kibishii cmd: %s\n", kibishiiInstallCmd)
if err != nil {

4
test/e2e/util/providers/azure_utils.go
View File

@ -364,10 +364,10 @@ func (s AzureStorage) IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupN
snapshotCountFound := 0
backupNameInSnapshot := ""
if err != nil {
errors.Wrap(err, fmt.Sprintf("Fail to list snapshots %s\n", envVars[resourceGroupEnvVar]))
return errors.Wrap(err, fmt.Sprintf("Fail to list snapshots %s\n", envVars[resourceGroupEnvVar]))
}
if result.Value == nil {
errors.New(fmt.Sprintf("No snapshots in Azure resource group %s\n", envVars[resourceGroupEnvVar]))
return errors.New(fmt.Sprintf("No snapshots in Azure resource group %s\n", envVars[resourceGroupEnvVar]))
}
for _, v := range *result.Value {
if snapshotCheck.EnableCSI {

10
test/e2e/util/providers/common.go
View File

@ -46,7 +46,7 @@ func ObjectsShouldBeInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bsl
func ObjectsShouldNotBeInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string, retryTimes int) error {
var err error
var exist bool
fmt.Printf("|| VERIFICATION || - %s %s should not exist in storage %s\n", subPrefix, backupName, bslPrefix)
fmt.Printf("|| VERIFICATION || - %s %s should not exist in object store %s\n", subPrefix, backupName, bslPrefix)
for i := 0; i < retryTimes; i++ {
exist, err = IsObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix)
if err != nil {
@ -114,9 +114,9 @@ func SnapshotsShouldNotExistInCloud(cloudProvider, cloudCredentialsFile, bslBuck
snapshotCheckPoint.ExpectCount = 0
err := IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName, snapshotCheckPoint)
if err != nil {
return errors.Wrapf(err, fmt.Sprintf("|| UNEXPECTED ||Snapshots %s is existed in cloud after backup as expected", backupName))
return errors.Wrapf(err, fmt.Sprintf("|| UNEXPECTED ||Snapshots %s exist in cloud after backup as expected", backupName))
}
fmt.Printf("|| EXPECTED || - Snapshots are not existed in cloud, backup %s\n", backupName)
fmt.Printf("|| EXPECTED || - Snapshots do not exist in cloud, backup %s\n", backupName)
return nil
}
@ -124,9 +124,9 @@ func SnapshotsShouldBeCreatedInCloud(cloudProvider, cloudCredentialsFile, bslBuc
fmt.Printf("|| VERIFICATION || - Snapshots should exist in cloud, backup %s\n", backupName)
err := IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName, snapshotCheckPoint)
if err != nil {
return errors.Wrapf(err, fmt.Sprintf("|| UNEXPECTED ||Snapshots %s are not existed in cloud after backup as expected", backupName))
return errors.Wrapf(err, fmt.Sprintf("|| UNEXPECTED ||Snapshots %s do not exist in cloud after backup as expected", backupName))
}
fmt.Printf("|| EXPECTED || - Snapshots are existed in cloud, backup %s\n", backupName)
fmt.Printf("|| EXPECTED || - Snapshots exist in cloud, backup %s\n", backupName)
return nil
}

8
test/e2e/util/velero/install.go
View File

@ -87,7 +87,10 @@ func VeleroInstall(ctx context.Context, veleroCfg *VerleroConfig, useVolumeSnaps
return errors.WithMessagef(err, "Failed to get Velero InstallOptions for plugin provider %s", veleroCfg.ObjectStoreProvider)
}
veleroInstallOptions.UseVolumeSnapshots = useVolumeSnapshots
if !veleroCfg.UseRestic {
veleroInstallOptions.UseNodeAgent = !useVolumeSnapshots
}
veleroInstallOptions.UseRestic = veleroCfg.UseRestic
veleroInstallOptions.Image = veleroCfg.VeleroImage
veleroInstallOptions.Namespace = veleroCfg.VeleroNamespace
veleroInstallOptions.UploaderType = veleroCfg.UploaderType
@ -175,6 +178,9 @@ func installVeleroServer(ctx context.Context, cli string, options *installOption
if options.UseNodeAgent {
args = append(args, "--use-node-agent")
}
if options.UseRestic {
args = append(args, "--use-restic")
}
if options.UseVolumeSnapshots {
args = append(args, "--use-volume-snapshots")
}
@ -285,7 +291,7 @@ func createVelereResources(ctx context.Context, cli, namespace string, args []st
cmd.Stderr = os.Stderr
fmt.Printf("Running cmd %q \n", cmd.String())
if err = cmd.Run(); err != nil {
return errors.Wrapf(err, "failed to apply velere resources")
return errors.Wrapf(err, "failed to apply Velero resources")
}
return nil

30
test/e2e/util/velero/velero_utils.go
View File

@ -323,7 +323,7 @@ func VeleroBackupNamespace(ctx context.Context, veleroCLI, veleroNamespace strin
if backupCfg.UseVolumeSnapshots {
args = append(args, "--snapshot-volumes")
} else {
if backupCfg.UseRestic {
if backupCfg.UseResticIfFSBackup {
args = append(args, "--default-volumes-to-restic")
} else {
args = append(args, "--default-volumes-to-fs-backup")
@ -399,6 +399,7 @@ func VeleroRestoreExec(ctx context.Context, veleroCLI, veleroNamespace, restoreN
if err := VeleroCmdExec(ctx, veleroCLI, args); err != nil {
return err
}
return checkRestorePhase(ctx, veleroCLI, veleroNamespace, restoreName, velerov1api.RestorePhaseCompleted)
}
@ -436,20 +437,15 @@ func VeleroScheduleCreate(ctx context.Context, veleroCLI string, veleroNamespace
func VeleroCmdExec(ctx context.Context, veleroCLI string, args []string) error {
cmd := exec.CommandContext(ctx, veleroCLI, args...)
var errBuf, outBuf bytes.Buffer
cmd.Stderr = io.MultiWriter(os.Stderr, &errBuf)
cmd.Stdout = io.MultiWriter(os.Stdout, &outBuf)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
fmt.Printf("velero cmd =%v\n", cmd)
err := cmd.Run()
retAll := outBuf.String() + " " + errBuf.String()
if strings.Contains(strings.ToLower(retAll), "failed") {
return errors.Wrap(err, fmt.Sprintf("velero cmd =%v return with failure\n", cmd))
if strings.Contains(fmt.Sprint(cmd.Stdout), "Failed") {
return errors.New(fmt.Sprintf("velero cmd =%v return with failure\n", cmd))
}
if err != nil {
return err
}
return nil
}
func VeleroBackupLogs(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string) error {
args := []string{
@ -511,6 +507,16 @@ func VeleroCreateBackupLocation(ctx context.Context,
return VeleroCmdExec(ctx, veleroCLI, args)
}
func VeleroVersion(ctx context.Context, veleroCLI, veleroNamespace string) error {
args := []string{
"version", "--namespace", veleroNamespace,
}
if err := VeleroCmdExec(ctx, veleroCLI, args); err != nil {
return err
}
return nil
}
func getProviderPlugins(ctx context.Context, veleroCLI, objectStoreProvider, providerPlugins, feature string) ([]string, error) {
// Fetch the plugins for the provider before checking for the object store provider below.
var plugins []string
@ -787,14 +793,13 @@ func GetBackup(ctx context.Context, veleroCLI string, backupName string) (string
func IsBackupExist(ctx context.Context, veleroCLI string, backupName string) (bool, error) {
out, outerr, err := GetBackup(ctx, veleroCLI, backupName)
if err != nil {
if err != nil {
if strings.Contains(outerr, "not found") {
fmt.Printf("Backup CR %s was not found\n", backupName)
return false, nil
}
return false, err
}
}
fmt.Printf("Backup <%s> exist locally according to output \n[%s]\n", backupName, out)
return true, nil
}
@ -807,6 +812,7 @@ func WaitBackupDeleted(ctx context.Context, veleroCLI string, backupName string,
if exist {
return false, nil
} else {
fmt.Printf("Backup %s does not exist\n", backupName)
return true, nil
}
}