diff --git a/.github/workflows/e2e-test-kind.yaml b/.github/workflows/e2e-test-kind.yaml index 3f079d333..1a260c1b0 100644 --- a/.github/workflows/e2e-test-kind.yaml +++ b/.github/workflows/e2e-test-kind.yaml @@ -117,12 +117,17 @@ jobs: aws_access_key_id=minio aws_secret_access_key=minio123 EOF + + # Match kubectl version to k8s server version + curl -LO https://dl.k8s.io/release/v${{ matrix.k8s }}/bin/linux/amd64/kubectl + sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl + GOPATH=~/go CLOUD_PROVIDER=kind \ OBJECT_STORE_PROVIDER=aws BSL_CONFIG=region=minio,s3ForcePathStyle="true",s3Url=http://$(hostname -i):9000 \ CREDS_FILE=/tmp/credential BSL_BUCKET=bucket \ ADDITIONAL_OBJECT_STORE_PROVIDER=aws ADDITIONAL_BSL_CONFIG=region=minio,s3ForcePathStyle="true",s3Url=http://$(hostname -i):9000 \ ADDITIONAL_CREDS_FILE=/tmp/credential ADDITIONAL_BSL_BUCKET=additional-bucket \ - GINKGO_FOCUS='Basic\].+\[ClusterResource' VELERO_IMAGE=velero:pr-test \ + GINKGO_FOCUS='Basic\]\[ClusterResource' VELERO_IMAGE=velero:pr-test \ make -C test/e2e run timeout-minutes: 30 - name: Upload debug bundle diff --git a/pkg/cmd/cli/install/install.go b/pkg/cmd/cli/install/install.go index f9cffa48f..a7782aa1e 100644 --- a/pkg/cmd/cli/install/install.go +++ b/pkg/cmd/cli/install/install.go @@ -43,29 +43,31 @@ import ( // InstallOptions collects all the options for installing Velero into a Kubernetes cluster. type InstallOptions struct { - Namespace string - Image string - BucketName string - Prefix string - ProviderName string - PodAnnotations flag.Map - PodLabels flag.Map - ServiceAccountAnnotations flag.Map - VeleroPodCPURequest string - VeleroPodMemRequest string - VeleroPodCPULimit string - VeleroPodMemLimit string - NodeAgentPodCPURequest string - NodeAgentPodMemRequest string - NodeAgentPodCPULimit string - NodeAgentPodMemLimit string - RestoreOnly bool - SecretFile string - NoSecret bool - DryRun bool - BackupStorageConfig flag.Map - VolumeSnapshotConfig flag.Map - UseNodeAgent bool + Namespace string + Image string + BucketName string + Prefix string + ProviderName string + PodAnnotations flag.Map + PodLabels flag.Map + ServiceAccountAnnotations flag.Map + VeleroPodCPURequest string + VeleroPodMemRequest string + VeleroPodCPULimit string + VeleroPodMemLimit string + NodeAgentPodCPURequest string + NodeAgentPodMemRequest string + NodeAgentPodCPULimit string + NodeAgentPodMemLimit string + RestoreOnly bool + SecretFile string + NoSecret bool + DryRun bool + BackupStorageConfig flag.Map + VolumeSnapshotConfig flag.Map + UseNodeAgent bool + //TODO remove UseRestic when migration test out of using it + UseRestic bool Wait bool UseVolumeSnapshots bool DefaultRepoMaintenanceFrequency time.Duration diff --git a/test/e2e/Makefile b/test/e2e/Makefile index 8f703cc00..6b76b8ad1 100644 --- a/test/e2e/Makefile +++ b/test/e2e/Makefile @@ -48,6 +48,7 @@ OUTPUT_DIR := _output/$(GOOS)/$(GOARCH)/bin GINKGO_FOCUS ?= GINKGO_SKIP ?= SKIP_STR := $(foreach var, $(subst ., ,$(GINKGO_SKIP)),-skip "$(var)") +FOCUS_STR := $(foreach var, $(subst ., ,$(GINKGO_FOCUS)),-focus "$(var)") VELERO_CLI ?=$$(pwd)/../../_output/bin/$(GOOS)/$(GOARCH)/velero VELERO_IMAGE ?= velero/velero:main VELERO_VERSION ?= $(VERSION) @@ -110,7 +111,7 @@ run: ginkgo (echo "Bucket to store the backups from E2E tests is required, please re-run with BSL_BUCKET="; exit 1 ) @[ "${CLOUD_PROVIDER}" ] && echo "Using cloud provider ${CLOUD_PROVIDER}" || \ (echo "Cloud provider for target cloud/plug-in provider is required, please rerun with CLOUD_PROVIDER="; exit 1) - @$(GINKGO) -v -focus="$(GINKGO_FOCUS)" $(SKIP_STR) . -- -velerocli=$(VELERO_CLI) \ + @$(GINKGO) -v $(FOCUS_STR) $(SKIP_STR) . -- -velerocli=$(VELERO_CLI) \ -velero-image=$(VELERO_IMAGE) \ -plugins=$(PLUGINS) \ -velero-version=$(VELERO_VERSION) \ diff --git a/test/e2e/backups/schedule.go b/test/e2e/backups/schedule.go index 8cabe529c..a30ba761e 100644 --- a/test/e2e/backups/schedule.go +++ b/test/e2e/backups/schedule.go @@ -25,7 +25,7 @@ type ScheduleBackup struct { verifyTimes int } -var ScheduleBackupTest func() = TestFunc(&ScheduleBackup{TestCase: TestCase{NSBaseName: "ns", NSIncluded: &[]string{"ns1"}}}) +var ScheduleBackupTest func() = TestFunc(&ScheduleBackup{TestCase: TestCase{NSBaseName: "schedule-test-ns", NSIncluded: &[]string{"ns1"}}}) func (n *ScheduleBackup) Init() error { n.Client = TestClientInstance @@ -45,7 +45,6 @@ func (n *ScheduleBackup) StartRun() error { n.RestoreName = n.RestoreName + "restore-ns-mapping-" + UUIDgen.String() n.ScheduleArgs = []string{ - "schedule", "create", "--namespace", VeleroCfg.VeleroNamespace, n.ScheduleName, "--include-namespaces", strings.Join(*n.NSIncluded, ","), "--schedule=*/" + fmt.Sprintf("%v", n.Period) + " * * * *", } @@ -78,7 +77,10 @@ func (n *ScheduleBackup) Backup() error { now := time.Now().Minute() triggerNow := now % n.Period if triggerNow == 0 { - Expect(VeleroCmdExec(n.Ctx, VeleroCfg.VeleroCLI, n.ScheduleArgs)).To(Succeed()) + Expect(VeleroScheduleCreate(n.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, n.ScheduleName, n.ScheduleArgs)).To(Succeed(), func() string { + RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, "", "") + return "Fail to restore workload" + }) break } } diff --git a/test/e2e/basic/namespace-mapping.go b/test/e2e/basic/namespace-mapping.go index 3a8d56cce..e0b61fcf1 100644 --- a/test/e2e/basic/namespace-mapping.go +++ b/test/e2e/basic/namespace-mapping.go @@ -21,17 +21,24 @@ type NamespaceMapping struct { kibishiiData *KibishiiData } -var OneNamespaceMappingTest func() = TestFunc(&NamespaceMapping{TestCase: TestCase{NSBaseName: "ns", NSIncluded: &[]string{"ns1"}}}) -var MultiNamespacesMappingTest func() = TestFunc(&NamespaceMapping{TestCase: TestCase{NSBaseName: "ns", NSIncluded: &[]string{"ns1", "ns2"}}}) +const NamespaceBaseName string = "ns-mp-" + +var OneNamespaceMappingResticTest func() = TestFunc(&NamespaceMapping{TestCase: TestCase{NSBaseName: NamespaceBaseName, NSIncluded: &[]string{NamespaceBaseName + "1"}, UseVolumeSnapshots: false}}) +var MultiNamespacesMappingResticTest func() = TestFunc(&NamespaceMapping{TestCase: TestCase{NSBaseName: NamespaceBaseName, NSIncluded: &[]string{NamespaceBaseName + "1", NamespaceBaseName + "2"}, UseVolumeSnapshots: false}}) +var OneNamespaceMappingSnapshotTest func() = TestFunc(&NamespaceMapping{TestCase: TestCase{NSBaseName: NamespaceBaseName, NSIncluded: &[]string{NamespaceBaseName + "1"}, UseVolumeSnapshots: true}}) +var MultiNamespacesMappingSnapshotTest func() = TestFunc(&NamespaceMapping{TestCase: TestCase{NSBaseName: NamespaceBaseName, NSIncluded: &[]string{NamespaceBaseName + "1", NamespaceBaseName + "2"}, UseVolumeSnapshots: true}}) func (n *NamespaceMapping) Init() error { n.Client = TestClientInstance n.kibishiiData = &KibishiiData{2, 10, 10, 1024, 1024, 0, 2} - + backupType := "restic" + if n.UseVolumeSnapshots { + backupType = "snapshot" + } n.TestMsg = &TestMSG{ - Desc: "Backup resources with include namespace test", - FailedMSG: "Failed to backup with namespace include", - Text: fmt.Sprintf("should backup namespaces %s", *n.NSIncluded), + Desc: fmt.Sprintf("Restore namespace %s with namespace mapping by %s test", *n.NSIncluded, backupType), + FailedMSG: "Failed to restore with namespace mapping", + Text: fmt.Sprintf("should restore namespace %s with namespace mapping by %s", *n.NSIncluded, backupType), } return nil } @@ -49,15 +56,20 @@ func (n *NamespaceMapping) StartRun() error { n.BackupName = n.BackupName + ns n.RestoreName = n.RestoreName + ns } - n.BackupName = n.BackupName + "backup-ns-mapping-" + UUIDgen.String() - n.RestoreName = n.RestoreName + "restore-ns-mapping-" + UUIDgen.String() + n.BackupName = n.BackupName + UUIDgen.String() + n.RestoreName = n.RestoreName + UUIDgen.String() n.MappedNamespaceList = mappedNSList fmt.Println(mappedNSList) n.BackupArgs = []string{ "create", "--namespace", VeleroCfg.VeleroNamespace, "backup", n.BackupName, - "--include-namespaces", strings.Join(*n.NSIncluded, ","), - "--default-volumes-to-fs-backup", "--wait", + "--include-namespaces", strings.Join(*n.NSIncluded, ","), "--wait", + } + if n.UseVolumeSnapshots { + n.BackupArgs = append(n.BackupArgs, "--snapshot-volumes") + } else { + n.BackupArgs = append(n.BackupArgs, "--snapshot-volumes=false") + n.BackupArgs = append(n.BackupArgs, "--default-volumes-to-fs-backup") } n.RestoreArgs = []string{ "create", "--namespace", VeleroCfg.VeleroNamespace, "restore", n.RestoreName, diff --git a/test/e2e/basic/resources-check/namespaces.go b/test/e2e/basic/resources-check/namespaces.go index 7c1d6ad4f..67da3481b 100644 --- a/test/e2e/basic/resources-check/namespaces.go +++ b/test/e2e/basic/resources-check/namespaces.go @@ -121,6 +121,7 @@ func (m *MultiNSBackup) Verify() error { } func (m *MultiNSBackup) Destroy() error { + m.Ctx, _ = context.WithTimeout(context.Background(), 60*time.Minute) err := CleanupNamespaces(m.Ctx, m.Client, m.NSBaseName) if err != nil { return errors.Wrap(err, "Could cleanup retrieve namespaces") diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 1025d0216..ef35b7f5d 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -37,6 +37,7 @@ import ( . "github.com/vmware-tanzu/velero/test/e2e/privilegesmgmt" . "github.com/vmware-tanzu/velero/test/e2e/pv-backup" . "github.com/vmware-tanzu/velero/test/e2e/resource-filtering" + . "github.com/vmware-tanzu/velero/test/e2e/scale" . "github.com/vmware-tanzu/velero/test/e2e/upgrade" @@ -126,8 +127,10 @@ var _ = Describe("[Migration][Snapshot]", MigrationWithSnapshots) var _ = Describe("[Schedule][OrederedResources] Backup resources should follow the specific order in schedule", ScheduleOrderedResources) -var _ = Describe("[NamespaceMapping][Single] Backup resources should follow the specific order in schedule", OneNamespaceMappingTest) -var _ = Describe("[NamespaceMapping][Multiple] Backup resources should follow the specific order in schedule", MultiNamespacesMappingTest) +var _ = Describe("[NamespaceMapping][Single][Restic] Backup resources should follow the specific order in schedule", OneNamespaceMappingResticTest) +var _ = Describe("[NamespaceMapping][Multiple][Restic] Backup resources should follow the specific order in schedule", MultiNamespacesMappingResticTest) +var _ = Describe("[NamespaceMapping][Single][Snapshot] Backup resources should follow the specific order in schedule", OneNamespaceMappingSnapshotTest) +var _ = Describe("[NamespaceMapping][Multiple][Snapshot] Backup resources should follow the specific order in schedule", MultiNamespacesMappingSnapshotTest) var _ = Describe("[pv-backup][Opt-In] Backup resources should follow the specific order in schedule", OptInPVBackupTest) var _ = Describe("[pv-backup][Opt-Out] Backup resources should follow the specific order in schedule", OptOutPVBackupTest) diff --git a/test/e2e/migration/migration.go b/test/e2e/migration/migration.go index 3e81d8dd1..e957ace26 100644 --- a/test/e2e/migration/migration.go +++ b/test/e2e/migration/migration.go @@ -126,6 +126,8 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) OriginVeleroCfg.Plugins = "" //TODO: Remove this once origin Velero version is 1.10 and upper OriginVeleroCfg.UploaderType = "" + OriginVeleroCfg.UseNodeAgent = false + OriginVeleroCfg.UseRestic = !useVolumeSnapshots } fmt.Println(OriginVeleroCfg) Expect(VeleroInstall(context.Background(), &OriginVeleroCfg, useVolumeSnapshots)).To(Succeed()) @@ -157,9 +159,9 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) BackupStorageClassCfg.IncludeResources = "StorageClass" BackupStorageClassCfg.IncludeClusterResources = true //TODO Remove UseRestic parameter once minor version is 1.10 or upper - BackupStorageClassCfg.UseRestic = true + BackupStorageClassCfg.UseResticIfFSBackup = true if veleroCLI2Version.VeleroVersion == "self" { - BackupStorageClassCfg.UseRestic = false + BackupStorageClassCfg.UseResticIfFSBackup = false } Expect(VeleroBackupNamespace(context.Background(), OriginVeleroCfg.VeleroCLI, @@ -175,9 +177,9 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) BackupCfg.BackupLocation = "" BackupCfg.Selector = "" //TODO Remove UseRestic parameter once minor version is 1.10 or upper - BackupCfg.UseRestic = true + BackupCfg.UseResticIfFSBackup = true if veleroCLI2Version.VeleroVersion == "self" { - BackupCfg.UseRestic = false + BackupCfg.UseResticIfFSBackup = false } Expect(VeleroBackupNamespace(context.Background(), OriginVeleroCfg.VeleroCLI, OriginVeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string { @@ -235,6 +237,8 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) VeleroCfg.ObjectStoreProvider = "" VeleroCfg.ClientToInstallVelero = VeleroCfg.StandbyClient + VeleroCfg.UseNodeAgent = !useVolumeSnapshots + VeleroCfg.UseRestic = false Expect(VeleroInstall(context.Background(), &VeleroCfg, useVolumeSnapshots)).To(Succeed()) }) diff --git a/test/e2e/privilegesmgmt/ssr.go b/test/e2e/privilegesmgmt/ssr.go index 526efb1e8..428cacc81 100644 --- a/test/e2e/privilegesmgmt/ssr.go +++ b/test/e2e/privilegesmgmt/ssr.go @@ -62,16 +62,14 @@ func SSRTest() { Expect(CreateNamespace(ctx, *VeleroCfg.ClientToInstallVelero, testNS)).To(Succeed(), fmt.Sprintf("Failed to create %s namespace", testNS)) - By(fmt.Sprintf("Get version in %s namespace", testNS)) - cmd := []string{"version", "-n", testNS} - Expect(VeleroCmdExec(context.Background(), VeleroCfg.VeleroCLI, cmd)).To(Succeed(), - fmt.Sprintf("Failed to create an ssr object in the %s namespace", testNS)) - - By(fmt.Sprintf("Get version in %s namespace", VeleroCfg.VeleroNamespace)) - cmd = []string{"version", "-n", VeleroCfg.VeleroNamespace} - Expect(VeleroCmdExec(context.Background(), VeleroCfg.VeleroCLI, cmd)).To(Succeed(), - fmt.Sprintf("Failed to create an ssr object in %s namespace", VeleroCfg.VeleroNamespace)) - + By(fmt.Sprintf("Get version in %s namespace", testNS), func() { + Expect(VeleroVersion(context.Background(), VeleroCfg.VeleroCLI, testNS)).To(Succeed(), + fmt.Sprintf("Failed to create an ssr object in the %s namespace", testNS)) + }) + By(fmt.Sprintf("Get version in %s namespace", VeleroCfg.VeleroNamespace), func() { + Expect(VeleroVersion(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To(Succeed(), + fmt.Sprintf("Failed to create an ssr object in %s namespace", VeleroCfg.VeleroNamespace)) + }) ssrListResp := new(v1.ServerStatusRequestList) By(fmt.Sprintf("Check ssr object in %s namespace", VeleroCfg.VeleroNamespace)) err = waitutil.PollImmediate(5*time.Second, time.Minute, diff --git a/test/e2e/pv-backup/pv-backup-filter.go b/test/e2e/pv-backup/pv-backup-filter.go index 3de103894..4cef609d3 100644 --- a/test/e2e/pv-backup/pv-backup-filter.go +++ b/test/e2e/pv-backup/pv-backup-filter.go @@ -49,8 +49,8 @@ func (p *PVBackupFiltering) StartRun() error { if err != nil { return err } - p.BackupName = p.BackupName + "backup-opt-in-" + UUIDgen.String() - p.RestoreName = p.RestoreName + "restore-opt-in-" + UUIDgen.String() + p.BackupName = p.BackupName + "backup-" + p.id + "-" + UUIDgen.String() + p.RestoreName = p.RestoreName + "restore-" + p.id + "-" + UUIDgen.String() p.BackupArgs = []string{ "create", "--namespace", VeleroCfg.VeleroNamespace, "backup", p.BackupName, "--include-namespaces", strings.Join(*p.NSIncluded, ","), @@ -84,7 +84,7 @@ func (p *PVBackupFiltering) CreateResources() error { for j := 0; j <= VOLUME_COUNT_PER_POD-1; j++ { volume := fmt.Sprintf("volume-%s-%d-%d", p.id, i, j) volumes = append(volumes, volume) - //Volumes cherry pick policy for opt-in/out annotation to pods + //Volumes cherry-pick policy for opt-in/out annotation to apply if j%2 == 0 { volumeToAnnotationList = append(volumeToAnnotationList, volume) } diff --git a/test/e2e/test/test.go b/test/e2e/test/test.go index ee9218a0c..4a754efb8 100644 --- a/test/e2e/test/test.go +++ b/test/e2e/test/test.go @@ -20,6 +20,7 @@ import ( "context" "flag" "fmt" + "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -47,6 +48,7 @@ type VeleroBackupRestoreTest interface { Verify() error Clean() error GetTestMsg() *TestMSG + GetTestCase() *TestCase } type TestMSG struct { @@ -56,16 +58,17 @@ type TestMSG struct { } type TestCase struct { - BackupName string - RestoreName string - NSBaseName string - BackupArgs []string - RestoreArgs []string - NamespacesTotal int - TestMsg *TestMSG - Client TestClient - Ctx context.Context - NSIncluded *[]string + BackupName string + RestoreName string + NSBaseName string + BackupArgs []string + RestoreArgs []string + NamespacesTotal int + TestMsg *TestMSG + Client TestClient + Ctx context.Context + NSIncluded *[]string + UseVolumeSnapshots bool } var TestClientInstance TestClient @@ -79,7 +82,7 @@ func TestFunc(test VeleroBackupRestoreTest) func() { BeforeEach(func() { flag.Parse() if VeleroCfg.InstallVelero { - Expect(VeleroInstall(context.Background(), &VeleroCfg, false)).To(Succeed()) + Expect(VeleroInstall(context.Background(), &VeleroCfg, test.GetTestCase().UseVolumeSnapshots)).To(Succeed()) } }) AfterEach(func() { @@ -101,16 +104,17 @@ func TestFuncWithMultiIt(tests []VeleroBackupRestoreTest) func() { By("Create test client instance", func() { TestClientInstance = *VeleroCfg.ClientToInstallVelero }) - + var useVolumeSnapshots bool for k := range tests { Expect(tests[k].Init()).To(Succeed(), fmt.Sprintf("Failed to instantiate test %s case", tests[k].GetTestMsg().Desc)) + useVolumeSnapshots = tests[k].GetTestCase().UseVolumeSnapshots } BeforeEach(func() { flag.Parse() if VeleroCfg.InstallVelero { if countIt == 0 { - Expect(VeleroInstall(context.Background(), &VeleroCfg, false)).To(Succeed()) + Expect(VeleroInstall(context.Background(), &VeleroCfg, useVolumeSnapshots)).To(Succeed()) } countIt++ } @@ -148,7 +152,7 @@ func (t *TestCase) StartRun() error { } func (t *TestCase) Backup() error { - if err := VeleroCmdExec(t.Ctx, VeleroCfg.VeleroCLI, t.BackupArgs); err != nil { + if err := VeleroBackupExec(t.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, t.BackupName, t.BackupArgs); err != nil { RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, t.BackupName, "") return errors.Wrapf(err, "Failed to backup resources") } @@ -164,9 +168,17 @@ func (t *TestCase) Destroy() error { } func (t *TestCase) Restore() error { + // the snapshots of AWS may be still in pending status when do the restore, wait for a while + // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 + // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed + if t.UseVolumeSnapshots { + fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...") + time.Sleep(5 * time.Minute) + } + By("Start to restore ......", func() { - Expect(VeleroCmdExec(t.Ctx, VeleroCfg.VeleroCLI, t.RestoreArgs)).To(Succeed(), func() string { - RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, t.BackupName, "") + Expect(VeleroRestoreExec(t.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, t.RestoreName, t.RestoreArgs)).To(Succeed(), func() string { + RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, "", t.RestoreName) return "Fail to restore workload" }) }) @@ -193,6 +205,9 @@ func (t *TestCase) GetTestMsg() *TestMSG { return t.TestMsg } +func (t *TestCase) GetTestCase() *TestCase { + return t +} func RunTestCase(test VeleroBackupRestoreTest) error { fmt.Printf("Running test case %s\n", test.GetTestMsg().Desc) if test == nil { diff --git a/test/e2e/types.go b/test/e2e/types.go index 068d957eb..521517070 100644 --- a/test/e2e/types.go +++ b/test/e2e/types.go @@ -64,6 +64,8 @@ type VerleroConfig struct { DefaultClient *TestClient StandbyClient *TestClient UploaderType string + UseNodeAgent bool + UseRestic bool } type SnapshotCheckPoint struct { @@ -87,7 +89,7 @@ type BackupConfig struct { ExcludeResources string IncludeClusterResources bool OrderedResources string - UseRestic bool + UseResticIfFSBackup bool } type VeleroCLI2Version struct { diff --git a/test/e2e/upgrade/upgrade.go b/test/e2e/upgrade/upgrade.go index a46231b46..120deb978 100644 --- a/test/e2e/upgrade/upgrade.go +++ b/test/e2e/upgrade/upgrade.go @@ -113,6 +113,8 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC tmpCfgForOldVeleroInstall.ResticHelperImage = "" tmpCfgForOldVeleroInstall.Plugins = "" tmpCfgForOldVeleroInstall.UploaderType = "" + tmpCfgForOldVeleroInstall.UseNodeAgent = false + tmpCfgForOldVeleroInstall.UseRestic = !useVolumeSnapshots Expect(VeleroInstall(context.Background(), &tmpCfgForOldVeleroInstall, useVolumeSnapshots)).To(Succeed()) @@ -144,8 +146,8 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC BackupCfg.BackupLocation = "" BackupCfg.UseVolumeSnapshots = useVolumeSnapshots BackupCfg.Selector = "" - //TODO: pay attention to this param - BackupCfg.UseRestic = true + //TODO: pay attention to this param, remove it when restic is not the default backup tool any more. + BackupCfg.UseResticIfFSBackup = true Expect(VeleroBackupNamespace(oneHourTimeout, tmpCfg.UpgradeFromVeleroCLI, tmpCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string { RunDebug(context.Background(), tmpCfg.UpgradeFromVeleroCLI, tmpCfg.VeleroNamespace, @@ -196,6 +198,8 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC By(fmt.Sprintf("Upgrade Velero by CLI %s", tmpCfg.VeleroCLI), func() { tmpCfg.GCFrequency = "" + tmpCfg.UseNodeAgent = !useVolumeSnapshots + tmpCfg.UseRestic = false Expect(VeleroInstall(context.Background(), &tmpCfg, useVolumeSnapshots)).To(Succeed()) Expect(CheckVeleroVersion(context.Background(), tmpCfg.VeleroCLI, tmpCfg.VeleroVersion)).To(Succeed()) diff --git a/test/e2e/util/kibishii/kibishii_utils.go b/test/e2e/util/kibishii/kibishii_utils.go index b87f3e03b..016fa072d 100644 --- a/test/e2e/util/kibishii/kibishii_utils.go +++ b/test/e2e/util/kibishii/kibishii_utils.go @@ -120,12 +120,11 @@ func RunKibishiiTests(client TestClient, veleroCfg VerleroConfig, backupName, re if err := DeleteNamespace(oneHourTimeout, client, kibishiiNamespace, true); err != nil { return errors.Wrapf(err, "failed to delete namespace %s", kibishiiNamespace) } - time.Sleep(5 * time.Minute) // the snapshots of AWS may be still in pending status when do the restore, wait for a while // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed - if providerName == "aws" && useVolumeSnapshots { + if useVolumeSnapshots { fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...") time.Sleep(5 * time.Minute) } @@ -138,7 +137,6 @@ func RunKibishiiTests(client TestClient, veleroCfg VerleroConfig, backupName, re if err := KibishiiVerifyAfterRestore(client, kibishiiNamespace, oneHourTimeout, DefaultKibishiiData); err != nil { return errors.Wrapf(err, "Error verifying kibishii after restore") } - fmt.Printf("kibishii test completed successfully\n") return nil } @@ -151,8 +149,7 @@ func installKibishii(ctx context.Context, namespace string, cloudPlatform, veler } // We use kustomize to generate YAML for Kibishii from the checked-in yaml directories kibishiiInstallCmd := exec.CommandContext(ctx, "kubectl", "apply", "-n", namespace, "-k", - kibishiiDirectory+cloudPlatform) - + kibishiiDirectory+cloudPlatform, "--timeout=90s") _, stderr, err := veleroexec.RunCommand(kibishiiInstallCmd) fmt.Printf("Install Kibishii cmd: %s\n", kibishiiInstallCmd) if err != nil { diff --git a/test/e2e/util/providers/azure_utils.go b/test/e2e/util/providers/azure_utils.go index 8c579c74f..1dce4cb5b 100644 --- a/test/e2e/util/providers/azure_utils.go +++ b/test/e2e/util/providers/azure_utils.go @@ -364,10 +364,10 @@ func (s AzureStorage) IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupN snapshotCountFound := 0 backupNameInSnapshot := "" if err != nil { - errors.Wrap(err, fmt.Sprintf("Fail to list snapshots %s\n", envVars[resourceGroupEnvVar])) + return errors.Wrap(err, fmt.Sprintf("Fail to list snapshots %s\n", envVars[resourceGroupEnvVar])) } if result.Value == nil { - errors.New(fmt.Sprintf("No snapshots in Azure resource group %s\n", envVars[resourceGroupEnvVar])) + return errors.New(fmt.Sprintf("No snapshots in Azure resource group %s\n", envVars[resourceGroupEnvVar])) } for _, v := range *result.Value { if snapshotCheck.EnableCSI { diff --git a/test/e2e/util/providers/common.go b/test/e2e/util/providers/common.go index 61ffd1bf9..8c37d79d9 100644 --- a/test/e2e/util/providers/common.go +++ b/test/e2e/util/providers/common.go @@ -46,7 +46,7 @@ func ObjectsShouldBeInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bsl func ObjectsShouldNotBeInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string, retryTimes int) error { var err error var exist bool - fmt.Printf("|| VERIFICATION || - %s %s should not exist in storage %s\n", subPrefix, backupName, bslPrefix) + fmt.Printf("|| VERIFICATION || - %s %s should not exist in object store %s\n", subPrefix, backupName, bslPrefix) for i := 0; i < retryTimes; i++ { exist, err = IsObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix) if err != nil { @@ -114,9 +114,9 @@ func SnapshotsShouldNotExistInCloud(cloudProvider, cloudCredentialsFile, bslBuck snapshotCheckPoint.ExpectCount = 0 err := IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName, snapshotCheckPoint) if err != nil { - return errors.Wrapf(err, fmt.Sprintf("|| UNEXPECTED ||Snapshots %s is existed in cloud after backup as expected", backupName)) + return errors.Wrapf(err, fmt.Sprintf("|| UNEXPECTED ||Snapshots %s exist in cloud after backup as expected", backupName)) } - fmt.Printf("|| EXPECTED || - Snapshots are not existed in cloud, backup %s\n", backupName) + fmt.Printf("|| EXPECTED || - Snapshots do not exist in cloud, backup %s\n", backupName) return nil } @@ -124,9 +124,9 @@ func SnapshotsShouldBeCreatedInCloud(cloudProvider, cloudCredentialsFile, bslBuc fmt.Printf("|| VERIFICATION || - Snapshots should exist in cloud, backup %s\n", backupName) err := IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName, snapshotCheckPoint) if err != nil { - return errors.Wrapf(err, fmt.Sprintf("|| UNEXPECTED ||Snapshots %s are not existed in cloud after backup as expected", backupName)) + return errors.Wrapf(err, fmt.Sprintf("|| UNEXPECTED ||Snapshots %s do not exist in cloud after backup as expected", backupName)) } - fmt.Printf("|| EXPECTED || - Snapshots are existed in cloud, backup %s\n", backupName) + fmt.Printf("|| EXPECTED || - Snapshots exist in cloud, backup %s\n", backupName) return nil } diff --git a/test/e2e/util/velero/install.go b/test/e2e/util/velero/install.go index fc0389bc5..d3730b0bb 100644 --- a/test/e2e/util/velero/install.go +++ b/test/e2e/util/velero/install.go @@ -87,7 +87,10 @@ func VeleroInstall(ctx context.Context, veleroCfg *VerleroConfig, useVolumeSnaps return errors.WithMessagef(err, "Failed to get Velero InstallOptions for plugin provider %s", veleroCfg.ObjectStoreProvider) } veleroInstallOptions.UseVolumeSnapshots = useVolumeSnapshots - veleroInstallOptions.UseNodeAgent = !useVolumeSnapshots + if !veleroCfg.UseRestic { + veleroInstallOptions.UseNodeAgent = !useVolumeSnapshots + } + veleroInstallOptions.UseRestic = veleroCfg.UseRestic veleroInstallOptions.Image = veleroCfg.VeleroImage veleroInstallOptions.Namespace = veleroCfg.VeleroNamespace veleroInstallOptions.UploaderType = veleroCfg.UploaderType @@ -175,6 +178,9 @@ func installVeleroServer(ctx context.Context, cli string, options *installOption if options.UseNodeAgent { args = append(args, "--use-node-agent") } + if options.UseRestic { + args = append(args, "--use-restic") + } if options.UseVolumeSnapshots { args = append(args, "--use-volume-snapshots") } @@ -285,7 +291,7 @@ func createVelereResources(ctx context.Context, cli, namespace string, args []st cmd.Stderr = os.Stderr fmt.Printf("Running cmd %q \n", cmd.String()) if err = cmd.Run(); err != nil { - return errors.Wrapf(err, "failed to apply velere resources") + return errors.Wrapf(err, "failed to apply Velero resources") } return nil diff --git a/test/e2e/util/velero/velero_utils.go b/test/e2e/util/velero/velero_utils.go index f8495865c..ded37dd46 100644 --- a/test/e2e/util/velero/velero_utils.go +++ b/test/e2e/util/velero/velero_utils.go @@ -323,7 +323,7 @@ func VeleroBackupNamespace(ctx context.Context, veleroCLI, veleroNamespace strin if backupCfg.UseVolumeSnapshots { args = append(args, "--snapshot-volumes") } else { - if backupCfg.UseRestic { + if backupCfg.UseResticIfFSBackup { args = append(args, "--default-volumes-to-restic") } else { args = append(args, "--default-volumes-to-fs-backup") @@ -399,6 +399,7 @@ func VeleroRestoreExec(ctx context.Context, veleroCLI, veleroNamespace, restoreN if err := VeleroCmdExec(ctx, veleroCLI, args); err != nil { return err } + return checkRestorePhase(ctx, veleroCLI, veleroNamespace, restoreName, velerov1api.RestorePhaseCompleted) } @@ -436,19 +437,14 @@ func VeleroScheduleCreate(ctx context.Context, veleroCLI string, veleroNamespace func VeleroCmdExec(ctx context.Context, veleroCLI string, args []string) error { cmd := exec.CommandContext(ctx, veleroCLI, args...) - var errBuf, outBuf bytes.Buffer - cmd.Stderr = io.MultiWriter(os.Stderr, &errBuf) - cmd.Stdout = io.MultiWriter(os.Stdout, &outBuf) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr fmt.Printf("velero cmd =%v\n", cmd) err := cmd.Run() - retAll := outBuf.String() + " " + errBuf.String() - if strings.Contains(strings.ToLower(retAll), "failed") { - return errors.Wrap(err, fmt.Sprintf("velero cmd =%v return with failure\n", cmd)) + if strings.Contains(fmt.Sprint(cmd.Stdout), "Failed") { + return errors.New(fmt.Sprintf("velero cmd =%v return with failure\n", cmd)) } - if err != nil { - return err - } - return nil + return err } func VeleroBackupLogs(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string) error { @@ -511,6 +507,16 @@ func VeleroCreateBackupLocation(ctx context.Context, return VeleroCmdExec(ctx, veleroCLI, args) } +func VeleroVersion(ctx context.Context, veleroCLI, veleroNamespace string) error { + args := []string{ + "version", "--namespace", veleroNamespace, + } + if err := VeleroCmdExec(ctx, veleroCLI, args); err != nil { + return err + } + return nil +} + func getProviderPlugins(ctx context.Context, veleroCLI, objectStoreProvider, providerPlugins, feature string) ([]string, error) { // Fetch the plugins for the provider before checking for the object store provider below. var plugins []string @@ -788,12 +794,11 @@ func GetBackup(ctx context.Context, veleroCLI string, backupName string) (string func IsBackupExist(ctx context.Context, veleroCLI string, backupName string) (bool, error) { out, outerr, err := GetBackup(ctx, veleroCLI, backupName) if err != nil { - if err != nil { - if strings.Contains(outerr, "not found") { - return false, nil - } - return false, err + if strings.Contains(outerr, "not found") { + fmt.Printf("Backup CR %s was not found\n", backupName) + return false, nil } + return false, err } fmt.Printf("Backup <%s> exist locally according to output \n[%s]\n", backupName, out) return true, nil @@ -807,6 +812,7 @@ func WaitBackupDeleted(ctx context.Context, veleroCLI string, backupName string, if exist { return false, nil } else { + fmt.Printf("Backup %s does not exist\n", backupName) return true, nil } }