206 lines
7.4 KiB
Go
206 lines
7.4 KiB
Go
/*
|
|
*
|
|
* Copyright the Velero contributors.
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
* /
|
|
*/
|
|
|
|
package backups
|
|
|
|
import (
|
|
"context"
|
|
"flag"
|
|
"fmt"
|
|
"math/rand"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
. "github.com/onsi/ginkgo"
|
|
. "github.com/onsi/gomega"
|
|
|
|
. "github.com/vmware-tanzu/velero/test/e2e"
|
|
. "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
|
|
|
|
. "github.com/vmware-tanzu/velero/test/e2e/util/kibishii"
|
|
. "github.com/vmware-tanzu/velero/test/e2e/util/providers"
|
|
. "github.com/vmware-tanzu/velero/test/e2e/util/velero"
|
|
)
|
|
|
|
type TTL struct {
|
|
testNS string
|
|
backupName string
|
|
restoreName string
|
|
ttl time.Duration
|
|
}
|
|
|
|
func (b *TTL) Init() {
|
|
rand.Seed(time.Now().UnixNano())
|
|
UUIDgen, _ = uuid.NewRandom()
|
|
b.testNS = "backup-ttl-test-" + UUIDgen.String()
|
|
b.backupName = "backup-ttl-test-" + UUIDgen.String()
|
|
b.restoreName = "restore-ttl-test-" + UUIDgen.String()
|
|
b.ttl = 10 * time.Minute
|
|
|
|
}
|
|
|
|
func TTLTest() {
|
|
var err error
|
|
var veleroCfg VeleroConfig
|
|
useVolumeSnapshots := true
|
|
test := new(TTL)
|
|
veleroCfg = VeleroCfg
|
|
client := *veleroCfg.ClientToInstallVelero
|
|
|
|
BeforeEach(func() {
|
|
flag.Parse()
|
|
veleroCfg = VeleroCfg
|
|
if veleroCfg.InstallVelero {
|
|
// Make sure GCFrequency is shorter than backup TTL
|
|
veleroCfg.GCFrequency = "4m0s"
|
|
veleroCfg.UseVolumeSnapshots = useVolumeSnapshots
|
|
Expect(VeleroInstall(context.Background(), &veleroCfg, false)).To(Succeed())
|
|
}
|
|
})
|
|
|
|
AfterEach(func() {
|
|
veleroCfg.GCFrequency = ""
|
|
if !veleroCfg.Debug {
|
|
By("Clean backups after test", func() {
|
|
DeleteBackups(context.Background(), *veleroCfg.ClientToInstallVelero)
|
|
})
|
|
if veleroCfg.InstallVelero {
|
|
Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).To(Succeed())
|
|
}
|
|
ctx, ctxCancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
|
defer ctxCancel()
|
|
Expect(DeleteNamespace(ctx, client, test.testNS, false)).To(Succeed(), fmt.Sprintf("Failed to delete the namespace %s", test.testNS))
|
|
}
|
|
})
|
|
|
|
It("Backups in object storage should be synced to a new Velero successfully", func() {
|
|
test.Init()
|
|
ctx, ctxCancel := context.WithTimeout(context.Background(), 1*time.Hour)
|
|
defer ctxCancel()
|
|
By(fmt.Sprintf("Prepare workload as target to backup by creating namespace %s namespace", test.testNS), func() {
|
|
Expect(CreateNamespace(ctx, client, test.testNS)).To(Succeed(),
|
|
fmt.Sprintf("Failed to create %s namespace", test.testNS))
|
|
})
|
|
|
|
By("Deploy sample workload of Kibishii", func() {
|
|
Expect(KibishiiPrepareBeforeBackup(ctx, client, veleroCfg.CloudProvider,
|
|
test.testNS, veleroCfg.RegistryCredentialFile, veleroCfg.Features,
|
|
veleroCfg.KibishiiDirectory, useVolumeSnapshots, DefaultKibishiiData)).To(Succeed())
|
|
})
|
|
|
|
var BackupCfg BackupConfig
|
|
BackupCfg.BackupName = test.backupName
|
|
BackupCfg.Namespace = test.testNS
|
|
BackupCfg.BackupLocation = ""
|
|
BackupCfg.UseVolumeSnapshots = useVolumeSnapshots
|
|
BackupCfg.Selector = ""
|
|
BackupCfg.TTL = test.ttl
|
|
|
|
By(fmt.Sprintf("Backup the workload in %s namespace", test.testNS), func() {
|
|
Expect(VeleroBackupNamespace(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string {
|
|
RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, test.backupName, "")
|
|
return "Fail to backup workload"
|
|
})
|
|
})
|
|
|
|
var snapshotCheckPoint SnapshotCheckPoint
|
|
if useVolumeSnapshots {
|
|
if veleroCfg.CloudProvider == "vsphere" {
|
|
// TODO - remove after upload progress monitoring is implemented
|
|
By("Waiting for vSphere uploads to complete", func() {
|
|
Expect(WaitForVSphereUploadCompletion(ctx, time.Hour,
|
|
test.testNS, 2)).To(Succeed())
|
|
})
|
|
}
|
|
snapshotCheckPoint, err = GetSnapshotCheckPoint(client, veleroCfg, 2, test.testNS, test.backupName, KibishiiPVCNameList)
|
|
Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
|
|
|
|
Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider,
|
|
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLConfig,
|
|
test.backupName, snapshotCheckPoint)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
|
|
}
|
|
|
|
By(fmt.Sprintf("Simulating a disaster by removing namespace %s\n", BackupCfg.BackupName), func() {
|
|
Expect(DeleteNamespace(ctx, client, BackupCfg.BackupName, true)).To(Succeed(),
|
|
fmt.Sprintf("Failed to delete namespace %s", BackupCfg.BackupName))
|
|
})
|
|
|
|
if veleroCfg.CloudProvider == "aws" && useVolumeSnapshots {
|
|
fmt.Println("Waiting 7 minutes to make sure the snapshots are ready...")
|
|
time.Sleep(7 * time.Minute)
|
|
}
|
|
|
|
By(fmt.Sprintf("Restore %s", test.testNS), func() {
|
|
Expect(VeleroRestore(ctx, veleroCfg.VeleroCLI,
|
|
veleroCfg.VeleroNamespace, test.restoreName, test.backupName, "")).To(Succeed(), func() string {
|
|
RunDebug(ctx, veleroCfg.VeleroCLI,
|
|
veleroCfg.VeleroNamespace, "", test.restoreName)
|
|
return "Fail to restore workload"
|
|
})
|
|
})
|
|
|
|
By("Associated Restores should be created", func() {
|
|
Expect(ObjectsShouldBeInBucket(veleroCfg.CloudProvider,
|
|
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,
|
|
veleroCfg.BSLPrefix, veleroCfg.BSLConfig, test.restoreName,
|
|
RestoreObjectsPrefix)).NotTo(HaveOccurred(), "Fail to get restore object")
|
|
|
|
})
|
|
|
|
By("Check TTL was set correctly", func() {
|
|
ttl, err := GetBackupTTL(ctx, veleroCfg.VeleroNamespace, test.backupName)
|
|
Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
|
|
t, _ := time.ParseDuration(strings.ReplaceAll(ttl, "'", ""))
|
|
fmt.Println(t.Round(time.Minute).String())
|
|
Expect(t).To(Equal(test.ttl))
|
|
})
|
|
|
|
By(fmt.Sprintf("Waiting %s minutes for removing backup ralated resources by GC", test.ttl.String()), func() {
|
|
time.Sleep(test.ttl)
|
|
})
|
|
|
|
By("Check if backups are deleted by GC", func() {
|
|
Expect(WaitBackupDeleted(ctx, veleroCfg.VeleroCLI, test.backupName, time.Minute*10)).To(Succeed(), fmt.Sprintf("Backup %s was not deleted by GC", test.backupName))
|
|
})
|
|
|
|
By("Backup file from cloud object storage should be deleted", func() {
|
|
Expect(ObjectsShouldNotBeInBucket(veleroCfg.CloudProvider,
|
|
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,
|
|
veleroCfg.BSLPrefix, veleroCfg.BSLConfig, test.backupName,
|
|
BackupObjectsPrefix, 5)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
|
|
})
|
|
|
|
By("PersistentVolume snapshots should be deleted", func() {
|
|
if useVolumeSnapshots {
|
|
Expect(SnapshotsShouldNotExistInCloud(veleroCfg.CloudProvider,
|
|
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLConfig,
|
|
test.backupName, snapshotCheckPoint)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
|
|
}
|
|
})
|
|
|
|
By("Associated Restores should be deleted", func() {
|
|
Expect(ObjectsShouldNotBeInBucket(veleroCfg.CloudProvider,
|
|
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,
|
|
veleroCfg.BSLPrefix, veleroCfg.BSLConfig, test.restoreName,
|
|
RestoreObjectsPrefix, 5)).NotTo(HaveOccurred(), "Fail to get restore object")
|
|
|
|
})
|
|
})
|
|
}
|