vmware-tanzu
/
velero
mirror of https://github.com/vmware-tanzu/velero.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

update backup deletion controller for snapshot locations 

Signed-off-by: Steve Kriss <steve@heptio.com>
pull/948/head
Steve Kriss 2018-10-16 15:49:37 -06:00
parent 38c72b8cc2
commit 4a03370f1d
3 changed files with 89 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pkg/cmd/server/server.go
View File

@ -697,13 +697,13 @@ func (s *server) runControllers(config *api.Config, defaultVolumeSnapshotLocatio
s.sharedInformerFactory.Ark().V1().DeleteBackupRequests(),
s.arkClient.ArkV1(), // deleteBackupRequestClient
s.arkClient.ArkV1(), // backupClient
nil,
s.sharedInformerFactory.Ark().V1().Restores(),
s.arkClient.ArkV1(), // restoreClient
backupTracker,
s.resticManager,
s.sharedInformerFactory.Ark().V1().PodVolumeBackups(),
s.sharedInformerFactory.Ark().V1().BackupStorageLocations(),
s.sharedInformerFactory.Ark().V1().VolumeSnapshotLocations(),
newPluginManager,
)
wg.Add(1)

81
pkg/controller/backup_deletion_controller.go
View File

@ -51,13 +51,13 @@ type backupDeletionController struct {
deleteBackupRequestClient arkv1client.DeleteBackupRequestsGetter
deleteBackupRequestLister listers.DeleteBackupRequestLister
backupClient arkv1client.BackupsGetter
blockStore cloudprovider.BlockStore
restoreLister listers.RestoreLister
restoreClient arkv1client.RestoresGetter
backupTracker BackupTracker
resticMgr restic.RepositoryManager
podvolumeBackupLister listers.PodVolumeBackupLister
backupLocationLister listers.BackupStorageLocationLister
snapshotLocationLister listers.VolumeSnapshotLocationLister
processRequestFunc func(*v1.DeleteBackupRequest) error
clock clock.Clock
newPluginManager func(logrus.FieldLogger) plugin.Manager
@ -70,13 +70,13 @@ func NewBackupDeletionController(
deleteBackupRequestInformer informers.DeleteBackupRequestInformer,
deleteBackupRequestClient arkv1client.DeleteBackupRequestsGetter,
backupClient arkv1client.BackupsGetter,
blockStore cloudprovider.BlockStore,
restoreInformer informers.RestoreInformer,
restoreClient arkv1client.RestoresGetter,
backupTracker BackupTracker,
resticMgr restic.RepositoryManager,
podvolumeBackupInformer informers.PodVolumeBackupInformer,
backupLocationInformer informers.BackupStorageLocationInformer,
snapshotLocationInformer informers.VolumeSnapshotLocationInformer,
newPluginManager func(logrus.FieldLogger) plugin.Manager,
) Interface {
c := &backupDeletionController{
@ -84,13 +84,13 @@ func NewBackupDeletionController(
deleteBackupRequestClient: deleteBackupRequestClient,
deleteBackupRequestLister: deleteBackupRequestInformer.Lister(),
backupClient: backupClient,
blockStore: blockStore,
restoreLister: restoreInformer.Lister(),
restoreClient: restoreClient,
backupTracker: backupTracker,
resticMgr: resticMgr,
podvolumeBackupLister: podvolumeBackupInformer.Lister(),
backupLocationLister: backupLocationInformer.Lister(),
snapshotLocationLister: snapshotLocationInformer.Lister(),
// use variables to refer to these functions so they can be
// replaced with fakes for testing.
@ -107,6 +107,7 @@ func NewBackupDeletionController(
restoreInformer.Informer().HasSynced,
podvolumeBackupInformer.Informer().HasSynced,
backupLocationInformer.Informer().HasSynced,
snapshotLocationInformer.Informer().HasSynced,
)
c.processRequestFunc = c.processRequest
@ -223,17 +224,6 @@ func (c *backupDeletionController) processRequest(req *v1.DeleteBackupRequest) e
}
}
// If the backup includes snapshots but we don't currently have a PVProvider, we don't
// want to orphan the snapshots so skip deletion.
if c.blockStore == nil && len(backup.Status.VolumeBackups) > 0 {
req, err = c.patchDeleteBackupRequest(req, func(r *v1.DeleteBackupRequest) {
r.Status.Phase = v1.DeleteBackupRequestPhaseProcessed
r.Status.Errors = []string{"unable to delete backup because it includes PV snapshots and Ark is not configured with a PersistentVolumeProvider"}
})
return err
}
// Set backup status to Deleting
backup, err = c.patchBackup(backup, func(b *v1.Backup) {
b.Status.Phase = v1.BackupPhaseDeleting
@ -245,11 +235,36 @@ func (c *backupDeletionController) processRequest(req *v1.DeleteBackupRequest) e
var errs []string
pluginManager := c.newPluginManager(log)
defer pluginManager.CleanupClients()
backupStore, backupStoreErr := c.backupStoreForBackup(backup, pluginManager, log)
if backupStoreErr != nil {
errs = append(errs, backupStoreErr.Error())
// TODO need to not proceed since backupStore will be nil
}
log.Info("Removing PV snapshots")
for _, volumeBackup := range backup.Status.VolumeBackups {
log.WithField("snapshotID", volumeBackup.SnapshotID).Info("Removing snapshot associated with backup")
if err := c.blockStore.DeleteSnapshot(volumeBackup.SnapshotID); err != nil {
errs = append(errs, errors.Wrapf(err, "error deleting snapshot %s", volumeBackup.SnapshotID).Error())
if snapshots, err := backupStore.GetBackupVolumeSnapshots(backup.Name); err != nil {
errs = append(errs, errors.Wrap(err, "error getting backup's volume snapshots").Error())
} else {
blockStores := make(map[string]cloudprovider.BlockStore)
for _, snapshot := range snapshots {
log.WithField("providerSnapshotID", snapshot.Status.ProviderSnapshotID).Info("Removing snapshot associated with backup")
blockStore, ok := blockStores[snapshot.Spec.Location]
if !ok {
if blockStore, err = blockStoreForSnapshotLocation(backup.Namespace, snapshot.Spec.Location, c.snapshotLocationLister, pluginManager); err != nil {
errs = append(errs, err.Error())
continue
}
blockStores[snapshot.Spec.Location] = blockStore
}
if err := blockStore.DeleteSnapshot(snapshot.Status.ProviderSnapshotID); err != nil {
errs = append(errs, errors.Wrapf(err, "error deleting snapshot %s", snapshot.Status.ProviderSnapshotID).Error())
}
}
}
@ -261,14 +276,6 @@ func (c *backupDeletionController) processRequest(req *v1.DeleteBackupRequest) e
}
log.Info("Removing backup from backup storage")
pluginManager := c.newPluginManager(log)
defer pluginManager.CleanupClients()
backupStore, backupStoreErr := c.backupStoreForBackup(backup, pluginManager, log)
if backupStoreErr != nil {
errs = append(errs, backupStoreErr.Error())
}
if err := backupStore.DeleteBackup(backup.Name); err != nil {
errs = append(errs, err.Error())
}
@ -328,6 +335,28 @@ func (c *backupDeletionController) processRequest(req *v1.DeleteBackupRequest) e
return nil
}
func blockStoreForSnapshotLocation(
namespace, snapshotLocationName string,
snapshotLocationLister listers.VolumeSnapshotLocationLister,
pluginManager plugin.Manager,
) (cloudprovider.BlockStore, error) {
snapshotLocation, err := snapshotLocationLister.VolumeSnapshotLocations(namespace).Get(snapshotLocationName)
if err != nil {
return nil, errors.Wrapf(err, "error getting volume snapshot location %s", snapshotLocationName)
}
blockStore, err := pluginManager.GetBlockStore(snapshotLocation.Spec.Provider)
if err != nil {
return nil, errors.Wrapf(err, "error getting block store for provider %s", snapshotLocation.Spec.Provider)
}
if err = blockStore.Init(snapshotLocation.Spec.Config); err != nil {
return nil, errors.Wrapf(err, "error initializing block store for volume snapshot location %s", snapshotLocationName)
}
return blockStore, nil
}
func (c *backupDeletionController) backupStoreForBackup(backup *v1.Backup, pluginManager plugin.Manager, log logrus.FieldLogger) (persistence.BackupStore, error) {
backupLocation, err := c.backupLocationLister.BackupStorageLocations(backup.Namespace).Get(backup.Spec.StorageLocation)
if err != nil {

76
pkg/controller/backup_deletion_controller_test.go
View File

@ -42,6 +42,7 @@ import (
"github.com/heptio/ark/pkg/plugin"
pluginmocks "github.com/heptio/ark/pkg/plugin/mocks"
arktest "github.com/heptio/ark/pkg/util/test"
"github.com/heptio/ark/pkg/volume"
)
func TestBackupDeletionControllerProcessQueueItem(t *testing.T) {
@ -53,13 +54,13 @@ func TestBackupDeletionControllerProcessQueueItem(t *testing.T) {
sharedInformers.Ark().V1().DeleteBackupRequests(),
client.ArkV1(), // deleteBackupRequestClient
client.ArkV1(), // backupClient
nil, // blockStore
sharedInformers.Ark().V1().Restores(),
client.ArkV1(), // restoreClient
NewBackupTracker(),
nil, // restic repository manager
sharedInformers.Ark().V1().PodVolumeBackups(),
sharedInformers.Ark().V1().BackupStorageLocations(),
sharedInformers.Ark().V1().VolumeSnapshotLocations(),
nil, // new plugin manager func
).(*backupDeletionController)
@ -139,13 +140,13 @@ func setupBackupDeletionControllerTest(objects ...runtime.Object) *backupDeletio
sharedInformers.Ark().V1().DeleteBackupRequests(),
client.ArkV1(), // deleteBackupRequestClient
client.ArkV1(), // backupClient
blockStore,
sharedInformers.Ark().V1().Restores(),
client.ArkV1(), // restoreClient
NewBackupTracker(),
nil, // restic repository manager
sharedInformers.Ark().V1().PodVolumeBackups(),
sharedInformers.Ark().V1().BackupStorageLocations(),
sharedInformers.Ark().V1().VolumeSnapshotLocations(),
func(logrus.FieldLogger) plugin.Manager { return pluginManager },
).(*backupDeletionController),
@ -323,47 +324,8 @@ func TestBackupDeletionControllerProcessRequest(t *testing.T) {
assert.Equal(t, expectedActions, td.client.Actions())
})
t.Run("no block store, backup has snapshots", func(t *testing.T) {
td := setupBackupDeletionControllerTest()
td.controller.blockStore = nil
td.client.PrependReactor("get", "backups", func(action core.Action) (bool, runtime.Object, error) {
backup := arktest.NewTestBackup().WithName("backup-1").WithSnapshot("pv-1", "snap-1").Backup
return true, backup, nil
})
td.client.PrependReactor("patch", "deletebackuprequests", func(action core.Action) (bool, runtime.Object, error) {
return true, td.req, nil
})
err := td.controller.processRequest(td.req)
require.NoError(t, err)
expectedActions := []core.Action{
core.NewPatchAction(
v1.SchemeGroupVersion.WithResource("deletebackuprequests"),
td.req.Namespace,
td.req.Name,
[]byte(`{"status":{"phase":"InProgress"}}`),
),
core.NewGetAction(
v1.SchemeGroupVersion.WithResource("backups"),
td.req.Namespace,
td.req.Spec.BackupName,
),
core.NewPatchAction(
v1.SchemeGroupVersion.WithResource("deletebackuprequests"),
td.req.Namespace,
td.req.Name,
[]byte(`{"status":{"errors":["unable to delete backup because it includes PV snapshots and Ark is not configured with a PersistentVolumeProvider"],"phase":"Processed"}}`),
),
}
assert.Equal(t, expectedActions, td.client.Actions())
})
t.Run("full delete, no errors", func(t *testing.T) {
backup := arktest.NewTestBackup().WithName("foo").WithSnapshot("pv-1", "snap-1").Backup
backup := arktest.NewTestBackup().WithName("foo").Backup
backup.UID = "uid"
backup.Spec.StorageLocation = "primary"
@ -393,6 +355,17 @@ func TestBackupDeletionControllerProcessRequest(t *testing.T) {
}
require.NoError(t, td.sharedInformers.Ark().V1().BackupStorageLocations().Informer().GetStore().Add(location))
snapshotLocation := &v1.VolumeSnapshotLocation{
ObjectMeta: metav1.ObjectMeta{
Namespace: backup.Namespace,
Name: "vsl-1",
},
Spec: v1.VolumeSnapshotLocationSpec{
Provider: "provider-1",
},
}
require.NoError(t, td.sharedInformers.Ark().V1().VolumeSnapshotLocations().Informer().GetStore().Add(snapshotLocation))
// Clear out req labels to make sure the controller adds them
td.req.Labels = make(map[string]string)
@ -409,6 +382,23 @@ func TestBackupDeletionControllerProcessRequest(t *testing.T) {
return true, backup, nil
})
snapshots := []*volume.Snapshot{
{
Spec: volume.SnapshotSpec{
Location: "vsl-1",
},
Status: volume.SnapshotStatus{
ProviderSnapshotID: "snap-1",
},
},
}
pluginManager := &pluginmocks.Manager{}
pluginManager.On("GetBlockStore", "provider-1").Return(td.blockStore, nil)
pluginManager.On("CleanupClients")
td.controller.newPluginManager = func(logrus.FieldLogger) plugin.Manager { return pluginManager }
td.backupStore.On("GetBackupVolumeSnapshots", td.req.Spec.BackupName).Return(snapshots, nil)
td.backupStore.On("DeleteBackup", td.req.Spec.BackupName).Return(nil)
td.backupStore.On("DeleteRestore", "restore-1").Return(nil)
td.backupStore.On("DeleteRestore", "restore-2").Return(nil)
@ -593,13 +583,13 @@ func TestBackupDeletionControllerDeleteExpiredRequests(t *testing.T) {
sharedInformers.Ark().V1().DeleteBackupRequests(),
client.ArkV1(), // deleteBackupRequestClient
client.ArkV1(), // backupClient
nil, // blockStore
sharedInformers.Ark().V1().Restores(),
client.ArkV1(), // restoreClient
NewBackupTracker(),
nil,
sharedInformers.Ark().V1().PodVolumeBackups(),
sharedInformers.Ark().V1().BackupStorageLocations(),
sharedInformers.Ark().V1().VolumeSnapshotLocations(),
nil, // new plugin manager func
).(*backupDeletionController)