vmware-tanzu
/
velero
mirror of https://github.com/vmware-tanzu/velero.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

convert restorers to plugins 

Signed-off-by: Steve Kriss <steve@heptio.com>
pull/213/head
Steve Kriss 2017-11-21 09:24:43 -08:00
parent 932b8259ae
commit 179b95c81d
37 changed files with 1648 additions and 1406 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/faq.md
View File

@ -22,4 +22,4 @@ Examples of cases where Ark is useful:
Yes, with some exceptions. For example, when Ark restores pods it deletes the `nodeName` from the
pod so that it can be scheduled onto a new node. You can see some more examples of the differences
in [pod_restorer.go](https://github.com/heptio/ark/blob/master/pkg/restore/restorers/pod_restorer.go)
in [pod_action.go](https://github.com/heptio/ark/blob/master/pkg/restore/pod_action.go)

6
pkg/backup/backup.go
View File

@ -69,12 +69,6 @@ type resolvedAction struct {
selector labels.Selector
}
// LogSetter is an interface for a type that allows a FieldLogger
// to be set on it.
type LogSetter interface {
SetLog(logrus.FieldLogger)
}
func (i *itemKey) String() string {
return fmt.Sprintf("resource=%s,namespace=%s,name=%s", i.resource, i.namespace, i.name)
}

22
pkg/backup/item_action.go
View File

@ -1,3 +1,19 @@
/*
Copyright 2017 the Heptio Ark contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package backup
import (
@ -12,9 +28,9 @@ type ItemAction interface {
// AppliesTo returns information about which resources this action should be invoked for.
AppliesTo() (ResourceSelector, error)
// Execute allows the ItemAction to perform arbitrary logic with the item being backed up and the
// backup itself. Implementations may return additional ResourceIdentifiers that indicate specific
// items that also need to be backed up.
// Execute allows the ItemAction to perform arbitrary logic with the item being backed up.
// Implementations may return additional ResourceIdentifiers that indicate specific items
// that also need to be backed up.
Execute(item runtime.Unstructured, backup *api.Backup) (runtime.Unstructured, []ResourceIdentifier, error)
}

3
pkg/backup/item_backupper.go
View File

@ -37,6 +37,7 @@ import (
"github.com/heptio/ark/pkg/discovery"
"github.com/heptio/ark/pkg/util/collections"
kubeutil "github.com/heptio/ark/pkg/util/kube"
"github.com/heptio/ark/pkg/util/logging"
)
type itemBackupperFactory interface {
@ -187,7 +188,7 @@ func (ib *defaultItemBackupper) backupItem(logger logrus.FieldLogger, obj runtim
log.Info("Executing custom action")
if logSetter, ok := action.ItemAction.(LogSetter); ok {
if logSetter, ok := action.ItemAction.(logging.LogSetter); ok {
logSetter.SetLog(log)
}

24
pkg/cmd/server/plugin/plugin.go
View File

@ -27,6 +27,7 @@ import (
"github.com/heptio/ark/pkg/cloudprovider/azure"
"github.com/heptio/ark/pkg/cloudprovider/gcp"
arkplugin "github.com/heptio/ark/pkg/plugin"
"github.com/heptio/ark/pkg/restore"
)
func NewCommand() *cobra.Command {
@ -44,8 +45,14 @@ func NewCommand() *cobra.Command {
"azure": azure.NewBlockStore(),
}
backupActions := map[string]backup.ItemAction{
"backup_pv": backup.NewBackupPVAction(logger),
backupItemActions := map[string]backup.ItemAction{
"pv": backup.NewBackupPVAction(logger),
}
restoreItemActions := map[string]restore.ItemAction{
"job": restore.NewJobAction(logger),
"pod": restore.NewPodAction(logger),
"svc": restore.NewServiceAction(logger),
}
c := &cobra.Command{
@ -86,13 +93,22 @@ func NewCommand() *cobra.Command {
string(arkplugin.PluginKindBlockStore): arkplugin.NewBlockStorePlugin(blockStore),
}
case arkplugin.PluginKindBackupItemAction.String():
action, found := backupActions[name]
action, found := backupItemActions[name]
if !found {
logger.Fatalf("Unrecognized plugin name")
}
serveConfig.Plugins = map[string]plugin.Plugin{
arkplugin.PluginKindBackupItemAction.String(): arkplugin.NewBackupItemActionPlugin(action),
kind: arkplugin.NewBackupItemActionPlugin(action),
}
case arkplugin.PluginKindRestoreItemAction.String():
action, found := restoreItemActions[name]
if !found {
logger.Fatalf("Unrecognized plugin name")
}
serveConfig.Plugins = map[string]plugin.Plugin{
kind: arkplugin.NewRestoreItemActionPlugin(action),
}
default:
logger.Fatalf("Unsupported plugin kind")

13
pkg/cmd/server/server.go
View File

@ -54,7 +54,6 @@ import (
informers "github.com/heptio/ark/pkg/generated/informers/externalversions"
"github.com/heptio/ark/pkg/plugin"
"github.com/heptio/ark/pkg/restore"
"github.com/heptio/ark/pkg/restore/restorers"
"github.com/heptio/ark/pkg/util/kube"
"github.com/heptio/ark/pkg/util/logging"
)
@ -510,6 +509,7 @@ func (s *server) runControllers(config *api.Config) error {
s.sharedInformerFactory.Ark().V1().Backups(),
s.snapshotService != nil,
s.logger,
s.pluginManager,
)
wg.Add(1)
go func() {
@ -569,20 +569,11 @@ func newRestorer(
kubeClient kubernetes.Interface,
logger *logrus.Logger,
) (restore.Restorer, error) {
restorers := map[string]restorers.ResourceRestorer{
"persistentvolumes": restorers.NewPersistentVolumeRestorer(snapshotService),
"persistentvolumeclaims": restorers.NewPersistentVolumeClaimRestorer(),
"services": restorers.NewServiceRestorer(),
"namespaces": restorers.NewNamespaceRestorer(),
"pods": restorers.NewPodRestorer(logger),
"jobs": restorers.NewJobRestorer(logger),
}
return restore.NewKubernetesRestorer(
discoveryHelper,
client.NewDynamicFactory(clientPool),
restorers,
backupService,
snapshotService,
resourcePriorities,
backupClient,
kubeClient.CoreV1().Namespaces(),

2
pkg/controller/backup_controller.go
View File

@ -320,7 +320,7 @@ func (controller *backupController) runBackup(backup *api.Backup, bucket string)
err = kuberrs.NewAggregate(errs)
}()
actions, err := controller.pluginManager.GetBackupItemActions(backup.Name, controller.logger, controller.logger.Level)
actions, err := controller.pluginManager.GetBackupItemActions(backup.Name)
if err != nil {
return err
}

219
pkg/controller/backup_controller_test.go
View File

@ -25,7 +25,6 @@ import (
"k8s.io/apimachinery/pkg/util/clock"
core "k8s.io/client-go/testing"
"github.com/sirupsen/logrus"
testlogger "github.com/sirupsen/logrus/hooks/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
@ -37,6 +36,7 @@ import (
"github.com/heptio/ark/pkg/generated/clientset/versioned/fake"
"github.com/heptio/ark/pkg/generated/clientset/versioned/scheme"
informers "github.com/heptio/ark/pkg/generated/informers/externalversions"
"github.com/heptio/ark/pkg/restore"
. "github.com/heptio/ark/pkg/util/test"
)
@ -49,94 +49,6 @@ func (b *fakeBackupper) Backup(backup *v1.Backup, data, log io.Writer, actions [
return args.Error(0)
}
// Manager is an autogenerated mock type for the Manager type
type Manager struct {
mock.Mock
}
// CloseBackupItemActions provides a mock function with given fields: backupName
func (_m *Manager) CloseBackupItemActions(backupName string) error {
ret := _m.Called(backupName)
var r0 error
if rf, ok := ret.Get(0).(func(string) error); ok {
r0 = rf(backupName)
} else {
r0 = ret.Error(0)
}
return r0
}
// GetBackupItemActions provides a mock function with given fields: backupName, logger, level
func (_m *Manager) GetBackupItemActions(backupName string, logger logrus.FieldLogger, level logrus.Level) ([]backup.ItemAction, error) {
ret := _m.Called(backupName, logger, level)
var r0 []backup.ItemAction
if rf, ok := ret.Get(0).(func(string, logrus.FieldLogger, logrus.Level) []backup.ItemAction); ok {
r0 = rf(backupName, logger, level)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]backup.ItemAction)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(string, logrus.FieldLogger, logrus.Level) error); ok {
r1 = rf(backupName, logger, level)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// GetBlockStore provides a mock function with given fields: name
func (_m *Manager) GetBlockStore(name string) (cloudprovider.BlockStore, error) {
ret := _m.Called(name)
var r0 cloudprovider.BlockStore
if rf, ok := ret.Get(0).(func(string) cloudprovider.BlockStore); ok {
r0 = rf(name)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(cloudprovider.BlockStore)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(string) error); ok {
r1 = rf(name)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// GetObjectStore provides a mock function with given fields: name
func (_m *Manager) GetObjectStore(name string) (cloudprovider.ObjectStore, error) {
ret := _m.Called(name)
var r0 cloudprovider.ObjectStore
if rf, ok := ret.Get(0).(func(string) cloudprovider.ObjectStore); ok {
r0 = rf(name)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(cloudprovider.ObjectStore)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(string) error); ok {
r1 = rf(name)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
func TestProcessBackup(t *testing.T) {
tests := []struct {
name string
@ -243,7 +155,7 @@ func TestProcessBackup(t *testing.T) {
cloudBackups = &BackupService{}
sharedInformers = informers.NewSharedInformerFactory(client, 0)
logger, _ = testlogger.NewNullLogger()
pluginManager = &Manager{}
pluginManager = &MockManager{}
)
c := NewBackupController(
@ -284,7 +196,7 @@ func TestProcessBackup(t *testing.T) {
cloudBackups.On("UploadBackup", "bucket", backup.Name, mock.Anything, mock.Anything, mock.Anything).Return(nil)
pluginManager.On("GetBackupItemActions", backup.Name, logger, logger.Level).Return(nil, nil)
pluginManager.On("GetBackupItemActions", backup.Name).Return(nil, nil)
pluginManager.On("CloseBackupItemActions", backup.Name).Return(nil)
}
@ -353,3 +265,128 @@ func TestProcessBackup(t *testing.T) {
})
}
}
// MockManager is an autogenerated mock type for the Manager type
type MockManager struct {
mock.Mock
}
// CloseBackupItemActions provides a mock function with given fields: backupName
func (_m *MockManager) CloseBackupItemActions(backupName string) error {
ret := _m.Called(backupName)
var r0 error
if rf, ok := ret.Get(0).(func(string) error); ok {
r0 = rf(backupName)
} else {
r0 = ret.Error(0)
}
return r0
}
// GetBackupItemActions provides a mock function with given fields: backupName, logger, level
func (_m *MockManager) GetBackupItemActions(backupName string) ([]backup.ItemAction, error) {
ret := _m.Called(backupName)
var r0 []backup.ItemAction
if rf, ok := ret.Get(0).(func(string) []backup.ItemAction); ok {
r0 = rf(backupName)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]backup.ItemAction)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(string) error); ok {
r1 = rf(backupName)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// CloseRestoreItemActions provides a mock function with given fields: restoreName
func (_m *MockManager) CloseRestoreItemActions(restoreName string) error {
ret := _m.Called(restoreName)
var r0 error
if rf, ok := ret.Get(0).(func(string) error); ok {
r0 = rf(restoreName)
} else {
r0 = ret.Error(0)
}
return r0
}
// GetRestoreItemActions provides a mock function with given fields: restoreName, logger, level
func (_m *MockManager) GetRestoreItemActions(restoreName string) ([]restore.ItemAction, error) {
ret := _m.Called(restoreName)
var r0 []restore.ItemAction
if rf, ok := ret.Get(0).(func(string) []restore.ItemAction); ok {
r0 = rf(restoreName)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]restore.ItemAction)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(string) error); ok {
r1 = rf(restoreName)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// GetBlockStore provides a mock function with given fields: name
func (_m *MockManager) GetBlockStore(name string) (cloudprovider.BlockStore, error) {
ret := _m.Called(name)
var r0 cloudprovider.BlockStore
if rf, ok := ret.Get(0).(func(string) cloudprovider.BlockStore); ok {
r0 = rf(name)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(cloudprovider.BlockStore)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(string) error); ok {
r1 = rf(name)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// GetObjectStore provides a mock function with given fields: name
func (_m *MockManager) GetObjectStore(name string) (cloudprovider.ObjectStore, error) {
ret := _m.Called(name)
var r0 cloudprovider.ObjectStore
if rf, ok := ret.Get(0).(func(string) cloudprovider.ObjectStore); ok {
r0 = rf(name)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(cloudprovider.ObjectStore)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(string) error); ok {
r1 = rf(name)
} else {
r1 = ret.Error(1)
}
return r0, r1
}

19
pkg/controller/restore_controller.go
View File

@ -41,6 +41,7 @@ import (
arkv1client "github.com/heptio/ark/pkg/generated/clientset/versioned/typed/ark/v1"
informers "github.com/heptio/ark/pkg/generated/informers/externalversions/ark/v1"
listers "github.com/heptio/ark/pkg/generated/listers/ark/v1"
"github.com/heptio/ark/pkg/plugin"
"github.com/heptio/ark/pkg/restore"
"github.com/heptio/ark/pkg/util/collections"
kubeutil "github.com/heptio/ark/pkg/util/kube"
@ -63,7 +64,8 @@ type restoreController struct {
restoreListerSynced cache.InformerSynced
syncHandler func(restoreName string) error
queue workqueue.RateLimitingInterface
logger *logrus.Logger
logger logrus.FieldLogger
pluginManager plugin.Manager
}
func NewRestoreController(
@ -75,7 +77,8 @@ func NewRestoreController(
bucket string,
backupInformer informers.BackupInformer,
pvProviderExists bool,
logger *logrus.Logger,
logger logrus.FieldLogger,
pluginManager plugin.Manager,
) Interface {
c := &restoreController{
restoreClient: restoreClient,
@ -90,6 +93,7 @@ func NewRestoreController(
restoreListerSynced: restoreInformer.Informer().HasSynced,
queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "restore"),
logger: logger,
pluginManager: pluginManager,
}
c.syncHandler = c.processRestore
@ -391,8 +395,15 @@ func (controller *restoreController) runRestore(restore *api.Restore, bucket str
}
}()
actions, err := controller.pluginManager.GetRestoreItemActions(restore.Name)
if err != nil {
restoreErrors.Ark = append(restoreErrors.Ark, err.Error())
return
}
defer controller.pluginManager.CloseRestoreItemActions(restore.Name)
logContext.Info("starting restore")
restoreWarnings, restoreErrors = controller.restorer.Restore(restore, backup, backupFile, logFile)
restoreWarnings, restoreErrors = controller.restorer.Restore(restore, backup, backupFile, logFile, actions)
logContext.Info("restore completed")
// Try to upload the log file. This is best-effort. If we fail, we'll add to the ark errors.
@ -431,7 +442,7 @@ func (controller *restoreController) runRestore(restore *api.Restore, bucket str
return
}
func downloadToTempFile(backupName string, backupService cloudprovider.BackupService, bucket string, logger *logrus.Logger) (*os.File, error) {
func downloadToTempFile(backupName string, backupService cloudprovider.BackupService, bucket string, logger logrus.FieldLogger) (*os.File, error) {
readCloser, err := backupService.DownloadBackup(bucket, backupName)
if err != nil {
return nil, err

69
pkg/controller/restore_controller_test.go
View File

@ -23,7 +23,6 @@ import (
"io/ioutil"
"testing"
testlogger "github.com/sirupsen/logrus/hooks/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
@ -35,7 +34,8 @@ import (
api "github.com/heptio/ark/pkg/apis/ark/v1"
"github.com/heptio/ark/pkg/generated/clientset/versioned/fake"
informers "github.com/heptio/ark/pkg/generated/informers/externalversions"
. "github.com/heptio/ark/pkg/util/test"
"github.com/heptio/ark/pkg/restore"
arktest "github.com/heptio/ark/pkg/util/test"
)
func TestFetchBackup(t *testing.T) {
@ -51,14 +51,14 @@ func TestFetchBackup(t *testing.T) {
{
name: "lister has backup",
backupName: "backup-1",
informerBackups: []*api.Backup{NewTestBackup().WithName("backup-1").Backup},
expectedRes: NewTestBackup().WithName("backup-1").Backup,
informerBackups: []*api.Backup{arktest.NewTestBackup().WithName("backup-1").Backup},
expectedRes: arktest.NewTestBackup().WithName("backup-1").Backup,
},
{
name: "backupSvc has backup",
backupName: "backup-1",
backupServiceBackup: NewTestBackup().WithName("backup-1").Backup,
expectedRes: NewTestBackup().WithName("backup-1").Backup,
backupServiceBackup: arktest.NewTestBackup().WithName("backup-1").Backup,
expectedRes: arktest.NewTestBackup().WithName("backup-1").Backup,
},
{
name: "no backup",
@ -74,8 +74,9 @@ func TestFetchBackup(t *testing.T) {
client = fake.NewSimpleClientset()
restorer = &fakeRestorer{}
sharedInformers = informers.NewSharedInformerFactory(client, 0)
backupSvc = &BackupService{}
logger, _ = testlogger.NewNullLogger()
backupSvc = &arktest.BackupService{}
logger = arktest.NewLogger()
pluginManager = &MockManager{}
)
c := NewRestoreController(
@ -88,6 +89,7 @@ func TestFetchBackup(t *testing.T) {
sharedInformers.Ark().V1().Backups(),
false,
logger,
pluginManager,
).(*restoreController)
for _, itm := range test.informerBackups {
@ -135,23 +137,23 @@ func TestProcessRestore(t *testing.T) {
},
{
name: "restore with phase InProgress does not get processed",
restore: NewTestRestore("foo", "bar", api.RestorePhaseInProgress).Restore,
restore: arktest.NewTestRestore("foo", "bar", api.RestorePhaseInProgress).Restore,
expectedErr: false,
},
{
name: "restore with phase Completed does not get processed",
restore: NewTestRestore("foo", "bar", api.RestorePhaseCompleted).Restore,
restore: arktest.NewTestRestore("foo", "bar", api.RestorePhaseCompleted).Restore,
expectedErr: false,
},
{
name: "restore with phase FailedValidation does not get processed",
restore: NewTestRestore("foo", "bar", api.RestorePhaseFailedValidation).Restore,
restore: arktest.NewTestRestore("foo", "bar", api.RestorePhaseFailedValidation).Restore,
expectedErr: false,
},
{
name: "restore with both namespace in both includedNamespaces and excludedNamespaces fails validation",
restore: NewRestore("foo", "bar", "backup-1", "another-1", "*", api.RestorePhaseNew).WithExcludedNamespace("another-1").Restore,
backup: NewTestBackup().WithName("backup-1").Backup,
backup: arktest.NewTestBackup().WithName("backup-1").Backup,
expectedErr: false,
expectedRestoreUpdates: []*api.Restore{
NewRestore("foo", "bar", "backup-1", "another-1", "*", api.RestorePhaseFailedValidation).WithExcludedNamespace("another-1").
@ -162,7 +164,7 @@ func TestProcessRestore(t *testing.T) {
{
name: "restore with resource in both includedResources and excludedResources fails validation",
restore: NewRestore("foo", "bar", "backup-1", "*", "a-resource", api.RestorePhaseNew).WithExcludedResource("a-resource").Restore,
backup: NewTestBackup().WithName("backup-1").Backup,
backup: arktest.NewTestBackup().WithName("backup-1").Backup,
expectedErr: false,
expectedRestoreUpdates: []*api.Restore{
NewRestore("foo", "bar", "backup-1", "*", "a-resource", api.RestorePhaseFailedValidation).WithExcludedResource("a-resource").
@ -182,21 +184,21 @@ func TestProcessRestore(t *testing.T) {
},
{
name: "restore with non-existent backup name fails",
restore: NewTestRestore("foo", "bar", api.RestorePhaseNew).WithBackup("backup-1").WithIncludedNamespace("ns-1").Restore,
expectedErr: false,
backupServiceGetBackupError: errors.New("no backup here"),
name: "restore with non-existent backup name fails",
restore: arktest.NewTestRestore("foo", "bar", api.RestorePhaseNew).WithBackup("backup-1").WithIncludedNamespace("ns-1").Restore,
expectedErr: false,
expectedRestoreUpdates: []*api.Restore{
NewRestore("foo", "bar", "backup-1", "ns-1", "", api.RestorePhaseInProgress).Restore,
NewRestore("foo", "bar", "backup-1", "ns-1", "", api.RestorePhaseCompleted).
WithErrors(1).
Restore,
},
backupServiceGetBackupError: errors.New("no backup here"),
},
{
name: "restorer throwing an error causes the restore to fail",
restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", api.RestorePhaseNew).Restore,
backup: NewTestBackup().WithName("backup-1").Backup,
backup: arktest.NewTestBackup().WithName("backup-1").Backup,
restorerError: errors.New("blarg"),
expectedErr: false,
expectedRestoreUpdates: []*api.Restore{
@ -210,7 +212,7 @@ func TestProcessRestore(t *testing.T) {
{
name: "valid restore gets executed",
restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", api.RestorePhaseNew).Restore,
backup: NewTestBackup().WithName("backup-1").Backup,
backup: arktest.NewTestBackup().WithName("backup-1").Backup,
expectedErr: false,
expectedRestoreUpdates: []*api.Restore{
NewRestore("foo", "bar", "backup-1", "ns-1", "", api.RestorePhaseInProgress).Restore,
@ -221,7 +223,7 @@ func TestProcessRestore(t *testing.T) {
{
name: "valid restore with RestorePVs=true gets executed when allowRestoreSnapshots=true",
restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", api.RestorePhaseNew).WithRestorePVs(true).Restore,
backup: NewTestBackup().WithName("backup-1").Backup,
backup: arktest.NewTestBackup().WithName("backup-1").Backup,
allowRestoreSnapshots: true,
expectedErr: false,
expectedRestoreUpdates: []*api.Restore{
@ -233,7 +235,7 @@ func TestProcessRestore(t *testing.T) {
{
name: "restore with RestorePVs=true fails validation when allowRestoreSnapshots=false",
restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", api.RestorePhaseNew).WithRestorePVs(true).Restore,
backup: NewTestBackup().WithName("backup-1").Backup,
backup: arktest.NewTestBackup().WithName("backup-1").Backup,
expectedErr: false,
expectedRestoreUpdates: []*api.Restore{
NewRestore("foo", "bar", "backup-1", "ns-1", "", api.RestorePhaseFailedValidation).
@ -245,7 +247,7 @@ func TestProcessRestore(t *testing.T) {
{
name: "restoration of nodes is not supported",
restore: NewRestore("foo", "bar", "backup-1", "ns-1", "nodes", api.RestorePhaseNew).Restore,
backup: NewTestBackup().WithName("backup-1").Backup,
backup: arktest.NewTestBackup().WithName("backup-1").Backup,
expectedErr: false,
expectedRestoreUpdates: []*api.Restore{
NewRestore("foo", "bar", "backup-1", "ns-1", "nodes", api.RestorePhaseFailedValidation).
@ -262,8 +264,9 @@ func TestProcessRestore(t *testing.T) {
client = fake.NewSimpleClientset()
restorer = &fakeRestorer{}
sharedInformers = informers.NewSharedInformerFactory(client, 0)
backupSvc = &BackupService{}
logger, _ = testlogger.NewNullLogger()
backupSvc = &arktest.BackupService{}
logger = arktest.NewLogger()
pluginManager = &MockManager{}
)
defer restorer.AssertExpectations(t)
@ -279,6 +282,7 @@ func TestProcessRestore(t *testing.T) {
sharedInformers.Ark().V1().Backups(),
test.allowRestoreSnapshots,
logger,
pluginManager,
).(*restoreController)
if test.restore != nil {
@ -331,6 +335,11 @@ func TestProcessRestore(t *testing.T) {
backupSvc.On("GetBackup", "bucket", test.restore.Spec.BackupName).Return(nil, test.backupServiceGetBackupError)
}
if test.restore != nil {
pluginManager.On("GetRestoreItemActions", test.restore.Name).Return(nil, nil)
pluginManager.On("CloseRestoreItemActions", test.restore.Name).Return(nil)
}
err = c.processRestore(key)
backupSvc.AssertExpectations(t)
restorer.AssertExpectations(t)
@ -367,8 +376,8 @@ func TestProcessRestore(t *testing.T) {
}
}
func NewRestore(ns, name, backup, includeNS, includeResource string, phase api.RestorePhase) *TestRestore {
restore := NewTestRestore(ns, name, phase).WithBackup(backup)
func NewRestore(ns, name, backup, includeNS, includeResource string, phase api.RestorePhase) *arktest.TestRestore {
restore := arktest.NewTestRestore(ns, name, phase).WithBackup(backup)
if includeNS != "" {
restore = restore.WithIncludedNamespace(includeNS)
@ -390,7 +399,13 @@ type fakeRestorer struct {
calledWithArg api.Restore
}
func (r *fakeRestorer) Restore(restore *api.Restore, backup *api.Backup, backupReader io.Reader, logger io.Writer) (api.RestoreResult, api.RestoreResult) {
func (r *fakeRestorer) Restore(
restore *api.Restore,
backup *api.Backup,
backupReader io.Reader,
logger io.Writer,
actions []restore.ItemAction,
) (api.RestoreResult, api.RestoreResult) {
res := r.Called(restore, backup, backupReader, logger)
r.calledWithArg = *restore

103
pkg/plugin/generated/BackupItemAction.pb.go
View File

@ -8,10 +8,10 @@ It is generated from these files:
BackupItemAction.proto
BlockStore.proto
ObjectStore.proto
RestoreItemAction.proto
Shared.proto
It has these top-level messages:
AppliesToResponse
ExecuteRequest
ExecuteResponse
ResourceIdentifier
@ -36,8 +36,11 @@ It has these top-level messages:
DeleteObjectRequest
CreateSignedURLRequest
CreateSignedURLResponse
RestoreExecuteRequest
RestoreExecuteResponse
Empty
InitRequest
AppliesToResponse
*/
package generated
@ -61,54 +64,6 @@ var _ = math.Inf
// proto package needs to be updated.
const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
type AppliesToResponse struct {
IncludedNamespaces []string `protobuf:"bytes,1,rep,name=includedNamespaces" json:"includedNamespaces,omitempty"`
ExcludedNamespaces []string `protobuf:"bytes,2,rep,name=excludedNamespaces" json:"excludedNamespaces,omitempty"`
IncludedResources []string `protobuf:"bytes,3,rep,name=includedResources" json:"includedResources,omitempty"`
ExcludedResources []string `protobuf:"bytes,4,rep,name=excludedResources" json:"excludedResources,omitempty"`
Selector string `protobuf:"bytes,5,opt,name=selector" json:"selector,omitempty"`
}
func (m *AppliesToResponse) Reset() { *m = AppliesToResponse{} }
func (m *AppliesToResponse) String() string { return proto.CompactTextString(m) }
func (*AppliesToResponse) ProtoMessage() {}
func (*AppliesToResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} }
func (m *AppliesToResponse) GetIncludedNamespaces() []string {
if m != nil {
return m.IncludedNamespaces
}
return nil
}
func (m *AppliesToResponse) GetExcludedNamespaces() []string {
if m != nil {
return m.ExcludedNamespaces
}
return nil
}
func (m *AppliesToResponse) GetIncludedResources() []string {
if m != nil {
return m.IncludedResources
}
return nil
}
func (m *AppliesToResponse) GetExcludedResources() []string {
if m != nil {
return m.ExcludedResources
}
return nil
}
func (m *AppliesToResponse) GetSelector() string {
if m != nil {
return m.Selector
}
return ""
}
type ExecuteRequest struct {
Item []byte `protobuf:"bytes,1,opt,name=item,proto3" json:"item,omitempty"`
Backup []byte `protobuf:"bytes,2,opt,name=backup,proto3" json:"backup,omitempty"`
@ -117,7 +72,7 @@ type ExecuteRequest struct {
func (m *ExecuteRequest) Reset() { *m = ExecuteRequest{} }
func (m *ExecuteRequest) String() string { return proto.CompactTextString(m) }
func (*ExecuteRequest) ProtoMessage() {}
func (*ExecuteRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} }
func (*ExecuteRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} }
func (m *ExecuteRequest) GetItem() []byte {
if m != nil {
@ -141,7 +96,7 @@ type ExecuteResponse struct {
func (m *ExecuteResponse) Reset() { *m = ExecuteResponse{} }
func (m *ExecuteResponse) String() string { return proto.CompactTextString(m) }
func (*ExecuteResponse) ProtoMessage() {}
func (*ExecuteResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} }
func (*ExecuteResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} }
func (m *ExecuteResponse) GetItem() []byte {
if m != nil {
@ -167,7 +122,7 @@ type ResourceIdentifier struct {
func (m *ResourceIdentifier) Reset() { *m = ResourceIdentifier{} }
func (m *ResourceIdentifier) String() string { return proto.CompactTextString(m) }
func (*ResourceIdentifier) ProtoMessage() {}
func (*ResourceIdentifier) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} }
func (*ResourceIdentifier) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} }
func (m *ResourceIdentifier) GetGroup() string {
if m != nil {
@ -198,7 +153,6 @@ func (m *ResourceIdentifier) GetName() string {
}
func init() {
proto.RegisterType((*AppliesToResponse)(nil), "generated.AppliesToResponse")
proto.RegisterType((*ExecuteRequest)(nil), "generated.ExecuteRequest")
proto.RegisterType((*ExecuteResponse)(nil), "generated.ExecuteResponse")
proto.RegisterType((*ResourceIdentifier)(nil), "generated.ResourceIdentifier")
@ -312,28 +266,23 @@ var _BackupItemAction_serviceDesc = grpc.ServiceDesc{
func init() { proto.RegisterFile("BackupItemAction.proto", fileDescriptor0) }
var fileDescriptor0 = []byte{
// 366 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x92, 0xcb, 0x4e, 0xeb, 0x30,
0x10, 0x86, 0x95, 0xde, 0xce, 0xc9, 0x9c, 0xea, 0xb4, 0xb5, 0x50, 0x15, 0xa2, 0x22, 0x55, 0x59,
0x75, 0x81, 0xb2, 0x28, 0x4b, 0x58, 0x50, 0xa4, 0x0a, 0x75, 0xc3, 0xc2, 0xf0, 0x02, 0x69, 0x32,
0x94, 0x88, 0xc4, 0x36, 0xb6, 0x23, 0x95, 0xc7, 0xe0, 0x39, 0x79, 0x09, 0x64, 0xe7, 0xd2, 0xd2,
0x74, 0x97, 0x99, 0xff, 0x9b, 0x89, 0xe7, 0x9f, 0x81, 0xe9, 0x43, 0x14, 0xbf, 0x17, 0x62, 0xa3,
0x31, 0x5f, 0xc5, 0x3a, 0xe5, 0x2c, 0x14, 0x92, 0x6b, 0x4e, 0xdc, 0x1d, 0x32, 0x94, 0x91, 0xc6,
0xc4, 0x1f, 0x3e, 0xbf, 0x45, 0x12, 0x93, 0x52, 0x08, 0xbe, 0x1d, 0x98, 0xac, 0x84, 0xc8, 0x52,
0x54, 0x2f, 0x9c, 0xa2, 0x12, 0x9c, 0x29, 0x24, 0x21, 0x90, 0x94, 0xc5, 0x59, 0x91, 0x60, 0xf2,
0x14, 0xe5, 0xa8, 0x44, 0x14, 0xa3, 0xf2, 0x9c, 0x79, 0x77, 0xe1, 0xd2, 0x33, 0x8a, 0xe1, 0x71,
0xdf, 0xe2, 0x3b, 0x25, 0xdf, 0x56, 0xc8, 0x35, 0x4c, 0xea, 0x2e, 0x14, 0x15, 0x2f, 0xa4, 0xc1,
0xbb, 0x16, 0x6f, 0x0b, 0x86, 0xae, 0x7b, 0x1c, 0xe8, 0x5e, 0x49, 0xb7, 0x04, 0xe2, 0xc3, 0x5f,
0x85, 0x19, 0xc6, 0x9a, 0x4b, 0xaf, 0x3f, 0x77, 0x16, 0x2e, 0x6d, 0xe2, 0xe0, 0x0e, 0xfe, 0xaf,
0xf7, 0x18, 0x17, 0x1a, 0x29, 0x7e, 0x14, 0xa8, 0x34, 0x21, 0xd0, 0x4b, 0x35, 0xe6, 0x9e, 0x33,
0x77, 0x16, 0x43, 0x6a, 0xbf, 0xc9, 0x14, 0x06, 0x5b, 0x6b, 0xa3, 0xd7, 0xb1, 0xd9, 0x2a, 0x0a,
0x18, 0x8c, 0x9a, 0xea, 0xca, 0xa8, 0x73, 0xe5, 0x8f, 0x30, 0x8a, 0x92, 0x24, 0x35, 0xee, 0x47,
0x99, 0xd9, 0x44, 0xe9, 0xc4, 0xbf, 0xe5, 0x55, 0xd8, 0x6c, 0x21, 0xac, 0xdf, 0xbb, 0x49, 0x90,
0xe9, 0xf4, 0x35, 0x45, 0x49, 0x4f, 0xab, 0x82, 0x3d, 0x90, 0x36, 0x46, 0x2e, 0xa0, 0xbf, 0x93,
0xbc, 0x10, 0xf6, 0x9f, 0x2e, 0x2d, 0x03, 0x33, 0xb5, 0xac, 0x58, 0xfb, 0x6a, 0x97, 0x36, 0x31,
0x99, 0x81, 0xcb, 0x6a, 0xef, 0xbd, 0xae, 0x15, 0x0f, 0x09, 0x33, 0x82, 0x09, 0xbc, 0x9e, 0x15,
0xec, 0xf7, 0xf2, 0xcb, 0x81, 0xf1, 0xe9, 0x25, 0x91, 0x5b, 0x70, 0x9b, 0x4b, 0x21, 0xe3, 0xa3,
0x59, 0xd6, 0xb9, 0xd0, 0x9f, 0xfe, 0xec, 0x28, 0xd3, 0xbe, 0xa8, 0x7b, 0xf8, 0x53, 0x79, 0x47,
0x2e, 0x8f, 0x4b, 0x7f, 0x6d, 0xc3, 0xf7, 0xcf, 0x49, 0x65, 0x87, 0xed, 0xc0, 0x1e, 0xec, 0xcd,
0x4f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xcf, 0x20, 0xf8, 0x53, 0xe3, 0x02, 0x00, 0x00,
// 288 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x51, 0xc1, 0x4e, 0xc2, 0x40,
0x10, 0x4d, 0x01, 0xd1, 0x8e, 0x44, 0xc8, 0xc4, 0x90, 0xda, 0x60, 0x42, 0x7a, 0xe2, 0xd4, 0x03,
0x1e, 0xf5, 0x20, 0x26, 0xc4, 0x70, 0x5d, 0xfd, 0x81, 0xa5, 0x1d, 0x71, 0x23, 0xdd, 0x5d, 0x77,
0xb7, 0x09, 0x7e, 0x86, 0x7f, 0x6c, 0xba, 0x6d, 0x6a, 0x45, 0x6e, 0xfb, 0xe6, 0xcd, 0x9b, 0x7d,
0xf3, 0x06, 0xa6, 0x4f, 0x3c, 0xfb, 0x28, 0xf5, 0xc6, 0x51, 0xb1, 0xca, 0x9c, 0x50, 0x32, 0xd5,
0x46, 0x39, 0x85, 0xe1, 0x8e, 0x24, 0x19, 0xee, 0x28, 0x8f, 0x47, 0x2f, 0xef, 0xdc, 0x50, 0x5e,
0x13, 0xc9, 0x03, 0x5c, 0xad, 0x0f, 0x94, 0x95, 0x8e, 0x18, 0x7d, 0x96, 0x64, 0x1d, 0x22, 0x0c,
0x84, 0xa3, 0x22, 0x0a, 0xe6, 0xc1, 0x62, 0xc4, 0xfc, 0x1b, 0xa7, 0x30, 0xdc, 0xfa, 0xc1, 0x51,
0xcf, 0x57, 0x1b, 0x94, 0x48, 0x18, 0xb7, 0x6a, 0xab, 0x95, 0xb4, 0x74, 0x52, 0xfe, 0x0c, 0x63,
0x9e, 0xe7, 0xa2, 0xf2, 0xc3, 0xf7, 0x95, 0x37, 0x1b, 0xf5, 0xe6, 0xfd, 0xc5, 0xe5, 0xf2, 0x36,
0x6d, 0x7d, 0xa5, 0x8c, 0xac, 0x2a, 0x4d, 0x46, 0x9b, 0x9c, 0xa4, 0x13, 0x6f, 0x82, 0x0c, 0x3b,
0x56, 0x25, 0x07, 0xc0, 0xff, 0x6d, 0x78, 0x0d, 0x67, 0x3b, 0xa3, 0x4a, 0xed, 0xff, 0x0c, 0x59,
0x0d, 0x30, 0x86, 0x0b, 0xd3, 0xf4, 0x7a, 0xd7, 0x21, 0x6b, 0x31, 0xce, 0x20, 0x94, 0xbc, 0x20,
0xab, 0x79, 0x46, 0x51, 0xdf, 0x93, 0xbf, 0x85, 0x6a, 0x85, 0x0a, 0x44, 0x03, 0x4f, 0xf8, 0xf7,
0xf2, 0x3b, 0x80, 0xc9, 0x71, 0xb6, 0x78, 0x0f, 0xe1, 0x4a, 0xeb, 0xbd, 0x20, 0xfb, 0xaa, 0x70,
0xd2, 0xd9, 0x65, 0x5d, 0x68, 0xf7, 0x15, 0xcf, 0x3a, 0x95, 0xb6, 0xaf, 0x0d, 0xea, 0x11, 0xce,
0x9b, 0xec, 0xf0, 0xa6, 0x2b, 0xfd, 0x73, 0x8d, 0x38, 0x3e, 0x45, 0xd5, 0x13, 0xb6, 0x43, 0x7f,
0xc2, 0xbb, 0x9f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x44, 0x09, 0x4d, 0x36, 0xf5, 0x01, 0x00, 0x00,
}

196
pkg/plugin/generated/RestoreItemAction.pb.go Normal file
View File

@ -0,0 +1,196 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// source: RestoreItemAction.proto
package generated
import proto "github.com/golang/protobuf/proto"
import fmt "fmt"
import math "math"
import (
context "golang.org/x/net/context"
grpc "google.golang.org/grpc"
)
// Reference imports to suppress errors if they are not otherwise used.
var _ = proto.Marshal
var _ = fmt.Errorf
var _ = math.Inf
type RestoreExecuteRequest struct {
Item []byte `protobuf:"bytes,1,opt,name=item,proto3" json:"item,omitempty"`
Restore []byte `protobuf:"bytes,2,opt,name=restore,proto3" json:"restore,omitempty"`
}
func (m *RestoreExecuteRequest) Reset() { *m = RestoreExecuteRequest{} }
func (m *RestoreExecuteRequest) String() string { return proto.CompactTextString(m) }
func (*RestoreExecuteRequest) ProtoMessage() {}
func (*RestoreExecuteRequest) Descriptor() ([]byte, []int) { return fileDescriptor3, []int{0} }
func (m *RestoreExecuteRequest) GetItem() []byte {
if m != nil {
return m.Item
}
return nil
}
func (m *RestoreExecuteRequest) GetRestore() []byte {
if m != nil {
return m.Restore
}
return nil
}
type RestoreExecuteResponse struct {
Item []byte `protobuf:"bytes,1,opt,name=item,proto3" json:"item,omitempty"`
Warning string `protobuf:"bytes,2,opt,name=warning" json:"warning,omitempty"`
}
func (m *RestoreExecuteResponse) Reset() { *m = RestoreExecuteResponse{} }
func (m *RestoreExecuteResponse) String() string { return proto.CompactTextString(m) }
func (*RestoreExecuteResponse) ProtoMessage() {}
func (*RestoreExecuteResponse) Descriptor() ([]byte, []int) { return fileDescriptor3, []int{1} }
func (m *RestoreExecuteResponse) GetItem() []byte {
if m != nil {
return m.Item
}
return nil
}
func (m *RestoreExecuteResponse) GetWarning() string {
if m != nil {
return m.Warning
}
return ""
}
func init() {
proto.RegisterType((*RestoreExecuteRequest)(nil), "generated.RestoreExecuteRequest")
proto.RegisterType((*RestoreExecuteResponse)(nil), "generated.RestoreExecuteResponse")
}
// Reference imports to suppress errors if they are not otherwise used.
var _ context.Context
var _ grpc.ClientConn
// This is a compile-time assertion to ensure that this generated file
// is compatible with the grpc package it is being compiled against.
const _ = grpc.SupportPackageIsVersion4
// Client API for RestoreItemAction service
type RestoreItemActionClient interface {
AppliesTo(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*AppliesToResponse, error)
Execute(ctx context.Context, in *RestoreExecuteRequest, opts ...grpc.CallOption) (*RestoreExecuteResponse, error)
}
type restoreItemActionClient struct {
cc *grpc.ClientConn
}
func NewRestoreItemActionClient(cc *grpc.ClientConn) RestoreItemActionClient {
return &restoreItemActionClient{cc}
}
func (c *restoreItemActionClient) AppliesTo(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*AppliesToResponse, error) {
out := new(AppliesToResponse)
err := grpc.Invoke(ctx, "/generated.RestoreItemAction/AppliesTo", in, out, c.cc, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *restoreItemActionClient) Execute(ctx context.Context, in *RestoreExecuteRequest, opts ...grpc.CallOption) (*RestoreExecuteResponse, error) {
out := new(RestoreExecuteResponse)
err := grpc.Invoke(ctx, "/generated.RestoreItemAction/Execute", in, out, c.cc, opts...)
if err != nil {
return nil, err
}
return out, nil
}
// Server API for RestoreItemAction service
type RestoreItemActionServer interface {
AppliesTo(context.Context, *Empty) (*AppliesToResponse, error)
Execute(context.Context, *RestoreExecuteRequest) (*RestoreExecuteResponse, error)
}
func RegisterRestoreItemActionServer(s *grpc.Server, srv RestoreItemActionServer) {
s.RegisterService(&_RestoreItemAction_serviceDesc, srv)
}
func _RestoreItemAction_AppliesTo_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(Empty)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(RestoreItemActionServer).AppliesTo(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/generated.RestoreItemAction/AppliesTo",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(RestoreItemActionServer).AppliesTo(ctx, req.(*Empty))
}
return interceptor(ctx, in, info, handler)
}
func _RestoreItemAction_Execute_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(RestoreExecuteRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(RestoreItemActionServer).Execute(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/generated.RestoreItemAction/Execute",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(RestoreItemActionServer).Execute(ctx, req.(*RestoreExecuteRequest))
}
return interceptor(ctx, in, info, handler)
}
var _RestoreItemAction_serviceDesc = grpc.ServiceDesc{
ServiceName: "generated.RestoreItemAction",
HandlerType: (*RestoreItemActionServer)(nil),
Methods: []grpc.MethodDesc{
{
MethodName: "AppliesTo",
Handler: _RestoreItemAction_AppliesTo_Handler,
},
{
MethodName: "Execute",
Handler: _RestoreItemAction_Execute_Handler,
},
},
Streams: []grpc.StreamDesc{},
Metadata: "RestoreItemAction.proto",
}
func init() { proto.RegisterFile("RestoreItemAction.proto", fileDescriptor3) }
var fileDescriptor3 = []byte{
// 210 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x12, 0x0f, 0x4a, 0x2d, 0x2e,
0xc9, 0x2f, 0x4a, 0xf5, 0x2c, 0x49, 0xcd, 0x75, 0x4c, 0x2e, 0xc9, 0xcc, 0xcf, 0xd3, 0x2b, 0x28,
0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x4c, 0x4f, 0xcd, 0x4b, 0x2d, 0x4a, 0x2c, 0x49, 0x4d, 0x91, 0xe2,
0x09, 0xce, 0x48, 0x2c, 0x4a, 0x4d, 0x81, 0x48, 0x28, 0xb9, 0x72, 0x89, 0x42, 0xf5, 0xb8, 0x56,
0xa4, 0x26, 0x97, 0x96, 0xa4, 0x06, 0xa5, 0x16, 0x96, 0xa6, 0x16, 0x97, 0x08, 0x09, 0x71, 0xb1,
0x64, 0x96, 0xa4, 0xe6, 0x4a, 0x30, 0x2a, 0x30, 0x6a, 0xf0, 0x04, 0x81, 0xd9, 0x42, 0x12, 0x5c,
0xec, 0x45, 0x10, 0xc5, 0x12, 0x4c, 0x60, 0x61, 0x18, 0x57, 0xc9, 0x8d, 0x4b, 0x0c, 0xdd, 0x98,
0xe2, 0x82, 0xfc, 0xbc, 0xe2, 0x54, 0x5c, 0xe6, 0x94, 0x27, 0x16, 0xe5, 0x65, 0xe6, 0xa5, 0x83,
0xcd, 0xe1, 0x0c, 0x82, 0x71, 0x8d, 0x16, 0x30, 0x72, 0x09, 0x62, 0xf8, 0x41, 0xc8, 0x9a, 0x8b,
0xd3, 0xb1, 0xa0, 0x20, 0x27, 0x33, 0xb5, 0x38, 0x24, 0x5f, 0x48, 0x40, 0x0f, 0xee, 0x17, 0x3d,
0xd7, 0xdc, 0x82, 0x92, 0x4a, 0x29, 0x19, 0x24, 0x11, 0xb8, 0x3a, 0xb8, 0x03, 0xfc, 0xb8, 0xd8,
0xa1, 0x6e, 0x12, 0x52, 0x40, 0x52, 0x88, 0xd5, 0xd7, 0x52, 0x8a, 0x78, 0x54, 0x40, 0xcc, 0x4b,
0x62, 0x03, 0x07, 0x9c, 0x31, 0x20, 0x00, 0x00, 0xff, 0xff, 0xb9, 0x08, 0x09, 0x74, 0x6c, 0x01,
0x00, 0x00,
}

86
pkg/plugin/generated/Shared.pb.go
View File

@ -18,7 +18,7 @@ type Empty struct {
func (m *Empty) Reset() { *m = Empty{} }
func (m *Empty) String() string { return proto.CompactTextString(m) }
func (*Empty) ProtoMessage() {}
func (*Empty) Descriptor() ([]byte, []int) { return fileDescriptor3, []int{0} }
func (*Empty) Descriptor() ([]byte, []int) { return fileDescriptor4, []int{0} }
type InitRequest struct {
Config map[string]string `protobuf:"bytes,1,rep,name=config" json:"config,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
@ -27,7 +27,7 @@ type InitRequest struct {
func (m *InitRequest) Reset() { *m = InitRequest{} }
func (m *InitRequest) String() string { return proto.CompactTextString(m) }
func (*InitRequest) ProtoMessage() {}
func (*InitRequest) Descriptor() ([]byte, []int) { return fileDescriptor3, []int{1} }
func (*InitRequest) Descriptor() ([]byte, []int) { return fileDescriptor4, []int{1} }
func (m *InitRequest) GetConfig() map[string]string {
if m != nil {
@ -36,23 +36,79 @@ func (m *InitRequest) GetConfig() map[string]string {
return nil
}
type AppliesToResponse struct {
IncludedNamespaces []string `protobuf:"bytes,1,rep,name=includedNamespaces" json:"includedNamespaces,omitempty"`
ExcludedNamespaces []string `protobuf:"bytes,2,rep,name=excludedNamespaces" json:"excludedNamespaces,omitempty"`
IncludedResources []string `protobuf:"bytes,3,rep,name=includedResources" json:"includedResources,omitempty"`
ExcludedResources []string `protobuf:"bytes,4,rep,name=excludedResources" json:"excludedResources,omitempty"`
Selector string `protobuf:"bytes,5,opt,name=selector" json:"selector,omitempty"`
}
func (m *AppliesToResponse) Reset() { *m = AppliesToResponse{} }
func (m *AppliesToResponse) String() string { return proto.CompactTextString(m) }
func (*AppliesToResponse) ProtoMessage() {}
func (*AppliesToResponse) Descriptor() ([]byte, []int) { return fileDescriptor4, []int{2} }
func (m *AppliesToResponse) GetIncludedNamespaces() []string {
if m != nil {
return m.IncludedNamespaces
}
return nil
}
func (m *AppliesToResponse) GetExcludedNamespaces() []string {
if m != nil {
return m.ExcludedNamespaces
}
return nil
}
func (m *AppliesToResponse) GetIncludedResources() []string {
if m != nil {
return m.IncludedResources
}
return nil
}
func (m *AppliesToResponse) GetExcludedResources() []string {
if m != nil {
return m.ExcludedResources
}
return nil
}
func (m *AppliesToResponse) GetSelector() string {
if m != nil {
return m.Selector
}
return ""
}
func init() {
proto.RegisterType((*Empty)(nil), "generated.Empty")
proto.RegisterType((*InitRequest)(nil), "generated.InitRequest")
proto.RegisterType((*AppliesToResponse)(nil), "generated.AppliesToResponse")
}
func init() { proto.RegisterFile("Shared.proto", fileDescriptor3) }
func init() { proto.RegisterFile("Shared.proto", fileDescriptor4) }
var fileDescriptor3 = []byte{
// 156 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x09, 0xce, 0x48, 0x2c,
0x4a, 0x4d, 0xd1, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x4c, 0x4f, 0xcd, 0x4b, 0x2d, 0x4a,
0x2c, 0x49, 0x4d, 0x51, 0x62, 0xe7, 0x62, 0x75, 0xcd, 0x2d, 0x28, 0xa9, 0x54, 0x6a, 0x61, 0xe4,
0xe2, 0xf6, 0xcc, 0xcb, 0x2c, 0x09, 0x4a, 0x2d, 0x2c, 0x4d, 0x2d, 0x2e, 0x11, 0xb2, 0xe2, 0x62,
0x4b, 0xce, 0xcf, 0x4b, 0xcb, 0x4c, 0x97, 0x60, 0x54, 0x60, 0xd6, 0xe0, 0x36, 0x52, 0xd2, 0x83,
0x6b, 0xd2, 0x43, 0x52, 0xa7, 0xe7, 0x0c, 0x56, 0xe4, 0x9a, 0x57, 0x52, 0x54, 0x19, 0x04, 0xd5,
0x21, 0x65, 0xc9, 0xc5, 0x8d, 0x24, 0x2c, 0x24, 0xc0, 0xc5, 0x9c, 0x9d, 0x5a, 0x29, 0xc1, 0xa8,
0xc0, 0xa8, 0xc1, 0x19, 0x04, 0x62, 0x0a, 0x89, 0x70, 0xb1, 0x96, 0x25, 0xe6, 0x94, 0xa6, 0x4a,
0x30, 0x81, 0xc5, 0x20, 0x1c, 0x2b, 0x26, 0x0b, 0xc6, 0x24, 0x36, 0xb0, 0x0b, 0x8d, 0x01, 0x01,
0x00, 0x00, 0xff, 0xff, 0x85, 0xab, 0x54, 0x37, 0xb1, 0x00, 0x00, 0x00,
var fileDescriptor4 = []byte{
// 257 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0xd1, 0xb1, 0x4e, 0xc3, 0x30,
0x10, 0x06, 0x60, 0xb9, 0x21, 0x85, 0x5c, 0x18, 0xa8, 0xc5, 0x10, 0x75, 0xaa, 0x32, 0x75, 0x40,
0x19, 0x60, 0x81, 0x6e, 0x08, 0x75, 0x60, 0x61, 0x30, 0xbc, 0x80, 0x49, 0x7e, 0x4a, 0x44, 0x6a,
0x1b, 0xdb, 0x41, 0x64, 0xe7, 0x6d, 0x79, 0x09, 0x14, 0x87, 0x96, 0x4a, 0x61, 0xf3, 0xdd, 0xff,
0xdd, 0xc9, 0x96, 0xe9, 0xf4, 0xf1, 0x55, 0x5a, 0x54, 0x85, 0xb1, 0xda, 0x6b, 0x9e, 0x6c, 0xa0,
0x60, 0xa5, 0x47, 0x95, 0x1f, 0x53, 0xbc, 0xde, 0x1a, 0xdf, 0xe5, 0x5f, 0x8c, 0xd2, 0x7b, 0x55,
0x7b, 0x81, 0xf7, 0x16, 0xce, 0xf3, 0x15, 0x4d, 0x4b, 0xad, 0x5e, 0xea, 0x4d, 0xc6, 0x16, 0xd1,
0x32, 0xbd, 0xcc, 0x8b, 0xfd, 0x50, 0x71, 0xe0, 0x8a, 0xbb, 0x80, 0xd6, 0xca, 0xdb, 0x4e, 0xfc,
0x4e, 0xcc, 0x6f, 0x28, 0x3d, 0x68, 0xf3, 0x33, 0x8a, 0xde, 0xd0, 0x65, 0x6c, 0xc1, 0x96, 0x89,
0xe8, 0x8f, 0xfc, 0x9c, 0xe2, 0x0f, 0xd9, 0xb4, 0xc8, 0x26, 0xa1, 0x37, 0x14, 0xab, 0xc9, 0x35,
0xcb, 0xbf, 0x19, 0xcd, 0x6e, 0x8d, 0x69, 0x6a, 0xb8, 0x27, 0x2d, 0xe0, 0x8c, 0x56, 0x0e, 0xbc,
0x20, 0x5e, 0xab, 0xb2, 0x69, 0x2b, 0x54, 0x0f, 0x72, 0x0b, 0x67, 0x64, 0x09, 0x17, 0x2e, 0x96,
0x88, 0x7f, 0x92, 0xde, 0xe3, 0x73, 0xe4, 0x27, 0x83, 0x1f, 0x27, 0xfc, 0x82, 0x66, 0xbb, 0x2d,
0x02, 0x4e, 0xb7, 0xb6, 0xe7, 0x51, 0xe0, 0xe3, 0xa0, 0xd7, 0xbb, 0x1d, 0x7f, 0xfa, 0x68, 0xd0,
0xa3, 0x80, 0xcf, 0xe9, 0xc4, 0xa1, 0x41, 0xe9, 0xb5, 0xcd, 0xe2, 0xf0, 0xdc, 0x7d, 0xfd, 0x3c,
0x0d, 0xff, 0x71, 0xf5, 0x13, 0x00, 0x00, 0xff, 0xff, 0x19, 0xd7, 0x88, 0x92, 0x9f, 0x01, 0x00,
0x00,
}

98
pkg/plugin/manager.go
View File

@ -23,13 +23,13 @@ import (
"path/filepath"
"strings"
"github.com/hashicorp/go-hclog"
plugin "github.com/hashicorp/go-plugin"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/heptio/ark/pkg/backup"
"github.com/heptio/ark/pkg/cloudprovider"
"github.com/heptio/ark/pkg/restore"
)
// PluginKind is a type alias for a string that describes
@ -71,6 +71,10 @@ const (
// a Backup ItemAction plugin.
PluginKindBackupItemAction PluginKind = "backupitemaction"
// PluginKindRestoreItemAction is the Kind string for
// a Restore ItemAction plugin.
PluginKindRestoreItemAction PluginKind = "restoreitemaction"
pluginDir = "/plugins"
)
@ -79,6 +83,7 @@ var AllPluginKinds = []PluginKind{
PluginKindBlockStore,
PluginKindCloudProvider,
PluginKindBackupItemAction,
PluginKindRestoreItemAction,
}
type pluginInfo struct {
@ -104,15 +109,26 @@ type Manager interface {
// (mainly because each one outputs to a per-backup log),
// and should be terminated upon completion of the backup with
// CloseBackupItemActions().
GetBackupItemActions(backupName string, logger logrus.FieldLogger, level logrus.Level) ([]backup.ItemAction, error)
GetBackupItemActions(backupName string) ([]backup.ItemAction, error)
// CloseBackupItemActions terminates the plugin sub-processes that
// are hosting BackupItemAction plugins for the given backup name.
CloseBackupItemActions(backupName string) error
// GetRestoreItemActions returns all restore.ItemAction plugins.
// These plugin instances should ONLY be used for a single restore
// (mainly because each one outputs to a per-restore log),
// and should be terminated upon completion of the restore with
// CloseRestoreItemActions().
GetRestoreItemActions(restoreName string) ([]restore.ItemAction, error)
// CloseRestoreItemActions terminates the plugin sub-processes that
// are hosting RestoreItemAction plugins for the given restore name.
CloseRestoreItemActions(restoreName string) error
}
type manager struct {
logger hclog.Logger
logger *logrusAdapter
pluginRegistry *registry
clientStore *clientStore
}
@ -162,7 +178,11 @@ func (m *manager) registerPlugins() error {
for _, provider := range []string{"aws", "gcp", "azure"} {
m.pluginRegistry.register(provider, "/ark", []string{"plugin", "cloudprovider", provider}, PluginKindObjectStore, PluginKindBlockStore)
}
m.pluginRegistry.register("backup_pv", "/ark", []string{"plugin", string(PluginKindBackupItemAction), "backup_pv"}, PluginKindBackupItemAction)
m.pluginRegistry.register("pv", "/ark", []string{"plugin", string(PluginKindBackupItemAction), "pv"}, PluginKindBackupItemAction)
m.pluginRegistry.register("job", "/ark", []string{"plugin", string(PluginKindRestoreItemAction), "job"}, PluginKindRestoreItemAction)
m.pluginRegistry.register("pod", "/ark", []string{"plugin", string(PluginKindRestoreItemAction), "pod"}, PluginKindRestoreItemAction)
m.pluginRegistry.register("svc", "/ark", []string{"plugin", string(PluginKindRestoreItemAction), "svc"}, PluginKindRestoreItemAction)
// second, register external plugins (these will override internal plugins, if applicable)
if _, err := os.Stat(pluginDir); err != nil {
@ -272,7 +292,7 @@ func (m *manager) getCloudProviderPlugin(name string, kind PluginKind) (interfac
// (mainly because each one outputs to a per-backup log),
// and should be terminated upon completion of the backup with
// CloseBackupActions().
func (m *manager) GetBackupItemActions(backupName string, logger logrus.FieldLogger, level logrus.Level) ([]backup.ItemAction, error) {
func (m *manager) GetBackupItemActions(backupName string) ([]backup.ItemAction, error) {
clients, err := m.clientStore.list(PluginKindBackupItemAction, backupName)
if err != nil {
pluginInfo, err := m.pluginRegistry.list(PluginKindBackupItemAction)
@ -280,14 +300,12 @@ func (m *manager) GetBackupItemActions(backupName string, logger logrus.FieldLog
return nil, err
}
// create clients for each, using the provided logger
log := &logrusAdapter{impl: logger, level: level}
// create clients for each
for _, plugin := range pluginInfo {
client := newClientBuilder(baseConfig()).
withCommand(plugin.commandName, plugin.commandArgs...).
withPlugin(PluginKindBackupItemAction, &BackupItemActionPlugin{log: log}).
withLogger(log).
withPlugin(PluginKindBackupItemAction, &BackupItemActionPlugin{log: m.logger}).
withLogger(m.logger).
client()
m.clientStore.add(client, PluginKindBackupItemAction, plugin.name, backupName)
@ -300,12 +318,14 @@ func (m *manager) GetBackupItemActions(backupName string, logger logrus.FieldLog
for _, client := range clients {
plugin, err := getPluginInstance(client, PluginKindBackupItemAction)
if err != nil {
m.CloseBackupItemActions(backupName)
return nil, err
}
backupAction, ok := plugin.(backup.ItemAction)
if !ok {
return nil, errors.New("could not convert gRPC client to backup.BackupAction")
m.CloseBackupItemActions(backupName)
return nil, errors.New("could not convert gRPC client to backup.ItemAction")
}
backupActions = append(backupActions, backupAction)
@ -317,7 +337,59 @@ func (m *manager) GetBackupItemActions(backupName string, logger logrus.FieldLog
// CloseBackupItemActions terminates the plugin sub-processes that
// are hosting BackupItemAction plugins for the given backup name.
func (m *manager) CloseBackupItemActions(backupName string) error {
clients, err := m.clientStore.list(PluginKindBackupItemAction, backupName)
return closeAll(m.clientStore, PluginKindBackupItemAction, backupName)
}
func (m *manager) GetRestoreItemActions(restoreName string) ([]restore.ItemAction, error) {
clients, err := m.clientStore.list(PluginKindRestoreItemAction, restoreName)
if err != nil {
pluginInfo, err := m.pluginRegistry.list(PluginKindRestoreItemAction)
if err != nil {
return nil, err
}
// create clients for each
for _, plugin := range pluginInfo {
client := newClientBuilder(baseConfig()).
withCommand(plugin.commandName, plugin.commandArgs...).
withPlugin(PluginKindRestoreItemAction, &RestoreItemActionPlugin{log: m.logger}).
withLogger(m.logger).
client()
m.clientStore.add(client, PluginKindRestoreItemAction, plugin.name, restoreName)
clients = append(clients, client)
}
}
var itemActions []restore.ItemAction
for _, client := range clients {
plugin, err := getPluginInstance(client, PluginKindRestoreItemAction)
if err != nil {
m.CloseRestoreItemActions(restoreName)
return nil, err
}
itemAction, ok := plugin.(restore.ItemAction)
if !ok {
m.CloseRestoreItemActions(restoreName)
return nil, errors.New("could not convert gRPC client to restore.ItemAction")
}
itemActions = append(itemActions, itemAction)
}
return itemActions, nil
}
// CloseRestoreItemActions terminates the plugin sub-processes that
// are hosting RestoreItemAction plugins for the given restore name.
func (m *manager) CloseRestoreItemActions(restoreName string) error {
return closeAll(m.clientStore, PluginKindRestoreItemAction, restoreName)
}
func closeAll(store *clientStore, kind PluginKind, scope string) error {
clients, err := store.list(kind, scope)
if err != nil {
return err
}
@ -326,7 +398,7 @@ func (m *manager) CloseBackupItemActions(backupName string) error {
client.Kill()
}
m.clientStore.deleteAll(PluginKindBackupItemAction, backupName)
store.deleteAll(kind, scope)
return nil
}

8
pkg/plugin/proto/BackupItemAction.proto
View File

@ -3,14 +3,6 @@ package generated;
import "Shared.proto";
message AppliesToResponse {
repeated string includedNamespaces = 1;
repeated string excludedNamespaces = 2;
repeated string includedResources = 3;
repeated string excludedResources = 4;
string selector = 5;
}
message ExecuteRequest {
bytes item = 1;
bytes backup = 2;

19
pkg/plugin/proto/RestoreItemAction.proto Normal file
View File

@ -0,0 +1,19 @@
syntax = "proto3";
package generated;
import "Shared.proto";
message RestoreExecuteRequest {
bytes item = 1;
bytes restore = 2;
}
message RestoreExecuteResponse {
bytes item = 1;
string warning = 2;
}
service RestoreItemAction {
rpc AppliesTo(Empty) returns (AppliesToResponse);
rpc Execute(RestoreExecuteRequest) returns (RestoreExecuteResponse);
}

8
pkg/plugin/proto/Shared.proto
View File

@ -5,4 +5,12 @@ message Empty {}
message InitRequest {
map<string, string> config = 1;
}
message AppliesToResponse {
repeated string includedNamespaces = 1;
repeated string excludedNamespaces = 2;
repeated string includedResources = 3;
repeated string excludedResources = 4;
string selector = 5;
}

177
pkg/plugin/restore_item_action.go Normal file
View File

@ -0,0 +1,177 @@
/*
Copyright 2017 the Heptio Ark contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package plugin
import (
"encoding/json"
"github.com/hashicorp/go-plugin"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"golang.org/x/net/context"
"google.golang.org/grpc"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
api "github.com/heptio/ark/pkg/apis/ark/v1"
proto "github.com/heptio/ark/pkg/plugin/generated"
"github.com/heptio/ark/pkg/restore"
)
// RestoreItemActionPlugin is an implementation of go-plugin's Plugin
// interface with support for gRPC for the restore/ItemAction
// interface.
type RestoreItemActionPlugin struct {
plugin.NetRPCUnsupportedPlugin
impl restore.ItemAction
log *logrusAdapter
}
// NewRestoreItemActionPlugin constructs a RestoreItemActionPlugin.
func NewRestoreItemActionPlugin(itemAction restore.ItemAction) *RestoreItemActionPlugin {
return &RestoreItemActionPlugin{
impl: itemAction,
}
}
// GRPCServer registers a RestoreItemAction gRPC server.
func (p *RestoreItemActionPlugin) GRPCServer(s *grpc.Server) error {
proto.RegisterRestoreItemActionServer(s, &RestoreItemActionGRPCServer{impl: p.impl})
return nil
}
// GRPCClient returns a RestoreItemAction gRPC client.
func (p *RestoreItemActionPlugin) GRPCClient(c *grpc.ClientConn) (interface{}, error) {
return &RestoreItemActionGRPCClient{grpcClient: proto.NewRestoreItemActionClient(c), log: p.log}, nil
}
// RestoreItemActionGRPCClient implements the backup/ItemAction interface and uses a
// gRPC client to make calls to the plugin server.
type RestoreItemActionGRPCClient struct {
grpcClient proto.RestoreItemActionClient
log *logrusAdapter
}
func (c *RestoreItemActionGRPCClient) AppliesTo() (restore.ResourceSelector, error) {
res, err := c.grpcClient.AppliesTo(context.Background(), &proto.Empty{})
if err != nil {
return restore.ResourceSelector{}, err
}
return restore.ResourceSelector{
IncludedNamespaces: res.IncludedNamespaces,
ExcludedNamespaces: res.ExcludedNamespaces,
IncludedResources: res.IncludedResources,
ExcludedResources: res.ExcludedResources,
LabelSelector: res.Selector,
}, nil
}
func (c *RestoreItemActionGRPCClient) Execute(item runtime.Unstructured, restore *api.Restore) (runtime.Unstructured, error, error) {
itemJSON, err := json.Marshal(item.UnstructuredContent())
if err != nil {
return nil, nil, err
}
restoreJSON, err := json.Marshal(restore)
if err != nil {
return nil, nil, err
}
req := &proto.RestoreExecuteRequest{
Item: itemJSON,
Restore: restoreJSON,
}
res, err := c.grpcClient.Execute(context.Background(), req)
if err != nil {
return nil, nil, err
}
var updatedItem unstructured.Unstructured
if err := json.Unmarshal(res.Item, &updatedItem); err != nil {
return nil, nil, err
}
var warning error
if res.Warning != "" {
warning = errors.New(res.Warning)
}
return &updatedItem, warning, nil
}
func (c *RestoreItemActionGRPCClient) SetLog(log logrus.FieldLogger) {
c.log.impl = log
}
// RestoreItemActionGRPCServer implements the proto-generated RestoreItemActionServer interface, and accepts
// gRPC calls and forwards them to an implementation of the pluggable interface.
type RestoreItemActionGRPCServer struct {
impl restore.ItemAction
}
func (s *RestoreItemActionGRPCServer) AppliesTo(ctx context.Context, req *proto.Empty) (*proto.AppliesToResponse, error) {
appliesTo, err := s.impl.AppliesTo()
if err != nil {
return nil, err
}
return &proto.AppliesToResponse{
IncludedNamespaces: appliesTo.IncludedNamespaces,
ExcludedNamespaces: appliesTo.ExcludedNamespaces,
IncludedResources: appliesTo.IncludedResources,
ExcludedResources: appliesTo.ExcludedResources,
Selector: appliesTo.LabelSelector,
}, nil
}
func (s *RestoreItemActionGRPCServer) Execute(ctx context.Context, req *proto.RestoreExecuteRequest) (*proto.RestoreExecuteResponse, error) {
var (
item unstructured.Unstructured
restore api.Restore
)
if err := json.Unmarshal(req.Item, &item); err != nil {
return nil, err
}
if err := json.Unmarshal(req.Restore, &restore); err != nil {
return nil, err
}
res, warning, err := s.impl.Execute(&item, &restore)
if err != nil {
return nil, err
}
updatedItem, err := json.Marshal(res)
if err != nil {
return nil, err
}
var warnMessage string
if warning != nil {
warnMessage = warning.Error()
}
return &proto.RestoreExecuteResponse{
Item: updatedItem,
Warning: warnMessage,
}, nil
}

43
pkg/restore/item_action.go Normal file
View File

@ -0,0 +1,43 @@
/*
Copyright 2017 the Heptio Ark contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package restore
import (
"k8s.io/apimachinery/pkg/runtime"
api "github.com/heptio/ark/pkg/apis/ark/v1"
)
// ItemAction is an actor that performs an operation on an individual item being restored.
type ItemAction interface {
// AppliesTo returns information about which resources this action should be invoked for.
AppliesTo() (ResourceSelector, error)
// Execute allows the ItemAction to perform arbitrary logic with the item being restored.
Execute(obj runtime.Unstructured, restore *api.Restore) (res runtime.Unstructured, warning error, err error)
}
// ResourceSelector is a collection of included/excluded namespaces,
// included/excluded resources, and a label-selector that can be used
// to match a set of items from a cluster.
type ResourceSelector struct {
IncludedNamespaces []string
ExcludedNamespaces []string
IncludedResources []string
ExcludedResources []string
LabelSelector string
}

40
pkg/restore/restorers/job_restorer.go → pkg/restore/job_action.go
View File

@ -1,5 +1,5 @@
/*
Copyright 2017 Heptio Inc.
Copyright 2017 the Heptio Ark contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package restorers
package restore
import (
"github.com/sirupsen/logrus"
@ -25,39 +25,33 @@ import (
"github.com/heptio/ark/pkg/util/collections"
)
type jobRestorer struct {
logger *logrus.Logger
type jobAction struct {
logger logrus.FieldLogger
}
var _ ResourceRestorer = &jobRestorer{}
func NewJobRestorer(logger *logrus.Logger) ResourceRestorer {
return &jobRestorer{
func NewJobAction(logger logrus.FieldLogger) ItemAction {
return &jobAction{
logger: logger,
}
}
func (r *jobRestorer) Handles(obj runtime.Unstructured, restore *api.Restore) bool {
return true
func (a *jobAction) AppliesTo() (ResourceSelector, error) {
return ResourceSelector{
IncludedResources: []string{"jobs"},
}, nil
}
func (r *jobRestorer) Prepare(obj runtime.Unstructured, restore *api.Restore, backup *api.Backup) (runtime.Unstructured, error, error) {
r.logger.Debug("resetting metadata and status")
_, err := resetMetadataAndStatus(obj, true)
if err != nil {
return nil, nil, err
}
func (a *jobAction) Execute(obj runtime.Unstructured, restore *api.Restore) (runtime.Unstructured, error, error) {
fieldDeletions := map[string]string{
"spec.selector.matchLabels": "controller-uid",
"spec.template.metadata.labels": "controller-uid",
}
for k, v := range fieldDeletions {
r.logger.Debugf("Getting %s", k)
a.logger.Debugf("Getting %s", k)
labels, err := collections.GetMap(obj.UnstructuredContent(), k)
if err != nil {
r.logger.WithError(err).Debugf("Unable to get %s", k)
a.logger.WithError(err).Debugf("Unable to get %s", k)
} else {
delete(labels, v)
}
@ -65,11 +59,3 @@ func (r *jobRestorer) Prepare(obj runtime.Unstructured, restore *api.Restore, ba
return obj, nil, nil
}
func (r *jobRestorer) Wait() bool {
return false
}
func (r *jobRestorer) Ready(obj runtime.Unstructured) bool {
return true
}

21
pkg/restore/restorers/job_restorer_test.go → pkg/restore/job_action_test.go
View File

@ -1,5 +1,5 @@
/*
Copyright 2017 Heptio Inc.
Copyright 2017 the Heptio Ark contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@ -14,29 +14,25 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package restorers
package restore
import (
"testing"
testlogger "github.com/sirupsen/logrus/hooks/test"
"github.com/stretchr/testify/assert"
"k8s.io/apimachinery/pkg/runtime"
arktest "github.com/heptio/ark/pkg/util/test"
)
func TestJobRestorerPrepare(t *testing.T) {
func TestJobActionExecute(t *testing.T) {
tests := []struct {
name string
obj runtime.Unstructured
expectedErr bool
expectedRes runtime.Unstructured
}{
{
name: "no metadata should error",
obj: NewTestUnstructured().Unstructured,
expectedErr: true,
},
{
name: "missing spec.selector and/or spec.template should not error",
obj: NewTestUnstructured().WithName("job-1").
@ -127,12 +123,9 @@ func TestJobRestorerPrepare(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
var (
logger, _ = testlogger.NewNullLogger()
restorer = NewJobRestorer(logger)
)
action := NewJobAction(arktest.NewLogger())
res, _, err := restorer.Prepare(test.obj, nil, nil)
res, _, err := action.Execute(test.obj, nil)
if assert.Equal(t, test.expectedErr, err != nil) {
assert.Equal(t, test.expectedRes, res)

58
pkg/restore/restorers/pod_restorer.go → pkg/restore/pod_action.go
View File

@ -1,5 +1,5 @@
/*
Copyright 2017 Heptio Inc.
Copyright 2017 the Heptio Ark contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package restorers
package restore
import (
"regexp"
@ -27,56 +27,50 @@ import (
"github.com/heptio/ark/pkg/util/collections"
)
type podRestorer struct {
logger *logrus.Logger
type podAction struct {
logger logrus.FieldLogger
}
var _ ResourceRestorer = &podRestorer{}
func NewPodRestorer(logger *logrus.Logger) ResourceRestorer {
return &podRestorer{
func NewPodAction(logger logrus.FieldLogger) ItemAction {
return &podAction{
logger: logger,
}
}
func (nsr *podRestorer) Handles(obj runtime.Unstructured, restore *api.Restore) bool {
return true
func (a *podAction) AppliesTo() (ResourceSelector, error) {
return ResourceSelector{
IncludedResources: []string{"pods"},
}, nil
}
var (
defaultTokenRegex = regexp.MustCompile("default-token-.*")
)
func (r *podRestorer) Prepare(obj runtime.Unstructured, restore *api.Restore, backup *api.Backup) (runtime.Unstructured, error, error) {
r.logger.Debug("resetting metadata and status")
_, err := resetMetadataAndStatus(obj, true)
if err != nil {
return nil, nil, err
}
r.logger.Debug("getting spec")
func (a *podAction) Execute(obj runtime.Unstructured, restore *api.Restore) (runtime.Unstructured, error, error) {
a.logger.Debug("getting spec")
spec, err := collections.GetMap(obj.UnstructuredContent(), "spec")
if err != nil {
return nil, nil, err
}
r.logger.Debug("deleting spec.NodeName")
a.logger.Debug("deleting spec.NodeName")
delete(spec, "nodeName")
newVolumes := make([]interface{}, 0)
r.logger.Debug("iterating over volumes")
a.logger.Debug("iterating over volumes")
err = collections.ForEach(spec, "volumes", func(volume map[string]interface{}) error {
name, err := collections.GetString(volume, "name")
if err != nil {
return err
}
r.logger.WithField("volumeName", name).Debug("Checking volume")
a.logger.WithField("volumeName", name).Debug("Checking volume")
if !defaultTokenRegex.MatchString(name) {
r.logger.WithField("volumeName", name).Debug("Preserving volume")
a.logger.WithField("volumeName", name).Debug("Preserving volume")
newVolumes = append(newVolumes, volume)
} else {
r.logger.WithField("volumeName", name).Debug("Excluding volume")
a.logger.WithField("volumeName", name).Debug("Excluding volume")
}
return nil
@ -85,10 +79,10 @@ func (r *podRestorer) Prepare(obj runtime.Unstructured, restore *api.Restore, ba
return nil, nil, err
}
r.logger.Debug("Setting spec.volumes")
a.logger.Debug("Setting spec.volumes")
spec["volumes"] = newVolumes
r.logger.Debug("iterating over containers")
a.logger.Debug("iterating over containers")
err = collections.ForEach(spec, "containers", func(container map[string]interface{}) error {
var newVolumeMounts []interface{}
err := collections.ForEach(container, "volumeMounts", func(volumeMount map[string]interface{}) error {
@ -97,12 +91,12 @@ func (r *podRestorer) Prepare(obj runtime.Unstructured, restore *api.Restore, ba
return err
}
r.logger.WithField("volumeMount", name).Debug("Checking volumeMount")
a.logger.WithField("volumeMount", name).Debug("Checking volumeMount")
if !defaultTokenRegex.MatchString(name) {
r.logger.WithField("volumeMount", name).Debug("Preserving volumeMount")
a.logger.WithField("volumeMount", name).Debug("Preserving volumeMount")
newVolumeMounts = append(newVolumeMounts, volumeMount)
} else {
r.logger.WithField("volumeMount", name).Debug("Excluding volumeMount")
a.logger.WithField("volumeMount", name).Debug("Excluding volumeMount")
}
return nil
@ -121,11 +115,3 @@ func (r *podRestorer) Prepare(obj runtime.Unstructured, restore *api.Restore, ba
return obj, nil, nil
}
func (nsr *podRestorer) Wait() bool {
return false
}
func (nsr *podRestorer) Ready(obj runtime.Unstructured) bool {
return true
}

15
pkg/restore/restorers/pod_restorer_test.go → pkg/restore/pod_action_test.go
View File

@ -1,5 +1,5 @@
/*
Copyright 2017 Heptio Inc.
Copyright 2017 the Heptio Ark contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@ -14,18 +14,18 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package restorers
package restore
import (
"testing"
testlogger "github.com/sirupsen/logrus/hooks/test"
arktest "github.com/heptio/ark/pkg/util/test"
"github.com/stretchr/testify/assert"
"k8s.io/apimachinery/pkg/runtime"
)
func TestPodRestorerPrepare(t *testing.T) {
func TestPodActionExecute(t *testing.T) {
tests := []struct {
name string
obj runtime.Unstructured
@ -97,12 +97,9 @@ func TestPodRestorerPrepare(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
var (
logger, _ = testlogger.NewNullLogger()
restorer = NewPodRestorer(logger)
)
action := NewPodAction(arktest.NewLogger())
res, _, err := restorer.Prepare(test.obj, nil, nil)
res, _, err := action.Execute(test.obj, nil)
if assert.Equal(t, test.expectedErr, err != nil) {
assert.Equal(t, test.expectedRes, res)

10
pkg/restore/resource_waiter.go
View File

@ -36,14 +36,16 @@ const objectCreateWaitTimeout = 30 * time.Second
// of this struct is to construct it, register all of the desired items to wait for via
// RegisterItem, and then to Wait() for them to become ready or the timeout to be exceeded.
type resourceWaiter struct {
itemWatch watch.Interface
watchChan <-chan watch.Event
items sets.String
readyFunc func(runtime.Unstructured) bool
}
func newResourceWaiter(watchChan <-chan watch.Event, readyFunc func(runtime.Unstructured) bool) *resourceWaiter {
func newResourceWaiter(itemWatch watch.Interface, readyFunc func(runtime.Unstructured) bool) *resourceWaiter {
return &resourceWaiter{
watchChan: watchChan,
itemWatch: itemWatch,
watchChan: itemWatch.ResultChan(),
items: sets.NewString(),
readyFunc: readyFunc,
}
@ -82,3 +84,7 @@ func (rw *resourceWaiter) Wait() error {
}
}
}
func (rw *resourceWaiter) Stop() {
rw.itemWatch.Stop()
}

443
pkg/restore/restore.go
View File

@ -20,13 +20,14 @@ import (
"archive/tar"
"compress/gzip"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"sort"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"k8s.io/api/core/v1"
@ -34,6 +35,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/sets"
corev1 "k8s.io/client-go/kubernetes/typed/core/v1"
@ -43,7 +45,6 @@ import (
"github.com/heptio/ark/pkg/cloudprovider"
"github.com/heptio/ark/pkg/discovery"
arkv1client "github.com/heptio/ark/pkg/generated/clientset/versioned/typed/ark/v1"
"github.com/heptio/ark/pkg/restore/restorers"
"github.com/heptio/ark/pkg/util/collections"
"github.com/heptio/ark/pkg/util/kube"
"github.com/heptio/ark/pkg/util/logging"
@ -52,11 +53,9 @@ import (
// Restorer knows how to restore a backup.
type Restorer interface {
// Restore restores the backup data from backupReader, returning warnings and errors.
Restore(restore *api.Restore, backup *api.Backup, backupReader io.Reader, logFile io.Writer) (api.RestoreResult, api.RestoreResult)
Restore(restore *api.Restore, backup *api.Backup, backupReader io.Reader, logFile io.Writer, actions []ItemAction) (api.RestoreResult, api.RestoreResult)
}
var _ Restorer = &kubernetesRestorer{}
type gvString string
type kindString string
@ -64,8 +63,8 @@ type kindString string
type kubernetesRestorer struct {
discoveryHelper discovery.Helper
dynamicFactory client.DynamicFactory
restorers map[schema.GroupResource]restorers.ResourceRestorer
backupService cloudprovider.BackupService
snapshotService cloudprovider.SnapshotService
backupClient arkv1client.BackupsGetter
namespaceClient corev1.NamespaceInterface
resourcePriorities []string
@ -75,7 +74,7 @@ type kubernetesRestorer struct {
// prioritizeResources returns an ordered, fully-resolved list of resources to restore based on
// the provided discovery helper, resource priorities, and included/excluded resources.
func prioritizeResources(helper discovery.Helper, priorities []string, includedResources *collections.IncludesExcludes, logger *logrus.Logger) ([]schema.GroupResource, error) {
func prioritizeResources(helper discovery.Helper, priorities []string, includedResources *collections.IncludesExcludes, logger logrus.FieldLogger) ([]schema.GroupResource, error) {
var ret []schema.GroupResource
// set keeps track of resolved GroupResource names
@ -110,7 +109,7 @@ func prioritizeResources(helper discovery.Helper, priorities []string, includedR
gr := groupVersion.WithResource(resource.Name).GroupResource()
if !includedResources.ShouldInclude(gr.String()) {
logger.WithField("groupResource", gr.String()).Debug("Not including resource")
logger.WithField("groupResource", gr.String()).Info("Not including resource")
continue
}
@ -135,27 +134,18 @@ func prioritizeResources(helper discovery.Helper, priorities []string, includedR
func NewKubernetesRestorer(
discoveryHelper discovery.Helper,
dynamicFactory client.DynamicFactory,
customRestorers map[string]restorers.ResourceRestorer,
backupService cloudprovider.BackupService,
snapshotService cloudprovider.SnapshotService,
resourcePriorities []string,
backupClient arkv1client.BackupsGetter,
namespaceClient corev1.NamespaceInterface,
logger *logrus.Logger,
) (Restorer, error) {
r := make(map[schema.GroupResource]restorers.ResourceRestorer)
for gr, restorer := range customRestorers {
gvr, _, err := discoveryHelper.ResourceFor(schema.ParseGroupResource(gr).WithVersion(""))
if err != nil {
return nil, err
}
r[gvr.GroupResource()] = restorer
}
return &kubernetesRestorer{
discoveryHelper: discoveryHelper,
dynamicFactory: dynamicFactory,
restorers: r,
backupService: backupService,
snapshotService: snapshotService,
backupClient: backupClient,
namespaceClient: namespaceClient,
resourcePriorities: resourcePriorities,
@ -167,7 +157,7 @@ func NewKubernetesRestorer(
// Restore executes a restore into the target Kubernetes cluster according to the restore spec
// and using data from the provided backup/backup reader. Returns a warnings and errors RestoreResult,
// respectively, summarizing info about the restore.
func (kr *kubernetesRestorer) Restore(restore *api.Restore, backup *api.Backup, backupReader io.Reader, logFile io.Writer) (api.RestoreResult, api.RestoreResult) {
func (kr *kubernetesRestorer) Restore(restore *api.Restore, backup *api.Backup, backupReader io.Reader, logFile io.Writer, actions []ItemAction) (api.RestoreResult, api.RestoreResult) {
// metav1.LabelSelectorAsSelector converts a nil LabelSelector to a
// Nothing Selector, i.e. a selector that matches nothing. We want
// a selector that matches everything. This can be accomplished by
@ -182,27 +172,6 @@ func (kr *kubernetesRestorer) Restore(restore *api.Restore, backup *api.Backup,
return api.RestoreResult{}, api.RestoreResult{Ark: []string{err.Error()}}
}
// get resource includes-excludes
resourceIncludesExcludes := collections.GenerateIncludesExcludes(
restore.Spec.IncludedResources,
restore.Spec.ExcludedResources,
func(item string) string {
gvr, _, err := kr.discoveryHelper.ResourceFor(schema.ParseGroupResource(item).WithVersion(""))
if err != nil {
kr.logger.WithError(err).WithField("resource", item).Error("Unable to resolve resource")
return ""
}
gr := gvr.GroupResource()
return gr.String()
},
)
prioritizedResources, err := prioritizeResources(kr.discoveryHelper, kr.resourcePriorities, resourceIncludesExcludes, kr.logger)
if err != nil {
return api.RestoreResult{}, api.RestoreResult{Ark: []string{err.Error()}}
}
gzippedLog := gzip.NewWriter(logFile)
defer gzippedLog.Close()
@ -211,6 +180,18 @@ func (kr *kubernetesRestorer) Restore(restore *api.Restore, backup *api.Backup,
log.Hooks.Add(&logging.ErrorLocationHook{})
log.Hooks.Add(&logging.LogLocationHook{})
// get resource includes-excludes
resourceIncludesExcludes := getResourceIncludesExcludes(kr.discoveryHelper, restore.Spec.IncludedResources, restore.Spec.ExcludedResources)
prioritizedResources, err := prioritizeResources(kr.discoveryHelper, kr.resourcePriorities, resourceIncludesExcludes, log)
if err != nil {
return api.RestoreResult{}, api.RestoreResult{Ark: []string{err.Error()}}
}
resolvedActions, err := resolveActions(actions, kr.discoveryHelper)
if err != nil {
return api.RestoreResult{}, api.RestoreResult{Ark: []string{err.Error()}}
}
ctx := &context{
backup: backup,
backupReader: backupReader,
@ -221,23 +202,88 @@ func (kr *kubernetesRestorer) Restore(restore *api.Restore, backup *api.Backup,
dynamicFactory: kr.dynamicFactory,
fileSystem: kr.fileSystem,
namespaceClient: kr.namespaceClient,
restorers: kr.restorers,
actions: resolvedActions,
snapshotService: kr.snapshotService,
waitForPVs: true,
}
return ctx.execute()
}
// getResourceIncludesExcludes takes the lists of resources to include and exclude, uses the
// discovery helper to resolve them to fully-qualified group-resource names, and returns an
// IncludesExcludes list.
func getResourceIncludesExcludes(helper discovery.Helper, includes, excludes []string) *collections.IncludesExcludes {
resources := collections.GenerateIncludesExcludes(
includes,
excludes,
func(item string) string {
gvr, _, err := helper.ResourceFor(schema.ParseGroupResource(item).WithVersion(""))
if err != nil {
return ""
}
gr := gvr.GroupResource()
return gr.String()
},
)
return resources
}
type resolvedAction struct {
ItemAction
resourceIncludesExcludes *collections.IncludesExcludes
namespaceIncludesExcludes *collections.IncludesExcludes
selector labels.Selector
}
func resolveActions(actions []ItemAction, helper discovery.Helper) ([]resolvedAction, error) {
var resolved []resolvedAction
for _, action := range actions {
resourceSelector, err := action.AppliesTo()
if err != nil {
return nil, err
}
resources := getResourceIncludesExcludes(helper, resourceSelector.IncludedResources, resourceSelector.ExcludedResources)
namespaces := collections.NewIncludesExcludes().Includes(resourceSelector.IncludedNamespaces...).Excludes(resourceSelector.ExcludedNamespaces...)
selector := labels.Everything()
if resourceSelector.LabelSelector != "" {
if selector, err = labels.Parse(resourceSelector.LabelSelector); err != nil {
return nil, err
}
}
res := resolvedAction{
ItemAction: action,
resourceIncludesExcludes: resources,
namespaceIncludesExcludes: namespaces,
selector: selector,
}
resolved = append(resolved, res)
}
return resolved, nil
}
type context struct {
backup *api.Backup
backupReader io.Reader
restore *api.Restore
prioritizedResources []schema.GroupResource
selector labels.Selector
logger *logrus.Logger
logger logrus.FieldLogger
dynamicFactory client.DynamicFactory
fileSystem FileSystem
namespaceClient corev1.NamespaceInterface
restorers map[schema.GroupResource]restorers.ResourceRestorer
actions []resolvedAction
snapshotService cloudprovider.SnapshotService
waitForPVs bool
}
func (ctx *context) infof(msg string, args ...interface{}) {
@ -262,7 +308,9 @@ func (ctx *context) execute() (api.RestoreResult, api.RestoreResult) {
func (ctx *context) restoreFromDir(dir string) (api.RestoreResult, api.RestoreResult) {
warnings, errs := api.RestoreResult{}, api.RestoreResult{}
namespaceFilter := collections.NewIncludesExcludes().Includes(ctx.restore.Spec.IncludedNamespaces...).Excludes(ctx.restore.Spec.ExcludedNamespaces...)
namespaceFilter := collections.NewIncludesExcludes().
Includes(ctx.restore.Spec.IncludedNamespaces...).
Excludes(ctx.restore.Spec.ExcludedNamespaces...)
// Make sure the top level "resources" dir exists:
resourcesDir := filepath.Join(dir, api.ResourcesDir)
@ -273,6 +321,7 @@ func (ctx *context) restoreFromDir(dir string) (api.RestoreResult, api.RestoreRe
}
if !rde {
addArkError(&errs, errors.New("backup does not contain top level resources directory"))
return warnings, errs
}
resourceDirs, err := ctx.fileSystem.ReadDir(resourcesDir)
@ -288,6 +337,8 @@ func (ctx *context) restoreFromDir(dir string) (api.RestoreResult, api.RestoreRe
resourceDirsMap[rscName] = rscDir
}
existingNamespaces := sets.NewString()
for _, resource := range ctx.prioritizedResources {
rscDir := resourceDirsMap[resource.String()]
if rscDir == nil {
@ -343,15 +394,21 @@ func (ctx *context) restoreFromDir(dir string) (api.RestoreResult, api.RestoreRe
mappedNsName = target
}
// ensure namespace exists
ns := &v1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: mappedNsName,
},
}
if _, err := kube.EnsureNamespaceExists(ns, ctx.namespaceClient); err != nil {
addArkError(&errs, err)
continue
// if we don't know whether this namespace exists yet, attempt to create
// it in order to ensure it exists. Try to get it from the backup tarball
// (in order to get any backed-up metadata), but if we don't find it there,
// create a blank one.
if !existingNamespaces.Has(mappedNsName) {
logger := ctx.logger.WithField("namespace", nsName)
ns := getNamespace(logger, filepath.Join(dir, api.ResourcesDir, "namespaces", api.ClusterScopedDir, nsName+".json"), mappedNsName)
if _, err := kube.EnsureNamespaceExists(ns, ctx.namespaceClient); err != nil {
addArkError(&errs, err)
continue
}
// keep track of namespaces that we know exist so we don't
// have to try to create them multiple times
existingNamespaces.Insert(mappedNsName)
}
w, e := ctx.restoreResource(resource.String(), mappedNsName, nsPath)
@ -363,6 +420,42 @@ func (ctx *context) restoreFromDir(dir string) (api.RestoreResult, api.RestoreRe
return warnings, errs
}
// getNamespace returns a namespace API object that we should attempt to
// create before restoring anything into it. It will come from the backup
// tarball if it exists, else will be a new one. If from the tarball, it
// will retain its labels, annotations, and spec.
func getNamespace(logger logrus.FieldLogger, path, remappedName string) *v1.Namespace {
var nsBytes []byte
var err error
if nsBytes, err = ioutil.ReadFile(path); err != nil {
return &v1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: remappedName,
},
}
}
var backupNS v1.Namespace
if err := json.Unmarshal(nsBytes, &backupNS); err != nil {
logger.Warnf("Error unmarshalling namespace from backup, creating new one.")
return &v1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: remappedName,
},
}
}
return &v1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: remappedName,
Labels: backupNS.Labels,
Annotations: backupNS.Annotations,
},
Spec: backupNS.Spec,
}
}
// merge combines two RestoreResult objects into one
// by appending the corresponding lists to one another.
func merge(a, b *api.RestoreResult) {
@ -421,12 +514,26 @@ func (ctx *context) restoreResource(resource, namespace, resourcePath string) (a
}
var (
resourceClient client.Dynamic
restorer restorers.ResourceRestorer
waiter *resourceWaiter
groupResource = schema.ParseGroupResource(resource)
resourceClient client.Dynamic
waiter *resourceWaiter
groupResource = schema.ParseGroupResource(resource)
applicableActions []resolvedAction
)
// pre-filter the actions based on namespace & resource includes/excludes since
// these will be the same for all items being restored below
for _, action := range ctx.actions {
if !action.resourceIncludesExcludes.ShouldInclude(groupResource.String()) {
continue
}
if namespace != "" && !action.namespaceIncludesExcludes.ShouldInclude(namespace) {
continue
}
applicableActions = append(applicableActions, action)
}
for _, file := range files {
fullPath := filepath.Join(resourcePath, file.Name())
obj, err := ctx.unmarshal(fullPath)
@ -439,8 +546,13 @@ func (ctx *context) restoreResource(resource, namespace, resourcePath string) (a
continue
}
if restorer == nil {
// initialize client & restorer for this Resource. we need
if hasControllerOwner(obj.GetOwnerReferences()) {
ctx.infof("%s/%s has a controller owner - skipping", obj.GetNamespace(), obj.GetName())
continue
}
if resourceClient == nil {
// initialize client for this Resource. we need
// metadata from an object to do this.
ctx.infof("Getting client for %v", obj.GroupVersionKind())
@ -455,72 +567,88 @@ func (ctx *context) restoreResource(resource, namespace, resourcePath string) (a
addArkError(&errs, fmt.Errorf("error getting resource client for namespace %q, resource %q: %v", namespace, &groupResource, err))
return warnings, errs
}
}
restorer = ctx.restorers[groupResource]
if restorer == nil {
ctx.infof("Using default restorer for %v", &groupResource)
restorer = restorers.NewBasicRestorer(true)
} else {
ctx.infof("Using custom restorer for %v", &groupResource)
if groupResource.Group == "" && groupResource.Resource == "persistentvolumes" {
// restore the PV from snapshot (if applicable)
updatedObj, warning, err := ctx.executePVAction(obj)
if warning != nil {
addToResult(&warnings, namespace, fmt.Errorf("warning executing PVAction for %s: %v", fullPath, warning))
}
if err != nil {
addToResult(&errs, namespace, fmt.Errorf("error executing PVAction for %s: %v", fullPath, err))
continue
}
obj = updatedObj
if restorer.Wait() {
itmWatch, err := resourceClient.Watch(metav1.ListOptions{})
// wait for the PV to be ready
if ctx.waitForPVs {
pvWatch, err := resourceClient.Watch(metav1.ListOptions{})
if err != nil {
addArkError(&errs, fmt.Errorf("error watching for namespace %q, resource %q: %v", namespace, &groupResource, err))
addToResult(&errs, namespace, fmt.Errorf("error watching for namespace %q, resource %q: %v", namespace, &groupResource, err))
return warnings, errs
}
watchChan := itmWatch.ResultChan()
defer itmWatch.Stop()
waiter = newResourceWaiter(watchChan, restorer.Ready)
waiter = newResourceWaiter(pvWatch, isPVReady)
defer waiter.Stop()
}
}
if !restorer.Handles(obj, ctx.restore) {
continue
for _, action := range applicableActions {
if !action.selector.Matches(labels.Set(obj.GetLabels())) {
continue
}
ctx.infof("Executing item action for %v", &groupResource)
if logSetter, ok := action.ItemAction.(logging.LogSetter); ok {
logSetter.SetLog(ctx.logger)
}
updatedObj, warning, err := action.Execute(obj, ctx.restore)
if warning != nil {
addToResult(&warnings, namespace, fmt.Errorf("warning preparing %s: %v", fullPath, warning))
}
if err != nil {
addToResult(&errs, namespace, fmt.Errorf("error preparing %s: %v", fullPath, err))
continue
}
unstructuredObj, ok := updatedObj.(*unstructured.Unstructured)
if !ok {
addToResult(&errs, namespace, fmt.Errorf("%s: unexpected type %T", fullPath, updatedObj))
continue
}
obj = unstructuredObj
}
if hasControllerOwner(obj.GetOwnerReferences()) {
ctx.infof("%s/%s has a controller owner - skipping", obj.GetNamespace(), obj.GetName())
continue
}
preparedObj, warning, err := restorer.Prepare(obj, ctx.restore, ctx.backup)
if warning != nil {
addToResult(&warnings, namespace, fmt.Errorf("warning preparing %s: %v", fullPath, warning))
}
if err != nil {
addToResult(&errs, namespace, fmt.Errorf("error preparing %s: %v", fullPath, err))
continue
}
unstructuredObj, ok := preparedObj.(*unstructured.Unstructured)
if !ok {
addToResult(&errs, namespace, fmt.Errorf("%s: unexpected type %T", fullPath, preparedObj))
// clear out non-core metadata fields & status
if obj, err = resetMetadataAndStatus(obj, true); err != nil {
addToResult(&errs, namespace, err)
continue
}
// necessary because we may have remapped the namespace
unstructuredObj.SetNamespace(namespace)
obj.SetNamespace(namespace)
// add an ark-restore label to each resource for easy ID
addLabel(unstructuredObj, api.RestoreLabelKey, ctx.restore.Name)
addLabel(obj, api.RestoreLabelKey, ctx.restore.Name)
ctx.infof("Restoring %s: %v", obj.GroupVersionKind().Kind, unstructuredObj.GetName())
_, err = resourceClient.Create(unstructuredObj)
ctx.infof("Restoring %s: %v", obj.GroupVersionKind().Kind, obj.GetName())
_, err = resourceClient.Create(obj)
if apierrors.IsAlreadyExists(err) {
addToResult(&warnings, namespace, err)
continue
}
if err != nil {
ctx.infof("error restoring %s: %v", unstructuredObj.GetName(), err)
ctx.infof("error restoring %s: %v", obj.GetName(), err)
addToResult(&errs, namespace, fmt.Errorf("error restoring %s: %v", fullPath, err))
continue
}
if waiter != nil {
waiter.RegisterItem(unstructuredObj.GetName())
waiter.RegisterItem(obj.GetName())
}
}
@ -533,6 +661,127 @@ func (ctx *context) restoreResource(resource, namespace, resourcePath string) (a
return warnings, errs
}
func (ctx *context) executePVAction(obj *unstructured.Unstructured) (*unstructured.Unstructured, error, error) {
// we need to remove annotations from PVs since they potentially contain
// information about dynamic provisioners which will confuse the controllers.
metadata, err := collections.GetMap(obj.UnstructuredContent(), "metadata")
if err != nil {
return nil, nil, err
}
delete(metadata, "annotations")
spec, err := collections.GetMap(obj.UnstructuredContent(), "spec")
if err != nil {
return nil, nil, err
}
delete(spec, "claimRef")
delete(spec, "storageClassName")
// restore the PV from snapshot (if applicable)
return ctx.restoreVolumeFromSnapshot(obj)
}
func (ctx *context) restoreVolumeFromSnapshot(obj *unstructured.Unstructured) (*unstructured.Unstructured, error, error) {
spec, err := collections.GetMap(obj.UnstructuredContent(), "spec")
if err != nil {
return nil, nil, err
}
// if it's an unsupported volume type for snapshot restores, don't try to
// do a snapshot restore
if sourceType, _ := kube.GetPVSource(spec); sourceType == "" {
return obj, nil, nil
}
var (
pvName = obj.GetName()
restoreFromSnapshot = false
restore = ctx.restore
backup = ctx.backup
)
if restore.Spec.RestorePVs != nil && *restore.Spec.RestorePVs {
// when RestorePVs = yes, it's an error if we don't have a snapshot service
if ctx.snapshotService == nil {
return nil, nil, errors.New("PV restorer is not configured for PV snapshot restores")
}
// if there are no snapshots in the backup, return without error
if backup.Status.VolumeBackups == nil {
return obj, nil, nil
}
// if there are snapshots, and this is a supported PV type, but there's no
// snapshot for this PV, it's an error
if backup.Status.VolumeBackups[pvName] == nil {
return nil, nil, errors.Errorf("no snapshot found to restore volume %s from", pvName)
}
restoreFromSnapshot = true
}
if restore.Spec.RestorePVs == nil && ctx.snapshotService != nil {
// when RestorePVs = Auto, don't error if the backup doesn't have snapshots
if backup.Status.VolumeBackups == nil || backup.Status.VolumeBackups[pvName] == nil {
return obj, nil, nil
}
restoreFromSnapshot = true
}
if restoreFromSnapshot {
backupInfo := backup.Status.VolumeBackups[pvName]
ctx.infof("restoring PersistentVolume %s from SnapshotID %s", pvName, backupInfo.SnapshotID)
volumeID, err := ctx.snapshotService.CreateVolumeFromSnapshot(backupInfo.SnapshotID, backupInfo.Type, backupInfo.AvailabilityZone, backupInfo.Iops)
if err != nil {
return nil, nil, err
}
ctx.infof("successfully restored PersistentVolume %s from snapshot", pvName)
if err := kube.SetVolumeID(spec, volumeID); err != nil {
return nil, nil, err
}
}
var warning error
if ctx.snapshotService == nil && len(backup.Status.VolumeBackups) > 0 {
warning = errors.New("unable to restore PV snapshots: Ark server is not configured with a PersistentVolumeProvider")
}
return obj, warning, nil
}
func isPVReady(obj runtime.Unstructured) bool {
phase, err := collections.GetString(obj.UnstructuredContent(), "status.phase")
if err != nil {
return false
}
return phase == string(v1.VolumeAvailable)
}
func resetMetadataAndStatus(obj *unstructured.Unstructured, keepAnnotations bool) (*unstructured.Unstructured, error) {
metadata, err := collections.GetMap(obj.UnstructuredContent(), "metadata")
if err != nil {
return nil, err
}
for k := range metadata {
if k == "name" || k == "namespace" || k == "labels" || (k == "annotations" && keepAnnotations) {
continue
}
delete(metadata, k)
}
// this should never be backed up anyway, but remove it just
// in case.
delete(obj.UnstructuredContent(), "status")
return obj, nil
}
// addLabel applies the specified key/value to an object as a label.
func addLabel(obj *unstructured.Unstructured, key string, val string) {
labels := obj.GetLabels()

384
pkg/restore/restore_test.go
View File

@ -22,6 +22,7 @@ import (
"os"
"testing"
"github.com/pkg/errors"
"github.com/sirupsen/logrus/hooks/test"
testlogger "github.com/sirupsen/logrus/hooks/test"
"github.com/spf13/afero"
@ -37,9 +38,9 @@ import (
corev1 "k8s.io/client-go/kubernetes/typed/core/v1"
api "github.com/heptio/ark/pkg/apis/ark/v1"
"github.com/heptio/ark/pkg/restore/restorers"
"github.com/heptio/ark/pkg/cloudprovider"
"github.com/heptio/ark/pkg/util/collections"
. "github.com/heptio/ark/pkg/util/test"
arktest "github.com/heptio/ark/pkg/util/test"
)
func TestPrioritizeResources(t *testing.T) {
@ -95,7 +96,7 @@ func TestPrioritizeResources(t *testing.T) {
helperResourceList = append(helperResourceList, resourceList)
}
helper := NewFakeDiscoveryHelper(true, nil)
helper := arktest.NewFakeDiscoveryHelper(true, nil)
helper.ResourceList = helperResourceList
includesExcludes := collections.NewIncludesExcludes().Includes(test.includes...).Excludes(test.excludes...)
@ -303,12 +304,12 @@ func TestNamespaceRemapping(t *testing.T) {
expectedObjs = toUnstructured(newTestConfigMap().WithNamespace("ns-2").WithArkLabel("").ConfigMap)
)
resourceClient := &FakeDynamicClient{}
resourceClient := &arktest.FakeDynamicClient{}
for i := range expectedObjs {
resourceClient.On("Create", &expectedObjs[i]).Return(&expectedObjs[i], nil)
}
dynamicFactory := &FakeDynamicFactory{}
dynamicFactory := &arktest.FakeDynamicFactory{}
resource := metav1.APIResource{Name: "configmaps", Namespaced: true}
gv := schema.GroupVersion{Group: "", Version: "v1"}
dynamicFactory.On("ClientForGroupVersionResource", gv, resource, expectedNS).Return(resourceClient, nil)
@ -354,7 +355,7 @@ func TestRestoreResourceForNamespace(t *testing.T) {
labelSelector labels.Selector
includeClusterResources *bool
fileSystem *fakeFileSystem
restorers map[schema.GroupResource]restorers.ResourceRestorer
actions []resolvedAction
expectedErrors api.RestoreResult
expectedObjs []unstructured.Unstructured
}{
@ -442,8 +443,15 @@ func TestRestoreResourceForNamespace(t *testing.T) {
resourcePath: "configmaps",
labelSelector: labels.NewSelector(),
fileSystem: newFakeFileSystem().WithFile("configmaps/cm-1.json", newTestConfigMap().ToJSON()),
restorers: map[schema.GroupResource]restorers.ResourceRestorer{{Resource: "configmaps"}: newFakeCustomRestorer()},
expectedObjs: toUnstructured(newTestConfigMap().WithLabels(map[string]string{"fake-restorer": "foo"}).WithArkLabel("my-restore").ConfigMap),
actions: []resolvedAction{
{
ItemAction: newFakeAction("configmaps"),
resourceIncludesExcludes: collections.NewIncludesExcludes().Includes("configmaps"),
namespaceIncludesExcludes: collections.NewIncludesExcludes(),
selector: labels.Everything(),
},
},
expectedObjs: toUnstructured(newTestConfigMap().WithLabels(map[string]string{"fake-restorer": "foo"}).WithArkLabel("my-restore").ConfigMap),
},
{
name: "custom restorer for different group/resource is not used",
@ -451,8 +459,15 @@ func TestRestoreResourceForNamespace(t *testing.T) {
resourcePath: "configmaps",
labelSelector: labels.NewSelector(),
fileSystem: newFakeFileSystem().WithFile("configmaps/cm-1.json", newTestConfigMap().ToJSON()),
restorers: map[schema.GroupResource]restorers.ResourceRestorer{{Resource: "foo-resource"}: newFakeCustomRestorer()},
expectedObjs: toUnstructured(newTestConfigMap().WithArkLabel("my-restore").ConfigMap),
actions: []resolvedAction{
{
ItemAction: newFakeAction("foo-resource"),
resourceIncludesExcludes: collections.NewIncludesExcludes().Includes("foo-resource"),
namespaceIncludesExcludes: collections.NewIncludesExcludes(),
selector: labels.Everything(),
},
},
expectedObjs: toUnstructured(newTestConfigMap().WithArkLabel("my-restore").ConfigMap),
},
{
name: "cluster-scoped resources are skipped when IncludeClusterResources=false",
@ -511,24 +526,23 @@ func TestRestoreResourceForNamespace(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
resourceClient := &FakeDynamicClient{}
resourceClient := &arktest.FakeDynamicClient{}
for i := range test.expectedObjs {
resourceClient.On("Create", &test.expectedObjs[i]).Return(&test.expectedObjs[i], nil)
}
dynamicFactory := &FakeDynamicFactory{}
resource := metav1.APIResource{Name: "configmaps", Namespaced: true}
dynamicFactory := &arktest.FakeDynamicFactory{}
gv := schema.GroupVersion{Group: "", Version: "v1"}
resource := metav1.APIResource{Name: "configmaps", Namespaced: true}
dynamicFactory.On("ClientForGroupVersionResource", gv, resource, test.namespace).Return(resourceClient, nil)
pvResource := metav1.APIResource{Name: "persistentvolumes", Namespaced: false}
dynamicFactory.On("ClientForGroupVersionResource", gv, pvResource, test.namespace).Return(resourceClient, nil)
log, _ := testlogger.NewNullLogger()
ctx := &context{
dynamicFactory: dynamicFactory,
restorers: test.restorers,
actions: test.actions,
fileSystem: test.fileSystem,
selector: test.labelSelector,
restore: &api.Restore{
@ -541,7 +555,7 @@ func TestRestoreResourceForNamespace(t *testing.T) {
},
},
backup: &api.Backup{},
logger: log,
logger: arktest.NewLogger(),
}
warnings, errors := ctx.restoreResource(test.resourcePath, test.namespace, test.resourcePath)
@ -617,6 +631,312 @@ func TestHasControllerOwner(t *testing.T) {
}
}
func TestResetMetadataAndStatus(t *testing.T) {
tests := []struct {
name string
obj *unstructured.Unstructured
keepAnnotations bool
expectedErr bool
expectedRes *unstructured.Unstructured
}{
{
name: "no metadata causes error",
obj: NewTestUnstructured().Unstructured,
keepAnnotations: false,
expectedErr: true,
},
{
name: "don't keep annotations",
obj: NewTestUnstructured().WithMetadata("name", "namespace", "labels", "annotations").Unstructured,
keepAnnotations: false,
expectedErr: false,
expectedRes: NewTestUnstructured().WithMetadata("name", "namespace", "labels").Unstructured,
},
{
name: "keep annotations",
obj: NewTestUnstructured().WithMetadata("name", "namespace", "labels", "annotations").Unstructured,
keepAnnotations: true,
expectedErr: false,
expectedRes: NewTestUnstructured().WithMetadata("name", "namespace", "labels", "annotations").Unstructured,
},
{
name: "don't keep extraneous metadata",
obj: NewTestUnstructured().WithMetadata("foo").Unstructured,
keepAnnotations: false,
expectedErr: false,
expectedRes: NewTestUnstructured().WithMetadata().Unstructured,
},
{
name: "don't keep status",
obj: NewTestUnstructured().WithMetadata().WithStatus().Unstructured,
keepAnnotations: false,
expectedErr: false,
expectedRes: NewTestUnstructured().WithMetadata().Unstructured,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
res, err := resetMetadataAndStatus(test.obj, test.keepAnnotations)
if assert.Equal(t, test.expectedErr, err != nil) {
assert.Equal(t, test.expectedRes, res)
}
})
}
}
func TestRestoreVolumeFromSnapshot(t *testing.T) {
iops := int64(1000)
tests := []struct {
name string
obj *unstructured.Unstructured
restore *api.Restore
backup *api.Backup
volumeMap map[api.VolumeBackupInfo]string
noSnapshotService bool
expectedWarn bool
expectedErr bool
expectedRes *unstructured.Unstructured
}{
{
name: "no name should error",
obj: NewTestUnstructured().WithMetadata().Unstructured,
restore: arktest.NewDefaultTestRestore().Restore,
expectedErr: true,
},
{
name: "no spec should error",
obj: NewTestUnstructured().WithName("pv-1").Unstructured,
restore: arktest.NewDefaultTestRestore().Restore,
expectedErr: true,
},
{
name: "when RestorePVs=false, should not error if there is no PV->BackupInfo map",
obj: NewTestUnstructured().WithName("pv-1").WithSpec().Unstructured,
restore: arktest.NewDefaultTestRestore().WithRestorePVs(false).Restore,
backup: &api.Backup{Status: api.BackupStatus{}},
expectedErr: false,
expectedRes: NewTestUnstructured().WithName("pv-1").WithSpec().Unstructured,
},
{
name: "when RestorePVs=true, return without error if there is no PV->BackupInfo map",
obj: NewTestUnstructured().WithName("pv-1").WithSpecField("awsElasticBlockStore", make(map[string]interface{})).Unstructured,
restore: arktest.NewDefaultTestRestore().WithRestorePVs(true).Restore,
backup: &api.Backup{Status: api.BackupStatus{}},
expectedErr: false,
expectedRes: NewTestUnstructured().WithName("pv-1").WithSpecField("awsElasticBlockStore", make(map[string]interface{})).Unstructured,
},
{
name: "when RestorePVs=true, error if there is PV->BackupInfo map but no entry for this PV",
obj: NewTestUnstructured().WithName("pv-1").WithSpecField("awsElasticBlockStore", make(map[string]interface{})).Unstructured,
restore: arktest.NewDefaultTestRestore().WithRestorePVs(true).Restore,
backup: &api.Backup{Status: api.BackupStatus{VolumeBackups: map[string]*api.VolumeBackupInfo{"another-pv": {}}}},
expectedErr: true,
},
{
name: "when RestorePVs=true, AWS volume ID should be set correctly",
obj: NewTestUnstructured().WithName("pv-1").WithSpecField("awsElasticBlockStore", make(map[string]interface{})).Unstructured,
restore: arktest.NewDefaultTestRestore().WithRestorePVs(true).Restore,
backup: &api.Backup{Status: api.BackupStatus{VolumeBackups: map[string]*api.VolumeBackupInfo{"pv-1": {SnapshotID: "snap-1"}}}},
volumeMap: map[api.VolumeBackupInfo]string{{SnapshotID: "snap-1"}: "volume-1"},
expectedErr: false,
expectedRes: NewTestUnstructured().WithName("pv-1").WithSpecField("awsElasticBlockStore", map[string]interface{}{"volumeID": "volume-1"}).Unstructured,
},
{
name: "when RestorePVs=true, GCE pdName should be set correctly",
obj: NewTestUnstructured().WithName("pv-1").WithSpecField("gcePersistentDisk", make(map[string]interface{})).Unstructured,
restore: arktest.NewDefaultTestRestore().WithRestorePVs(true).Restore,
backup: &api.Backup{Status: api.BackupStatus{VolumeBackups: map[string]*api.VolumeBackupInfo{"pv-1": {SnapshotID: "snap-1"}}}},
volumeMap: map[api.VolumeBackupInfo]string{{SnapshotID: "snap-1"}: "volume-1"},
expectedErr: false,
expectedRes: NewTestUnstructured().WithName("pv-1").WithSpecField("gcePersistentDisk", map[string]interface{}{"pdName": "volume-1"}).Unstructured,
},
{
name: "when RestorePVs=true, Azure pdName should be set correctly",
obj: NewTestUnstructured().WithName("pv-1").WithSpecField("azureDisk", make(map[string]interface{})).Unstructured,
restore: arktest.NewDefaultTestRestore().WithRestorePVs(true).Restore,
backup: &api.Backup{Status: api.BackupStatus{VolumeBackups: map[string]*api.VolumeBackupInfo{"pv-1": {SnapshotID: "snap-1"}}}},
volumeMap: map[api.VolumeBackupInfo]string{{SnapshotID: "snap-1"}: "volume-1"},
expectedErr: false,
expectedRes: NewTestUnstructured().WithName("pv-1").WithSpecField("azureDisk", map[string]interface{}{"diskName": "volume-1"}).Unstructured,
},
{
name: "when RestorePVs=true, unsupported PV source should not get snapshot restored",
obj: NewTestUnstructured().WithName("pv-1").WithSpecField("unsupportedPVSource", make(map[string]interface{})).Unstructured,
restore: arktest.NewDefaultTestRestore().WithRestorePVs(true).Restore,
backup: &api.Backup{Status: api.BackupStatus{VolumeBackups: map[string]*api.VolumeBackupInfo{"pv-1": {SnapshotID: "snap-1"}}}},
volumeMap: map[api.VolumeBackupInfo]string{{SnapshotID: "snap-1"}: "volume-1"},
expectedErr: false,
expectedRes: NewTestUnstructured().WithName("pv-1").WithSpecField("unsupportedPVSource", make(map[string]interface{})).Unstructured,
},
{
name: "volume type and IOPS are correctly passed to CreateVolume",
obj: NewTestUnstructured().WithName("pv-1").WithSpecField("awsElasticBlockStore", make(map[string]interface{})).Unstructured,
restore: arktest.NewDefaultTestRestore().WithRestorePVs(true).Restore,
backup: &api.Backup{Status: api.BackupStatus{VolumeBackups: map[string]*api.VolumeBackupInfo{"pv-1": {SnapshotID: "snap-1", Type: "gp", Iops: &iops}}}},
volumeMap: map[api.VolumeBackupInfo]string{{SnapshotID: "snap-1", Type: "gp", Iops: &iops}: "volume-1"},
expectedErr: false,
expectedRes: NewTestUnstructured().WithName("pv-1").WithSpecField("awsElasticBlockStore", map[string]interface{}{"volumeID": "volume-1"}).Unstructured,
},
{
name: "When no SnapshotService, warn if backup has snapshots that will not be restored",
obj: NewTestUnstructured().WithName("pv-1").WithSpecField("awsElasticBlockStore", make(map[string]interface{})).Unstructured,
restore: arktest.NewDefaultTestRestore().Restore,
backup: &api.Backup{Status: api.BackupStatus{VolumeBackups: map[string]*api.VolumeBackupInfo{"pv-1": {SnapshotID: "snap-1"}}}},
volumeMap: map[api.VolumeBackupInfo]string{{SnapshotID: "snap-1"}: "volume-1"},
noSnapshotService: true,
expectedErr: false,
expectedWarn: true,
expectedRes: NewTestUnstructured().WithName("pv-1").WithSpecField("awsElasticBlockStore", make(map[string]interface{})).Unstructured,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
var snapshotService cloudprovider.SnapshotService
if !test.noSnapshotService {
snapshotService = &arktest.FakeSnapshotService{RestorableVolumes: test.volumeMap}
}
ctx := &context{
restore: test.restore,
backup: test.backup,
snapshotService: snapshotService,
logger: arktest.NewLogger(),
}
res, warn, err := ctx.restoreVolumeFromSnapshot(test.obj)
assert.Equal(t, test.expectedWarn, warn != nil)
if assert.Equal(t, test.expectedErr, err != nil) {
assert.Equal(t, test.expectedRes, res)
}
})
}
}
func TestIsPVReady(t *testing.T) {
tests := []struct {
name string
obj *unstructured.Unstructured
expected bool
}{
{
name: "no status returns not ready",
obj: NewTestUnstructured().Unstructured,
expected: false,
},
{
name: "no status.phase returns not ready",
obj: NewTestUnstructured().WithStatus().Unstructured,
expected: false,
},
{
name: "empty status.phase returns not ready",
obj: NewTestUnstructured().WithStatusField("phase", "").Unstructured,
expected: false,
},
{
name: "non-Available status.phase returns not ready",
obj: NewTestUnstructured().WithStatusField("phase", "foo").Unstructured,
expected: false,
},
{
name: "Available status.phase returns ready",
obj: NewTestUnstructured().WithStatusField("phase", "Available").Unstructured,
expected: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
assert.Equal(t, test.expected, isPVReady(test.obj))
})
}
}
type testUnstructured struct {
*unstructured.Unstructured
}
func NewTestUnstructured() *testUnstructured {
obj := &testUnstructured{
Unstructured: &unstructured.Unstructured{
Object: make(map[string]interface{}),
},
}
return obj
}
func (obj *testUnstructured) WithMetadata(fields ...string) *testUnstructured {
return obj.withMap("metadata", fields...)
}
func (obj *testUnstructured) WithSpec(fields ...string) *testUnstructured {
return obj.withMap("spec", fields...)
}
func (obj *testUnstructured) WithStatus(fields ...string) *testUnstructured {
return obj.withMap("status", fields...)
}
func (obj *testUnstructured) WithMetadataField(field string, value interface{}) *testUnstructured {
return obj.withMapEntry("metadata", field, value)
}
func (obj *testUnstructured) WithSpecField(field string, value interface{}) *testUnstructured {
return obj.withMapEntry("spec", field, value)
}
func (obj *testUnstructured) WithStatusField(field string, value interface{}) *testUnstructured {
return obj.withMapEntry("status", field, value)
}
func (obj *testUnstructured) WithAnnotations(fields ...string) *testUnstructured {
annotations := make(map[string]interface{})
for _, field := range fields {
annotations[field] = "foo"
}
obj = obj.WithMetadataField("annotations", annotations)
return obj
}
func (obj *testUnstructured) WithName(name string) *testUnstructured {
return obj.WithMetadataField("name", name)
}
func (obj *testUnstructured) withMap(name string, fields ...string) *testUnstructured {
m := make(map[string]interface{})
obj.Object[name] = m
for _, field := range fields {
m[field] = "foo"
}
return obj
}
func (obj *testUnstructured) withMapEntry(mapName, field string, value interface{}) *testUnstructured {
var m map[string]interface{}
if res, ok := obj.Unstructured.Object[mapName]; !ok {
m = make(map[string]interface{})
obj.Unstructured.Object[mapName] = m
} else {
m = res.(map[string]interface{})
}
m[field] = value
return obj
}
func toUnstructured(objs ...runtime.Object) []unstructured.Unstructured {
res := make([]unstructured.Unstructured, 0, len(objs))
@ -802,17 +1122,21 @@ func (fs *fakeFileSystem) DirExists(path string) (bool, error) {
return afero.DirExists(fs.fs, path)
}
type fakeCustomRestorer struct {
restorers.ResourceRestorer
type fakeAction struct {
resource string
}
func newFakeCustomRestorer() *fakeCustomRestorer {
return &fakeCustomRestorer{
ResourceRestorer: restorers.NewBasicRestorer(true),
}
func newFakeAction(resource string) *fakeAction {
return &fakeAction{resource}
}
func (r *fakeCustomRestorer) Prepare(obj runtime.Unstructured, restore *api.Restore, backup *api.Backup) (runtime.Unstructured, error, error) {
func (r *fakeAction) AppliesTo() (ResourceSelector, error) {
return ResourceSelector{
IncludedResources: []string{r.resource},
}, nil
}
func (r *fakeAction) Execute(obj runtime.Unstructured, restore *api.Restore) (runtime.Unstructured, error, error) {
metadata, err := collections.GetMap(obj.UnstructuredContent(), "metadata")
if err != nil {
return nil, nil, err
@ -824,8 +1148,18 @@ func (r *fakeCustomRestorer) Prepare(obj runtime.Unstructured, restore *api.Rest
metadata["labels"].(map[string]interface{})["fake-restorer"] = "foo"
unstructuredObj, ok := obj.(*unstructured.Unstructured)
if !ok {
return nil, nil, errors.New("Unexpected type")
}
// want the baseline functionality too
return r.ResourceRestorer.Prepare(obj, restore, backup)
res, err := resetMetadataAndStatus(unstructuredObj, true)
if err != nil {
return nil, nil, err
}
return res, nil, nil
}
type fakeNamespaceClient struct {

75
pkg/restore/restorers/namespace_restorer.go
View File

@ -1,75 +0,0 @@
/*
Copyright 2017 Heptio Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package restorers
import (
"k8s.io/apimachinery/pkg/runtime"
api "github.com/heptio/ark/pkg/apis/ark/v1"
"github.com/heptio/ark/pkg/util/collections"
)
type namespaceRestorer struct{}
var _ ResourceRestorer = &namespaceRestorer{}
func NewNamespaceRestorer() ResourceRestorer {
return &namespaceRestorer{}
}
func (nsr *namespaceRestorer) Handles(obj runtime.Unstructured, restore *api.Restore) bool {
nsName, err := collections.GetString(obj.UnstructuredContent(), "metadata.name")
if err != nil {
return false
}
return collections.NewIncludesExcludes().
Includes(restore.Spec.IncludedNamespaces...).
Excludes(restore.Spec.ExcludedNamespaces...).
ShouldInclude(nsName)
}
func (nsr *namespaceRestorer) Prepare(obj runtime.Unstructured, restore *api.Restore, backup *api.Backup) (runtime.Unstructured, error, error) {
updated, err := resetMetadataAndStatus(obj, true)
if err != nil {
return nil, nil, err
}
metadata, err := collections.GetMap(obj.UnstructuredContent(), "metadata")
if err != nil {
return nil, nil, err
}
currentName, err := collections.GetString(obj.UnstructuredContent(), "metadata.name")
if err != nil {
return nil, nil, err
}
if newName, mapped := restore.Spec.NamespaceMapping[currentName]; mapped {
metadata["name"] = newName
}
return updated, nil, nil
}
func (nsr *namespaceRestorer) Wait() bool {
return false
}
func (nsr *namespaceRestorer) Ready(obj runtime.Unstructured) bool {
return true
}

125
pkg/restore/restorers/namespace_restorer_test.go
View File

@ -1,125 +0,0 @@
/*
Copyright 2017 Heptio Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package restorers
import (
"testing"
"github.com/stretchr/testify/assert"
"k8s.io/apimachinery/pkg/runtime"
api "github.com/heptio/ark/pkg/apis/ark/v1"
testutil "github.com/heptio/ark/pkg/util/test"
)
func TestHandles(t *testing.T) {
tests := []struct {
name string
obj runtime.Unstructured
restore *api.Restore
expect bool
}{
{
name: "restorable NS",
obj: NewTestUnstructured().WithName("ns-1").Unstructured,
restore: testutil.NewDefaultTestRestore().WithIncludedNamespace("ns-1").Restore,
expect: true,
},
{
name: "restorable NS via wildcard",
obj: NewTestUnstructured().WithName("ns-1").Unstructured,
restore: testutil.NewDefaultTestRestore().WithIncludedNamespace("*").Restore,
expect: true,
},
{
name: "non-restorable NS",
obj: NewTestUnstructured().WithName("ns-1").Unstructured,
restore: testutil.NewDefaultTestRestore().WithIncludedNamespace("ns-2").Restore,
expect: false,
},
{
name: "namespace is explicitly excluded",
obj: NewTestUnstructured().WithName("ns-1").Unstructured,
restore: testutil.NewDefaultTestRestore().WithIncludedNamespace("*").WithExcludedNamespace("ns-1").Restore,
expect: false,
},
{
name: "namespace obj doesn't have name",
obj: NewTestUnstructured().WithMetadata().Unstructured,
restore: testutil.NewDefaultTestRestore().WithIncludedNamespace("ns-1").Restore,
expect: false,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
restorer := NewNamespaceRestorer()
assert.Equal(t, test.expect, restorer.Handles(test.obj, test.restore))
})
}
}
func TestPrepare(t *testing.T) {
tests := []struct {
name string
obj runtime.Unstructured
restore *api.Restore
expectedErr bool
expectedRes runtime.Unstructured
}{
{
name: "standard non-mapped namespace",
obj: NewTestUnstructured().WithStatus().WithName("ns-1").Unstructured,
restore: testutil.NewDefaultTestRestore().Restore,
expectedErr: false,
expectedRes: NewTestUnstructured().WithName("ns-1").Unstructured,
},
{
name: "standard mapped namespace",
obj: NewTestUnstructured().WithStatus().WithName("ns-1").Unstructured,
restore: testutil.NewDefaultTestRestore().WithMappedNamespace("ns-1", "ns-2").Restore,
expectedErr: false,
expectedRes: NewTestUnstructured().WithName("ns-2").Unstructured,
},
{
name: "object without name results in error",
obj: NewTestUnstructured().WithMetadata().WithStatus().Unstructured,
restore: testutil.NewDefaultTestRestore().Restore,
expectedErr: true,
},
{
name: "annotations are kept",
obj: NewTestUnstructured().WithName("ns-1").WithAnnotations().Unstructured,
restore: testutil.NewDefaultTestRestore().Restore,
expectedErr: false,
expectedRes: NewTestUnstructured().WithName("ns-1").WithAnnotations().Unstructured,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
restorer := NewNamespaceRestorer()
res, _, err := restorer.Prepare(test.obj, test.restore, nil)
if assert.Equal(t, test.expectedErr, err != nil) {
assert.Equal(t, test.expectedRes, res)
}
})
}
}

129
pkg/restore/restorers/pv_restorer.go
View File

@ -1,129 +0,0 @@
/*
Copyright 2017 Heptio Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package restorers
import (
"github.com/pkg/errors"
"k8s.io/apimachinery/pkg/runtime"
api "github.com/heptio/ark/pkg/apis/ark/v1"
"github.com/heptio/ark/pkg/cloudprovider"
"github.com/heptio/ark/pkg/util/collections"
kubeutil "github.com/heptio/ark/pkg/util/kube"
)
type persistentVolumeRestorer struct {
snapshotService cloudprovider.SnapshotService
}
var _ ResourceRestorer = &persistentVolumeRestorer{}
func NewPersistentVolumeRestorer(snapshotService cloudprovider.SnapshotService) ResourceRestorer {
return &persistentVolumeRestorer{
snapshotService: snapshotService,
}
}
func (sr *persistentVolumeRestorer) Handles(obj runtime.Unstructured, restore *api.Restore) bool {
return true
}
func (sr *persistentVolumeRestorer) Prepare(obj runtime.Unstructured, restore *api.Restore, backup *api.Backup) (runtime.Unstructured, error, error) {
if _, err := resetMetadataAndStatus(obj, false); err != nil {
return nil, nil, err
}
spec, err := collections.GetMap(obj.UnstructuredContent(), "spec")
if err != nil {
return nil, nil, err
}
delete(spec, "claimRef")
delete(spec, "storageClassName")
pvName, err := collections.GetString(obj.UnstructuredContent(), "metadata.name")
if err != nil {
return nil, nil, err
}
// if it's an unsupported volume type for snapshot restores, we're done
if sourceType, _ := kubeutil.GetPVSource(spec); sourceType == "" {
return obj, nil, nil
}
restoreFromSnapshot := false
if restore.Spec.RestorePVs != nil && *restore.Spec.RestorePVs {
// when RestorePVs = yes, it's an error if we don't have a snapshot service
if sr.snapshotService == nil {
return nil, nil, errors.New("PV restorer is not configured for PV snapshot restores")
}
// if there are no snapshots in the backup, return without error
if backup.Status.VolumeBackups == nil {
return obj, nil, nil
}
// if there are snapshots, and this is a supported PV type, but there's no
// snapshot for this PV, it's an error
if backup.Status.VolumeBackups[pvName] == nil {
return nil, nil, errors.Errorf("no snapshot found to restore volume %s from", pvName)
}
restoreFromSnapshot = true
}
if restore.Spec.RestorePVs == nil && sr.snapshotService != nil {
// when RestorePVs = Auto, don't error if the backup doesn't have snapshots
if backup.Status.VolumeBackups == nil || backup.Status.VolumeBackups[pvName] == nil {
return obj, nil, nil
}
restoreFromSnapshot = true
}
if restoreFromSnapshot {
backupInfo := backup.Status.VolumeBackups[pvName]
volumeID, err := sr.snapshotService.CreateVolumeFromSnapshot(backupInfo.SnapshotID, backupInfo.Type, backupInfo.AvailabilityZone, backupInfo.Iops)
if err != nil {
return nil, nil, err
}
if err := kubeutil.SetVolumeID(spec, volumeID); err != nil {
return nil, nil, err
}
}
var warning error
if sr.snapshotService == nil && len(backup.Status.VolumeBackups) > 0 {
warning = errors.New("unable to restore PV snapshots: Ark server is not configured with a PersistentVolumeProvider")
}
return obj, warning, nil
}
func (sr *persistentVolumeRestorer) Wait() bool {
return true
}
func (sr *persistentVolumeRestorer) Ready(obj runtime.Unstructured) bool {
phase, err := collections.GetString(obj.UnstructuredContent(), "status.phase")
return err == nil && phase == "Available"
}

213
pkg/restore/restorers/pv_restorer_test.go
View File

@ -1,213 +0,0 @@
/*
Copyright 2017 Heptio Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package restorers
import (
"testing"
"github.com/stretchr/testify/assert"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
api "github.com/heptio/ark/pkg/apis/ark/v1"
"github.com/heptio/ark/pkg/cloudprovider"
. "github.com/heptio/ark/pkg/util/test"
)
func TestPVRestorerPrepare(t *testing.T) {
iops := int64(1000)
tests := []struct {
name string
obj runtime.Unstructured
restore *api.Restore
backup *api.Backup
volumeMap map[api.VolumeBackupInfo]string
noSnapshotService bool
expectedWarn bool
expectedErr bool
expectedRes runtime.Unstructured
}{
{
name: "no name should error",
obj: NewTestUnstructured().WithMetadata().Unstructured,
restore: NewDefaultTestRestore().Restore,
expectedErr: true,
},
{
name: "no spec should error",
obj: NewTestUnstructured().WithName("pv-1").Unstructured,
restore: NewDefaultTestRestore().Restore,
expectedErr: true,
},
{
name: "when RestorePVs=false, should not error if there is no PV->BackupInfo map",
obj: NewTestUnstructured().WithName("pv-1").WithSpec().Unstructured,
restore: NewDefaultTestRestore().WithRestorePVs(false).Restore,
backup: &api.Backup{Status: api.BackupStatus{}},
expectedErr: false,
expectedRes: NewTestUnstructured().WithName("pv-1").WithSpec().Unstructured,
},
{
name: "when RestorePVs=true, return without error if there is no PV->BackupInfo map",
obj: NewTestUnstructured().WithName("pv-1").WithSpecField("awsElasticBlockStore", make(map[string]interface{})).Unstructured,
restore: NewDefaultTestRestore().WithRestorePVs(true).Restore,
backup: &api.Backup{Status: api.BackupStatus{}},
expectedErr: false,
expectedRes: NewTestUnstructured().WithName("pv-1").WithSpecField("awsElasticBlockStore", make(map[string]interface{})).Unstructured,
},
{
name: "when RestorePVs=true, error if there is PV->BackupInfo map but no entry for this PV",
obj: NewTestUnstructured().WithName("pv-1").WithSpecField("awsElasticBlockStore", make(map[string]interface{})).Unstructured,
restore: NewDefaultTestRestore().WithRestorePVs(true).Restore,
backup: &api.Backup{Status: api.BackupStatus{VolumeBackups: map[string]*api.VolumeBackupInfo{"another-pv": {}}}},
expectedErr: true,
},
{
name: "claimRef and storageClassName (only) should be cleared from spec",
obj: NewTestUnstructured().
WithName("pv-1").
WithSpecField("claimRef", "foo").
WithSpecField("storageClassName", "foo").
WithSpecField("foo", "bar").
Unstructured,
restore: NewDefaultTestRestore().WithRestorePVs(false).Restore,
expectedErr: false,
expectedRes: NewTestUnstructured().
WithName("pv-1").
WithSpecField("foo", "bar").
Unstructured,
},
{
name: "when RestorePVs=true, AWS volume ID should be set correctly",
obj: NewTestUnstructured().WithName("pv-1").WithSpecField("awsElasticBlockStore", make(map[string]interface{})).Unstructured,
restore: NewDefaultTestRestore().WithRestorePVs(true).Restore,
backup: &api.Backup{Status: api.BackupStatus{VolumeBackups: map[string]*api.VolumeBackupInfo{"pv-1": {SnapshotID: "snap-1"}}}},
volumeMap: map[api.VolumeBackupInfo]string{{SnapshotID: "snap-1"}: "volume-1"},
expectedErr: false,
expectedRes: NewTestUnstructured().WithName("pv-1").WithSpecField("awsElasticBlockStore", map[string]interface{}{"volumeID": "volume-1"}).Unstructured,
},
{
name: "when RestorePVs=true, GCE pdName should be set correctly",
obj: NewTestUnstructured().WithName("pv-1").WithSpecField("gcePersistentDisk", make(map[string]interface{})).Unstructured,
restore: NewDefaultTestRestore().WithRestorePVs(true).Restore,
backup: &api.Backup{Status: api.BackupStatus{VolumeBackups: map[string]*api.VolumeBackupInfo{"pv-1": {SnapshotID: "snap-1"}}}},
volumeMap: map[api.VolumeBackupInfo]string{{SnapshotID: "snap-1"}: "volume-1"},
expectedErr: false,
expectedRes: NewTestUnstructured().WithName("pv-1").WithSpecField("gcePersistentDisk", map[string]interface{}{"pdName": "volume-1"}).Unstructured,
},
{
name: "when RestorePVs=true, Azure pdName should be set correctly",
obj: NewTestUnstructured().WithName("pv-1").WithSpecField("azureDisk", make(map[string]interface{})).Unstructured,
restore: NewDefaultTestRestore().WithRestorePVs(true).Restore,
backup: &api.Backup{Status: api.BackupStatus{VolumeBackups: map[string]*api.VolumeBackupInfo{"pv-1": {SnapshotID: "snap-1"}}}},
volumeMap: map[api.VolumeBackupInfo]string{{SnapshotID: "snap-1"}: "volume-1"},
expectedErr: false,
expectedRes: NewTestUnstructured().WithName("pv-1").WithSpecField("azureDisk", map[string]interface{}{"diskName": "volume-1"}).Unstructured,
},
{
name: "when RestorePVs=true, unsupported PV source should not get snapshot restored",
obj: NewTestUnstructured().WithName("pv-1").WithSpecField("unsupportedPVSource", make(map[string]interface{})).Unstructured,
restore: NewDefaultTestRestore().WithRestorePVs(true).Restore,
backup: &api.Backup{Status: api.BackupStatus{VolumeBackups: map[string]*api.VolumeBackupInfo{"pv-1": {SnapshotID: "snap-1"}}}},
volumeMap: map[api.VolumeBackupInfo]string{{SnapshotID: "snap-1"}: "volume-1"},
expectedErr: false,
expectedRes: NewTestUnstructured().WithName("pv-1").WithSpecField("unsupportedPVSource", make(map[string]interface{})).Unstructured,
},
{
name: "volume type and IOPS are correctly passed to CreateVolume",
obj: NewTestUnstructured().WithName("pv-1").WithSpecField("awsElasticBlockStore", make(map[string]interface{})).Unstructured,
restore: NewDefaultTestRestore().WithRestorePVs(true).Restore,
backup: &api.Backup{Status: api.BackupStatus{VolumeBackups: map[string]*api.VolumeBackupInfo{"pv-1": {SnapshotID: "snap-1", Type: "gp", Iops: &iops}}}},
volumeMap: map[api.VolumeBackupInfo]string{{SnapshotID: "snap-1", Type: "gp", Iops: &iops}: "volume-1"},
expectedErr: false,
expectedRes: NewTestUnstructured().WithName("pv-1").WithSpecField("awsElasticBlockStore", map[string]interface{}{"volumeID": "volume-1"}).Unstructured,
},
{
name: "When no SnapshotService, warn if backup has snapshots that will not be restored",
obj: NewTestUnstructured().WithName("pv-1").WithSpecField("awsElasticBlockStore", make(map[string]interface{})).Unstructured,
restore: NewDefaultTestRestore().Restore,
backup: &api.Backup{Status: api.BackupStatus{VolumeBackups: map[string]*api.VolumeBackupInfo{"pv-1": {SnapshotID: "snap-1"}}}},
volumeMap: map[api.VolumeBackupInfo]string{{SnapshotID: "snap-1"}: "volume-1"},
noSnapshotService: true,
expectedErr: false,
expectedWarn: true,
expectedRes: NewTestUnstructured().WithName("pv-1").WithSpecField("awsElasticBlockStore", make(map[string]interface{})).Unstructured,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
var snapshotService cloudprovider.SnapshotService
if !test.noSnapshotService {
snapshotService = &FakeSnapshotService{RestorableVolumes: test.volumeMap}
}
restorer := NewPersistentVolumeRestorer(snapshotService)
res, warn, err := restorer.Prepare(test.obj, test.restore, test.backup)
assert.Equal(t, test.expectedWarn, warn != nil)
if assert.Equal(t, test.expectedErr, err != nil) {
assert.Equal(t, test.expectedRes, res)
}
})
}
}
func TestPVRestorerReady(t *testing.T) {
tests := []struct {
name string
obj *unstructured.Unstructured
expected bool
}{
{
name: "no status returns not ready",
obj: NewTestUnstructured().Unstructured,
expected: false,
},
{
name: "no status.phase returns not ready",
obj: NewTestUnstructured().WithStatus().Unstructured,
expected: false,
},
{
name: "empty status.phase returns not ready",
obj: NewTestUnstructured().WithStatusField("phase", "").Unstructured,
expected: false,
},
{
name: "non-Available status.phase returns not ready",
obj: NewTestUnstructured().WithStatusField("phase", "foo").Unstructured,
expected: false,
},
{
name: "Available status.phase returns ready",
obj: NewTestUnstructured().WithStatusField("phase", "Available").Unstructured,
expected: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
restorer := NewPersistentVolumeRestorer(nil)
assert.Equal(t, test.expected, restorer.Ready(test.obj))
})
}
}

52
pkg/restore/restorers/pvc_restorer.go
View File

@ -1,52 +0,0 @@
/*
Copyright 2017 Heptio Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package restorers
import (
"k8s.io/apimachinery/pkg/runtime"
api "github.com/heptio/ark/pkg/apis/ark/v1"
"github.com/heptio/ark/pkg/util/collections"
)
type persistentVolumeClaimRestorer struct{}
var _ ResourceRestorer = &persistentVolumeClaimRestorer{}
func NewPersistentVolumeClaimRestorer() ResourceRestorer {
return &persistentVolumeClaimRestorer{}
}
func (sr *persistentVolumeClaimRestorer) Handles(obj runtime.Unstructured, restore *api.Restore) bool {
return true
}
func (sr *persistentVolumeClaimRestorer) Prepare(obj runtime.Unstructured, restore *api.Restore, backup *api.Backup) (runtime.Unstructured, error, error) {
res, err := resetMetadataAndStatus(obj, true)
return res, nil, err
}
func (sr *persistentVolumeClaimRestorer) Wait() bool {
return true
}
func (sr *persistentVolumeClaimRestorer) Ready(obj runtime.Unstructured) bool {
phase, err := collections.GetString(obj.UnstructuredContent(), "status.phase")
return err == nil && phase == "Bound"
}

67
pkg/restore/restorers/pvc_restorer_test.go
View File

@ -1,67 +0,0 @@
/*
Copyright 2017 Heptio Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package restorers
import (
"testing"
"github.com/stretchr/testify/assert"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
)
func TestPVCRestorerReady(t *testing.T) {
tests := []struct {
name string
obj *unstructured.Unstructured
expected bool
}{
{
name: "no status returns not ready",
obj: NewTestUnstructured().Unstructured,
expected: false,
},
{
name: "no status.phase returns not ready",
obj: NewTestUnstructured().WithStatus().Unstructured,
expected: false,
},
{
name: "empty status.phase returns not ready",
obj: NewTestUnstructured().WithStatusField("phase", "").Unstructured,
expected: false,
},
{
name: "non-Available status.phase returns not ready",
obj: NewTestUnstructured().WithStatusField("phase", "foo").Unstructured,
expected: false,
},
{
name: "Bound status.phase returns ready",
obj: NewTestUnstructured().WithStatusField("phase", "Bound").Unstructured,
expected: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
restorer := NewPersistentVolumeClaimRestorer()
assert.Equal(t, test.expected, restorer.Ready(test.obj))
})
}
}

85
pkg/restore/restorers/resource_restorer.go
View File

@ -1,85 +0,0 @@
/*
Copyright 2017 Heptio Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package restorers
import (
"k8s.io/apimachinery/pkg/runtime"
api "github.com/heptio/ark/pkg/apis/ark/v1"
"github.com/heptio/ark/pkg/util/collections"
)
// ResourceRestorer exposes the operations necessary to prepare Kubernetes resources
// for restore and confirm their readiness following restoration via Ark.
type ResourceRestorer interface {
// Handles returns true if the Restorer should restore this object.
Handles(obj runtime.Unstructured, restore *api.Restore) bool
// Prepare gets an item ready to be restored.
Prepare(obj runtime.Unstructured, restore *api.Restore, backup *api.Backup) (res runtime.Unstructured, warning error, err error)
// Wait returns true if restoration should wait for all of this restorer's resources to be ready before moving on to the next restorer.
Wait() bool
// Ready returns true if the given item is considered ready by the system. Only used if Wait() returns true.
Ready(obj runtime.Unstructured) bool
}
func resetMetadataAndStatus(obj runtime.Unstructured, keepAnnotations bool) (runtime.Unstructured, error) {
metadata, err := collections.GetMap(obj.UnstructuredContent(), "metadata")
if err != nil {
return nil, err
}
for k := range metadata {
if k != "name" && k != "namespace" && k != "labels" && (!keepAnnotations || k != "annotations") {
delete(metadata, k)
}
}
delete(obj.UnstructuredContent(), "status")
return obj, nil
}
var _ ResourceRestorer = &basicRestorer{}
type basicRestorer struct {
saveAnnotations bool
}
func (br *basicRestorer) Handles(obj runtime.Unstructured, restore *api.Restore) bool {
return true
}
func (br *basicRestorer) Prepare(obj runtime.Unstructured, restore *api.Restore, backup *api.Backup) (runtime.Unstructured, error, error) {
obj, err := resetMetadataAndStatus(obj, br.saveAnnotations)
return obj, err, nil
}
func (br *basicRestorer) Wait() bool {
return false
}
func (br *basicRestorer) Ready(obj runtime.Unstructured) bool {
return true
}
func NewBasicRestorer(saveAnnotations bool) ResourceRestorer {
return &basicRestorer{saveAnnotations: saveAnnotations}
}

160
pkg/restore/restorers/resource_restorer_test.go
View File

@ -1,160 +0,0 @@
/*
Copyright 2017 Heptio Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package restorers
import (
"testing"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"github.com/stretchr/testify/assert"
)
func TestResetMetadataAndStatus(t *testing.T) {
tests := []struct {
name string
obj runtime.Unstructured
keepAnnotations bool
expectedErr bool
expectedRes runtime.Unstructured
}{
{
name: "no metadata causes error",
obj: NewTestUnstructured(),
keepAnnotations: false,
expectedErr: true,
},
{
name: "don't keep annotations",
obj: NewTestUnstructured().WithMetadata("name", "namespace", "labels", "annotations").Unstructured,
keepAnnotations: false,
expectedErr: false,
expectedRes: NewTestUnstructured().WithMetadata("name", "namespace", "labels").Unstructured,
},
{
name: "keep annotations",
obj: NewTestUnstructured().WithMetadata("name", "namespace", "labels", "annotations").Unstructured,
keepAnnotations: true,
expectedErr: false,
expectedRes: NewTestUnstructured().WithMetadata("name", "namespace", "labels", "annotations").Unstructured,
},
{
name: "don't keep extraneous metadata",
obj: NewTestUnstructured().WithMetadata("foo").Unstructured,
keepAnnotations: false,
expectedErr: false,
expectedRes: NewTestUnstructured().WithMetadata().Unstructured,
},
{
name: "don't keep status",
obj: NewTestUnstructured().WithMetadata().WithStatus().Unstructured,
keepAnnotations: false,
expectedErr: false,
expectedRes: NewTestUnstructured().WithMetadata().Unstructured,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
res, err := resetMetadataAndStatus(test.obj, test.keepAnnotations)
if assert.Equal(t, test.expectedErr, err != nil) {
assert.Equal(t, test.expectedRes, res)
}
})
}
}
type testUnstructured struct {
*unstructured.Unstructured
}
func NewTestUnstructured() *testUnstructured {
obj := &testUnstructured{
Unstructured: &unstructured.Unstructured{
Object: make(map[string]interface{}),
},
}
return obj
}
func (obj *testUnstructured) WithMetadata(fields ...string) *testUnstructured {
return obj.withMap("metadata", fields...)
}
func (obj *testUnstructured) WithSpec(fields ...string) *testUnstructured {
return obj.withMap("spec", fields...)
}
func (obj *testUnstructured) WithStatus(fields ...string) *testUnstructured {
return obj.withMap("status", fields...)
}
func (obj *testUnstructured) WithMetadataField(field string, value interface{}) *testUnstructured {
return obj.withMapEntry("metadata", field, value)
}
func (obj *testUnstructured) WithSpecField(field string, value interface{}) *testUnstructured {
return obj.withMapEntry("spec", field, value)
}
func (obj *testUnstructured) WithStatusField(field string, value interface{}) *testUnstructured {
return obj.withMapEntry("status", field, value)
}
func (obj *testUnstructured) WithAnnotations(fields ...string) *testUnstructured {
annotations := make(map[string]interface{})
for _, field := range fields {
annotations[field] = "foo"
}
obj = obj.WithMetadataField("annotations", annotations)
return obj
}
func (obj *testUnstructured) WithName(name string) *testUnstructured {
return obj.WithMetadataField("name", name)
}
func (obj *testUnstructured) withMap(name string, fields ...string) *testUnstructured {
m := make(map[string]interface{})
obj.Object[name] = m
for _, field := range fields {
m[field] = "foo"
}
return obj
}
func (obj *testUnstructured) withMapEntry(mapName, field string, value interface{}) *testUnstructured {
var m map[string]interface{}
if res, ok := obj.Unstructured.Object[mapName]; !ok {
m = make(map[string]interface{})
obj.Unstructured.Object[mapName] = m
} else {
m = res.(map[string]interface{})
}
m[field] = value
return obj
}

40
pkg/restore/restorers/service_restorer.go → pkg/restore/service_action.go
View File

@ -1,5 +1,5 @@
/*
Copyright 2017 Heptio Inc.
Copyright 2017 the Heptio Ark contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@ -14,32 +14,34 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package restorers
package restore
import (
"github.com/sirupsen/logrus"
"k8s.io/apimachinery/pkg/runtime"
api "github.com/heptio/ark/pkg/apis/ark/v1"
"github.com/heptio/ark/pkg/util/collections"
)
type serviceRestorer struct{}
var _ ResourceRestorer = &serviceRestorer{}
func NewServiceRestorer() ResourceRestorer {
return &serviceRestorer{}
type serviceAction struct {
log logrus.FieldLogger
}
func (sr *serviceRestorer) Handles(obj runtime.Unstructured, restore *api.Restore) bool {
return true
}
func (sr *serviceRestorer) Prepare(obj runtime.Unstructured, restore *api.Restore, backup *api.Backup) (runtime.Unstructured, error, error) {
if _, err := resetMetadataAndStatus(obj, true); err != nil {
return nil, nil, err
func NewServiceAction(log logrus.FieldLogger) ItemAction {
return &serviceAction{
log: log,
}
}
func (a *serviceAction) AppliesTo() (ResourceSelector, error) {
return ResourceSelector{
IncludedResources: []string{"services"},
}, nil
}
func (a *serviceAction) Execute(obj runtime.Unstructured, restore *api.Restore) (runtime.Unstructured, error, error) {
spec, err := collections.GetMap(obj.UnstructuredContent(), "spec")
if err != nil {
return nil, nil, err
@ -62,11 +64,3 @@ func (sr *serviceRestorer) Prepare(obj runtime.Unstructured, restore *api.Restor
return obj, nil, nil
}
func (sr *serviceRestorer) Wait() bool {
return false
}
func (sr *serviceRestorer) Ready(obj runtime.Unstructured) bool {
return true
}

11
pkg/restore/restorers/service_restorer_test.go → pkg/restore/service_action_test.go
View File

@ -1,5 +1,5 @@
/*
Copyright 2017 Heptio Inc.
Copyright 2017 the Heptio Ark contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@ -14,17 +14,18 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package restorers
package restore
import (
"testing"
arktest "github.com/heptio/ark/pkg/util/test"
"github.com/stretchr/testify/assert"
"k8s.io/apimachinery/pkg/runtime"
)
func TestServiceRestorerPrepare(t *testing.T) {
func TestServiceActionExecute(t *testing.T) {
tests := []struct {
name string
obj runtime.Unstructured
@ -66,9 +67,9 @@ func TestServiceRestorerPrepare(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
restorer := NewServiceRestorer()
action := NewServiceAction(arktest.NewLogger())
res, _, err := restorer.Prepare(test.obj, nil, nil)
res, _, err := action.Execute(test.obj, nil)
if assert.Equal(t, test.expectedErr, err != nil) {
assert.Equal(t, test.expectedRes, res)

9
pkg/util/logging/log_setter.go Normal file
View File

@ -0,0 +1,9 @@
package logging
import "github.com/sirupsen/logrus"
// LogSetter is an interface for a type that allows a FieldLogger
// to be set on it.
type LogSetter interface {
SetLog(logrus.FieldLogger)
}