416 lines
15 KiB
Go
416 lines
15 KiB
Go
/*
|
|
Copyright 2017 Heptio Inc.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package controller
|
|
|
|
import (
|
|
"bytes"
|
|
"errors"
|
|
"io"
|
|
"io/ioutil"
|
|
"testing"
|
|
|
|
testlogger "github.com/sirupsen/logrus/hooks/test"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/mock"
|
|
|
|
"k8s.io/apimachinery/pkg/runtime"
|
|
"k8s.io/client-go/kubernetes/scheme"
|
|
core "k8s.io/client-go/testing"
|
|
"k8s.io/client-go/tools/cache"
|
|
|
|
api "github.com/heptio/ark/pkg/apis/ark/v1"
|
|
"github.com/heptio/ark/pkg/generated/clientset/fake"
|
|
informers "github.com/heptio/ark/pkg/generated/informers/externalversions"
|
|
. "github.com/heptio/ark/pkg/util/test"
|
|
)
|
|
|
|
func TestFetchBackup(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
backupName string
|
|
informerBackups []*api.Backup
|
|
backupServiceBackup *api.Backup
|
|
backupServiceError error
|
|
expectedRes *api.Backup
|
|
expectedErr bool
|
|
}{
|
|
{
|
|
name: "lister has backup",
|
|
backupName: "backup-1",
|
|
informerBackups: []*api.Backup{NewTestBackup().WithName("backup-1").Backup},
|
|
expectedRes: NewTestBackup().WithName("backup-1").Backup,
|
|
},
|
|
{
|
|
name: "backupSvc has backup",
|
|
backupName: "backup-1",
|
|
backupServiceBackup: NewTestBackup().WithName("backup-1").Backup,
|
|
expectedRes: NewTestBackup().WithName("backup-1").Backup,
|
|
},
|
|
{
|
|
name: "no backup",
|
|
backupName: "backup-1",
|
|
backupServiceError: errors.New("no backup here"),
|
|
expectedErr: true,
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
var (
|
|
client = fake.NewSimpleClientset()
|
|
restorer = &fakeRestorer{}
|
|
sharedInformers = informers.NewSharedInformerFactory(client, 0)
|
|
backupSvc = &BackupService{}
|
|
logger, _ = testlogger.NewNullLogger()
|
|
)
|
|
|
|
c := NewRestoreController(
|
|
sharedInformers.Ark().V1().Restores(),
|
|
client.ArkV1(),
|
|
client.ArkV1(),
|
|
restorer,
|
|
backupSvc,
|
|
"bucket",
|
|
sharedInformers.Ark().V1().Backups(),
|
|
false,
|
|
logger,
|
|
).(*restoreController)
|
|
|
|
for _, itm := range test.informerBackups {
|
|
sharedInformers.Ark().V1().Backups().Informer().GetStore().Add(itm)
|
|
}
|
|
|
|
if test.backupServiceBackup != nil || test.backupServiceError != nil {
|
|
backupSvc.On("GetBackup", "bucket", test.backupName).Return(test.backupServiceBackup, test.backupServiceError)
|
|
}
|
|
|
|
backup, err := c.fetchBackup("bucket", test.backupName)
|
|
|
|
if assert.Equal(t, test.expectedErr, err != nil) {
|
|
assert.Equal(t, test.expectedRes, backup)
|
|
}
|
|
|
|
backupSvc.AssertExpectations(t)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestProcessRestore(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
restoreKey string
|
|
restore *api.Restore
|
|
backup *api.Backup
|
|
restorerError error
|
|
allowRestoreSnapshots bool
|
|
expectedErr bool
|
|
expectedRestoreUpdates []*api.Restore
|
|
expectedRestorerCall *api.Restore
|
|
backupServiceGetBackupError error
|
|
uploadLogError error
|
|
}{
|
|
{
|
|
name: "invalid key returns error",
|
|
restoreKey: "invalid/key/value",
|
|
expectedErr: true,
|
|
},
|
|
{
|
|
name: "missing restore returns error",
|
|
restoreKey: "foo/bar",
|
|
expectedErr: true,
|
|
},
|
|
{
|
|
name: "restore with phase InProgress does not get processed",
|
|
restore: NewTestRestore("foo", "bar", api.RestorePhaseInProgress).Restore,
|
|
expectedErr: false,
|
|
},
|
|
{
|
|
name: "restore with phase Completed does not get processed",
|
|
restore: NewTestRestore("foo", "bar", api.RestorePhaseCompleted).Restore,
|
|
expectedErr: false,
|
|
},
|
|
{
|
|
name: "restore with phase FailedValidation does not get processed",
|
|
restore: NewTestRestore("foo", "bar", api.RestorePhaseFailedValidation).Restore,
|
|
expectedErr: false,
|
|
},
|
|
{
|
|
name: "restore with both namespace in both includedNamespaces and excludedNamespaces fails validation",
|
|
restore: NewRestore("foo", "bar", "backup-1", "another-1", "*", api.RestorePhaseNew).WithExcludedNamespace("another-1").Restore,
|
|
backup: NewTestBackup().WithName("backup-1").Backup,
|
|
expectedErr: false,
|
|
expectedRestoreUpdates: []*api.Restore{
|
|
NewRestore("foo", "bar", "backup-1", "another-1", "*", api.RestorePhaseFailedValidation).WithExcludedNamespace("another-1").
|
|
WithValidationError("Invalid included/excluded namespace lists: excludes list cannot contain an item in the includes list: another-1").
|
|
Restore,
|
|
},
|
|
},
|
|
{
|
|
name: "restore with resource in both includedResources and excludedResources fails validation",
|
|
restore: NewRestore("foo", "bar", "backup-1", "*", "a-resource", api.RestorePhaseNew).WithExcludedResource("a-resource").Restore,
|
|
backup: NewTestBackup().WithName("backup-1").Backup,
|
|
expectedErr: false,
|
|
expectedRestoreUpdates: []*api.Restore{
|
|
NewRestore("foo", "bar", "backup-1", "*", "a-resource", api.RestorePhaseFailedValidation).WithExcludedResource("a-resource").
|
|
WithValidationError("Invalid included/excluded resource lists: excludes list cannot contain an item in the includes list: a-resource").
|
|
Restore,
|
|
},
|
|
},
|
|
{
|
|
name: "new restore with empty backup name fails validation",
|
|
restore: NewRestore("foo", "bar", "", "ns-1", "", api.RestorePhaseNew).Restore,
|
|
expectedErr: false,
|
|
expectedRestoreUpdates: []*api.Restore{
|
|
NewRestore("foo", "bar", "", "ns-1", "*", api.RestorePhaseFailedValidation).
|
|
WithValidationError("BackupName must be non-empty and correspond to the name of a backup in object storage.").
|
|
Restore,
|
|
},
|
|
},
|
|
|
|
{
|
|
name: "restore with non-existent backup name fails",
|
|
restore: NewTestRestore("foo", "bar", api.RestorePhaseNew).WithBackup("backup-1").WithIncludedNamespace("ns-1").Restore,
|
|
expectedErr: false,
|
|
backupServiceGetBackupError: errors.New("no backup here"),
|
|
expectedRestoreUpdates: []*api.Restore{
|
|
NewRestore("foo", "bar", "backup-1", "ns-1", "*", api.RestorePhaseInProgress).Restore,
|
|
NewRestore("foo", "bar", "backup-1", "ns-1", "*", api.RestorePhaseCompleted).
|
|
WithErrors(api.RestoreResult{
|
|
Ark: []string{"no backup here"},
|
|
}).
|
|
Restore,
|
|
},
|
|
},
|
|
{
|
|
name: "restorer throwing an error causes the restore to fail",
|
|
restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", api.RestorePhaseNew).Restore,
|
|
backup: NewTestBackup().WithName("backup-1").Backup,
|
|
restorerError: errors.New("blarg"),
|
|
expectedErr: false,
|
|
expectedRestoreUpdates: []*api.Restore{
|
|
NewRestore("foo", "bar", "backup-1", "ns-1", "*", api.RestorePhaseInProgress).Restore,
|
|
NewRestore("foo", "bar", "backup-1", "ns-1", "*", api.RestorePhaseCompleted).
|
|
WithErrors(api.RestoreResult{
|
|
Namespaces: map[string][]string{
|
|
"ns-1": {"blarg"},
|
|
},
|
|
}).
|
|
Restore,
|
|
},
|
|
expectedRestorerCall: NewRestore("foo", "bar", "backup-1", "ns-1", "*", api.RestorePhaseInProgress).Restore,
|
|
},
|
|
{
|
|
name: "valid restore gets executed",
|
|
restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", api.RestorePhaseNew).Restore,
|
|
backup: NewTestBackup().WithName("backup-1").Backup,
|
|
expectedErr: false,
|
|
expectedRestoreUpdates: []*api.Restore{
|
|
NewRestore("foo", "bar", "backup-1", "ns-1", "*", api.RestorePhaseInProgress).Restore,
|
|
NewRestore("foo", "bar", "backup-1", "ns-1", "*", api.RestorePhaseCompleted).Restore,
|
|
},
|
|
expectedRestorerCall: NewRestore("foo", "bar", "backup-1", "ns-1", "*", api.RestorePhaseInProgress).Restore,
|
|
},
|
|
{
|
|
name: "restore with no restorable namespaces gets defaulted to *",
|
|
restore: NewRestore("foo", "bar", "backup-1", "", "", api.RestorePhaseNew).Restore,
|
|
backup: NewTestBackup().WithName("backup-1").Backup,
|
|
expectedErr: false,
|
|
expectedRestoreUpdates: []*api.Restore{
|
|
NewRestore("foo", "bar", "backup-1", "*", "*", api.RestorePhaseInProgress).Restore,
|
|
NewRestore("foo", "bar", "backup-1", "*", "*", api.RestorePhaseCompleted).Restore,
|
|
},
|
|
expectedRestorerCall: NewRestore("foo", "bar", "backup-1", "*", "*", api.RestorePhaseInProgress).Restore,
|
|
},
|
|
{
|
|
name: "valid restore with RestorePVs=true gets executed when allowRestoreSnapshots=true",
|
|
restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", api.RestorePhaseNew).WithRestorePVs(true).Restore,
|
|
backup: NewTestBackup().WithName("backup-1").Backup,
|
|
allowRestoreSnapshots: true,
|
|
expectedErr: false,
|
|
expectedRestoreUpdates: []*api.Restore{
|
|
NewRestore("foo", "bar", "backup-1", "ns-1", "*", api.RestorePhaseInProgress).WithRestorePVs(true).Restore,
|
|
NewRestore("foo", "bar", "backup-1", "ns-1", "*", api.RestorePhaseCompleted).WithRestorePVs(true).Restore,
|
|
},
|
|
expectedRestorerCall: NewRestore("foo", "bar", "backup-1", "ns-1", "*", api.RestorePhaseInProgress).WithRestorePVs(true).Restore,
|
|
},
|
|
{
|
|
name: "restore with RestorePVs=true fails validation when allowRestoreSnapshots=false",
|
|
restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", api.RestorePhaseNew).WithRestorePVs(true).Restore,
|
|
backup: NewTestBackup().WithName("backup-1").Backup,
|
|
expectedErr: false,
|
|
expectedRestoreUpdates: []*api.Restore{
|
|
NewRestore("foo", "bar", "backup-1", "ns-1", "*", api.RestorePhaseFailedValidation).
|
|
WithRestorePVs(true).
|
|
WithValidationError("Server is not configured for PV snapshot restores").
|
|
Restore,
|
|
},
|
|
},
|
|
{
|
|
name: "restoration of nodes is not supported",
|
|
restore: NewRestore("foo", "bar", "backup-1", "ns-1", "nodes", api.RestorePhaseNew).Restore,
|
|
backup: NewTestBackup().WithName("backup-1").Backup,
|
|
expectedErr: false,
|
|
expectedRestoreUpdates: []*api.Restore{
|
|
NewRestore("foo", "bar", "backup-1", "ns-1", "nodes", api.RestorePhaseFailedValidation).
|
|
WithValidationError("nodes are a non-restorable resource").
|
|
WithValidationError("Invalid included/excluded resource lists: excludes list cannot contain an item in the includes list: nodes").
|
|
Restore,
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
var (
|
|
client = fake.NewSimpleClientset()
|
|
restorer = &fakeRestorer{}
|
|
sharedInformers = informers.NewSharedInformerFactory(client, 0)
|
|
backupSvc = &BackupService{}
|
|
logger, _ = testlogger.NewNullLogger()
|
|
)
|
|
|
|
defer restorer.AssertExpectations(t)
|
|
defer backupSvc.AssertExpectations(t)
|
|
|
|
c := NewRestoreController(
|
|
sharedInformers.Ark().V1().Restores(),
|
|
client.ArkV1(),
|
|
client.ArkV1(),
|
|
restorer,
|
|
backupSvc,
|
|
"bucket",
|
|
sharedInformers.Ark().V1().Backups(),
|
|
test.allowRestoreSnapshots,
|
|
logger,
|
|
).(*restoreController)
|
|
|
|
if test.restore != nil {
|
|
sharedInformers.Ark().V1().Restores().Informer().GetStore().Add(test.restore)
|
|
|
|
// this is necessary so the Update() call returns the appropriate object
|
|
client.PrependReactor("update", "restores", func(action core.Action) (bool, runtime.Object, error) {
|
|
obj := action.(core.UpdateAction).GetObject()
|
|
// need to deep copy so we can test the backup state for each call to update
|
|
copy, err := scheme.Scheme.DeepCopy(obj)
|
|
if err != nil {
|
|
return false, nil, err
|
|
}
|
|
ret := copy.(runtime.Object)
|
|
return true, ret, nil
|
|
})
|
|
}
|
|
|
|
if test.backup != nil {
|
|
sharedInformers.Ark().V1().Backups().Informer().GetStore().Add(test.backup)
|
|
}
|
|
|
|
var warnings, errors api.RestoreResult
|
|
if test.restorerError != nil {
|
|
errors.Namespaces = map[string][]string{"ns-1": {test.restorerError.Error()}}
|
|
}
|
|
if test.uploadLogError != nil {
|
|
errors.Ark = append(errors.Ark, "error uploading log file to object storage: "+test.uploadLogError.Error())
|
|
}
|
|
if test.expectedRestorerCall != nil {
|
|
downloadedBackup := ioutil.NopCloser(bytes.NewReader([]byte("hello world")))
|
|
backupSvc.On("DownloadBackup", mock.Anything, mock.Anything).Return(downloadedBackup, nil)
|
|
restorer.On("Restore", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(warnings, errors)
|
|
backupSvc.On("UploadRestoreLog", "bucket", test.restore.Spec.BackupName, test.restore.Name, mock.Anything).Return(test.uploadLogError)
|
|
}
|
|
|
|
var (
|
|
key = test.restoreKey
|
|
err error
|
|
)
|
|
if key == "" && test.restore != nil {
|
|
key, err = cache.MetaNamespaceKeyFunc(test.restore)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
}
|
|
|
|
if test.backupServiceGetBackupError != nil {
|
|
backupSvc.On("GetBackup", "bucket", test.restore.Spec.BackupName).Return(nil, test.backupServiceGetBackupError)
|
|
}
|
|
|
|
err = c.processRestore(key)
|
|
backupSvc.AssertExpectations(t)
|
|
restorer.AssertExpectations(t)
|
|
|
|
assert.Equal(t, test.expectedErr, err != nil, "got error %v", err)
|
|
|
|
if test.expectedRestoreUpdates != nil {
|
|
var expectedActions []core.Action
|
|
|
|
for _, upd := range test.expectedRestoreUpdates {
|
|
action := core.NewUpdateAction(
|
|
api.SchemeGroupVersion.WithResource("restores"),
|
|
upd.Namespace,
|
|
upd)
|
|
|
|
expectedActions = append(expectedActions, action)
|
|
}
|
|
|
|
assert.Equal(t, expectedActions, client.Actions())
|
|
}
|
|
|
|
if test.expectedRestorerCall == nil {
|
|
assert.Empty(t, restorer.Calls)
|
|
assert.Zero(t, restorer.calledWithArg)
|
|
} else {
|
|
assert.Equal(t, 1, len(restorer.Calls))
|
|
|
|
// explicitly capturing the argument passed to Restore myself because
|
|
// I want to validate the called arg as of the time of calling, but
|
|
// the mock stores the pointer, which gets modified after
|
|
assert.Equal(t, *test.expectedRestorerCall, restorer.calledWithArg)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func NewRestore(ns, name, backup, includeNS, includeResource string, phase api.RestorePhase) *TestRestore {
|
|
restore := NewTestRestore(ns, name, phase).WithBackup(backup)
|
|
|
|
if includeNS != "" {
|
|
restore = restore.WithIncludedNamespace(includeNS)
|
|
}
|
|
|
|
if includeResource != "" {
|
|
restore = restore.WithIncludedResource(includeResource)
|
|
}
|
|
|
|
for _, n := range nonRestorableResources {
|
|
restore.WithExcludedResource(n)
|
|
}
|
|
|
|
return restore
|
|
}
|
|
|
|
type fakeRestorer struct {
|
|
mock.Mock
|
|
calledWithArg api.Restore
|
|
}
|
|
|
|
func (r *fakeRestorer) Restore(restore *api.Restore, backup *api.Backup, backupReader io.Reader, logger io.Writer) (api.RestoreResult, api.RestoreResult) {
|
|
res := r.Called(restore, backup, backupReader, logger)
|
|
|
|
r.calledWithArg = *restore
|
|
|
|
return res.Get(0).(api.RestoreResult), res.Get(1).(api.RestoreResult)
|
|
}
|