vmware-tanzu
/
velero
mirror of https://github.com/vmware-tanzu/velero.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #78 from skriss/restore-resource-inc-exc 

Add --include-resources, --exclude-resources for restore operations
pull/82/head
Andy Goldstein 2017-09-12 15:53:21 -04:00 committed by GitHub
parent 5899beaec7 307cb71e02
commit d0b7880881
13 changed files with 340 additions and 188 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/cli-reference/ark_restore_create.md
View File

@ -15,8 +15,10 @@ ark restore create BACKUP [flags]
```
--exclude-namespaces stringArray namespaces to exclude from the restore
--exclude-resources stringArray resources to exclude from the restore, formatted as resource.group, such as storageclasses.storage.k8s.io
-h, --help help for create
--include-namespaces stringArray namespaces to include in the restore (use '*' for all namespaces) (default *)
--include-resources stringArray resources to include in the restore, formatted as resource.group, such as storageclasses.storage.k8s.io (use '*' for all resources)
--label-columns stringArray a comma-separated list of labels to be displayed as columns
--labels mapStringString labels to apply to the restore
--namespace-mappings mapStringString namespace mappings from name in the backup to desired restored name in the form src1:dst1,src2:dst2,...

8
pkg/apis/ark/v1/restore.go
View File

@ -32,6 +32,14 @@ type RestoreSpec struct {
// included in the restore.
ExcludedNamespaces []string `json:"excludedNamespaces"`
// IncludedResources is a slice of resource names to include
// in the restore. If empty, all resources in the backup are included.
IncludedResources []string `json:"includedResources"`
// ExcludedResources is a slice of resource names that are not
// included in the restore.
ExcludedResources []string `json:"excludedResources"`
// NamespaceMapping is a map of source namespace names
// to target namespace names to restore into. Any source
// namespaces not included in the map will be restored into

62
pkg/backup/backup.go
View File

@ -76,7 +76,7 @@ func NewKubernetesBackupper(
dynamicFactory client.DynamicFactory,
actions map[string]Action,
) (Backupper, error) {
resolvedActions, err := resolveActions(discoveryHelper.Mapper(), actions)
resolvedActions, err := resolveActions(discoveryHelper, actions)
if err != nil {
return nil, err
}
@ -91,11 +91,11 @@ func NewKubernetesBackupper(
// resolveActions resolves the string-based map of group-resources to actions and returns a map of
// schema.GroupResources to actions.
func resolveActions(mapper meta.RESTMapper, actions map[string]Action) (map[schema.GroupResource]Action, error) {
func resolveActions(helper discovery.Helper, actions map[string]Action) (map[schema.GroupResource]Action, error) {
ret := make(map[schema.GroupResource]Action)
for resource, action := range actions {
gr, err := resolveGroupResource(mapper, resource)
gr, err := helper.ResolveGroupResource(resource)
if err != nil {
return nil, err
}
@ -105,46 +105,22 @@ func resolveActions(mapper meta.RESTMapper, actions map[string]Action) (map[sche
return ret, nil
}
// resolveResources uses the RESTMapper to resolve resources to their fully-qualified group-resource
// names. fn is invoked for each resolved resource. resolveResources returns a list of any resources that failed to resolve.
func (ctx *backupContext) resolveResources(mapper meta.RESTMapper, resources []string, allowAll bool, fn func(string)) {
for _, resource := range resources {
if allowAll && resource == "*" {
fn("*")
return
}
gr, err := resolveGroupResource(mapper, resource)
if err != nil {
ctx.log("Unable to resolve resource %q: %v", resource, err)
continue
}
fn(gr.String())
}
}
// getResourceIncludesExcludes takes the lists of resources to include and exclude from the
// backup, uses the discovery helper to resolve them to fully-qualified group-resource names, and returns
// an IncludesExcludes list.
func (ctx *backupContext) getResourceIncludesExcludes(helper discovery.Helper, includes, excludes []string) *collections.IncludesExcludes {
return collections.GenerateIncludesExcludes(
includes,
excludes,
func(item string) (string, error) {
gr, err := helper.ResolveGroupResource(item)
if err != nil {
return "", err
}
// getResourceIncludesExcludes takes the lists of resources to include and exclude, uses the
// RESTMapper to resolve them to fully-qualified group-resource names, and returns an
// IncludesExcludes list.
func (ctx *backupContext) getResourceIncludesExcludes(mapper meta.RESTMapper, includes, excludes []string) *collections.IncludesExcludes {
resources := collections.NewIncludesExcludes()
ctx.resolveResources(mapper, includes, true, func(s string) { resources.Includes(s) })
ctx.resolveResources(mapper, excludes, false, func(s string) { resources.Excludes(s) })
ctx.log("Including resources: %v", strings.Join(resources.GetIncludes(), ", "))
ctx.log("Excluding resources: %v", strings.Join(resources.GetExcludes(), ", "))
return resources
}
// resolveGroupResource uses the RESTMapper to resolve resource to a fully-qualified
// schema.GroupResource. If the RESTMapper is unable to do so, an error is returned instead.
func resolveGroupResource(mapper meta.RESTMapper, resource string) (schema.GroupResource, error) {
gvr, err := mapper.ResourceFor(schema.ParseGroupResource(resource).WithVersion(""))
if err != nil {
return schema.GroupResource{}, err
}
return gvr.GroupResource(), nil
return gr.String(), nil
},
)
}
// getNamespaceIncludesExcludes returns an IncludesExcludes list containing which namespaces to
@ -206,7 +182,7 @@ func (kb *kubernetesBackupper) Backup(backup *api.Backup, backupFile, logFile io
ctx.log("Starting backup")
ctx.resourceIncludesExcludes = ctx.getResourceIncludesExcludes(kb.discoveryHelper.Mapper(), backup.Spec.IncludedResources, backup.Spec.ExcludedResources)
ctx.resourceIncludesExcludes = ctx.getResourceIncludesExcludes(kb.discoveryHelper, backup.Spec.IncludedResources, backup.Spec.ExcludedResources)
for _, group := range kb.discoveryHelper.Resources() {
ctx.log("Processing group %s", group.GroupVersion)

59
pkg/backup/backup_test.go
View File

@ -104,16 +104,18 @@ func TestResolveActions(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
mapper := &FakeMapper{
Resources: map[schema.GroupVersionResource]schema.GroupVersionResource{
schema.GroupVersionResource{Resource: "foo"}: schema.GroupVersionResource{Group: "somegroup", Resource: "foodies"},
schema.GroupVersionResource{Resource: "fie"}: schema.GroupVersionResource{Group: "somegroup", Resource: "fields"},
schema.GroupVersionResource{Resource: "bar"}: schema.GroupVersionResource{Group: "anothergroup", Resource: "barnacles"},
schema.GroupVersionResource{Resource: "baz"}: schema.GroupVersionResource{Group: "anothergroup", Resource: "bazaars"},
dh := &FakeDiscoveryHelper{
RESTMapper: &FakeMapper{
Resources: map[schema.GroupVersionResource]schema.GroupVersionResource{
schema.GroupVersionResource{Resource: "foo"}: schema.GroupVersionResource{Group: "somegroup", Resource: "foodies"},
schema.GroupVersionResource{Resource: "fie"}: schema.GroupVersionResource{Group: "somegroup", Resource: "fields"},
schema.GroupVersionResource{Resource: "bar"}: schema.GroupVersionResource{Group: "anothergroup", Resource: "barnacles"},
schema.GroupVersionResource{Resource: "baz"}: schema.GroupVersionResource{Group: "anothergroup", Resource: "bazaars"},
},
},
}
actual, err := resolveActions(mapper, test.input)
actual, err := resolveActions(dh, test.input)
gotError := err != nil
if e, a := test.expectError, gotError; e != a {
@ -176,12 +178,14 @@ func TestGetResourceIncludesExcludes(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
mapper := &FakeMapper{
Resources: map[schema.GroupVersionResource]schema.GroupVersionResource{
schema.GroupVersionResource{Resource: "foo"}: schema.GroupVersionResource{Group: "somegroup", Resource: "foodies"},
schema.GroupVersionResource{Resource: "fie"}: schema.GroupVersionResource{Group: "somegroup", Resource: "fields"},
schema.GroupVersionResource{Resource: "bar"}: schema.GroupVersionResource{Group: "anothergroup", Resource: "barnacles"},
schema.GroupVersionResource{Resource: "baz"}: schema.GroupVersionResource{Group: "anothergroup", Resource: "bazaars"},
dh := &FakeDiscoveryHelper{
RESTMapper: &FakeMapper{
Resources: map[schema.GroupVersionResource]schema.GroupVersionResource{
schema.GroupVersionResource{Resource: "foo"}: schema.GroupVersionResource{Group: "somegroup", Resource: "foodies"},
schema.GroupVersionResource{Resource: "fie"}: schema.GroupVersionResource{Group: "somegroup", Resource: "fields"},
schema.GroupVersionResource{Resource: "bar"}: schema.GroupVersionResource{Group: "anothergroup", Resource: "barnacles"},
schema.GroupVersionResource{Resource: "baz"}: schema.GroupVersionResource{Group: "anothergroup", Resource: "bazaars"},
},
},
}
@ -190,7 +194,7 @@ func TestGetResourceIncludesExcludes(t *testing.T) {
logger: &logger{w: b},
}
actual := ctx.getResourceIncludesExcludes(mapper, test.includes, test.excludes)
actual := ctx.getResourceIncludesExcludes(dh, test.includes, test.excludes)
sort.Strings(test.expectedIncludes)
actualIncludes := actual.GetIncludes()
@ -236,23 +240,6 @@ func TestGetNamespaceIncludesExcludes(t *testing.T) {
}
}
type fakeDiscoveryHelper struct {
resources []*metav1.APIResourceList
mapper meta.RESTMapper
}
func (dh *fakeDiscoveryHelper) Resources() []*metav1.APIResourceList {
return dh.resources
}
func (dh *fakeDiscoveryHelper) Mapper() meta.RESTMapper {
return dh.mapper
}
func (dh *fakeDiscoveryHelper) Refresh() error {
return nil
}
func TestBackupMethod(t *testing.T) {
// TODO ensure LabelSelector is passed through to the List() calls
backup := &v1.Backup{
@ -303,15 +290,15 @@ func TestBackupMethod(t *testing.T) {
ShortNames: []string{"csr"},
}
discoveryHelper := &fakeDiscoveryHelper{
mapper: &FakeMapper{
discoveryHelper := &FakeDiscoveryHelper{
RESTMapper: &FakeMapper{
Resources: map[schema.GroupVersionResource]schema.GroupVersionResource{
schema.GroupVersionResource{Resource: "cm"}: schema.GroupVersionResource{Group: "", Version: "v1", Resource: "configmaps"},
schema.GroupVersionResource{Resource: "csr"}: schema.GroupVersionResource{Group: "certificates.k8s.io", Version: "v1beta1", Resource: "certificatesigningrequests"},
schema.GroupVersionResource{Resource: "roles"}: schema.GroupVersionResource{Group: "rbac.authorization.k8s.io", Version: "v1beta1", Resource: "roles"},
},
},
resources: []*metav1.APIResourceList{
ResourceList: []*metav1.APIResourceList{
{
GroupVersion: "v1",
APIResources: []metav1.APIResource{configMapsResource, podsResource},
@ -829,8 +816,8 @@ func TestBackupResource(t *testing.T) {
}
}
discoveryHelper := &fakeDiscoveryHelper{
mapper: &FakeMapper{
discoveryHelper := &FakeDiscoveryHelper{
RESTMapper: &FakeMapper{
Resources: map[schema.GroupVersionResource]schema.GroupVersionResource{
schema.GroupVersionResource{Resource: "certificatesigningrequests"}: schema.GroupVersionResource{Group: "certificates.k8s.io", Version: "v1beta1", Resource: "certificatesigningrequests"},
schema.GroupVersionResource{Resource: "other"}: schema.GroupVersionResource{Group: "somegroup", Version: "someversion", Resource: "otherthings"},

6
pkg/cmd/cli/restore/create.go
View File

@ -59,6 +59,8 @@ type CreateOptions struct {
Labels flag.Map
IncludeNamespaces flag.StringArray
ExcludeNamespaces flag.StringArray
IncludeResources flag.StringArray
ExcludeResources flag.StringArray
NamespaceMappings flag.Map
Selector flag.LabelSelector
}
@ -77,6 +79,8 @@ func (o *CreateOptions) BindFlags(flags *pflag.FlagSet) {
flags.Var(&o.ExcludeNamespaces, "exclude-namespaces", "namespaces to exclude from the restore")
flags.Var(&o.NamespaceMappings, "namespace-mappings", "namespace mappings from name in the backup to desired restored name in the form src1:dst1,src2:dst2,...")
flags.Var(&o.Labels, "labels", "labels to apply to the restore")
flags.Var(&o.IncludeResources, "include-resources", "resources to include in the restore, formatted as resource.group, such as storageclasses.storage.k8s.io (use '*' for all resources)")
flags.Var(&o.ExcludeResources, "exclude-resources", "resources to exclude from the restore, formatted as resource.group, such as storageclasses.storage.k8s.io")
flags.VarP(&o.Selector, "selector", "l", "only restore resources matching this label selector")
f := flags.VarPF(&o.RestoreVolumes, "restore-volumes", "", "whether to restore volumes from snapshots")
// this allows the user to just specify "--restore-volumes" as shorthand for "--restore-volumes=true"
@ -117,6 +121,8 @@ func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error {
BackupName: o.BackupName,
IncludedNamespaces: o.IncludeNamespaces,
ExcludedNamespaces: o.ExcludeNamespaces,
IncludedResources: o.IncludeResources,
ExcludedResources: o.ExcludeResources,
NamespaceMapping: o.NamespaceMappings.Data(),
LabelSelector: o.Selector.LabelSelector,
RestorePVs: o.RestoreVolumes.Value,

8
pkg/controller/restore_controller.go
View File

@ -221,9 +221,13 @@ func (controller *restoreController) processRestore(key string) error {
return err
}
// defaulting
if len(restore.Spec.IncludedNamespaces) == 0 {
restore.Spec.IncludedNamespaces = []string{"*"}
}
if len(restore.Spec.IncludedResources) == 0 {
restore.Spec.IncludedResources = []string{"*"}
}
// validation
if restore.Status.ValidationErrors = controller.getValidationErrors(restore); len(restore.Status.ValidationErrors) > 0 {
@ -284,6 +288,10 @@ func (controller *restoreController) getValidationErrors(itm *api.Restore) []str
validationErrors = append(validationErrors, fmt.Sprintf("Invalid included/excluded namespace lists: %v", err))
}
for _, err := range collections.ValidateIncludesExcludes(itm.Spec.IncludedResources, itm.Spec.ExcludedResources) {
validationErrors = append(validationErrors, fmt.Sprintf("Invalid included/excluded resource lists: %v", err))
}
if !controller.pvProviderExists && itm.Spec.RestorePVs != nil && *itm.Spec.RestorePVs {
validationErrors = append(validationErrors, "Server is not configured for PV snapshot restores")
}

148
pkg/controller/restore_controller_test.go
View File

@ -119,7 +119,6 @@ func TestProcessRestore(t *testing.T) {
expectedRestoreUpdates []*api.Restore
expectedRestorerCall *api.Restore
backupServiceGetBackupError error
expectRestore bool
}{
{
name: "invalid key returns error",
@ -148,37 +147,45 @@ func TestProcessRestore(t *testing.T) {
},
{
name: "restore with both namespace in both includedNamespaces and excludedNamespaces fails validation",
restore: NewTestRestore("foo", "bar", api.RestorePhaseNew).WithBackup("backup-1").WithIncludedNamespace("another-1").WithExcludedNamespace("another-1").Restore,
restore: NewRestore("foo", "bar", "backup-1", "another-1", "*", api.RestorePhaseNew).WithExcludedNamespace("another-1").Restore,
backup: NewTestBackup().WithName("backup-1").Backup,
expectedErr: false,
expectedRestoreUpdates: []*api.Restore{
NewTestRestore("foo", "bar", api.RestorePhaseFailedValidation).
WithBackup("backup-1").
WithIncludedNamespace("another-1").
WithExcludedNamespace("another-1").
WithValidationError("Invalid included/excluded namespace lists: excludes list cannot contain an item in the includes list: another-1").Restore,
NewRestore("foo", "bar", "backup-1", "another-1", "*", api.RestorePhaseFailedValidation).WithExcludedNamespace("another-1").
WithValidationError("Invalid included/excluded namespace lists: excludes list cannot contain an item in the includes list: another-1").
Restore,
},
},
{
name: "restore with resource in both includedResources and excludedResources fails validation",
restore: NewRestore("foo", "bar", "backup-1", "*", "a-resource", api.RestorePhaseNew).WithExcludedResource("a-resource").Restore,
backup: NewTestBackup().WithName("backup-1").Backup,
expectedErr: false,
expectedRestoreUpdates: []*api.Restore{
NewRestore("foo", "bar", "backup-1", "*", "a-resource", api.RestorePhaseFailedValidation).WithExcludedResource("a-resource").
WithValidationError("Invalid included/excluded resource lists: excludes list cannot contain an item in the includes list: a-resource").
Restore,
},
},
{
name: "new restore with empty backup name fails validation",
restore: NewTestRestore("foo", "bar", api.RestorePhaseNew).WithIncludedNamespace("ns-1").Restore,
restore: NewRestore("foo", "bar", "", "ns-1", "", api.RestorePhaseNew).Restore,
expectedErr: false,
expectedRestoreUpdates: []*api.Restore{
NewTestRestore("foo", "bar", api.RestorePhaseFailedValidation).
WithIncludedNamespace("ns-1").
WithValidationError("BackupName must be non-empty and correspond to the name of a backup in object storage.").Restore,
NewRestore("foo", "bar", "", "ns-1", "*", api.RestorePhaseFailedValidation).
WithValidationError("BackupName must be non-empty and correspond to the name of a backup in object storage.").
Restore,
},
},
{
name: "restore with non-existent backup name fails",
restore: NewTestRestore("foo", "bar", api.RestorePhaseNew).WithBackup("backup-1").WithIncludedNamespace("ns-1").Restore,
expectedErr: false,
backupServiceGetBackupError: errors.New("no backup here"),
expectedRestoreUpdates: []*api.Restore{
NewTestRestore("foo", "bar", api.RestorePhaseInProgress).WithBackup("backup-1").WithIncludedNamespace("ns-1").Restore,
NewTestRestore("foo", "bar", api.RestorePhaseCompleted).
WithBackup("backup-1").
WithIncludedNamespace("ns-1").
NewRestore("foo", "bar", "backup-1", "ns-1", "*", api.RestorePhaseInProgress).Restore,
NewRestore("foo", "bar", "backup-1", "ns-1", "*", api.RestorePhaseCompleted).
WithErrors(api.RestoreResult{
Cluster: []string{"no backup here"},
}).
@ -187,69 +194,66 @@ func TestProcessRestore(t *testing.T) {
},
{
name: "restorer throwing an error causes the restore to fail",
restore: NewTestRestore("foo", "bar", api.RestorePhaseNew).WithBackup("backup-1").WithIncludedNamespace("ns-1").Restore,
restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", api.RestorePhaseNew).Restore,
backup: NewTestBackup().WithName("backup-1").Backup,
expectRestore: true,
restorerError: errors.New("blarg"),
expectedErr: false,
expectedRestoreUpdates: []*api.Restore{
NewTestRestore("foo", "bar", api.RestorePhaseInProgress).WithBackup("backup-1").WithIncludedNamespace("ns-1").Restore,
NewTestRestore("foo", "bar", api.RestorePhaseCompleted).
WithBackup("backup-1").
WithIncludedNamespace("ns-1").
NewRestore("foo", "bar", "backup-1", "ns-1", "*", api.RestorePhaseInProgress).Restore,
NewRestore("foo", "bar", "backup-1", "ns-1", "*", api.RestorePhaseCompleted).
WithErrors(api.RestoreResult{
Namespaces: map[string][]string{
"ns-1": {"blarg"},
},
}).Restore,
}).
Restore,
},
expectedRestorerCall: NewTestRestore("foo", "bar", api.RestorePhaseInProgress).WithBackup("backup-1").WithIncludedNamespace("ns-1").Restore,
expectedRestorerCall: NewRestore("foo", "bar", "backup-1", "ns-1", "*", api.RestorePhaseInProgress).Restore,
},
{
name: "valid restore gets executed",
restore: NewTestRestore("foo", "bar", api.RestorePhaseNew).WithBackup("backup-1").WithIncludedNamespace("ns-1").Restore,
backup: NewTestBackup().WithName("backup-1").Backup,
expectRestore: true,
expectedErr: false,
expectedRestoreUpdates: []*api.Restore{
NewTestRestore("foo", "bar", api.RestorePhaseInProgress).WithBackup("backup-1").WithIncludedNamespace("ns-1").Restore,
NewTestRestore("foo", "bar", api.RestorePhaseCompleted).WithBackup("backup-1").WithIncludedNamespace("ns-1").Restore,
},
expectedRestorerCall: NewTestRestore("foo", "bar", api.RestorePhaseInProgress).WithBackup("backup-1").WithIncludedNamespace("ns-1").Restore,
},
{
name: "restore with no restorable namespaces gets defaulted to *",
restore: NewTestRestore("foo", "bar", api.RestorePhaseNew).WithBackup("backup-1").Restore,
backup: NewTestBackup().WithName("backup-1").Backup,
expectRestore: true,
expectedErr: false,
expectedRestoreUpdates: []*api.Restore{
NewTestRestore("foo", "bar", api.RestorePhaseInProgress).WithBackup("backup-1").WithIncludedNamespace("*").Restore,
NewTestRestore("foo", "bar", api.RestorePhaseCompleted).WithBackup("backup-1").WithIncludedNamespace("*").Restore,
},
expectedRestorerCall: NewTestRestore("foo", "bar", api.RestorePhaseInProgress).WithBackup("backup-1").WithIncludedNamespace("*").Restore,
},
{
name: "valid restore with RestorePVs=true gets executed when allowRestoreSnapshots=true",
restore: NewTestRestore("foo", "bar", api.RestorePhaseNew).WithBackup("backup-1").WithIncludedNamespace("ns-1").WithRestorePVs(true).Restore,
backup: NewTestBackup().WithName("backup-1").Backup,
expectRestore: true,
allowRestoreSnapshots: true,
expectedErr: false,
expectedRestoreUpdates: []*api.Restore{
NewTestRestore("foo", "bar", api.RestorePhaseInProgress).WithBackup("backup-1").WithIncludedNamespace("ns-1").WithRestorePVs(true).Restore,
NewTestRestore("foo", "bar", api.RestorePhaseCompleted).WithBackup("backup-1").WithIncludedNamespace("ns-1").WithRestorePVs(true).Restore,
},
expectedRestorerCall: NewTestRestore("foo", "bar", api.RestorePhaseInProgress).WithBackup("backup-1").WithIncludedNamespace("ns-1").WithRestorePVs(true).Restore,
},
{
name: "restore with RestorePVs=true fails validation when allowRestoreSnapshots=false",
restore: NewTestRestore("foo", "bar", api.RestorePhaseNew).WithBackup("backup-1").WithIncludedNamespace("ns-1").WithRestorePVs(true).Restore,
name: "valid restore gets executed",
restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", api.RestorePhaseNew).Restore,
backup: NewTestBackup().WithName("backup-1").Backup,
expectedErr: false,
expectedRestoreUpdates: []*api.Restore{
NewTestRestore("foo", "bar", api.RestorePhaseFailedValidation).WithBackup("backup-1").WithIncludedNamespace("ns-1").WithRestorePVs(true).
WithValidationError("Server is not configured for PV snapshot restores").Restore,
NewRestore("foo", "bar", "backup-1", "ns-1", "*", api.RestorePhaseInProgress).Restore,
NewRestore("foo", "bar", "backup-1", "ns-1", "*", api.RestorePhaseCompleted).Restore,
},
expectedRestorerCall: NewRestore("foo", "bar", "backup-1", "ns-1", "*", api.RestorePhaseInProgress).Restore,
},
{
name: "restore with no restorable namespaces gets defaulted to *",
restore: NewRestore("foo", "bar", "backup-1", "", "", api.RestorePhaseNew).Restore,
backup: NewTestBackup().WithName("backup-1").Backup,
expectedErr: false,
expectedRestoreUpdates: []*api.Restore{
NewRestore("foo", "bar", "backup-1", "*", "*", api.RestorePhaseInProgress).Restore,
NewRestore("foo", "bar", "backup-1", "*", "*", api.RestorePhaseCompleted).Restore,
},
expectedRestorerCall: NewRestore("foo", "bar", "backup-1", "*", "*", api.RestorePhaseInProgress).Restore,
},
{
name: "valid restore with RestorePVs=true gets executed when allowRestoreSnapshots=true",
restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", api.RestorePhaseNew).WithRestorePVs(true).Restore,
backup: NewTestBackup().WithName("backup-1").Backup,
allowRestoreSnapshots: true,
expectedErr: false,
expectedRestoreUpdates: []*api.Restore{
NewRestore("foo", "bar", "backup-1", "ns-1", "*", api.RestorePhaseInProgress).WithRestorePVs(true).Restore,
NewRestore("foo", "bar", "backup-1", "ns-1", "*", api.RestorePhaseCompleted).WithRestorePVs(true).Restore,
},
expectedRestorerCall: NewRestore("foo", "bar", "backup-1", "ns-1", "*", api.RestorePhaseInProgress).WithRestorePVs(true).Restore,
},
{
name: "restore with RestorePVs=true fails validation when allowRestoreSnapshots=false",
restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", api.RestorePhaseNew).WithRestorePVs(true).Restore,
backup: NewTestBackup().WithName("backup-1").Backup,
expectedErr: false,
expectedRestoreUpdates: []*api.Restore{
NewRestore("foo", "bar", "backup-1", "ns-1", "*", api.RestorePhaseFailedValidation).
WithRestorePVs(true).
WithValidationError("Server is not configured for PV snapshot restores").
Restore,
},
},
}
@ -299,7 +303,7 @@ func TestProcessRestore(t *testing.T) {
if test.restorerError != nil {
errors.Namespaces = map[string][]string{"ns-1": {test.restorerError.Error()}}
}
if test.expectRestore {
if test.expectedRestorerCall != nil {
downloadedBackup := ioutil.NopCloser(bytes.NewReader([]byte("hello world")))
backupSvc.On("DownloadBackup", mock.Anything, mock.Anything).Return(downloadedBackup, nil)
restorer.On("Restore", mock.Anything, mock.Anything, mock.Anything).Return(warnings, errors)
@ -356,6 +360,20 @@ func TestProcessRestore(t *testing.T) {
}
}
func NewRestore(ns, name, backup, includeNS, includeResource string, phase api.RestorePhase) *TestRestore {
restore := NewTestRestore(ns, name, phase).WithBackup(backup)
if includeNS != "" {
restore = restore.WithIncludedNamespace(includeNS)
}
if includeResource != "" {
restore = restore.WithIncludedResource(includeResource)
}
return restore
}
type fakeRestorer struct {
mock.Mock
calledWithArg api.Restore

12
pkg/discovery/helper.go
View File

@ -44,6 +44,10 @@ type Helper interface {
// Refresh pulls an updated set of Ark-backuppable resources from the
// discovery API.
Refresh() error
// ResolveGroupResource uses the RESTMapper to resolve resource to a fully-qualified
// schema.GroupResource. If the RESTMapper is unable to do so, an error is returned instead.
ResolveGroupResource(resource string) (schema.GroupResource, error)
}
type helper struct {
@ -139,3 +143,11 @@ func (h *helper) Resources() []*metav1.APIResourceList {
defer h.lock.RUnlock()
return h.resources
}
func (h *helper) ResolveGroupResource(resource string) (schema.GroupResource, error) {
gvr, err := h.mapper.ResourceFor(schema.ParseGroupResource(resource).WithVersion(""))
if err != nil {
return schema.GroupResource{}, err
}
return gvr.GroupResource(), nil
}

56
pkg/restore/restore.go
View File

@ -30,7 +30,6 @@ import (
"github.com/golang/glog"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/labels"
@ -45,8 +44,8 @@ import (
"github.com/heptio/ark/pkg/discovery"
arkv1client "github.com/heptio/ark/pkg/generated/clientset/typed/ark/v1"
"github.com/heptio/ark/pkg/restore/restorers"
"github.com/heptio/ark/pkg/util/kube"
"github.com/heptio/ark/pkg/util/collections"
"github.com/heptio/ark/pkg/util/kube"
)
// Restorer knows how to restore a backup.
@ -75,7 +74,7 @@ type kubernetesRestorer struct {
// prioritizeResources takes a list of pre-prioritized resources and a full list of resources to restore,
// and returns an ordered list of GroupResource-resolved resources in the order that they should be
// restored.
func prioritizeResources(mapper meta.RESTMapper, priorities []string, resources []*metav1.APIResourceList) ([]schema.GroupResource, error) {
func prioritizeResources(helper discovery.Helper, priorities []string, includedResources *collections.IncludesExcludes) ([]schema.GroupResource, error) {
var ret []schema.GroupResource
// set keeps track of resolved GroupResource names
@ -83,19 +82,23 @@ func prioritizeResources(mapper meta.RESTMapper, priorities []string, resources
// start by resolving priorities into GroupResources and adding them to ret
for _, r := range priorities {
gr := schema.ParseGroupResource(r)
gvr, err := mapper.ResourceFor(gr.WithVersion(""))
gr, err := helper.ResolveGroupResource(r)
if err != nil {
return nil, err
}
gr = gvr.GroupResource()
if !includedResources.ShouldInclude(gr.String()) {
glog.Infof("Not including resource %v", gr)
continue
}
ret = append(ret, gr)
set.Insert(gr.String())
}
// go through everything we got from discovery and add anything not in "set" to byName
var byName []schema.GroupResource
for _, resourceGroup := range resources {
for _, resourceGroup := range helper.Resources() {
// will be something like storage.k8s.io/v1
groupVersion, err := schema.ParseGroupVersion(resourceGroup.GroupVersion)
if err != nil {
@ -104,6 +107,12 @@ func prioritizeResources(mapper meta.RESTMapper, priorities []string, resources
for _, resource := range resourceGroup.APIResources {
gr := groupVersion.WithResource(resource.Name).GroupResource()
if !includedResources.ShouldInclude(gr.String()) {
glog.Infof("Not including resource %v", gr)
continue
}
if !set.Has(gr.String()) {
byName = append(byName, gr)
}
@ -131,14 +140,13 @@ func NewKubernetesRestorer(
backupClient arkv1client.BackupsGetter,
namespaceClient corev1.NamespaceInterface,
) (Restorer, error) {
mapper := discoveryHelper.Mapper()
r := make(map[schema.GroupResource]restorers.ResourceRestorer)
for gr, restorer := range customRestorers {
gvr, err := mapper.ResourceFor(schema.ParseGroupResource(gr).WithVersion(""))
resolved, err := discoveryHelper.ResolveGroupResource(gr)
if err != nil {
return nil, err
}
r[gvr.GroupResource()] = restorer
r[resolved] = restorer
}
return &kubernetesRestorer{
@ -171,7 +179,21 @@ func (kr *kubernetesRestorer) Restore(restore *api.Restore, backup *api.Backup,
return api.RestoreResult{}, api.RestoreResult{Ark: []string{err.Error()}}
}
prioritizedResources, err := prioritizeResources(kr.discoveryHelper.Mapper(), kr.resourcePriorities, kr.discoveryHelper.Resources())
// get resource includes-excludes
resourceIncludesExcludes := collections.GenerateIncludesExcludes(
restore.Spec.IncludedResources,
restore.Spec.ExcludedResources,
func(item string) (string, error) {
gr, err := kr.discoveryHelper.ResolveGroupResource(item)
if err != nil {
return "", err
}
return gr.String(), nil
},
)
prioritizedResources, err := prioritizeResources(kr.discoveryHelper, kr.resourcePriorities, resourceIncludesExcludes)
if err != nil {
return api.RestoreResult{}, api.RestoreResult{Ark: []string{err.Error()}}
}
@ -389,7 +411,7 @@ func (kr *kubernetesRestorer) restoreResourceForNamespace(
if restorer == nil {
// initialize client & restorer for this Resource. we need
// metadata from an object to do this.
glog.Infof("Getting client for %s", obj.GroupVersionKind().String())
glog.Infof("Getting client for %v", obj.GroupVersionKind())
resource := metav1.APIResource{
Namespaced: len(namespace) > 0,
@ -399,22 +421,22 @@ func (kr *kubernetesRestorer) restoreResourceForNamespace(
var err error
resourceClient, err = kr.dynamicFactory.ClientForGroupVersionKind(obj.GroupVersionKind(), resource, namespace)
if err != nil {
addArkError(&errors, fmt.Errorf("error getting resource client for namespace %q, resource %q: %v", namespace, groupResource.String(), err))
addArkError(&errors, fmt.Errorf("error getting resource client for namespace %q, resource %q: %v", namespace, groupResource, err))
return warnings, errors
}
restorer = kr.restorers[groupResource]
if restorer == nil {
glog.Infof("Using default restorer for %s", groupResource.String())
glog.Infof("Using default restorer for %v", groupResource)
restorer = restorers.NewBasicRestorer(true)
} else {
glog.Infof("Using custom restorer for %s", groupResource.String())
glog.Infof("Using custom restorer for %v", groupResource)
}
if restorer.Wait() {
itmWatch, err := resourceClient.Watch(metav1.ListOptions{})
if err != nil {
addArkError(&errors, fmt.Errorf("error watching for namespace %q, resource %q: %v", namespace, groupResource.String(), err))
addArkError(&errors, fmt.Errorf("error watching for namespace %q, resource %q: %v", namespace, groupResource, err))
return warnings, errors
}
watchChan := itmWatch.ResultChan()
@ -473,7 +495,7 @@ func (kr *kubernetesRestorer) restoreResourceForNamespace(
if waiter != nil {
if err := waiter.Wait(); err != nil {
addArkError(&errors, fmt.Errorf("error waiting for all %s resources to be created in namespace %s: %v", groupResource.String(), namespace, err))
addArkError(&errors, fmt.Errorf("error waiting for all %v resources to be created in namespace %s: %v", groupResource, namespace, err))
}
}

85
pkg/restore/restore_test.go
View File

@ -24,6 +24,7 @@ import (
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@ -40,39 +41,71 @@ import (
)
func TestPrioritizeResources(t *testing.T) {
mapper := &FakeMapper{AutoReturnResource: true}
priorities := []string{"namespaces", "configmaps", "pods"}
resources := []*metav1.APIResourceList{
tests := []struct {
name string
apiResources map[string][]string
priorities []string
includes []string
excludes []string
expected []string
}{
{
GroupVersion: "v1",
APIResources: []metav1.APIResource{
{Name: "aaa"},
{Name: "bbb"},
{Name: "configmaps"},
{Name: "ddd"},
{Name: "namespaces"},
{Name: "ooo"},
{Name: "pods"},
{Name: "sss"},
name: "priorities & ordering are correctly applied",
apiResources: map[string][]string{
"v1": []string{"aaa", "bbb", "configmaps", "ddd", "namespaces", "ooo", "pods", "sss"},
},
priorities: []string{"namespaces", "configmaps", "pods"},
includes: []string{"*"},
expected: []string{"namespaces", "configmaps", "pods", "aaa", "bbb", "ddd", "ooo", "sss"},
},
{
name: "includes are correctly applied",
apiResources: map[string][]string{
"v1": []string{"aaa", "bbb", "configmaps", "ddd", "namespaces", "ooo", "pods", "sss"},
},
priorities: []string{"namespaces", "configmaps", "pods"},
includes: []string{"namespaces", "aaa", "sss"},
expected: []string{"namespaces", "aaa", "sss"},
},
{
name: "excludes are correctly applied",
apiResources: map[string][]string{
"v1": []string{"aaa", "bbb", "configmaps", "ddd", "namespaces", "ooo", "pods", "sss"},
},
priorities: []string{"namespaces", "configmaps", "pods"},
includes: []string{"*"},
excludes: []string{"ooo", "pods"},
expected: []string{"namespaces", "configmaps", "aaa", "bbb", "ddd", "sss"},
},
}
result, err := prioritizeResources(mapper, priorities, resources)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
helper := &FakeDiscoveryHelper{RESTMapper: &FakeMapper{AutoReturnResource: true}}
expected := []string{"namespaces", "configmaps", "pods", "aaa", "bbb", "ddd", "ooo", "sss"}
for i := range result {
if len(expected) < i+1 {
t.Errorf("result is too small: %v", result)
break
}
for gv, resources := range test.apiResources {
resourceList := &metav1.APIResourceList{GroupVersion: gv}
for _, resource := range resources {
resourceList.APIResources = append(resourceList.APIResources, metav1.APIResource{Name: resource})
}
helper.ResourceList = append(helper.ResourceList, resourceList)
}
if e, a := expected[i], result[i].Resource; e != a {
t.Errorf("index %d, expected %s, got %s", i, e, a)
}
includesExcludes := collections.NewIncludesExcludes().Includes(test.includes...).Excludes(test.excludes...)
result, err := prioritizeResources(helper, test.priorities, includesExcludes)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
require.Equal(t, len(test.expected), len(result))
for i := range result {
if e, a := test.expected[i], result[i].Resource; e != a {
t.Errorf("index %d, expected %s, got %s", i, e, a)
}
}
})
}
}

41
pkg/util/collections/includes_excludes.go
View File

@ -20,6 +20,8 @@ import (
"errors"
"fmt"
"github.com/golang/glog"
"k8s.io/apimachinery/pkg/util/sets"
)
@ -74,6 +76,8 @@ func (ie *IncludesExcludes) ShouldInclude(s string) bool {
return ie.includes.Has("*") || ie.includes.Has(s)
}
// ValidateIncludesExcludes checks provided lists of included and excluded
// items to ensure they are a valid set of IncludesExcludes data.
func ValidateIncludesExcludes(includesList, excludesList []string) []error {
// TODO we should not allow an IncludesExcludes object to be created that
// does not meet these criteria. Do a more significant refactoring to embed
@ -98,9 +102,44 @@ func ValidateIncludesExcludes(includesList, excludesList []string) []error {
for _, itm := range excludes.List() {
if includes.Has(itm) {
errs = append(errs, errors.New(fmt.Sprintf("excludes list cannot contain an item in the includes list: %v", itm)))
errs = append(errs, fmt.Errorf("excludes list cannot contain an item in the includes list: %v", itm))
}
}
return errs
}
// GenerateIncludesExcludes constructs an IncludesExcludes struct by taking the provided
// include/exclude slices, applying the specified mapping function to each item in them,
// and adding the output of the function to the new struct. If the mapping function returns
// an error for an item, it is omitted from the result.
func GenerateIncludesExcludes(includes, excludes []string, mapFunc func(string) (string, error)) *IncludesExcludes {
res := NewIncludesExcludes()
for _, item := range includes {
if item == "*" {
res.Includes(item)
continue
}
key, err := mapFunc(item)
if err != nil {
glog.Errorf("unable to include item %q: %v", item, err)
continue
}
res.Includes(key)
}
for _, item := range excludes {
key, err := mapFunc(item)
if err != nil {
glog.Errorf("unable to exclude item %q: %v", item, err)
continue
}
res.Excludes(key)
}
return res
}

31
pkg/util/test/fake_discovery_helper.go Normal file
View File

@ -0,0 +1,31 @@
package test
import (
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
)
type FakeDiscoveryHelper struct {
ResourceList []*metav1.APIResourceList
RESTMapper meta.RESTMapper
}
func (dh *FakeDiscoveryHelper) Mapper() meta.RESTMapper {
return dh.RESTMapper
}
func (dh *FakeDiscoveryHelper) Resources() []*metav1.APIResourceList {
return dh.ResourceList
}
func (dh *FakeDiscoveryHelper) Refresh() error {
return nil
}
func (dh *FakeDiscoveryHelper) ResolveGroupResource(resource string) (schema.GroupResource, error) {
gvr, err := dh.RESTMapper.ResourceFor(schema.ParseGroupResource(resource).WithVersion(""))
if err != nil {
return schema.GroupResource{}, err
}
return gvr.GroupResource(), nil
}

10
pkg/util/test/test_restore.go
View File

@ -82,3 +82,13 @@ func (r *TestRestore) WithMappedNamespace(from string, to string) *TestRestore {
r.Spec.NamespaceMapping[from] = to
return r
}
func (r *TestRestore) WithIncludedResource(resource string) *TestRestore {
r.Spec.IncludedResources = append(r.Spec.IncludedResources, resource)
return r
}
func (r *TestRestore) WithExcludedResource(resource string) *TestRestore {
r.Spec.ExcludedResources = append(r.Spec.ExcludedResources, resource)
return r
}