store backup resource list metadata in object storage (#1709)

* move backedUpItems to pkg/backup.Request struct

Signed-off-by: Adnan Abdulhussein <aadnan@vmware.com>

* construct resource itemKey field from gvk

Signed-off-by: Adnan Abdulhussein <aadnan@vmware.com>

* store backup resource list metadata in object storage

Signed-off-by: Adnan Abdulhussein <aadnan@vmware.com>

* remove debug log

Signed-off-by: Adnan Abdulhussein <aadnan@vmware.com>

* fix formatting

Signed-off-by: Adnan Abdulhussein <aadnan@vmware.com>

* add missing license blocks and split BackupInfo struct lines

Signed-off-by: Adnan Abdulhussein <aadnan@vmware.com>

* add test for checking BackedUpItems matches tarball contents

Signed-off-by: Adnan Abdulhussein <aadnan@vmware.com>

* add comment to explain test

Signed-off-by: Adnan Abdulhussein <aadnan@vmware.com>
pull/1727/head
Adnan Abdulhussein 2019-08-05 10:15:55 -07:00 committed by Steve Kriss
parent 635dd27e1a
commit 07525bd593
13 changed files with 323 additions and 42 deletions

View File

@ -60,12 +60,6 @@ type kubernetesBackupper struct {
resticTimeout time.Duration
}
type itemKey struct {
resource string
namespace string
name string
}
type resolvedAction struct {
velero.BackupItemAction
@ -240,6 +234,8 @@ func (kb *kubernetesBackupper) Backup(log logrus.FieldLogger, backupRequest *Req
return err
}
backupRequest.BackedUpItems = map[itemKey]struct{}{}
podVolumeTimeout := kb.resticTimeout
if val := backupRequest.Annotations[api.PodVolumeOperationTimeoutAnnotation]; val != "" {
parsed, err := time.ParseDuration(val)
@ -266,7 +262,6 @@ func (kb *kubernetesBackupper) Backup(log logrus.FieldLogger, backupRequest *Req
backupRequest,
kb.dynamicFactory,
kb.discoveryHelper,
make(map[itemKey]struct{}),
cohabitatingResources(),
kb.podCommandExecutor,
tw,

View File

@ -53,6 +53,56 @@ import (
"github.com/heptio/velero/pkg/volume"
)
func TestBackedUpItemsMatchesTarballContents(t *testing.T) {
// TODO: figure out if this can be replaced with the restmapper
// (https://github.com/kubernetes/apimachinery/blob/035e418f1ad9b6da47c4e01906a0cfe32f4ee2e7/pkg/api/meta/restmapper.go)
gvkToResource := map[string]string{
"v1/Pod": "pods",
"apps/v1/Deployment": "deployments.apps",
"v1/PersistentVolume": "persistentvolumes",
}
h := newHarness(t)
req := &Request{Backup: defaultBackup().Result()}
backupFile := bytes.NewBuffer([]byte{})
apiResources := []*test.APIResource{
test.Pods(
builder.ForPod("foo", "bar").Result(),
builder.ForPod("zoo", "raz").Result(),
),
test.Deployments(
builder.ForDeployment("foo", "bar").Result(),
builder.ForDeployment("zoo", "raz").Result(),
),
test.PVs(
builder.ForPersistentVolume("bar").Result(),
builder.ForPersistentVolume("baz").Result(),
),
}
for _, resource := range apiResources {
h.addItems(t, resource)
}
h.backupper.Backup(h.log, req, backupFile, nil, nil)
// go through BackedUpItems after the backup to assemble the list of files we
// expect to see in the tarball and compare to see if they match
var expectedFiles []string
for item := range req.BackedUpItems {
file := "resources/" + gvkToResource[item.resource]
if item.namespace != "" {
file = file + "/namespaces/" + item.namespace
} else {
file = file + "/cluster"
}
file = file + "/" + item.name + ".json"
expectedFiles = append(expectedFiles, file)
}
assertTarballContents(t, backupFile, append(expectedFiles, "metadata/version")...)
}
// TestBackupResourceFiltering runs backups with different combinations
// of resource filters (included/excluded resources, included/excluded
// namespaces, label selectors, "include cluster resources" flag), and

View File

@ -37,7 +37,6 @@ type groupBackupperFactory interface {
backupRequest *Request,
dynamicFactory client.DynamicFactory,
discoveryHelper discovery.Helper,
backedUpItems map[itemKey]struct{},
cohabitatingResources map[string]*cohabitatingResource,
podCommandExecutor podexec.PodCommandExecutor,
tarWriter tarWriter,
@ -54,7 +53,6 @@ func (f *defaultGroupBackupperFactory) newGroupBackupper(
backupRequest *Request,
dynamicFactory client.DynamicFactory,
discoveryHelper discovery.Helper,
backedUpItems map[itemKey]struct{},
cohabitatingResources map[string]*cohabitatingResource,
podCommandExecutor podexec.PodCommandExecutor,
tarWriter tarWriter,
@ -67,7 +65,6 @@ func (f *defaultGroupBackupperFactory) newGroupBackupper(
backupRequest: backupRequest,
dynamicFactory: dynamicFactory,
discoveryHelper: discoveryHelper,
backedUpItems: backedUpItems,
cohabitatingResources: cohabitatingResources,
podCommandExecutor: podCommandExecutor,
tarWriter: tarWriter,
@ -88,7 +85,6 @@ type defaultGroupBackupper struct {
backupRequest *Request
dynamicFactory client.DynamicFactory
discoveryHelper discovery.Helper
backedUpItems map[itemKey]struct{}
cohabitatingResources map[string]*cohabitatingResource
podCommandExecutor podexec.PodCommandExecutor
tarWriter tarWriter
@ -120,7 +116,6 @@ func (gb *defaultGroupBackupper) backupGroup(group *metav1.APIResourceList) erro
gb.backupRequest,
gb.dynamicFactory,
gb.discoveryHelper,
gb.backedUpItems,
gb.cohabitatingResources,
gb.podCommandExecutor,
gb.tarWriter,

View File

@ -19,6 +19,7 @@ package backup
import (
"archive/tar"
"encoding/json"
"fmt"
"path/filepath"
"time"
@ -46,7 +47,6 @@ import (
type itemBackupperFactory interface {
newItemBackupper(
backup *Request,
backedUpItems map[itemKey]struct{},
podCommandExecutor podexec.PodCommandExecutor,
tarWriter tarWriter,
dynamicFactory client.DynamicFactory,
@ -61,7 +61,6 @@ type defaultItemBackupperFactory struct{}
func (f *defaultItemBackupperFactory) newItemBackupper(
backupRequest *Request,
backedUpItems map[itemKey]struct{},
podCommandExecutor podexec.PodCommandExecutor,
tarWriter tarWriter,
dynamicFactory client.DynamicFactory,
@ -72,7 +71,6 @@ func (f *defaultItemBackupperFactory) newItemBackupper(
) ItemBackupper {
ib := &defaultItemBackupper{
backupRequest: backupRequest,
backedUpItems: backedUpItems,
tarWriter: tarWriter,
dynamicFactory: dynamicFactory,
discoveryHelper: discoveryHelper,
@ -97,7 +95,6 @@ type ItemBackupper interface {
type defaultItemBackupper struct {
backupRequest *Request
backedUpItems map[itemKey]struct{}
tarWriter tarWriter
dynamicFactory client.DynamicFactory
discoveryHelper discovery.Helper
@ -149,19 +146,18 @@ func (ib *defaultItemBackupper) backupItem(logger logrus.FieldLogger, obj runtim
log.Info("Skipping item because it's being deleted.")
return nil
}
key := itemKey{
resource: groupResource.String(),
resource: resourceKey(obj),
namespace: namespace,
name: name,
}
if _, exists := ib.backedUpItems[key]; exists {
if _, exists := ib.backupRequest.BackedUpItems[key]; exists {
log.Info("Skipping item because it's already been backed up.")
return nil
}
ib.backedUpItems[key] = struct{}{}
log.Debug(obj.GetObjectKind().GroupVersionKind().GroupVersion().String() + "/" + obj.GetObjectKind().GroupVersionKind().Kind)
ib.backupRequest.BackedUpItems[key] = struct{}{}
log.Info("Backing up item")
@ -485,3 +481,10 @@ func volumeSnapshot(backup *api.Backup, volumeName, volumeID, volumeType, az, lo
},
}
}
// resourceKey returns a string representing the object's GroupVersionKind (e.g.
// apps/v1/Deployment).
func resourceKey(obj runtime.Unstructured) string {
gvk := obj.GetObjectKind().GroupVersionKind()
return fmt.Sprintf("%s/%s", gvk.GroupVersion().String(), gvk.Kind)
}

View File

@ -0,0 +1,47 @@
/*
Copyright 2019 the Velero contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package backup
import (
"testing"
"github.com/stretchr/testify/assert"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"github.com/heptio/velero/pkg/builder"
)
func Test_resourceKey(t *testing.T) {
tests := []struct {
resource metav1.Object
want string
}{
{resource: builder.ForPod("default", "test").Result(), want: "v1/Pod"},
{resource: builder.ForDeployment("default", "test").Result(), want: "apps/v1/Deployment"},
{resource: builder.ForPersistentVolume("test").Result(), want: "v1/PersistentVolume"},
{resource: builder.ForRole("default", "test").Result(), want: "rbac.authorization.k8s.io/v1/Role"},
}
for _, tt := range tests {
t.Run(tt.want, func(t *testing.T) {
content, _ := runtime.DefaultUnstructuredConverter.ToUnstructured(tt.resource)
unstructured := &unstructured.Unstructured{Object: content}
assert.Equal(t, tt.want, resourceKey(unstructured))
})
}
}

View File

@ -1,11 +1,35 @@
/*
Copyright 2019 the Velero contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package backup
import (
"fmt"
velerov1api "github.com/heptio/velero/pkg/apis/velero/v1"
"github.com/heptio/velero/pkg/util/collections"
"github.com/heptio/velero/pkg/volume"
)
type itemKey struct {
resource string
namespace string
name string
}
// Request is a request for a backup, with all references to other objects
// materialized (e.g. backup/snapshot locations, includes/excludes, etc.)
type Request struct {
@ -20,4 +44,19 @@ type Request struct {
VolumeSnapshots []*volume.Snapshot
PodVolumeBackups []*velerov1api.PodVolumeBackup
BackedUpItems map[itemKey]struct{}
}
// BackupResourceList returns the list of backed up resources grouped by the API
// Version and Kind
func (r *Request) BackupResourceList() map[string][]string {
resources := map[string][]string{}
for i := range r.BackedUpItems {
entry := i.name
if i.namespace != "" {
entry = fmt.Sprintf("%s/%s", i.namespace, i.name)
}
resources[i.resource] = append(resources[i.resource], entry)
}
return resources
}

View File

@ -0,0 +1,58 @@
/*
Copyright 2019 the Velero contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package backup
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestRequest_BackupResourceList(t *testing.T) {
items := []itemKey{
{
resource: "apps/v1/Deployment",
name: "my-deploy",
namespace: "default",
},
{
resource: "v1/Pod",
name: "pod1",
namespace: "ns1",
},
{
resource: "v1/Pod",
name: "pod2",
namespace: "ns2",
},
{
resource: "v1/PersistentVolume",
name: "my-pv",
},
}
backedUpItems := map[itemKey]struct{}{}
for _, it := range items {
backedUpItems[it] = struct{}{}
}
req := Request{BackedUpItems: backedUpItems}
assert.Equal(t, req.BackupResourceList(), map[string][]string{
"apps/v1/Deployment": {"default/my-deploy"},
"v1/Pod": {"ns1/pod1", "ns2/pod2"},
"v1/PersistentVolume": {"my-pv"},
})
}

View File

@ -40,7 +40,6 @@ type resourceBackupperFactory interface {
backupRequest *Request,
dynamicFactory client.DynamicFactory,
discoveryHelper discovery.Helper,
backedUpItems map[itemKey]struct{},
cohabitatingResources map[string]*cohabitatingResource,
podCommandExecutor podexec.PodCommandExecutor,
tarWriter tarWriter,
@ -57,7 +56,6 @@ func (f *defaultResourceBackupperFactory) newResourceBackupper(
backupRequest *Request,
dynamicFactory client.DynamicFactory,
discoveryHelper discovery.Helper,
backedUpItems map[itemKey]struct{},
cohabitatingResources map[string]*cohabitatingResource,
podCommandExecutor podexec.PodCommandExecutor,
tarWriter tarWriter,
@ -70,7 +68,6 @@ func (f *defaultResourceBackupperFactory) newResourceBackupper(
backupRequest: backupRequest,
dynamicFactory: dynamicFactory,
discoveryHelper: discoveryHelper,
backedUpItems: backedUpItems,
cohabitatingResources: cohabitatingResources,
podCommandExecutor: podCommandExecutor,
tarWriter: tarWriter,
@ -91,7 +88,6 @@ type defaultResourceBackupper struct {
backupRequest *Request
dynamicFactory client.DynamicFactory
discoveryHelper discovery.Helper
backedUpItems map[itemKey]struct{}
cohabitatingResources map[string]*cohabitatingResource
podCommandExecutor podexec.PodCommandExecutor
tarWriter tarWriter
@ -156,7 +152,6 @@ func (rb *defaultResourceBackupper) backupResource(group *metav1.APIResourceList
itemBackupper := rb.itemBackupperFactory.newItemBackupper(
rb.backupRequest,
rb.backedUpItems,
rb.podCommandExecutor,
rb.tarWriter,
rb.dynamicFactory,

View File

@ -0,0 +1,57 @@
/*
Copyright 2019 the Velero contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package builder
import (
rbacv1api "k8s.io/api/rbac/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// RoleBuilder builds Role objects.
type RoleBuilder struct {
object *rbacv1api.Role
}
// ForRole is the constructor for a RoleBuilder.
func ForRole(ns, name string) *RoleBuilder {
return &RoleBuilder{
object: &rbacv1api.Role{
TypeMeta: metav1.TypeMeta{
APIVersion: rbacv1api.SchemeGroupVersion.String(),
Kind: "Role",
},
ObjectMeta: metav1.ObjectMeta{
Namespace: ns,
Name: name,
},
},
}
}
// Result returns the built Role.
func (b *RoleBuilder) Result() *rbacv1api.Role {
return b.object
}
// ObjectMeta applies functional options to the Role's ObjectMeta.
func (b *RoleBuilder) ObjectMeta(opts ...ObjectMetaOpt) *RoleBuilder {
for _, opt := range opts {
opt(b.object)
}
return b
}

View File

@ -570,7 +570,6 @@ func persistBackup(backup *pkgbackup.Request, backupContents, backupLog *os.File
volumeSnapshots := new(bytes.Buffer)
gzw := gzip.NewWriter(volumeSnapshots)
defer gzw.Close()
if err := json.NewEncoder(gzw).Encode(backup.VolumeSnapshots); err != nil {
errs = append(errs, errors.Wrap(err, "error encoding list of volume snapshots"))
@ -581,7 +580,6 @@ func persistBackup(backup *pkgbackup.Request, backupContents, backupLog *os.File
podVolumeBackups := new(bytes.Buffer)
gzw = gzip.NewWriter(podVolumeBackups)
defer gzw.Close()
if err := json.NewEncoder(gzw).Encode(backup.PodVolumeBackups); err != nil {
errs = append(errs, errors.Wrap(err, "error encoding pod volume backups"))
@ -590,11 +588,22 @@ func persistBackup(backup *pkgbackup.Request, backupContents, backupLog *os.File
errs = append(errs, errors.Wrap(err, "error closing gzip writer"))
}
backupResourceList := new(bytes.Buffer)
gzw = gzip.NewWriter(backupResourceList)
if err := json.NewEncoder(gzw).Encode(backup.BackupResourceList()); err != nil {
errs = append(errs, errors.Wrap(err, "error encoding backup resource list"))
}
if err := gzw.Close(); err != nil {
errs = append(errs, errors.Wrap(err, "error closing gzip writer"))
}
if len(errs) > 0 {
// Don't upload the JSON files or backup tarball if encoding to json fails.
backupJSON = nil
backupContents = nil
volumeSnapshots = nil
backupResourceList = nil
}
backupInfo := persistence.BackupInfo{
@ -604,6 +613,7 @@ func persistBackup(backup *pkgbackup.Request, backupContents, backupLog *os.File
Log: backupLog,
PodVolumeBackups: podVolumeBackups,
VolumeSnapshots: volumeSnapshots,
BackupResourceList: backupResourceList,
}
if err := backupStore.PutBackup(backupInfo); err != nil {
errs = append(errs, err)

View File

@ -37,7 +37,12 @@ import (
type BackupInfo struct {
Name string
Metadata, Contents, Log, PodVolumeBackups, VolumeSnapshots io.Reader
Metadata,
Contents,
Log,
PodVolumeBackups,
VolumeSnapshots,
BackupResourceList io.Reader
}
// BackupStore defines operations for creating, retrieving, and deleting
@ -231,6 +236,18 @@ func (s *objectBackupStore) PutBackup(info BackupInfo) error {
return kerrors.NewAggregate(errs)
}
if err := seekAndPutObject(s.objectStore, s.bucket, s.layout.getBackupResourceListKey(info.Name), info.BackupResourceList); err != nil {
errs := []error{err}
deleteErr := s.objectStore.DeleteObject(s.bucket, s.layout.getBackupContentsKey(info.Name))
errs = append(errs, deleteErr)
deleteErr = s.objectStore.DeleteObject(s.bucket, s.layout.getBackupMetadataKey(info.Name))
errs = append(errs, deleteErr)
return kerrors.NewAggregate(errs)
}
if err := s.putRevision(); err != nil {
s.logger.WithField("backup", info.Name).WithError(err).Warn("Error updating backup store revision")
}

View File

@ -91,6 +91,10 @@ func (l *ObjectStoreLayout) getBackupVolumeSnapshotsKey(backup string) string {
return path.Join(l.subdirs["backups"], backup, fmt.Sprintf("%s-volumesnapshots.json.gz", backup))
}
func (l *ObjectStoreLayout) getBackupResourceListKey(backup string) string {
return path.Join(l.subdirs["backups"], backup, fmt.Sprintf("%s-resource-list.json.gz", backup))
}
func (l *ObjectStoreLayout) getRestoreLogKey(restore string) string {
return path.Join(l.subdirs["restores"], restore, fmt.Sprintf("restore-%s-logs.gz", restore))
}

View File

@ -217,6 +217,7 @@ func TestPutBackup(t *testing.T) {
log io.Reader
podVolumeBackup io.Reader
snapshots io.Reader
resourceList io.Reader
expectedErr string
expectedKeys []string
}{
@ -227,6 +228,7 @@ func TestPutBackup(t *testing.T) {
log: newStringReadSeeker("log"),
podVolumeBackup: newStringReadSeeker("podVolumeBackup"),
snapshots: newStringReadSeeker("snapshots"),
resourceList: newStringReadSeeker("resourceList"),
expectedErr: "",
expectedKeys: []string{
"backups/backup-1/velero-backup.json",
@ -234,6 +236,7 @@ func TestPutBackup(t *testing.T) {
"backups/backup-1/backup-1-logs.gz",
"backups/backup-1/backup-1-podvolumebackups.json.gz",
"backups/backup-1/backup-1-volumesnapshots.json.gz",
"backups/backup-1/backup-1-resource-list.json.gz",
"metadata/revision",
},
},
@ -245,6 +248,7 @@ func TestPutBackup(t *testing.T) {
log: newStringReadSeeker("log"),
podVolumeBackup: newStringReadSeeker("podVolumeBackup"),
snapshots: newStringReadSeeker("snapshots"),
resourceList: newStringReadSeeker("resourceList"),
expectedErr: "",
expectedKeys: []string{
"prefix-1/backups/backup-1/velero-backup.json",
@ -252,6 +256,7 @@ func TestPutBackup(t *testing.T) {
"prefix-1/backups/backup-1/backup-1-logs.gz",
"prefix-1/backups/backup-1/backup-1-podvolumebackups.json.gz",
"prefix-1/backups/backup-1/backup-1-volumesnapshots.json.gz",
"prefix-1/backups/backup-1/backup-1-resource-list.json.gz",
"prefix-1/metadata/revision",
},
},
@ -262,6 +267,7 @@ func TestPutBackup(t *testing.T) {
log: newStringReadSeeker("log"),
podVolumeBackup: newStringReadSeeker("podVolumeBackup"),
snapshots: newStringReadSeeker("snapshots"),
resourceList: newStringReadSeeker("resourceList"),
expectedErr: "error readers return errors",
expectedKeys: []string{"backups/backup-1/backup-1-logs.gz"},
},
@ -271,6 +277,7 @@ func TestPutBackup(t *testing.T) {
contents: new(errorReader),
log: newStringReadSeeker("log"),
snapshots: newStringReadSeeker("snapshots"),
resourceList: newStringReadSeeker("resourceList"),
expectedErr: "error readers return errors",
expectedKeys: []string{"backups/backup-1/backup-1-logs.gz"},
},
@ -281,12 +288,14 @@ func TestPutBackup(t *testing.T) {
log: new(errorReader),
podVolumeBackup: newStringReadSeeker("podVolumeBackup"),
snapshots: newStringReadSeeker("snapshots"),
resourceList: newStringReadSeeker("resourceList"),
expectedErr: "",
expectedKeys: []string{
"backups/backup-1/velero-backup.json",
"backups/backup-1/backup-1.tar.gz",
"backups/backup-1/backup-1-podvolumebackups.json.gz",
"backups/backup-1/backup-1-volumesnapshots.json.gz",
"backups/backup-1/backup-1-resource-list.json.gz",
"metadata/revision",
},
},
@ -297,6 +306,7 @@ func TestPutBackup(t *testing.T) {
log: newStringReadSeeker("log"),
podVolumeBackup: newStringReadSeeker("podVolumeBackup"),
snapshots: newStringReadSeeker("snapshots"),
resourceList: newStringReadSeeker("resourceList"),
expectedErr: "",
expectedKeys: []string{"backups/backup-1/backup-1-logs.gz"},
},
@ -313,6 +323,7 @@ func TestPutBackup(t *testing.T) {
Log: tc.log,
PodVolumeBackups: tc.podVolumeBackup,
VolumeSnapshots: tc.snapshots,
BackupResourceList: tc.resourceList,
}
err := harness.PutBackup(backupInfo)