vmware-tanzu
/
velero
mirror of https://github.com/vmware-tanzu/velero.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #4832 from reasonerjt/fix-rm-csi-bak-tmp 

Make the vsc created by backup sync controller deletable
pull/4849/head
Wenkai Yin(尹文开) 2022-04-20 17:19:09 +08:00 committed by GitHub
parent de17605c62 4f9e445142
commit 9e786d681b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 129 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
changelogs/unreleased/4832-reasonerjt Normal file
View File

@ -0,0 +1 @@
Make the vsc created by backup sync controller deletable

9
pkg/cmd/server/server.go
View File

@ -535,10 +535,11 @@ func (s *server) initRestic() error {
return nil
}
func (s *server) getCSISnapshotListers() (snapshotv1listers.VolumeSnapshotLister, snapshotv1listers.VolumeSnapshotContentLister) {
func (s *server) getCSISnapshotListers() (snapshotv1listers.VolumeSnapshotLister, snapshotv1listers.VolumeSnapshotContentLister, snapshotv1listers.VolumeSnapshotClassLister) {
// Make empty listers that will only be populated if CSI is properly enabled.
var vsLister snapshotv1listers.VolumeSnapshotLister
var vscLister snapshotv1listers.VolumeSnapshotContentLister
var vsClassLister snapshotv1listers.VolumeSnapshotClassLister
var err error
// If CSI is enabled, check for the CSI groups and generate the listers
@ -556,11 +557,12 @@ func (s *server) getCSISnapshotListers() (snapshotv1listers.VolumeSnapshotLister
// Access the wrapped factory directly here since we've already done the feature flag check above to know it's safe.
vsLister = s.csiSnapshotterSharedInformerFactory.factory.Snapshot().V1().VolumeSnapshots().Lister()
vscLister = s.csiSnapshotterSharedInformerFactory.factory.Snapshot().V1().VolumeSnapshotContents().Lister()
vsClassLister = s.csiSnapshotterSharedInformerFactory.factory.Snapshot().V1().VolumeSnapshotClasses().Lister()
case err != nil:
cmd.CheckError(err)
}
}
return vsLister, vscLister
return vsLister, vscLister, vsClassLister
}
func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string) error {
@ -587,7 +589,7 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string
backupStoreGetter := persistence.NewObjectBackupStoreGetter(s.credentialFileStore)
csiVSLister, csiVSCLister := s.getCSISnapshotListers()
csiVSLister, csiVSCLister, csiVSClassLister := s.getCSISnapshotListers()
backupSyncControllerRunInfo := func() controllerRunInfo {
backupSyncContoller := controller.NewBackupSyncController(
@ -645,6 +647,7 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string
s.config.formatFlag.Parse(),
csiVSLister,
csiVSCLister,
csiVSClassLister,
backupStoreGetter,
)

34
pkg/controller/backup_controller.go
View File

@ -36,8 +36,11 @@ import (
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/clock"
kerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/client-go/tools/cache"
"github.com/vmware-tanzu/velero/pkg/util/csi"
snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1"
snapshotv1listers "github.com/kubernetes-csi/external-snapshotter/client/v4/listers/volumesnapshot/v1"
@ -85,6 +88,7 @@ type backupController struct {
formatFlag logging.Format
volumeSnapshotLister snapshotv1listers.VolumeSnapshotLister
volumeSnapshotContentLister snapshotv1listers.VolumeSnapshotContentLister
volumeSnapshotClassLister snapshotv1listers.VolumeSnapshotClassLister
}
func NewBackupController(
@ -106,6 +110,7 @@ func NewBackupController(
formatFlag logging.Format,
volumeSnapshotLister snapshotv1listers.VolumeSnapshotLister,
volumeSnapshotContentLister snapshotv1listers.VolumeSnapshotContentLister,
volumesnapshotClassLister snapshotv1listers.VolumeSnapshotClassLister,
backupStoreGetter persistence.ObjectBackupStoreGetter,
) Interface {
c := &backupController{
@ -128,6 +133,7 @@ func NewBackupController(
formatFlag: formatFlag,
volumeSnapshotLister: volumeSnapshotLister,
volumeSnapshotContentLister: volumeSnapshotContentLister,
volumeSnapshotClassLister: volumesnapshotClassLister,
backupStoreGetter: backupStoreGetter,
}
@ -604,9 +610,9 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error {
// This way, we only make the Lister call if the feature flag's on.
var volumeSnapshots []*snapshotv1api.VolumeSnapshot
var volumeSnapshotContents []*snapshotv1api.VolumeSnapshotContent
var volumeSnapshotClasses []*snapshotv1api.VolumeSnapshotClass
if features.IsEnabled(velerov1api.CSIFeatureFlag) {
selector := label.NewSelectorForBackup(backup.Name)
// Listers are wrapped in a nil check out of caution, since they may not be populated based on the
// EnableCSI feature flag. This is more to guard against programmer error, as they shouldn't be nil
// when EnableCSI is on.
@ -623,6 +629,23 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error {
backupLog.Error(err)
}
}
vsClassSet := sets.NewString()
for _, vsc := range volumeSnapshotContents {
// persist the volumesnapshotclasses referenced by vsc
if c.volumeSnapshotClassLister != nil &&
vsc.Spec.VolumeSnapshotClassName != nil &&
!vsClassSet.Has(*vsc.Spec.VolumeSnapshotClassName) {
if vsClass, err := c.volumeSnapshotClassLister.Get(*vsc.Spec.VolumeSnapshotClassName); err != nil {
backupLog.Error(err)
} else {
vsClassSet.Insert(*vsc.Spec.VolumeSnapshotClassName)
volumeSnapshotClasses = append(volumeSnapshotClasses, vsClass)
}
}
if err := csi.ResetVolumeSnapshotContent(vsc); err != nil {
backupLog.Error(err)
}
}
}
// Mark completion timestamp before serializing and uploading.
@ -673,7 +696,7 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error {
return err
}
if errs := persistBackup(backup, backupFile, logFile, backupStore, c.logger.WithField(Backup, kubeutil.NamespaceAndName(backup)), volumeSnapshots, volumeSnapshotContents); len(errs) > 0 {
if errs := persistBackup(backup, backupFile, logFile, backupStore, c.logger.WithField(Backup, kubeutil.NamespaceAndName(backup)), volumeSnapshots, volumeSnapshotContents, volumeSnapshotClasses); len(errs) > 0 {
fatalErrs = append(fatalErrs, errs...)
}
@ -720,6 +743,7 @@ func persistBackup(backup *pkgbackup.Request,
log logrus.FieldLogger,
csiVolumeSnapshots []*snapshotv1api.VolumeSnapshot,
csiVolumeSnapshotContents []*snapshotv1api.VolumeSnapshotContent,
csiVolumesnapshotClasses []*snapshotv1api.VolumeSnapshotClass,
) []error {
persistErrs := []error{}
backupJSON := new(bytes.Buffer)
@ -748,6 +772,10 @@ func persistBackup(backup *pkgbackup.Request,
if errs != nil {
persistErrs = append(persistErrs, errs...)
}
csiSnapshotClassesJSON, errs := encodeToJSONGzip(csiVolumesnapshotClasses, "csi volume snapshot classes list")
if errs != nil {
persistErrs = append(persistErrs, errs...)
}
backupResourceList, errs := encodeToJSONGzip(backup.BackupResourceList(), "backup resources list")
if errs != nil {
@ -762,6 +790,7 @@ func persistBackup(backup *pkgbackup.Request,
backupResourceList = nil
csiSnapshotJSON = nil
csiSnapshotContentsJSON = nil
csiSnapshotClassesJSON = nil
}
backupInfo := persistence.BackupInfo{
@ -774,6 +803,7 @@ func persistBackup(backup *pkgbackup.Request,
BackupResourceList: backupResourceList,
CSIVolumeSnapshots: csiSnapshotJSON,
CSIVolumeSnapshotContents: csiSnapshotContentsJSON,
CSIVolumeSnapshotClasses: csiSnapshotClassesJSON,
}
if err := backupStore.PutBackup(backupInfo); err != nil {
persistErrs = append(persistErrs, err)

16
pkg/controller/backup_sync_controller.go
View File

@ -283,6 +283,22 @@ func (c *backupSyncController) run() {
if features.IsEnabled(velerov1api.CSIFeatureFlag) {
// we are syncing these objects only to ensure that the storage snapshots are cleaned up
// on backup deletion or expiry.
log.Info("Syncing CSI volumesnapshotclasses in backup")
vsClasses, err := backupStore.GetCSIVolumeSnapshotClasses(backupName)
if err != nil {
log.WithError(errors.WithStack(err)).Error("Error getting CSI volumesnapclasses for this backup from backup store")
continue
}
for _, vsClass := range vsClasses {
vsClass.ResourceVersion = ""
created, err := c.csiSnapshotClient.SnapshotV1().VolumeSnapshotClasses().Create(context.TODO(), vsClass, metav1.CreateOptions{})
if err != nil {
log.WithError(errors.WithStack(err)).Errorf("Error syncing volumesnapshotclass %s into cluster", vsClass.Name)
continue
}
log.Infof("Created CSI volumesnapshotclass %s", created.Name)
}
log.Info("Syncing CSI volumesnapshotcontents in backup")
snapConts, err := backupStore.GetCSIVolumeSnapshotContents(backupName)
if err != nil {

5
pkg/persistence/mocks/backup_store.go
View File

@ -284,6 +284,11 @@ func (_m *BackupStore) GetCSIVolumeSnapshotContents(backup string) ([]*snapshotv
return nil, nil
}
func (_m *BackupStore) GetCSIVolumeSnapshotClasses(backup string) ([]*snapshotv1api.VolumeSnapshotClass, error) {
panic("Not implemented")
return nil, nil
}
func (_m *BackupStore) GetItemSnapshots(name string) ([]*volume.ItemSnapshot, error) {
panic("implement me")
}

24
pkg/persistence/object_store.go
View File

@ -47,7 +47,8 @@ type BackupInfo struct {
ItemSnapshots,
BackupResourceList,
CSIVolumeSnapshots,
CSIVolumeSnapshotContents io.Reader
CSIVolumeSnapshotContents,
CSIVolumeSnapshotClasses io.Reader
}
// BackupStore defines operations for creating, retrieving, and deleting
@ -65,6 +66,7 @@ type BackupStore interface {
GetBackupContents(name string) (io.ReadCloser, error)
GetCSIVolumeSnapshots(name string) ([]*snapshotv1api.VolumeSnapshot, error)
GetCSIVolumeSnapshotContents(name string) ([]*snapshotv1api.VolumeSnapshotContent, error)
GetCSIVolumeSnapshotClasses(name string) ([]*snapshotv1api.VolumeSnapshotClass, error)
// BackupExists checks if the backup metadata file exists in object storage.
BackupExists(bucket, backupName string) (bool, error)
@ -252,6 +254,7 @@ func (s *objectBackupStore) PutBackup(info BackupInfo) error {
s.layout.getBackupResourceListKey(info.Name): info.BackupResourceList,
s.layout.getCSIVolumeSnapshotKey(info.Name): info.CSIVolumeSnapshots,
s.layout.getCSIVolumeSnapshotContentsKey(info.Name): info.CSIVolumeSnapshotContents,
s.layout.getCSIVolumeSnapshotClassesKey(info.Name): info.CSIVolumeSnapshotClasses,
}
for key, reader := range backupObjs {
@ -371,6 +374,25 @@ func decode(jsongzReader io.Reader, into interface{}) error {
return nil
}
func (s *objectBackupStore) GetCSIVolumeSnapshotClasses(name string) ([]*snapshotv1api.VolumeSnapshotClass, error) {
res, err := tryGet(s.objectStore, s.bucket, s.layout.getCSIVolumeSnapshotClassesKey(name))
if err != nil {
return nil, err
}
if res == nil {
// this indicates that the no CSI volumesnapshots were prensent in the backup
return nil, nil
}
defer res.Close()
var csiVSClasses []*snapshotv1api.VolumeSnapshotClass
if err := decode(res, &csiVSClasses); err != nil {
return nil, err
}
return csiVSClasses, nil
}
func (s *objectBackupStore) GetCSIVolumeSnapshots(name string) ([]*snapshotv1api.VolumeSnapshot, error) {
res, err := tryGet(s.objectStore, s.bucket, s.layout.getCSIVolumeSnapshotKey(name))
if err != nil {

4
pkg/persistence/object_store_layout.go
View File

@ -111,3 +111,7 @@ func (l *ObjectStoreLayout) getCSIVolumeSnapshotKey(backup string) string {
func (l *ObjectStoreLayout) getCSIVolumeSnapshotContentsKey(backup string) string {
return path.Join(l.subdirs["backups"], backup, fmt.Sprintf("%s-csi-volumesnapshotcontents.json.gz", backup))
}
func (l *ObjectStoreLayout) getCSIVolumeSnapshotClassesKey(backup string) string {
return path.Join(l.subdirs["backups"], backup, fmt.Sprintf("%s-csi-volumesnapshotclasses.json.gz", backup))
}

42
pkg/util/csi/reset.go Normal file
View File

@ -0,0 +1,42 @@
/*
Copyright The Velero Contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package csi
import (
"fmt"
snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1"
corev1 "k8s.io/api/core/v1"
)
// ResetVolumeSnapshotContent make changes to the volumesnapshot content before it's persisted
// It will move the snapshot Handle to the source to avoid the snapshot-controller creating a snapshot when it's
// synced by the backup sync controller.
// It will return an error if the snapshot handle is not set, which should not happen when this func is called.
func ResetVolumeSnapshotContent(snapCont *snapshotv1api.VolumeSnapshotContent) error {
if snapCont.Status != nil && snapCont.Status.SnapshotHandle != nil && len(*snapCont.Status.SnapshotHandle) > 0 {
v := *snapCont.Status.SnapshotHandle
snapCont.Spec.Source = snapshotv1api.VolumeSnapshotContentSource{
SnapshotHandle: &v,
}
} else {
return fmt.Errorf("the volumesnapshotcontent '%s' does not have snapshothandle set", snapCont.Name)
}
snapCont.Spec.VolumeSnapshotRef = corev1.ObjectReference{}
return nil
}