diff --git a/changelogs/unreleased/8976-blackpiglet b/changelogs/unreleased/8976-blackpiglet new file mode 100644 index 000000000..c5aab6a51 --- /dev/null +++ b/changelogs/unreleased/8976-blackpiglet @@ -0,0 +1 @@ +Add BSL status check for backup/restore operations. diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index def8aae8c..e2b624b0a 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -56,6 +56,7 @@ import ( kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" "github.com/vmware-tanzu/velero/pkg/util/logging" "github.com/vmware-tanzu/velero/pkg/util/results" + veleroutil "github.com/vmware-tanzu/velero/pkg/util/velero" ) const ( @@ -424,6 +425,13 @@ func (b *backupReconciler) prepareBackupRequest(backup *velerov1api.Backup, logg request.Status.ValidationErrors = append(request.Status.ValidationErrors, fmt.Sprintf("backup can't be created because backup storage location %s is currently in read-only mode", request.StorageLocation.Name)) } + + if !veleroutil.BSLIsAvailable(*request.StorageLocation) { + request.Status.ValidationErrors = append( + request.Status.ValidationErrors, + fmt.Sprintf("backup can't be created because BackupStorageLocation %s is in Unavailable status.", request.StorageLocation.Name), + ) + } } // add the storage location as a label for easy filtering later. diff --git a/pkg/controller/backup_controller_test.go b/pkg/controller/backup_controller_test.go index a70bde8ec..b5ee3dbf1 100644 --- a/pkg/controller/backup_controller_test.go +++ b/pkg/controller/backup_controller_test.go @@ -155,7 +155,7 @@ func TestProcessBackupNonProcessedItems(t *testing.T) { } func TestProcessBackupValidationFailures(t *testing.T) { - defaultBackupLocation := builder.ForBackupStorageLocation("velero", "loc-1").Result() + defaultBackupLocation := builder.ForBackupStorageLocation("velero", "loc-1").Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result() tests := []struct { name string @@ -183,7 +183,7 @@ func TestProcessBackupValidationFailures(t *testing.T) { { name: "backup for read-only backup location fails validation", backup: defaultBackup().StorageLocation("read-only").Result(), - backupLocation: builder.ForBackupStorageLocation("velero", "read-only").AccessMode(velerov1api.BackupStorageLocationAccessModeReadOnly).Result(), + backupLocation: builder.ForBackupStorageLocation("velero", "read-only").AccessMode(velerov1api.BackupStorageLocationAccessModeReadOnly).Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result(), expectedErrs: []string{"backup can't be created because backup storage location read-only is currently in read-only mode"}, }, { @@ -203,6 +203,12 @@ func TestProcessBackupValidationFailures(t *testing.T) { backupLocation: defaultBackupLocation, expectedErrs: []string{"include-resources, exclude-resources and include-cluster-resources are old filter parameters.\ninclude-cluster-scoped-resources, exclude-cluster-scoped-resources, include-namespace-scoped-resources and exclude-namespace-scoped-resources are new filter parameters.\nThey cannot be used together"}, }, + { + name: "BSL in unavailable state", + backup: defaultBackup().StorageLocation("unavailable").Result(), + backupLocation: builder.ForBackupStorageLocation("velero", "unavailable").Phase(velerov1api.BackupStorageLocationPhaseUnavailable).Result(), + expectedErrs: []string{"backup can't be created because BackupStorageLocation unavailable is in Unavailable status."}, + }, } for _, test := range tests { @@ -655,7 +661,7 @@ func TestDefaultVolumesToResticDeprecation(t *testing.T) { } func TestProcessBackupCompletions(t *testing.T) { - defaultBackupLocation := builder.ForBackupStorageLocation("velero", "loc-1").Default(true).Bucket("store-1").Result() + defaultBackupLocation := builder.ForBackupStorageLocation("velero", "loc-1").Default(true).Bucket("store-1").Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result() now, err := time.Parse(time.RFC1123Z, time.RFC1123Z) require.NoError(t, err) @@ -715,7 +721,7 @@ func TestProcessBackupCompletions(t *testing.T) { { name: "backup with a specific backup location keeps it", backup: defaultBackup().StorageLocation("alt-loc").Result(), - backupLocation: builder.ForBackupStorageLocation("velero", "alt-loc").Bucket("store-1").Result(), + backupLocation: builder.ForBackupStorageLocation("velero", "alt-loc").Bucket("store-1").Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result(), defaultVolumesToFsBackup: false, expectedResult: &velerov1api.Backup{ TypeMeta: metav1.TypeMeta{ @@ -755,6 +761,7 @@ func TestProcessBackupCompletions(t *testing.T) { backupLocation: builder.ForBackupStorageLocation("velero", "read-write"). Bucket("store-1"). AccessMode(velerov1api.BackupStorageLocationAccessModeReadWrite). + Phase(velerov1api.BackupStorageLocationPhaseAvailable). Result(), defaultVolumesToFsBackup: true, expectedResult: &velerov1api.Backup{ @@ -1477,11 +1484,13 @@ func TestProcessBackupCompletions(t *testing.T) { } func TestValidateAndGetSnapshotLocations(t *testing.T) { + defaultBSL := builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "bsl").Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result() tests := []struct { name string backup *velerov1api.Backup locations []*velerov1api.VolumeSnapshotLocation defaultLocations map[string]string + bsl velerov1api.BackupStorageLocation expectedVolumeSnapshotLocationNames []string // adding these in the expected order will allow to test with better msgs in case of a test failure expectedErrors string expectedSuccess bool @@ -1495,6 +1504,7 @@ func TestValidateAndGetSnapshotLocations(t *testing.T) { builder.ForVolumeSnapshotLocation(velerov1api.DefaultNamespace, "some-name").Provider("fake-provider").Result(), }, expectedErrors: "a VolumeSnapshotLocation CRD for the location random-name with the name specified in the backup spec needs to be created before this snapshot can be executed. Error: volumesnapshotlocations.velero.io \"random-name\" not found", expectedSuccess: false, + bsl: *defaultBSL, }, { name: "duplicate locationName per provider: should filter out dups", @@ -1505,6 +1515,7 @@ func TestValidateAndGetSnapshotLocations(t *testing.T) { }, expectedVolumeSnapshotLocationNames: []string{"aws-us-west-1"}, expectedSuccess: true, + bsl: *defaultBSL, }, { name: "multiple non-dupe location names per provider should error", @@ -1516,6 +1527,7 @@ func TestValidateAndGetSnapshotLocations(t *testing.T) { }, expectedErrors: "more than one VolumeSnapshotLocation name specified for provider aws: aws-us-west-1; unexpected name was aws-us-east-1", expectedSuccess: false, + bsl: *defaultBSL, }, { name: "no location name for the provider exists, only one VSL for the provider: use it", @@ -1525,6 +1537,7 @@ func TestValidateAndGetSnapshotLocations(t *testing.T) { }, expectedVolumeSnapshotLocationNames: []string{"aws-us-east-1"}, expectedSuccess: true, + bsl: *defaultBSL, }, { name: "no location name for the provider exists, no default, more than one VSL for the provider: error", @@ -1534,6 +1547,7 @@ func TestValidateAndGetSnapshotLocations(t *testing.T) { builder.ForVolumeSnapshotLocation(velerov1api.DefaultNamespace, "aws-us-west-1").Provider("aws").Result(), }, expectedErrors: "provider aws has more than one possible volume snapshot location, and none were specified explicitly or as a default", + bsl: *defaultBSL, }, { name: "no location name for the provider exists, more than one VSL for the provider: the provider's default should be added", @@ -1545,11 +1559,13 @@ func TestValidateAndGetSnapshotLocations(t *testing.T) { }, expectedVolumeSnapshotLocationNames: []string{"aws-us-east-1"}, expectedSuccess: true, + bsl: *defaultBSL, }, { name: "no existing location name and no default location name given", backup: defaultBackup().Phase(velerov1api.BackupPhaseNew).Result(), expectedSuccess: true, + bsl: *defaultBSL, }, { name: "multiple location names for a provider, default location name for another provider", @@ -1561,6 +1577,7 @@ func TestValidateAndGetSnapshotLocations(t *testing.T) { }, expectedVolumeSnapshotLocationNames: []string{"aws-us-west-1", "some-name"}, expectedSuccess: true, + bsl: *defaultBSL, }, { name: "location name does not correspond to any existing location and snapshotvolume disabled; should return error", @@ -1572,6 +1589,7 @@ func TestValidateAndGetSnapshotLocations(t *testing.T) { }, expectedVolumeSnapshotLocationNames: nil, expectedErrors: "a VolumeSnapshotLocation CRD for the location random-name with the name specified in the backup spec needs to be created before this snapshot can be executed. Error: volumesnapshotlocations.velero.io \"random-name\" not found", expectedSuccess: false, + bsl: *defaultBSL, }, { name: "duplicate locationName per provider and snapshotvolume disabled; should return only one BSL", @@ -1582,6 +1600,7 @@ func TestValidateAndGetSnapshotLocations(t *testing.T) { }, expectedVolumeSnapshotLocationNames: []string{"aws-us-west-1"}, expectedSuccess: true, + bsl: *defaultBSL, }, { name: "no location name for the provider exists, only one VSL created and snapshotvolume disabled; should return the VSL", @@ -1591,6 +1610,7 @@ func TestValidateAndGetSnapshotLocations(t *testing.T) { }, expectedVolumeSnapshotLocationNames: []string{"aws-us-east-1"}, expectedSuccess: true, + bsl: *defaultBSL, }, { name: "multiple location names for a provider, no default location and backup has no location defined, but snapshotvolume disabled, should return error", @@ -1601,6 +1621,7 @@ func TestValidateAndGetSnapshotLocations(t *testing.T) { }, expectedVolumeSnapshotLocationNames: nil, expectedErrors: "provider aws has more than one possible volume snapshot location, and none were specified explicitly or as a default", + bsl: *defaultBSL, }, } diff --git a/pkg/controller/backup_deletion_controller.go b/pkg/controller/backup_deletion_controller.go index 3952c790d..edd176f3a 100644 --- a/pkg/controller/backup_deletion_controller.go +++ b/pkg/controller/backup_deletion_controller.go @@ -22,9 +22,8 @@ import ( "fmt" "time" - "github.com/vmware-tanzu/velero/pkg/util/csi" - jsonpatch "github.com/evanphx/json-patch/v5" + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" @@ -37,8 +36,6 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" - "github.com/vmware-tanzu/velero/internal/credentials" "github.com/vmware-tanzu/velero/internal/delete" "github.com/vmware-tanzu/velero/internal/volume" @@ -56,8 +53,10 @@ import ( repomanager "github.com/vmware-tanzu/velero/pkg/repository/manager" repotypes "github.com/vmware-tanzu/velero/pkg/repository/types" "github.com/vmware-tanzu/velero/pkg/util/boolptr" + "github.com/vmware-tanzu/velero/pkg/util/csi" "github.com/vmware-tanzu/velero/pkg/util/filesystem" "github.com/vmware-tanzu/velero/pkg/util/kube" + veleroutil "github.com/vmware-tanzu/velero/pkg/util/velero" ) const ( @@ -202,6 +201,11 @@ func (r *backupDeletionReconciler) Reconcile(ctx context.Context, req ctrl.Reque return ctrl.Result{}, err } + if !veleroutil.BSLIsAvailable(*location) { + err := r.patchDeleteBackupRequestWithError(ctx, dbr, fmt.Errorf("cannot delete backup because backup storage location %s is currently in Unavailable state", location.Name)) + return ctrl.Result{}, err + } + // if the request object has no labels defined, initialize an empty map since // we will be updating labels if dbr.Labels == nil { diff --git a/pkg/controller/backup_deletion_controller_test.go b/pkg/controller/backup_deletion_controller_test.go index d99a5c3c5..142c4af43 100644 --- a/pkg/controller/backup_deletion_controller_test.go +++ b/pkg/controller/backup_deletion_controller_test.go @@ -126,6 +126,9 @@ func TestBackupDeletionControllerReconcile(t *testing.T) { }, }, }, + Status: velerov1api.BackupStorageLocationStatus{ + Phase: velerov1api.BackupStorageLocationPhaseAvailable, + }, } dbr := defaultTestDbr() td := setupBackupDeletionControllerTest(t, dbr, location, backup) @@ -254,7 +257,7 @@ func TestBackupDeletionControllerReconcile(t *testing.T) { t.Run("backup storage location is in read-only mode", func(t *testing.T) { backup := builder.ForBackup(velerov1api.DefaultNamespace, "foo").StorageLocation("default").Result() - location := builder.ForBackupStorageLocation("velero", "default").AccessMode(velerov1api.BackupStorageLocationAccessModeReadOnly).Result() + location := builder.ForBackupStorageLocation("velero", "default").Phase(velerov1api.BackupStorageLocationPhaseAvailable).AccessMode(velerov1api.BackupStorageLocationAccessModeReadOnly).Result() td := setupBackupDeletionControllerTest(t, defaultTestDbr(), location, backup) @@ -268,6 +271,24 @@ func TestBackupDeletionControllerReconcile(t *testing.T) { assert.Len(t, res.Status.Errors, 1) assert.Equal(t, "cannot delete backup because backup storage location default is currently in read-only mode", res.Status.Errors[0]) }) + + t.Run("backup storage location is in unavailable state", func(t *testing.T) { + backup := builder.ForBackup(velerov1api.DefaultNamespace, "foo").StorageLocation("default").Result() + location := builder.ForBackupStorageLocation("velero", "default").Phase(velerov1api.BackupStorageLocationPhaseUnavailable).Result() + + td := setupBackupDeletionControllerTest(t, defaultTestDbr(), location, backup) + + _, err := td.controller.Reconcile(context.TODO(), td.req) + require.NoError(t, err) + + res := &velerov1api.DeleteBackupRequest{} + err = td.fakeClient.Get(ctx, td.req.NamespacedName, res) + require.NoError(t, err) + assert.Equal(t, "Processed", string(res.Status.Phase)) + assert.Len(t, res.Status.Errors, 1) + assert.Equal(t, "cannot delete backup because backup storage location default is currently in Unavailable state", res.Status.Errors[0]) + }) + t.Run("full delete, no errors", func(t *testing.T) { input := defaultTestDbr() @@ -297,6 +318,9 @@ func TestBackupDeletionControllerReconcile(t *testing.T) { }, }, }, + Status: velerov1api.BackupStorageLocationStatus{ + Phase: velerov1api.BackupStorageLocationPhaseAvailable, + }, } snapshotLocation := &velerov1api.VolumeSnapshotLocation{ @@ -416,6 +440,9 @@ func TestBackupDeletionControllerReconcile(t *testing.T) { }, }, }, + Status: velerov1api.BackupStorageLocationStatus{ + Phase: velerov1api.BackupStorageLocationPhaseAvailable, + }, } snapshotLocation := &velerov1api.VolumeSnapshotLocation{ @@ -518,6 +545,9 @@ func TestBackupDeletionControllerReconcile(t *testing.T) { }, }, }, + Status: velerov1api.BackupStorageLocationStatus{ + Phase: velerov1api.BackupStorageLocationPhaseAvailable, + }, } snapshotLocation := &velerov1api.VolumeSnapshotLocation{ @@ -600,6 +630,9 @@ func TestBackupDeletionControllerReconcile(t *testing.T) { }, }, }, + Status: velerov1api.BackupStorageLocationStatus{ + Phase: velerov1api.BackupStorageLocationPhaseAvailable, + }, } snapshotLocation := &velerov1api.VolumeSnapshotLocation{ diff --git a/pkg/controller/backup_sync_controller.go b/pkg/controller/backup_sync_controller.go index b80ae82e9..00eb9dafd 100644 --- a/pkg/controller/backup_sync_controller.go +++ b/pkg/controller/backup_sync_controller.go @@ -41,6 +41,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/persistence" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" "github.com/vmware-tanzu/velero/pkg/util/kube" + veleroutil "github.com/vmware-tanzu/velero/pkg/util/velero" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -92,6 +93,10 @@ func (b *backupSyncReconciler) Reconcile(ctx context.Context, req ctrl.Request) } return ctrl.Result{}, errors.Wrapf(err, "error getting BackupStorageLocation %s", req.String()) } + if !veleroutil.BSLIsAvailable(*location) { + log.Errorf("BackupStorageLocation is in unavailable state, skip syncing backup from it.") + return ctrl.Result{}, nil + } pluginManager := b.newPluginManager(log) defer pluginManager.CleanupClients() diff --git a/pkg/controller/backup_sync_controller_test.go b/pkg/controller/backup_sync_controller_test.go index 398165515..a4db93b16 100644 --- a/pkg/controller/backup_sync_controller_test.go +++ b/pkg/controller/backup_sync_controller_test.go @@ -62,6 +62,9 @@ func defaultLocation(namespace string) *velerov1api.BackupStorageLocation { }, Default: true, }, + Status: velerov1api.BackupStorageLocationStatus{ + Phase: velerov1api.BackupStorageLocationPhaseAvailable, + }, } } @@ -141,6 +144,9 @@ func defaultLocationWithLongerLocationName(namespace string) *velerov1api.Backup }, }, }, + Status: velerov1api.BackupStorageLocationStatus{ + Phase: velerov1api.BackupStorageLocationPhaseAvailable, + }, } } @@ -177,6 +183,21 @@ var _ = Describe("Backup Sync Reconciler", func() { namespace: "ns-1", location: defaultLocation("ns-1"), }, + { + name: "unavailable BSL", + namespace: "ns-1", + location: builder.ForBackupStorageLocation("ns-1", "default").Phase(velerov1api.BackupStorageLocationPhaseUnavailable).Result(), + cloudBackups: []*cloudBackupData{ + { + backup: builder.ForBackup("ns-1", "backup-1").Result(), + backupShouldSkipSync: true, + }, + { + backup: builder.ForBackup("ns-1", "backup-2").Result(), + backupShouldSkipSync: true, + }, + }, + }, { name: "normal case", namespace: "ns-1", diff --git a/pkg/controller/gc_controller.go b/pkg/controller/gc_controller.go index c162a1c28..bb3c60ae5 100644 --- a/pkg/controller/gc_controller.go +++ b/pkg/controller/gc_controller.go @@ -18,6 +18,7 @@ package controller import ( "context" + "fmt" "time" "github.com/pkg/errors" @@ -36,6 +37,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/constant" "github.com/vmware-tanzu/velero/pkg/label" "github.com/vmware-tanzu/velero/pkg/util/kube" + veleroutil "github.com/vmware-tanzu/velero/pkg/util/velero" ) const ( @@ -44,6 +46,7 @@ const ( gcFailureBSLNotFound = "BSLNotFound" gcFailureBSLCannotGet = "BSLCannotGet" gcFailureBSLReadOnly = "BSLReadOnly" + gcFailureBSLUnavailable = "BSLUnavailable" ) // gcReconciler creates DeleteBackupRequests for expired backups. @@ -144,12 +147,18 @@ func (c *gcReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Re } else { backup.Labels[garbageCollectionFailure] = gcFailureBSLCannotGet } + if err := c.Update(ctx, backup); err != nil { log.WithError(err).Error("error updating backup labels") } return ctrl.Result{}, errors.Wrap(err, "error getting backup storage location") } + if !veleroutil.BSLIsAvailable(*loc) { + log.Infof("BSL %s is unavailable, cannot gc backup", loc.Name) + return ctrl.Result{}, fmt.Errorf("bsl %s is unavailable, cannot gc backup", loc.Name) + } + if loc.Spec.AccessMode == velerov1api.BackupStorageLocationAccessModeReadOnly { log.Infof("Backup cannot be garbage-collected because backup storage location %s is currently in read-only mode", loc.Name) backup.Labels[garbageCollectionFailure] = gcFailureBSLReadOnly diff --git a/pkg/controller/gc_controller_test.go b/pkg/controller/gc_controller_test.go index 0241790c8..197deb050 100644 --- a/pkg/controller/gc_controller_test.go +++ b/pkg/controller/gc_controller_test.go @@ -46,7 +46,7 @@ func mockGCReconciler(fakeClient kbclient.Client, fakeClock *testclocks.FakeCloc func TestGCReconcile(t *testing.T) { fakeClock := testclocks.NewFakeClock(time.Now()) - defaultBackupLocation := builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "default").Result() + defaultBackupLocation := builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "default").Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result() tests := []struct { name string @@ -66,12 +66,12 @@ func TestGCReconcile(t *testing.T) { { name: "expired backup in read-only storage location is not deleted", backup: defaultBackup().Expiration(fakeClock.Now().Add(-time.Minute)).StorageLocation("read-only").Result(), - backupLocation: builder.ForBackupStorageLocation("velero", "read-only").AccessMode(velerov1api.BackupStorageLocationAccessModeReadOnly).Result(), + backupLocation: builder.ForBackupStorageLocation("velero", "read-only").AccessMode(velerov1api.BackupStorageLocationAccessModeReadOnly).Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result(), }, { name: "expired backup in read-write storage location is deleted", backup: defaultBackup().Expiration(fakeClock.Now().Add(-time.Minute)).StorageLocation("read-write").Result(), - backupLocation: builder.ForBackupStorageLocation("velero", "read-write").AccessMode(velerov1api.BackupStorageLocationAccessModeReadWrite).Result(), + backupLocation: builder.ForBackupStorageLocation("velero", "read-write").AccessMode(velerov1api.BackupStorageLocationAccessModeReadWrite).Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result(), }, { name: "expired backup with no pending deletion requests is deleted", @@ -118,6 +118,12 @@ func TestGCReconcile(t *testing.T) { }, }, }, + { + name: "BSL is unavailable", + backup: defaultBackup().Expiration(fakeClock.Now().Add(-time.Second)).StorageLocation("default").Result(), + backupLocation: builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "default").Phase(velerov1api.BackupStorageLocationPhaseUnavailable).Result(), + expectError: true, + }, } for _, test := range tests { diff --git a/pkg/controller/restore_controller.go b/pkg/controller/restore_controller.go index 248ccbc49..208a3ddca 100644 --- a/pkg/controller/restore_controller.go +++ b/pkg/controller/restore_controller.go @@ -58,6 +58,7 @@ import ( kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" "github.com/vmware-tanzu/velero/pkg/util/logging" "github.com/vmware-tanzu/velero/pkg/util/results" + veleroutil "github.com/vmware-tanzu/velero/pkg/util/velero" pkgrestoreUtil "github.com/vmware-tanzu/velero/pkg/util/velero/restore" ) @@ -393,6 +394,11 @@ func (r *restoreReconciler) validateAndComplete(restore *api.Restore) (backupInf return backupInfo{}, nil } + if !veleroutil.BSLIsAvailable(*info.location) { + restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, fmt.Sprintf("The BSL %s is unavailable, cannot retrieve the backup", info.location.Name)) + return backupInfo{}, nil + } + // Fill in the ScheduleName so it's easier to consume for metrics. if restore.Spec.ScheduleName == "" { restore.Spec.ScheduleName = info.backup.GetLabels()[api.ScheduleNameLabel] @@ -728,6 +734,10 @@ func (r *restoreReconciler) deleteExternalResources(restore *api.Restore) error return errors.Wrap(err, fmt.Sprintf("can't get backup info, backup: %s", restore.Spec.BackupName)) } + if !veleroutil.BSLIsAvailable(*backupInfo.location) { + return fmt.Errorf("bsl %s is unavailable, cannot get the backup info", backupInfo.location.Name) + } + // delete restore files in object storage pluginManager := r.newPluginManager(r.logger) defer pluginManager.CleanupClients() diff --git a/pkg/controller/restore_controller_test.go b/pkg/controller/restore_controller_test.go index 4fce9d312..2333943e1 100644 --- a/pkg/controller/restore_controller_test.go +++ b/pkg/controller/restore_controller_test.go @@ -66,7 +66,7 @@ func TestFetchBackupInfo(t *testing.T) { { name: "lister has backup", backupName: "backup-1", - informerLocations: []*velerov1api.BackupStorageLocation{builder.ForBackupStorageLocation("velero", "default").Provider("myCloud").Bucket("bucket").Result()}, + informerLocations: []*velerov1api.BackupStorageLocation{builder.ForBackupStorageLocation("velero", "default").Provider("myCloud").Bucket("bucket").Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result()}, informerBackups: []*velerov1api.Backup{defaultBackup().StorageLocation("default").Result()}, expectedRes: defaultBackup().StorageLocation("default").Result(), }, @@ -74,7 +74,7 @@ func TestFetchBackupInfo(t *testing.T) { name: "lister does not have a backup, but backupSvc does", backupName: "backup-1", backupStoreBackup: defaultBackup().StorageLocation("default").Result(), - informerLocations: []*velerov1api.BackupStorageLocation{builder.ForBackupStorageLocation("velero", "default").Provider("myCloud").Bucket("bucket").Result()}, + informerLocations: []*velerov1api.BackupStorageLocation{builder.ForBackupStorageLocation("velero", "default").Provider("myCloud").Bucket("bucket").Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result()}, informerBackups: []*velerov1api.Backup{defaultBackup().StorageLocation("default").Result()}, expectedRes: defaultBackup().StorageLocation("default").Result(), }, @@ -211,7 +211,7 @@ func TestProcessQueueItemSkips(t *testing.T) { } func TestRestoreReconcile(t *testing.T) { - defaultStorageLocation := builder.ForBackupStorageLocation("velero", "default").Provider("myCloud").Bucket("bucket").Result() + defaultStorageLocation := builder.ForBackupStorageLocation("velero", "default").Provider("myCloud").Bucket("bucket").Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result() now, err := time.Parse(time.RFC1123Z, time.RFC1123Z) require.NoError(t, err) @@ -464,6 +464,22 @@ func TestRestoreReconcile(t *testing.T) { expectedCompletedTime: ×tamp, expectedRestorerCall: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseInProgress).Result(), }, + { + name: "Restore creation is rejected when BSL is unavailable", + location: builder.ForBackupStorageLocation("velero", "default").Provider("myCloud").Bucket("bucket").Phase(velerov1api.BackupStorageLocationPhaseUnavailable).Result(), + restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseNew).Result(), + backup: defaultBackup().StorageLocation("default").Result(), + expectedErr: false, + expectedPhase: string(velerov1api.RestorePhaseNew), + expectedValidationErrors: []string{"The BSL default is unavailable, cannot retrieve the backup"}, + }, + { + name: "Restore deletion is rejected when BSL is unavailable.", + location: builder.ForBackupStorageLocation("velero", "default").Provider("myCloud").Bucket("bucket").Phase(velerov1api.BackupStorageLocationPhaseUnavailable).Result(), + restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseCompleted).ObjectMeta(builder.WithFinalizers(ExternalResourcesFinalizer), builder.WithDeletionTimestamp(timestamp.Time)).Result(), + backup: defaultBackup().StorageLocation("default").Result(), + expectedErr: true, + }, } formatFlag := logging.FormatText @@ -738,7 +754,7 @@ func TestValidateAndCompleteWhenScheduleNameSpecified(t *testing.T) { Result(), )) - location := builder.ForBackupStorageLocation("velero", "default").Provider("myCloud").Bucket("bucket").Result() + location := builder.ForBackupStorageLocation("velero", "default").Provider("myCloud").Bucket("bucket").Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result() require.NoError(t, r.kbClient.Create(context.Background(), location)) restore = &velerov1api.Restore{ @@ -797,7 +813,7 @@ func TestValidateAndCompleteWithResourceModifierSpecified(t *testing.T) { }, } - location := builder.ForBackupStorageLocation("velero", "default").Provider("myCloud").Bucket("bucket").Result() + location := builder.ForBackupStorageLocation("velero", "default").Provider("myCloud").Bucket("bucket").Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result() require.NoError(t, r.kbClient.Create(context.Background(), location)) require.NoError(t, r.kbClient.Create( diff --git a/pkg/util/velero/velero.go b/pkg/util/velero/velero.go index d52ecc59c..c0ab9771d 100644 --- a/pkg/util/velero/velero.go +++ b/pkg/util/velero/velero.go @@ -19,6 +19,8 @@ package velero import ( appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) // GetNodeSelectorFromVeleroServer get the node selector from the Velero server deployment @@ -105,3 +107,7 @@ func GetVeleroServerAnnotationValue(deployment *appsv1api.Deployment, key string return deployment.Spec.Template.Annotations[key] } + +func BSLIsAvailable(bsl velerov1api.BackupStorageLocation) bool { + return bsl.Status.Phase == velerov1api.BackupStorageLocationPhaseAvailable +} diff --git a/pkg/util/velero/velero_test.go b/pkg/util/velero/velero_test.go index 5e552c80c..06e9ed070 100644 --- a/pkg/util/velero/velero_test.go +++ b/pkg/util/velero/velero_test.go @@ -24,6 +24,9 @@ import ( appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/builder" ) func TestGetNodeSelectorFromVeleroServer(t *testing.T) { @@ -759,3 +762,11 @@ func TestGetVeleroServerLabelValue(t *testing.T) { }) } } + +func TestBSLIsAvailable(t *testing.T) { + availableBSL := builder.ForBackupStorageLocation("velero", "available").Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result() + unavailableBSL := builder.ForBackupStorageLocation("velero", "unavailable").Phase(velerov1api.BackupStorageLocationPhaseUnavailable).Result() + + assert.True(t, BSLIsAvailable(*availableBSL)) + assert.False(t, BSLIsAvailable(*unavailableBSL)) +}