add --allow-partially-failed flag to velero restore create (#1994)

* add --allow-partially-failed flag to velero restore create

Signed-off-by: Steve Kriss <krisss@vmware.com>

* remove extraneous client creation

Signed-off-by: Steve Kriss <krisss@vmware.com>

* add godoc to helper func

Signed-off-by: Steve Kriss <krisss@vmware.com>

* todo

Signed-off-by: Steve Kriss <krisss@vmware.com>

changelogs/unreleased/1994-skriss

@@ -0,0 +1 @@
+add `--allow-partially-failed` flag to `velero restore create` for use with `--from-schedule` to allow partially-failed backups to be restored

pkg/cmd/cli/restore/create.go

@@ -18,6 +18,7 @@ package restore
 
 import (
 	"fmt"
+	"sort"
 	"time"
 
 	"github.com/pkg/errors"
@@ -33,6 +34,7 @@ import (
 	"github.com/vmware-tanzu/velero/pkg/cmd/util/output"
 	veleroclient "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned"
 	v1 "github.com/vmware-tanzu/velero/pkg/generated/informers/externalversions/velero/v1"
+	"github.com/vmware-tanzu/velero/pkg/util/boolptr"
 )
 
 func NewCreateCommand(f client.Factory, use string) *cobra.Command {
@@ -50,6 +52,9 @@ func NewCreateCommand(f client.Factory, use string) *cobra.Command {
   # create a restore from the latest successful backup triggered by schedule "schedule-1"
   velero restore create --from-schedule schedule-1
 
+  # create a restore from the latest successful OR partially-failed backup triggered by schedule "schedule-1"
+  velero restore create --from-schedule schedule-1 --allow-partially-failed
+
   # create a restore for only persistentvolumeclaims and persistentvolumes within a backup
   velero restore create --from-backup backup-2 --include-resources persistentvolumeclaims,persistentvolumes
   `,
@@ -82,6 +87,7 @@ type CreateOptions struct {
 	Selector                flag.LabelSelector
 	IncludeClusterResources flag.OptionalBool
 	Wait                    bool
+	AllowPartiallyFailed    flag.OptionalBool
 
 	client veleroclient.Interface
 }
@@ -114,6 +120,9 @@ func (o *CreateOptions) BindFlags(flags *pflag.FlagSet) {
 	f = flags.VarPF(&o.IncludeClusterResources, "include-cluster-resources", "", "include cluster-scoped resources in the restore")
 	f.NoOptDefVal = "true"
 
+	f = flags.VarPF(&o.AllowPartiallyFailed, "allow-partially-failed", "", "if using --from-schedule, whether to consider PartiallyFailed backups when looking for the most recent one. This flag has no effect if not using --from-schedule.")
+	f.NoOptDefVal = "true"
+
 	flags.BoolVarP(&o.Wait, "wait", "w", o.Wait, "wait for the operation to complete")
 }
 
@@ -170,12 +179,67 @@ func (o *CreateOptions) Validate(c *cobra.Command, args []string, f client.Facto
 	return nil
 }
 
+// mostRecentBackup returns the backup with the most recent start timestamp that has a phase that's
+// in the provided list of allowed phases.
+func mostRecentBackup(backups []api.Backup, allowedPhases ...api.BackupPhase) *api.Backup {
+	// sort the backups in descending order of start time (i.e. most recent to least recent)
+	sort.Slice(backups, func(i, j int) bool {
+		// Use .After() because we want descending sort.
+
+		var iStartTime, jStartTime time.Time
+		if backups[i].Status.StartTimestamp != nil {
+			iStartTime = backups[i].Status.StartTimestamp.Time
+		}
+		if backups[j].Status.StartTimestamp != nil {
+			jStartTime = backups[j].Status.StartTimestamp.Time
+		}
+		return iStartTime.After(jStartTime)
+	})
+
+	// create a map of the allowed phases for easy lookup below
+	phases := map[api.BackupPhase]struct{}{}
+	for _, phase := range allowedPhases {
+		phases[phase] = struct{}{}
+	}
+
+	var res *api.Backup
+	for i, backup := range backups {
+		// if the backup's phase is one of the allowable ones, record
+		// the backup and break the loop so we can return it
+		if _, ok := phases[backup.Status.Phase]; ok {
+			res = &backups[i]
+			break
+		}
+	}
+
+	return res
+}
+
 func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error {
 	if o.client == nil {
 		// This should never happen
 		return errors.New("Velero client is not set; unable to proceed")
 	}
 
+	// if --allow-partially-failed was specified, look up the most recent Completed or
+	// PartiallyFailed backup for the provided schedule, and use that specific backup
+	// to restore from.
+	if o.ScheduleName != "" && boolptr.IsSetToTrue(o.AllowPartiallyFailed.Value) {
+		backups, err := o.client.VeleroV1().Backups(f.Namespace()).List(metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", api.ScheduleNameLabel, o.ScheduleName)})
+		if err != nil {
+			return err
+		}
+
+		// if we find a Completed or PartiallyFailed backup for the schedule, restore specifically from that backup. If we don't
+		// find one, proceed as-is -- the Velero server will handle validation.
+		if backup := mostRecentBackup(backups.Items, api.BackupPhaseCompleted, api.BackupPhasePartiallyFailed); backup != nil {
+			// TODO(sk): this is kind of a hack -- we should revisit this and probably
+			// move this logic to the server side or otherwise solve this problem.
+			o.BackupName = backup.Name
+			o.ScheduleName = ""
+		}
+	}
+
 	restore := &api.Restore{
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: f.Namespace(),