velero install: wait for restic daemonset to be ready (#1859)

* velero install: wait for restic daemonset to be ready

Signed-off-by: Steve Kriss <krisss@vmware.com>

* changelog

Signed-off-by: Steve Kriss <krisss@vmware.com>

changelogs/unreleased/1859-skriss

@@ -0,0 +1 @@
+velero install: if `--use-restic` and `--wait` are specified, wait up to a minute for restic daemonset to be ready

pkg/cmd/cli/install/install.go

@@ -238,10 +238,17 @@ func (o *InstallOptions) Run(c *cobra.Command, f client.Factory) error {
 	}
 
 	if o.Wait {
-		fmt.Println("Waiting for Velero to be ready.")
+		fmt.Println("Waiting for Velero deployment to be ready.")
 		if _, err = install.DeploymentIsReady(factory, o.Namespace); err != nil {
 			return errors.Wrap(err, errorMsg)
 		}
+
+		if o.UseRestic {
+			fmt.Println("Waiting for Velero restic daemonset to be ready.")
+			if _, err = install.DaemonSetIsReady(factory, o.Namespace); err != nil {
+				return errors.Wrap(err, errorMsg)
+			}
+		}
 	}
 	if o.SecretFile == "" {
 		fmt.Printf("\nNo secret file was specified, no Secret created.\n\n")

pkg/install/install.go

@@ -175,6 +175,53 @@ func DeploymentIsReady(factory client.DynamicFactory, namespace string) (bool, e
 	return isReady, err
 }
 
+// DaemonSetIsReady will poll the kubernetes API server to ensure the restic daemonset is ready, i.e. that
+// pods are scheduled and available on all of the the desired nodes.
+func DaemonSetIsReady(factory client.DynamicFactory, namespace string) (bool, error) {
+	gvk := schema.FromAPIVersionAndKind(appsv1.SchemeGroupVersion.String(), "DaemonSet")
+	apiResource := metav1.APIResource{
+		Name:       "daemonsets",
+		Namespaced: true,
+	}
+
+	c, err := factory.ClientForGroupVersionResource(gvk.GroupVersion(), apiResource, namespace)
+	if err != nil {
+		return false, errors.Wrapf(err, "Error creating client for daemonset polling")
+	}
+
+	// declare this variable out of scope so we can return it
+	var isReady bool
+	var readyObservations int32
+
+	err = wait.PollImmediate(time.Second, time.Minute, func() (bool, error) {
+		unstructuredDaemonSet, err := c.Get("restic", metav1.GetOptions{})
+		if apierrors.IsNotFound(err) {
+			return false, nil
+		} else if err != nil {
+			return false, errors.Wrap(err, "error waiting for daemonset to be ready")
+		}
+
+		daemonSet := new(appsv1.DaemonSet)
+		if err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstructuredDaemonSet.Object, daemonSet); err != nil {
+			return false, errors.Wrap(err, "error converting daemonset from unstructured")
+		}
+
+		if daemonSet.Status.NumberAvailable == daemonSet.Status.DesiredNumberScheduled {
+			readyObservations++
+		}
+
+		// Wait for 5 observations of the daemonset being "ready" to be consistent with our check for
+		// the deployment being ready.
+		if readyObservations > 4 {
+			isReady = true
+			return true, nil
+		} else {
+			return false, nil
+		}
+	})
+	return isReady, err
+}
+
 // GroupResources groups resources based on whether the resources are CustomResourceDefinitions or other types of kubernetes objects
 // This is useful to wait for readiness before creating CRD objects
 func GroupResources(resources *unstructured.UnstructuredList) *ResourceGroup {