apiserverHealthz: wait up to 8s before declaring failure

pkg/minikube/bootstrapper/bsutil/kverify/api_server.go

@@ -40,6 +40,7 @@ import (
 	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/cruntime"
+	"k8s.io/minikube/pkg/util/retry"
 )
 
 // WaitForAPIServerProcess waits for api server to be healthy returns error if it doesn't
@@ -93,7 +94,7 @@ func WaitForHealthyAPIServer(r cruntime.Manager, bs bootstrapper.Bootstrapper, c
 			time.Sleep(kconst.APICallRetryInterval * 5)
 		}
 
-		status, err := apiServerHealthz(hostname, port)
+		status, err := apiServerHealthzNow(hostname, port)
 		if err != nil {
 			glog.Warningf("status: %v", err)
 			return false, nil
@@ -187,13 +188,34 @@ func APIServerStatus(cr command.Runner, hostname string, port int) (state.State,
 	return apiServerHealthz(hostname, port)
 }
 
-// apiServerHealthz hits the /healthz endpoint and returns libmachine style state.State
+// apiServerHealthz checks apiserver in a patient and tolerant manner
 func apiServerHealthz(hostname string, port int) (state.State, error) {
+	var st state.State
+	var err error
+
+	check := func() error {
+		// etcd gets upset sometimes and causes healthz to report a failure. Be tolerant of it.
+		st, err = apiServerHealthzNow(hostname, port)
+		if err != nil {
+			return err
+		}
+		if st != state.Running {
+			return fmt.Errorf("state is %q", st)
+		}
+		return nil
+	}
+
+	err = retry.Local(check, 8*time.Second)
+	return st, err
+}
+
+// apiServerHealthzNow hits the /healthz endpoint and returns libmachine style state.State
+func apiServerHealthzNow(hostname string, port int) (state.State, error) {
 	url := fmt.Sprintf("https://%s/healthz", net.JoinHostPort(hostname, fmt.Sprint(port)))
 	glog.Infof("Checking apiserver healthz at %s ...", url)
 	// To avoid: x509: certificate signed by unknown authority
 	tr := &http.Transport{
-		Proxy:           nil, // To avoid connectiv issue if http(s)_proxy is set.
+		Proxy:           nil, // To avoid connectivy issue if http(s)_proxy is set.
 		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
 	}
 	client := &http.Client{Transport: tr}