From 8c5f6a213269bc0896dc21be184796ac1e391fc8 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Thu, 25 Jun 2020 15:14:51 -0700 Subject: [PATCH] apiserverHealthz: wait up to 8s before declaring failure --- .../bootstrapper/bsutil/kverify/api_server.go | 28 +++++++++++++++++-- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/api_server.go b/pkg/minikube/bootstrapper/bsutil/kverify/api_server.go index 753323f807..5f2806fbb4 100644 --- a/pkg/minikube/bootstrapper/bsutil/kverify/api_server.go +++ b/pkg/minikube/bootstrapper/bsutil/kverify/api_server.go @@ -40,6 +40,7 @@ import ( "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/cruntime" + "k8s.io/minikube/pkg/util/retry" ) // WaitForAPIServerProcess waits for api server to be healthy returns error if it doesn't @@ -93,7 +94,7 @@ func WaitForHealthyAPIServer(r cruntime.Manager, bs bootstrapper.Bootstrapper, c time.Sleep(kconst.APICallRetryInterval * 5) } - status, err := apiServerHealthz(hostname, port) + status, err := apiServerHealthzNow(hostname, port) if err != nil { glog.Warningf("status: %v", err) return false, nil @@ -187,13 +188,34 @@ func APIServerStatus(cr command.Runner, hostname string, port int) (state.State, return apiServerHealthz(hostname, port) } -// apiServerHealthz hits the /healthz endpoint and returns libmachine style state.State +// apiServerHealthz checks apiserver in a patient and tolerant manner func apiServerHealthz(hostname string, port int) (state.State, error) { + var st state.State + var err error + + check := func() error { + // etcd gets upset sometimes and causes healthz to report a failure. Be tolerant of it. + st, err = apiServerHealthzNow(hostname, port) + if err != nil { + return err + } + if st != state.Running { + return fmt.Errorf("state is %q", st) + } + return nil + } + + err = retry.Local(check, 8*time.Second) + return st, err +} + +// apiServerHealthzNow hits the /healthz endpoint and returns libmachine style state.State +func apiServerHealthzNow(hostname string, port int) (state.State, error) { url := fmt.Sprintf("https://%s/healthz", net.JoinHostPort(hostname, fmt.Sprint(port))) glog.Infof("Checking apiserver healthz at %s ...", url) // To avoid: x509: certificate signed by unknown authority tr := &http.Transport{ - Proxy: nil, // To avoid connectiv issue if http(s)_proxy is set. + Proxy: nil, // To avoid connectivy issue if http(s)_proxy is set. TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, } client := &http.Client{Transport: tr}