diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go index f4486196c3..0bc234c701 100644 --- a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go +++ b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go @@ -32,7 +32,9 @@ const ( // DefaultSAWaitKey is the name used in the flags for default service account DefaultSAWaitKey = "default_sa" // AppsRunning is the name used in the flags for waiting for k8s-apps to be running - AppsRunning = "apps_running" + AppsRunningKey = "apps_running" + // NodeReadyKey is the name used in the flags for waiting for the node status to be ready + NodeReadyKey = "node_ready" ) // vars related to the --wait flag @@ -40,13 +42,13 @@ var ( // DefaultComponents is map of the the default components to wait for DefaultComponents = map[string]bool{APIServerWaitKey: true, SystemPodsWaitKey: true} // NoWaitComponents is map of componets to wait for if specified 'none' or 'false' - NoComponents = map[string]bool{APIServerWaitKey: false, SystemPodsWaitKey: false, DefaultSAWaitKey: false, AppsRunning: false} + NoComponents = map[string]bool{APIServerWaitKey: false, SystemPodsWaitKey: false, DefaultSAWaitKey: false, AppsRunningKey: false, NodeReadyKey: false} // AllComponents is map for waiting for all components. - AllComponents = map[string]bool{APIServerWaitKey: true, SystemPodsWaitKey: true, DefaultSAWaitKey: true, AppsRunning: true} + AllComponents = map[string]bool{APIServerWaitKey: true, SystemPodsWaitKey: true, DefaultSAWaitKey: true, AppsRunningKey: true} // DefaultWaitList is list of all default components to wait for. only names to be used for start flags. DefaultWaitList = []string{APIServerWaitKey, SystemPodsWaitKey} // AllComponentsList list of all valid components keys to wait for. only names to be used used for start flags. - AllComponentsList = []string{APIServerWaitKey, SystemPodsWaitKey, DefaultSAWaitKey, AppsRunning} + AllComponentsList = []string{APIServerWaitKey, SystemPodsWaitKey, DefaultSAWaitKey, AppsRunningKey, NodeReadyKey} // AppsRunningList running list are valid k8s-app components to wait for them to be running AppsRunningList = []string{ "kube-dns", // coredns diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/node_ready.go b/pkg/minikube/bootstrapper/bsutil/kverify/node_ready.go new file mode 100644 index 0000000000..69e21ba162 --- /dev/null +++ b/pkg/minikube/bootstrapper/bsutil/kverify/node_ready.go @@ -0,0 +1,64 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package kverify verifies a running kubernetes cluster is healthy +package kverify + +import ( + "fmt" + "time" + + "github.com/golang/glog" + "github.com/pkg/errors" + v1 "k8s.io/api/core/v1" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants" +) + +// WaitForNodeReady waits for a node to be ready +func WaitForNodeReady(cs *kubernetes.Clientset, timeout time.Duration) error { + glog.Info("waiting for node to be ready ...") + start := time.Now() + defer func() { + glog.Infof("duration metric: took %s to wait for WaitForNodeReady...", time.Since(start)) + }() + checkReady := func() (bool, error) { + if time.Since(start) > timeout { + return false, fmt.Errorf("wait for node to be ready timed out") + } + ns, err := cs.CoreV1().Nodes().List(meta.ListOptions{}) + if err != nil { + glog.Infof("error listing nodes will retry: %v", err) + return false, nil + } + + for _, n := range ns.Items { + for _, c := range n.Status.Conditions { + if c.Type == v1.NodeReady && c.Status != v1.ConditionTrue { + glog.Infof("node %q has unwanted condition %q : Reason %q Message: %q. will try. ", n.Name, c.Type, c.Reason, c.Message) + return false, nil + } + } + } + return true, nil + } + if err := wait.PollImmediate(kconst.APICallRetryInterval, kconst.DefaultControlPlaneTimeout, checkReady); err != nil { + return errors.Wrapf(err, "wait node ready") + } + return nil +}