diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go index 15b89c6cdc..f4486196c3 100644 --- a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go +++ b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go @@ -18,22 +18,7 @@ limitations under the License. package kverify import ( - "fmt" - "os/exec" - "strings" "time" - - "github.com/docker/machine/libmachine/state" - "github.com/golang/glog" - core "k8s.io/api/core/v1" - meta "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/minikube/pkg/minikube/bootstrapper" - "k8s.io/minikube/pkg/minikube/command" - "k8s.io/minikube/pkg/minikube/config" - "k8s.io/minikube/pkg/minikube/cruntime" - "k8s.io/minikube/pkg/minikube/logs" ) // minLogCheckTime how long to wait before spamming error logs to console @@ -46,6 +31,8 @@ const ( SystemPodsWaitKey = "system_pods" // DefaultSAWaitKey is the name used in the flags for default service account DefaultSAWaitKey = "default_sa" + // AppsRunning is the name used in the flags for waiting for k8s-apps to be running + AppsRunning = "apps_running" ) // vars related to the --wait flag @@ -53,13 +40,22 @@ var ( // DefaultComponents is map of the the default components to wait for DefaultComponents = map[string]bool{APIServerWaitKey: true, SystemPodsWaitKey: true} // NoWaitComponents is map of componets to wait for if specified 'none' or 'false' - NoComponents = map[string]bool{APIServerWaitKey: false, SystemPodsWaitKey: false, DefaultSAWaitKey: false} + NoComponents = map[string]bool{APIServerWaitKey: false, SystemPodsWaitKey: false, DefaultSAWaitKey: false, AppsRunning: false} // AllComponents is map for waiting for all components. - AllComponents = map[string]bool{APIServerWaitKey: true, SystemPodsWaitKey: true, DefaultSAWaitKey: true} + AllComponents = map[string]bool{APIServerWaitKey: true, SystemPodsWaitKey: true, DefaultSAWaitKey: true, AppsRunning: true} // DefaultWaitList is list of all default components to wait for. only names to be used for start flags. DefaultWaitList = []string{APIServerWaitKey, SystemPodsWaitKey} // AllComponentsList list of all valid components keys to wait for. only names to be used used for start flags. - AllComponentsList = []string{APIServerWaitKey, SystemPodsWaitKey, DefaultSAWaitKey} + AllComponentsList = []string{APIServerWaitKey, SystemPodsWaitKey, DefaultSAWaitKey, AppsRunning} + // AppsRunningList running list are valid k8s-app components to wait for them to be running + AppsRunningList = []string{ + "kube-dns", // coredns + "etcd", + "kube-apiserver", + "kube-controller-manager", + "kube-proxy", + "kube-scheduler", + } ) // ShouldWait will return true if the config says need to wait @@ -71,95 +67,3 @@ func ShouldWait(wcs map[string]bool) bool { } return false } - -// ExpectedComponentsRunning returns whether or not all expected components are running -func ExpectedComponentsRunning(cs *kubernetes.Clientset) error { - expected := []string{ - "kube-dns", // coredns - "etcd", - "kube-apiserver", - "kube-controller-manager", - "kube-proxy", - "kube-scheduler", - } - - found := map[string]bool{} - - pods, err := cs.CoreV1().Pods("kube-system").List(meta.ListOptions{}) - if err != nil { - return err - } - - for _, pod := range pods.Items { - glog.Infof("found pod: %s", podStatusMsg(pod)) - if pod.Status.Phase != core.PodRunning { - continue - } - for k, v := range pod.ObjectMeta.Labels { - if k == "component" || k == "k8s-app" { - found[v] = true - } - } - } - - missing := []string{} - for _, e := range expected { - if !found[e] { - missing = append(missing, e) - } - } - if len(missing) > 0 { - return fmt.Errorf("missing components: %v", strings.Join(missing, ", ")) - } - return nil -} - -// podStatusMsg returns a human-readable pod status, for generating debug status -func podStatusMsg(pod core.Pod) string { - var sb strings.Builder - sb.WriteString(fmt.Sprintf("%q [%s] %s", pod.ObjectMeta.GetName(), pod.ObjectMeta.GetUID(), pod.Status.Phase)) - for i, c := range pod.Status.Conditions { - if c.Reason != "" { - if i == 0 { - sb.WriteString(": ") - } else { - sb.WriteString(" / ") - } - sb.WriteString(fmt.Sprintf("%s:%s", c.Type, c.Reason)) - } - if c.Message != "" { - sb.WriteString(fmt.Sprintf(" (%s)", c.Message)) - } - } - return sb.String() -} - -// announceProblems checks for problems, and slows polling down if any are found -func announceProblems(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg config.ClusterConfig, cr command.Runner) { - problems := logs.FindProblems(r, bs, cfg, cr) - if len(problems) > 0 { - logs.OutputProblems(problems, 5) - time.Sleep(kconst.APICallRetryInterval * 15) - } -} - -// KubeletStatus checks the kubelet status -func KubeletStatus(cr command.Runner) (state.State, error) { - glog.Infof("Checking kubelet status ...") - rr, err := cr.RunCmd(exec.Command("sudo", "systemctl", "is-active", "kubelet")) - if err != nil { - // Do not return now, as we still have parsing to do! - glog.Warningf("%s returned error: %v", rr.Command(), err) - } - s := strings.TrimSpace(rr.Stdout.String()) - glog.Infof("kubelet is-active: %s", s) - switch s { - case "active": - return state.Running, nil - case "inactive": - return state.Stopped, nil - case "activating": - return state.Starting, nil - } - return state.Error, nil -} diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/system_pods.go b/pkg/minikube/bootstrapper/bsutil/kverify/system_pods.go index 9993044ae7..f6e9a5f98f 100644 --- a/pkg/minikube/bootstrapper/bsutil/kverify/system_pods.go +++ b/pkg/minikube/bootstrapper/bsutil/kverify/system_pods.go @@ -19,9 +19,14 @@ package kverify import ( "fmt" + "os/exec" + "strings" "time" + "github.com/docker/machine/libmachine/state" "github.com/golang/glog" + "github.com/pkg/errors" + core "k8s.io/api/core/v1" meta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" @@ -30,6 +35,8 @@ import ( "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/cruntime" + "k8s.io/minikube/pkg/minikube/logs" + "k8s.io/minikube/pkg/util/retry" ) // WaitForSystemPods verifies essential pods for running kurnetes is running @@ -68,3 +75,98 @@ func WaitForSystemPods(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg con glog.Infof("duration metric: took %s to wait for pod list to return data ...", time.Since(pStart)) return nil } + +// ExpectAppsRunning returns whether or not all expected k8s-apps are running. (without waiting for them) +func ExpectAppsRunning(cs *kubernetes.Clientset, expected []string) error { + found := map[string]bool{} + + pods, err := cs.CoreV1().Pods("kube-system").List(meta.ListOptions{}) + if err != nil { + return err + } + + for _, pod := range pods.Items { + glog.Infof("found pod: %s", podStatusMsg(pod)) + if pod.Status.Phase != core.PodRunning { + continue + } + for k, v := range pod.ObjectMeta.Labels { + if k == "component" || k == "k8s-app" { + found[v] = true + } + } + } + + missing := []string{} + for _, e := range expected { + if !found[e] { + missing = append(missing, e) + } + } + if len(missing) > 0 { + return fmt.Errorf("missing components: %v", strings.Join(missing, ", ")) + } + return nil +} + +// WaitForAppsRunning waits for expected Apps To be running +func WaitForAppsRunning(cs *kubernetes.Clientset, expected []string, timeout time.Duration) error { + glog.Info("waiting for k8s-apps to be running ...") + start := time.Now() + checkRunning := func() error { return ExpectAppsRunning(cs, expected) } + if err := retry.Expo(checkRunning, 500*time.Millisecond, timeout); err != nil { + return errors.Wrap(err, "waitings for k8s app running") + } + glog.Infof("duration metric: took %s to wait for k8s-apps to be running ...", time.Since(start)) + return nil +} + +// podStatusMsg returns a human-readable pod status, for generating debug status +func podStatusMsg(pod core.Pod) string { + var sb strings.Builder + sb.WriteString(fmt.Sprintf("%q [%s] %s", pod.ObjectMeta.GetName(), pod.ObjectMeta.GetUID(), pod.Status.Phase)) + for i, c := range pod.Status.Conditions { + if c.Reason != "" { + if i == 0 { + sb.WriteString(": ") + } else { + sb.WriteString(" / ") + } + sb.WriteString(fmt.Sprintf("%s:%s", c.Type, c.Reason)) + } + if c.Message != "" { + sb.WriteString(fmt.Sprintf(" (%s)", c.Message)) + } + } + return sb.String() +} + +// announceProblems checks for problems, and slows polling down if any are found +func announceProblems(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg config.ClusterConfig, cr command.Runner) { + problems := logs.FindProblems(r, bs, cfg, cr) + if len(problems) > 0 { + logs.OutputProblems(problems, 5) + time.Sleep(kconst.APICallRetryInterval * 15) + } +} + +// KubeletStatus checks the kubelet status +func KubeletStatus(cr command.Runner) (state.State, error) { + glog.Infof("Checking kubelet status ...") + rr, err := cr.RunCmd(exec.Command("sudo", "systemctl", "is-active", "kubelet")) + if err != nil { + // Do not return now, as we still have parsing to do! + glog.Warningf("%s returned error: %v", rr.Command(), err) + } + s := strings.TrimSpace(rr.Stdout.String()) + glog.Infof("kubelet is-active: %s", s) + switch s { + case "active": + return state.Running, nil + case "inactive": + return state.Stopped, nil + case "activating": + return state.Starting, nil + } + return state.Error, nil +} diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index e8d76def91..9a65e7df8c 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -394,6 +394,17 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time return errors.Wrap(err, "waiting for default service account") } } + + if cfg.VerifyComponents[kverify.AppsRunning] { + client, err := k.client(hostname, port) + if err != nil { + return errors.Wrap(err, "get k8s client") + } + if err := kverify.WaitForAppsRunning(client, kverify.AppsRunningList, timeout); err != nil { + return errors.Wrap(err, "waiting for apps_running") + } + } + glog.Infof("duration metric: took %s to wait for : %+v ...", time.Since(start), cfg.VerifyComponents) return nil } @@ -416,7 +427,7 @@ func (k *Bootstrapper) needsReset(conf string, hostname string, port int, client return true } - if err := kverify.ExpectedComponentsRunning(client); err != nil { + if err := kverify.ExpectAppsRunning(client, kverify.AppsRunningList); err != nil { glog.Infof("needs reset: %v", err) return true }