resolve conflict

pkg/minikube/bootstrapper/bsutil/kverify/kubelet.go

@@ -0,0 +1,60 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package kverify verifies a running Kubernetes cluster is healthy
+package kverify
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/docker/machine/libmachine/state"
+	"k8s.io/klog/v2"
+	"k8s.io/minikube/pkg/minikube/command"
+	"k8s.io/minikube/pkg/minikube/sysinit"
+	"k8s.io/minikube/pkg/util/retry"
+)
+
+// KubeletStatus checks the kubelet status
+func KubeletStatus(cr command.Runner) state.State {
+	klog.Infof("Checking kubelet status ...")
+	active := sysinit.New(cr).Active("kubelet")
+	if active {
+		return state.Running
+	}
+	return state.Stopped
+}
+
+// WaitForKubelet will wait for Kubelet service to be running ...
+func WaitForKubelet(cr command.Runner, timeout time.Duration) error {
+	pStart := time.Now()
+	klog.Infof("waiting for kubelet to be running ....")
+	kr := func() error {
+		if st := KubeletStatus(cr); st != state.Running {
+			return fmt.Errorf("status %s", st)
+		}
+		return nil
+	}
+
+	if err := retry.Local(kr, timeout); err != nil {
+		return fmt.Errorf("not running: %s", err)
+	}
+
+	klog.Infof("duration metric: took %s WaitForKubelet to finish.", time.Since(pStart))
+
+	return nil
+
+}

pkg/minikube/bootstrapper/bsutil/kverify/kverify.go

@@ -35,6 +35,8 @@ const (
 	AppsRunningKey = "apps_running"
 	// NodeReadyKey is the name used in the flags for waiting for the node status to be ready
 	NodeReadyKey = "node_ready"
+	// NodeReadyKey is the name used in the flags for waiting for the node status to be ready
+	KubeletKey = "kubelet"
 )
 
 //  vars related to the --wait flag
@@ -42,13 +44,13 @@ var (
 	// DefaultComponents is map of the the default components to wait for
 	DefaultComponents = map[string]bool{APIServerWaitKey: true, SystemPodsWaitKey: true}
 	// NoWaitComponents is map of componets to wait for if specified 'none' or 'false'
-	NoComponents = map[string]bool{APIServerWaitKey: false, SystemPodsWaitKey: false, DefaultSAWaitKey: false, AppsRunningKey: false, NodeReadyKey: false}
+	NoComponents = map[string]bool{APIServerWaitKey: false, SystemPodsWaitKey: false, DefaultSAWaitKey: false, AppsRunningKey: false, NodeReadyKey: false, KubeletKey: false}
 	// AllComponents is map for waiting for all components.
-	AllComponents = map[string]bool{APIServerWaitKey: true, SystemPodsWaitKey: true, DefaultSAWaitKey: true, AppsRunningKey: true}
+	AllComponents = map[string]bool{APIServerWaitKey: true, SystemPodsWaitKey: true, DefaultSAWaitKey: true, AppsRunningKey: true, KubeletKey: true}
 	// DefaultWaitList is list of all default components to wait for. only names to be used for start flags.
 	DefaultWaitList = []string{APIServerWaitKey, SystemPodsWaitKey}
 	// AllComponentsList list of all valid components keys to wait for. only names to be used used for start flags.
-	AllComponentsList = []string{APIServerWaitKey, SystemPodsWaitKey, DefaultSAWaitKey, AppsRunningKey, NodeReadyKey}
+	AllComponentsList = []string{APIServerWaitKey, SystemPodsWaitKey, DefaultSAWaitKey, AppsRunningKey, NodeReadyKey, KubeletKey}
 	// AppsRunningList running list are valid k8s-app components to wait for them to be running
 	AppsRunningList = []string{
 		"kube-dns", // coredns

pkg/minikube/bootstrapper/bsutil/kverify/system_pods.go

@@ -22,7 +22,14 @@ import (
 	"strings"
 	"time"
 
+<<<<<<< HEAD
 	"github.com/docker/machine/libmachine/state"
+||||||| parent of a1b34164f (ensure kubelet is running and wait for it)
+	"github.com/docker/machine/libmachine/state"
+	"github.com/golang/glog"
+=======
+	"github.com/golang/glog"
+>>>>>>> a1b34164f (ensure kubelet is running and wait for it)
 	"github.com/pkg/errors"
 	core "k8s.io/api/core/v1"
 	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -34,7 +41,6 @@ import (
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/cruntime"
 	"k8s.io/minikube/pkg/minikube/logs"
-	"k8s.io/minikube/pkg/minikube/sysinit"
 	"k8s.io/minikube/pkg/util/retry"
 )
 
@@ -156,12 +162,4 @@ func announceProblems(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg conf
 	}
 }
 
-// KubeletStatus checks the kubelet status
-func KubeletStatus(cr command.Runner) state.State {
-	klog.Infof("Checking kubelet status ...")
-	active := sysinit.New(cr).Active("kubelet")
-	if active {
-		return state.Running
-	}
-	return state.Stopped
-}
+

pkg/minikube/bootstrapper/kubeadm/kubeadm.go

@@ -392,10 +392,13 @@ func (k *Bootstrapper) client(ip string, port int) (*kubernetes.Clientset, error
 // WaitForNode blocks until the node appears to be healthy
 func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, timeout time.Duration) error {
 	start := time.Now()
-
 	register.Reg.SetStep(register.VerifyingKubernetes)
 	out.T(style.HealthCheck, "Verifying Kubernetes components...")
-
+	// regardless if waiting or not, we make sure it is not stopped
+	// to solve corner cases when a container is hibernated and once coming back kubelet not running.
+	if err := k.ensureKubeletStarted(); err != nil {
+		klog.Warningf("Couldn't ensure kubelet is started this might cause issues: %v", err)
+	}
 	// TODO: #7706: for better performance we could use k.client inside minikube to avoid asking for external IP:PORT
 	cp, err := config.PrimaryControlPlane(&cfg)
 	if err != nil {
@@ -455,6 +458,12 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time
 			}
 		}
 	}
+	if cfg.VerifyComponents[kverify.KubeletKey] {
+		if err := kverify.WaitForKubelet(k.c, timeout); err != nil {
+			return errors.Wrap(err, "waiting for kubelet")
+		}
+
+	}
 
 	if cfg.VerifyComponents[kverify.NodeReadyKey] {
 		if err := kverify.WaitForNodeReady(client, timeout); err != nil {
@@ -471,6 +480,16 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time
 	return nil
 }
 
+// ensureKubeletStarted will start kubelet if whatever reason it is stopped.
+func (k *Bootstrapper) ensureKubeletStarted() error {
+	fmt.Println("inside ensureKubeletStarted")
+	if kverify.KubeletStatus(k.c) != state.Running {
+		fmt.Println("kubelet was not running for some reason , starting it...")
+		return sysinit.New(k.c).Start("kubelet")
+	}
+	return nil
+}
+
 // needsReconfigure returns whether or not the cluster needs to be reconfigured
 func (k *Bootstrapper) needsReconfigure(conf string, hostname string, port int, client *kubernetes.Clientset, version string) bool {
 	if rr, err := k.c.RunCmd(exec.Command("sudo", "diff", "-u", conf, conf+".new")); err != nil {

pkg/minikube/node/start.go

@@ -196,7 +196,7 @@ func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) {
 		}
 	}
 
-	klog.Infof("Will wait %s for node ...", waitTimeout)
+	klog.Infof("Will wait %s for node up to ", viper.GetDuration(waitTimeout))
 	if err := bs.WaitForNode(*starter.Cfg, *starter.Node, viper.GetDuration(waitTimeout)); err != nil {
 		return nil, errors.Wrapf(err, "wait %s for node", viper.GetDuration(waitTimeout))
 	}

site/content/en/docs/commands/start.md

@@ -90,7 +90,7 @@ minikube start [flags]
       --uuid string                       Provide VM UUID to restore MAC address (hyperkit driver only)
       --vm                                Filter to use only VM Drivers
       --vm-driver driver                  DEPRECATED, use driver instead.
-      --wait strings                      comma separated list of Kubernetes components to verify and wait for after starting a cluster. defaults to "apiserver,system_pods", available options: "apiserver,system_pods,default_sa,apps_running,node_ready" . other acceptable values are 'all' or 'none', 'true' and 'false' (default [apiserver,system_pods])
+      --wait strings                      comma separated list of Kubernetes components to verify and wait for after starting a cluster. defaults to "apiserver,system_pods", available options: "apiserver,system_pods,default_sa,apps_running,node_ready,kubelet" . other acceptable values are 'all' or 'none', 'true' and 'false' (default [apiserver,system_pods])
       --wait-timeout duration             max time to wait per Kubernetes or host to be healthy. (default 6m0s)
 ```