kubernetes
/
minikube
mirror of https://github.com/kubernetes/minikube.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #7460 from medyagh/wait-coredns 

add new wait component apps_running
pull/7470/head
Medya Ghazizadeh 2020-04-06 18:30:14 -07:00 committed by GitHub
parent f99a546c9c b880469cd7
commit a577ed0465
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 128 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

124
pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
View File

@ -18,22 +18,7 @@ limitations under the License.
package kverify
import (
"fmt"
"os/exec"
"strings"
"time"
"github.com/docker/machine/libmachine/state"
"github.com/golang/glog"
core "k8s.io/api/core/v1"
meta "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/minikube/pkg/minikube/bootstrapper"
"k8s.io/minikube/pkg/minikube/command"
"k8s.io/minikube/pkg/minikube/config"
"k8s.io/minikube/pkg/minikube/cruntime"
"k8s.io/minikube/pkg/minikube/logs"
)
// minLogCheckTime how long to wait before spamming error logs to console
@ -46,6 +31,8 @@ const (
SystemPodsWaitKey = "system_pods"
// DefaultSAWaitKey is the name used in the flags for default service account
DefaultSAWaitKey = "default_sa"
// AppsRunning is the name used in the flags for waiting for k8s-apps to be running
AppsRunning = "apps_running"
)
// vars related to the --wait flag
@ -53,13 +40,22 @@ var (
// DefaultComponents is map of the the default components to wait for
DefaultComponents = map[string]bool{APIServerWaitKey: true, SystemPodsWaitKey: true}
// NoWaitComponents is map of componets to wait for if specified 'none' or 'false'
NoComponents = map[string]bool{APIServerWaitKey: false, SystemPodsWaitKey: false, DefaultSAWaitKey: false}
NoComponents = map[string]bool{APIServerWaitKey: false, SystemPodsWaitKey: false, DefaultSAWaitKey: false, AppsRunning: false}
// AllComponents is map for waiting for all components.
AllComponents = map[string]bool{APIServerWaitKey: true, SystemPodsWaitKey: true, DefaultSAWaitKey: true}
AllComponents = map[string]bool{APIServerWaitKey: true, SystemPodsWaitKey: true, DefaultSAWaitKey: true, AppsRunning: true}
// DefaultWaitList is list of all default components to wait for. only names to be used for start flags.
DefaultWaitList = []string{APIServerWaitKey, SystemPodsWaitKey}
// AllComponentsList list of all valid components keys to wait for. only names to be used used for start flags.
AllComponentsList = []string{APIServerWaitKey, SystemPodsWaitKey, DefaultSAWaitKey}
AllComponentsList = []string{APIServerWaitKey, SystemPodsWaitKey, DefaultSAWaitKey, AppsRunning}
// AppsRunningList running list are valid k8s-app components to wait for them to be running
AppsRunningList = []string{
"kube-dns", // coredns
"etcd",
"kube-apiserver",
"kube-controller-manager",
"kube-proxy",
"kube-scheduler",
}
)
// ShouldWait will return true if the config says need to wait
@ -71,95 +67,3 @@ func ShouldWait(wcs map[string]bool) bool {
}
return false
}
// ExpectedComponentsRunning returns whether or not all expected components are running
func ExpectedComponentsRunning(cs *kubernetes.Clientset) error {
expected := []string{
"kube-dns", // coredns
"etcd",
"kube-apiserver",
"kube-controller-manager",
"kube-proxy",
"kube-scheduler",
}
found := map[string]bool{}
pods, err := cs.CoreV1().Pods("kube-system").List(meta.ListOptions{})
if err != nil {
return err
}
for _, pod := range pods.Items {
glog.Infof("found pod: %s", podStatusMsg(pod))
if pod.Status.Phase != core.PodRunning {
continue
}
for k, v := range pod.ObjectMeta.Labels {
if k == "component" || k == "k8s-app" {
found[v] = true
}
}
}
missing := []string{}
for _, e := range expected {
if !found[e] {
missing = append(missing, e)
}
}
if len(missing) > 0 {
return fmt.Errorf("missing components: %v", strings.Join(missing, ", "))
}
return nil
}
// podStatusMsg returns a human-readable pod status, for generating debug status
func podStatusMsg(pod core.Pod) string {
var sb strings.Builder
sb.WriteString(fmt.Sprintf("%q [%s] %s", pod.ObjectMeta.GetName(), pod.ObjectMeta.GetUID(), pod.Status.Phase))
for i, c := range pod.Status.Conditions {
if c.Reason != "" {
if i == 0 {
sb.WriteString(": ")
} else {
sb.WriteString(" / ")
}
sb.WriteString(fmt.Sprintf("%s:%s", c.Type, c.Reason))
}
if c.Message != "" {
sb.WriteString(fmt.Sprintf(" (%s)", c.Message))
}
}
return sb.String()
}
// announceProblems checks for problems, and slows polling down if any are found
func announceProblems(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg config.ClusterConfig, cr command.Runner) {
problems := logs.FindProblems(r, bs, cfg, cr)
if len(problems) > 0 {
logs.OutputProblems(problems, 5)
time.Sleep(kconst.APICallRetryInterval * 15)
}
}
// KubeletStatus checks the kubelet status
func KubeletStatus(cr command.Runner) (state.State, error) {
glog.Infof("Checking kubelet status ...")
rr, err := cr.RunCmd(exec.Command("sudo", "systemctl", "is-active", "kubelet"))
if err != nil {
// Do not return now, as we still have parsing to do!
glog.Warningf("%s returned error: %v", rr.Command(), err)
}
s := strings.TrimSpace(rr.Stdout.String())
glog.Infof("kubelet is-active: %s", s)
switch s {
case "active":
return state.Running, nil
case "inactive":
return state.Stopped, nil
case "activating":
return state.Starting, nil
}
return state.Error, nil
}

102
pkg/minikube/bootstrapper/bsutil/kverify/system_pods.go
View File

@ -19,9 +19,14 @@ package kverify
import (
"fmt"
"os/exec"
"strings"
"time"
"github.com/docker/machine/libmachine/state"
"github.com/golang/glog"
"github.com/pkg/errors"
core "k8s.io/api/core/v1"
meta "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes"
@ -30,6 +35,8 @@ import (
"k8s.io/minikube/pkg/minikube/command"
"k8s.io/minikube/pkg/minikube/config"
"k8s.io/minikube/pkg/minikube/cruntime"
"k8s.io/minikube/pkg/minikube/logs"
"k8s.io/minikube/pkg/util/retry"
)
// WaitForSystemPods verifies essential pods for running kurnetes is running
@ -68,3 +75,98 @@ func WaitForSystemPods(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg con
glog.Infof("duration metric: took %s to wait for pod list to return data ...", time.Since(pStart))
return nil
}
// ExpectAppsRunning returns whether or not all expected k8s-apps are running. (without waiting for them)
func ExpectAppsRunning(cs *kubernetes.Clientset, expected []string) error {
found := map[string]bool{}
pods, err := cs.CoreV1().Pods("kube-system").List(meta.ListOptions{})
if err != nil {
return err
}
for _, pod := range pods.Items {
glog.Infof("found pod: %s", podStatusMsg(pod))
if pod.Status.Phase != core.PodRunning {
continue
}
for k, v := range pod.ObjectMeta.Labels {
if k == "component" || k == "k8s-app" {
found[v] = true
}
}
}
missing := []string{}
for _, e := range expected {
if !found[e] {
missing = append(missing, e)
}
}
if len(missing) > 0 {
return fmt.Errorf("missing components: %v", strings.Join(missing, ", "))
}
return nil
}
// WaitForAppsRunning waits for expected Apps To be running
func WaitForAppsRunning(cs *kubernetes.Clientset, expected []string, timeout time.Duration) error {
glog.Info("waiting for k8s-apps to be running ...")
start := time.Now()
checkRunning := func() error { return ExpectAppsRunning(cs, expected) }
if err := retry.Expo(checkRunning, 500*time.Millisecond, timeout); err != nil {
return errors.Wrap(err, "waitings for k8s app running")
}
glog.Infof("duration metric: took %s to wait for k8s-apps to be running ...", time.Since(start))
return nil
}
// podStatusMsg returns a human-readable pod status, for generating debug status
func podStatusMsg(pod core.Pod) string {
var sb strings.Builder
sb.WriteString(fmt.Sprintf("%q [%s] %s", pod.ObjectMeta.GetName(), pod.ObjectMeta.GetUID(), pod.Status.Phase))
for i, c := range pod.Status.Conditions {
if c.Reason != "" {
if i == 0 {
sb.WriteString(": ")
} else {
sb.WriteString(" / ")
}
sb.WriteString(fmt.Sprintf("%s:%s", c.Type, c.Reason))
}
if c.Message != "" {
sb.WriteString(fmt.Sprintf(" (%s)", c.Message))
}
}
return sb.String()
}
// announceProblems checks for problems, and slows polling down if any are found
func announceProblems(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg config.ClusterConfig, cr command.Runner) {
problems := logs.FindProblems(r, bs, cfg, cr)
if len(problems) > 0 {
logs.OutputProblems(problems, 5)
time.Sleep(kconst.APICallRetryInterval * 15)
}
}
// KubeletStatus checks the kubelet status
func KubeletStatus(cr command.Runner) (state.State, error) {
glog.Infof("Checking kubelet status ...")
rr, err := cr.RunCmd(exec.Command("sudo", "systemctl", "is-active", "kubelet"))
if err != nil {
// Do not return now, as we still have parsing to do!
glog.Warningf("%s returned error: %v", rr.Command(), err)
}
s := strings.TrimSpace(rr.Stdout.String())
glog.Infof("kubelet is-active: %s", s)
switch s {
case "active":
return state.Running, nil
case "inactive":
return state.Stopped, nil
case "activating":
return state.Starting, nil
}
return state.Error, nil
}

13
pkg/minikube/bootstrapper/kubeadm/kubeadm.go
View File

@ -394,6 +394,17 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time
return errors.Wrap(err, "waiting for default service account")
}
}
if cfg.VerifyComponents[kverify.AppsRunning] {
client, err := k.client(hostname, port)
if err != nil {
return errors.Wrap(err, "get k8s client")
}
if err := kverify.WaitForAppsRunning(client, kverify.AppsRunningList, timeout); err != nil {
return errors.Wrap(err, "waiting for apps_running")
}
}
glog.Infof("duration metric: took %s to wait for : %+v ...", time.Since(start), cfg.VerifyComponents)
return nil
}
@ -416,7 +427,7 @@ func (k *Bootstrapper) needsReset(conf string, hostname string, port int, client
return true
}
if err := kverify.ExpectedComponentsRunning(client); err != nil {
if err := kverify.ExpectAppsRunning(client, kverify.AppsRunningList); err != nil {
glog.Infof("needs reset: %v", err)
return true
}