diff --git a/cmd/minikube/cmd/delete.go b/cmd/minikube/cmd/delete.go index 5efe7e2ee3..832aa2af2d 100644 --- a/cmd/minikube/cmd/delete.go +++ b/cmd/minikube/cmd/delete.go @@ -35,6 +35,7 @@ import ( "k8s.io/minikube/pkg/minikube/cluster" pkg_config "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/constants" + "k8s.io/minikube/pkg/minikube/cruntime" "k8s.io/minikube/pkg/minikube/driver" "k8s.io/minikube/pkg/minikube/exit" "k8s.io/minikube/pkg/minikube/kubeconfig" @@ -194,7 +195,7 @@ func deleteProfile(profile *pkg_config.Profile) error { } if err == nil && driver.BareMetal(cc.VMDriver) { - if err := uninstallKubernetes(api, cc.KubernetesConfig, viper.GetString(cmdcfg.Bootstrapper)); err != nil { + if err := uninstallKubernetes(api, profile.Name, cc.KubernetesConfig, viper.GetString(cmdcfg.Bootstrapper)); err != nil { deletionError, ok := err.(DeletionError) if ok { delErr := profileDeletionErr(profile.Name, fmt.Sprintf("%v", err)) @@ -276,12 +277,34 @@ func profileDeletionErr(profileName string, additionalInfo string) error { return fmt.Errorf("error deleting profile \"%s\": %s", profileName, additionalInfo) } -func uninstallKubernetes(api libmachine.API, kc pkg_config.KubernetesConfig, bsName string) error { +func uninstallKubernetes(api libmachine.API, profile string, kc pkg_config.KubernetesConfig, bsName string) error { out.T(out.Resetting, "Uninstalling Kubernetes {{.kubernetes_version}} using {{.bootstrapper_name}} ...", out.V{"kubernetes_version": kc.KubernetesVersion, "bootstrapper_name": bsName}) clusterBootstrapper, err := getClusterBootstrapper(api, bsName) if err != nil { return DeletionError{Err: fmt.Errorf("unable to get bootstrapper: %v", err), Errtype: Fatal} - } else if err = clusterBootstrapper.DeleteCluster(kc); err != nil { + } + + host, err := cluster.CheckIfHostExistsAndLoad(api, profile) + if err != nil { + exit.WithError("Error getting host", err) + } + r, err := machine.CommandRunner(host) + if err != nil { + exit.WithError("Failed to get command runner", err) + } + + cr, err := cruntime.New(cruntime.Config{Type: kc.ContainerRuntime, Runner: r}) + if err != nil { + exit.WithError("Failed runtime", err) + } + + // Unpause the cluster if necessary to avoid hung kubeadm + _, err = cluster.Unpause(cr, r, nil) + if err != nil { + glog.Errorf("unpause failed: %v", err) + } + + if err = clusterBootstrapper.DeleteCluster(kc); err != nil { return DeletionError{Err: fmt.Errorf("failed to delete cluster: %v", err), Errtype: Fatal} } return nil diff --git a/cmd/minikube/cmd/status.go b/cmd/minikube/cmd/status.go index ad050cf199..9af9a12a39 100644 --- a/cmd/minikube/cmd/status.go +++ b/cmd/minikube/cmd/status.go @@ -30,7 +30,7 @@ import ( "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/spf13/viper" - cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config" + "k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/kverify" "k8s.io/minikube/pkg/minikube/cluster" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/constants" @@ -142,24 +142,31 @@ func status(api libmachine.API, name string) (*Status, error) { return st, errors.Wrap(err, "host") } - // Nonexistent it is! if hs == state.None.String() { return st, nil } + st.Host = hs - if st.Host != state.Running.String() { + if st.Host != state.Running.String() && st.Host != state.Paused.String() { return st, nil } - bs, err := getClusterBootstrapper(api, viper.GetString(cmdcfg.Bootstrapper)) + host, err := cluster.CheckIfHostExistsAndLoad(api, name) if err != nil { - return st, errors.Wrap(err, "bootstrapper") + return st, err } - st.Kubelet, err = bs.GetKubeletStatus() + cr, err := machine.CommandRunner(host) + if err != nil { + return st, err + } + + stk, err := kverify.KubeletStatus(cr) if err != nil { glog.Warningf("kubelet err: %v", err) st.Kubelet = state.Error.String() + } else { + st.Kubelet = stk.String() } ip, err := cluster.GetHostDriverIP(api, name) @@ -175,10 +182,12 @@ func status(api libmachine.API, name string) (*Status, error) { port = constants.APIServerPort } - st.APIServer, err = bs.GetAPIServerStatus(ip, port) + sta, err := kverify.APIServerStatus(cr, ip, port) if err != nil { glog.Errorln("Error apiserver status:", err) st.APIServer = state.Error.String() + } else { + st.APIServer = sta.String() } st.Kubeconfig = Misconfigured diff --git a/pkg/drivers/none/none.go b/pkg/drivers/none/none.go index 7c431e8267..b5bfe49336 100644 --- a/pkg/drivers/none/none.go +++ b/pkg/drivers/none/none.go @@ -26,7 +26,7 @@ import ( "github.com/docker/machine/libmachine/state" "github.com/golang/glog" "github.com/pkg/errors" - "k8s.io/apimachinery/pkg/util/net" + knet "k8s.io/apimachinery/pkg/util/net" pkgdrivers "k8s.io/minikube/pkg/drivers" "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/cruntime" @@ -94,7 +94,7 @@ func (d *Driver) DriverName() string { // GetIP returns an IP or hostname that this host is available at func (d *Driver) GetIP() (string, error) { - ip, err := net.ChooseHostInterface() + ip, err := knet.ChooseHostInterface() if err != nil { return "", err } @@ -123,10 +123,13 @@ func (d *Driver) GetURL() (string, error) { // GetState returns the state that the host is in (running, stopped, etc) func (d *Driver) GetState() (state.State, error) { - if err := checkKubelet(d.exec); err != nil { - glog.Infof("kubelet not running: %v", err) - return state.Stopped, nil + _, err := d.GetIP() + if err != nil { + return state.Error, err } + + // If we got this far, it's safe to call the host as running like a VM would. + // Rely on other checks to determine if kubelet and apiserver are healthy. return state.Running, nil } @@ -251,13 +254,3 @@ func restartKubelet(cr command.Runner) error { } return nil } - -// checkKubelet returns an error if the kubelet is not running. -func checkKubelet(cr command.Runner) error { - glog.Infof("checking for running kubelet ...") - c := exec.Command("systemctl", "is-active", "--quiet", "service", "kubelet") - if _, err := cr.RunCmd(c); err != nil { - return errors.Wrap(err, "check kubelet") - } - return nil -} diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go index 00374eb23b..0a056adcf4 100644 --- a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go +++ b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go @@ -23,6 +23,8 @@ import ( "net" "net/http" "os/exec" + "path" + "strings" "time" "github.com/docker/machine/libmachine/state" @@ -101,12 +103,12 @@ func APIServerIsRunning(start time.Time, ip string, port int, timeout time.Durat return false, fmt.Errorf("cluster wait timed out during healthz check") } - status, err := APIServerStatus(net.ParseIP(ip), port) + status, err := apiServerHealthz(net.ParseIP(ip), port) if err != nil { glog.Warningf("status: %v", err) return false, nil } - if status != "Running" { + if status != state.Running { return false, nil } return true, nil @@ -119,9 +121,53 @@ func APIServerIsRunning(start time.Time, ip string, port int, timeout time.Durat return nil } -// APIServerStatus hits the /healthz endpoint and returns libmachine style state.State -func APIServerStatus(ip net.IP, apiserverPort int) (string, error) { - url := fmt.Sprintf("https://%s/healthz", net.JoinHostPort(ip.String(), fmt.Sprint(apiserverPort))) +// APIServerStatus returns apiserver status in libmachine style state.State +func APIServerStatus(cr command.Runner, ip net.IP, port int) (state.State, error) { + glog.Infof("Checking apiserver status ...") + // sudo, in case hidepid is set + rr, err := cr.RunCmd(exec.Command("sudo", "pgrep", "kube-apiserver")) + if err != nil { + return state.Stopped, nil + } + pids := strings.Split(strings.TrimSpace(rr.Stdout.String()), "\n") + pid := pids[len(pids)-1] + if len(pids) != 1 { + glog.Errorf("found %d apiserver pids: %v - choosing %s", len(pids), pids, pid) + } + + // Get the freezer cgroup entry for this pid + rr, err = cr.RunCmd(exec.Command("sudo", "egrep", "^[0-9]+:freezer:", path.Join("/proc", pid, "cgroup"))) + if err != nil { + glog.Warningf("unable to find freezer cgroup: %v", err) + return apiServerHealthz(ip, port) + + } + freezer := strings.TrimSpace(rr.Stdout.String()) + glog.Infof("apiserver freezer: %q", freezer) + fparts := strings.Split(freezer, ":") + if len(fparts) != 3 { + glog.Warningf("unable to parse freezer - found %d parts: %s", len(fparts), freezer) + return apiServerHealthz(ip, port) + } + + rr, err = cr.RunCmd(exec.Command("sudo", "cat", path.Join("/sys/fs/cgroup/freezer", fparts[2], "freezer.state"))) + if err != nil { + glog.Errorf("unable to get freezer state: %s", rr.Stderr.String()) + return apiServerHealthz(ip, port) + } + + fs := strings.TrimSpace(rr.Stdout.String()) + glog.Infof("freezer state: %q", fs) + if fs == "FREEZING" || fs == "FROZEN" { + return state.Paused, nil + } + return apiServerHealthz(ip, port) +} + +// apiServerHealthz hits the /healthz endpoint and returns libmachine style state.State +func apiServerHealthz(ip net.IP, port int) (state.State, error) { + url := fmt.Sprintf("https://%s/healthz", net.JoinHostPort(ip.String(), fmt.Sprint(port))) + glog.Infof("Checking apiserver healthz at %s ...", url) // To avoid: x509: certificate signed by unknown authority tr := &http.Transport{ Proxy: nil, // To avoid connectiv issue if http(s)_proxy is set. @@ -131,11 +177,31 @@ func APIServerStatus(ip net.IP, apiserverPort int) (string, error) { resp, err := client.Get(url) // Connection refused, usually. if err != nil { - return state.Stopped.String(), nil + return state.Stopped, nil } if resp.StatusCode != http.StatusOK { glog.Warningf("%s response: %v %+v", url, err, resp) - return state.Error.String(), nil + return state.Error, nil } - return state.Running.String(), nil + return state.Running, nil +} + +func KubeletStatus(cr command.Runner) (state.State, error) { + glog.Infof("Checking kubelet status ...") + rr, err := cr.RunCmd(exec.Command("sudo", "systemctl", "is-active", "kubelet")) + if err != nil { + // Do not return now, as we still have parsing to do! + glog.Warningf("%s returned error: %v", rr.Command(), err) + } + s := strings.TrimSpace(rr.Stdout.String()) + glog.Infof("kubelet is-active: %s", s) + switch s { + case "active": + return state.Running, nil + case "inactive": + return state.Stopped, nil + case "activating": + return state.Starting, nil + } + return state.Error, nil } diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 73aea4464e..e2f5074947 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -99,42 +99,12 @@ func (k *Bootstrapper) GetKubeletStatus() (string, error) { } // GetAPIServerStatus returns the api-server status -func (k *Bootstrapper) GetAPIServerStatus(ip net.IP, apiserverPort int) (string, error) { - // sudo, in case hidepid is set - rr, err := k.c.RunCmd(exec.Command("sudo", "pgrep", "kube-apiserver")) +func (k *Bootstrapper) GetAPIServerStatus(ip net.IP, port int) (string, error) { + s, err := kverify.APIServerStatus(k.c, ip, port) if err != nil { - return state.Stopped.String(), nil + return state.Error.String(), err } - pid := strings.TrimSpace(rr.Stdout.String()) - - // Get the freezer cgroup entry for this pid - rr, err = k.c.RunCmd(exec.Command("sudo", "egrep", "^[0-9]+:freezer:", path.Join("/proc", pid, "cgroup"))) - if err != nil { - glog.Warningf("unable to find freezer cgroup: %v", err) - return kverify.APIServerStatus(ip, apiserverPort) - - } - freezer := strings.TrimSpace(rr.Stdout.String()) - glog.Infof("apiserver freezer: %q", freezer) - fparts := strings.Split(freezer, ":") - if len(fparts) != 3 { - glog.Warningf("unable to parse freezer - found %d parts: %s", len(fparts), freezer) - return kverify.APIServerStatus(ip, apiserverPort) - } - - rr, err = k.c.RunCmd(exec.Command("sudo", "cat", path.Join("/sys/fs/cgroup/freezer", fparts[2], "freezer.state"))) - if err != nil { - glog.Errorf("unable to get freezer state: %s", rr.Stderr.String()) - return kverify.APIServerStatus(ip, apiserverPort) - } - - fs := strings.TrimSpace(rr.Stdout.String()) - glog.Infof("freezer state: %q", fs) - if fs == "FREEZING" || fs == "FROZEN" { - return state.Paused.String(), nil - } - - return kverify.APIServerStatus(ip, apiserverPort) + return s.String(), nil } // LogCommands returns a map of log type to a command which will display that log.