kubernetes
/
minikube
mirror of https://github.com/kubernetes/minikube.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Improve handling of paused none clusters.

pull/6452/head
Thomas Stromberg 2020-01-31 15:27:08 -08:00
parent 40f2f7c28d
commit dd4ad10b83
5 changed files with 128 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
cmd/minikube/cmd/delete.go
View File

@ -35,6 +35,7 @@ import (
"k8s.io/minikube/pkg/minikube/cluster"
pkg_config "k8s.io/minikube/pkg/minikube/config"
"k8s.io/minikube/pkg/minikube/constants"
"k8s.io/minikube/pkg/minikube/cruntime"
"k8s.io/minikube/pkg/minikube/driver"
"k8s.io/minikube/pkg/minikube/exit"
"k8s.io/minikube/pkg/minikube/kubeconfig"
@ -194,7 +195,7 @@ func deleteProfile(profile *pkg_config.Profile) error {
}
if err == nil && driver.BareMetal(cc.VMDriver) {
if err := uninstallKubernetes(api, cc.KubernetesConfig, viper.GetString(cmdcfg.Bootstrapper)); err != nil {
if err := uninstallKubernetes(api, profile.Name, cc.KubernetesConfig, viper.GetString(cmdcfg.Bootstrapper)); err != nil {
deletionError, ok := err.(DeletionError)
if ok {
delErr := profileDeletionErr(profile.Name, fmt.Sprintf("%v", err))
@ -276,12 +277,34 @@ func profileDeletionErr(profileName string, additionalInfo string) error {
return fmt.Errorf("error deleting profile \"%s\": %s", profileName, additionalInfo)
}
func uninstallKubernetes(api libmachine.API, kc pkg_config.KubernetesConfig, bsName string) error {
func uninstallKubernetes(api libmachine.API, profile string, kc pkg_config.KubernetesConfig, bsName string) error {
out.T(out.Resetting, "Uninstalling Kubernetes {{.kubernetes_version}} using {{.bootstrapper_name}} ...", out.V{"kubernetes_version": kc.KubernetesVersion, "bootstrapper_name": bsName})
clusterBootstrapper, err := getClusterBootstrapper(api, bsName)
if err != nil {
return DeletionError{Err: fmt.Errorf("unable to get bootstrapper: %v", err), Errtype: Fatal}
} else if err = clusterBootstrapper.DeleteCluster(kc); err != nil {
}
host, err := cluster.CheckIfHostExistsAndLoad(api, profile)
if err != nil {
exit.WithError("Error getting host", err)
}
r, err := machine.CommandRunner(host)
if err != nil {
exit.WithError("Failed to get command runner", err)
}
cr, err := cruntime.New(cruntime.Config{Type: kc.ContainerRuntime, Runner: r})
if err != nil {
exit.WithError("Failed runtime", err)
}
// Unpause the cluster if necessary to avoid hung kubeadm
_, err = cluster.Unpause(cr, r, nil)
if err != nil {
glog.Errorf("unpause failed: %v", err)
}
if err = clusterBootstrapper.DeleteCluster(kc); err != nil {
return DeletionError{Err: fmt.Errorf("failed to delete cluster: %v", err), Errtype: Fatal}
}
return nil

23
cmd/minikube/cmd/status.go
View File

@ -30,7 +30,7 @@ import (
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/spf13/viper"
cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config"
"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/kverify"
"k8s.io/minikube/pkg/minikube/cluster"
"k8s.io/minikube/pkg/minikube/config"
"k8s.io/minikube/pkg/minikube/constants"
@ -142,24 +142,31 @@ func status(api libmachine.API, name string) (*Status, error) {
return st, errors.Wrap(err, "host")
}
// Nonexistent it is!
if hs == state.None.String() {
return st, nil
}
st.Host = hs
if st.Host != state.Running.String() {
if st.Host != state.Running.String() && st.Host != state.Paused.String() {
return st, nil
}
bs, err := getClusterBootstrapper(api, viper.GetString(cmdcfg.Bootstrapper))
host, err := cluster.CheckIfHostExistsAndLoad(api, name)
if err != nil {
return st, errors.Wrap(err, "bootstrapper")
return st, err
}
st.Kubelet, err = bs.GetKubeletStatus()
cr, err := machine.CommandRunner(host)
if err != nil {
return st, err
}
stk, err := kverify.KubeletStatus(cr)
if err != nil {
glog.Warningf("kubelet err: %v", err)
st.Kubelet = state.Error.String()
} else {
st.Kubelet = stk.String()
}
ip, err := cluster.GetHostDriverIP(api, name)
@ -175,10 +182,12 @@ func status(api libmachine.API, name string) (*Status, error) {
port = constants.APIServerPort
}
st.APIServer, err = bs.GetAPIServerStatus(ip, port)
sta, err := kverify.APIServerStatus(cr, ip, port)
if err != nil {
glog.Errorln("Error apiserver status:", err)
st.APIServer = state.Error.String()
} else {
st.APIServer = sta.String()
}
st.Kubeconfig = Misconfigured

23
pkg/drivers/none/none.go
View File

@ -26,7 +26,7 @@ import (
"github.com/docker/machine/libmachine/state"
"github.com/golang/glog"
"github.com/pkg/errors"
"k8s.io/apimachinery/pkg/util/net"
knet "k8s.io/apimachinery/pkg/util/net"
pkgdrivers "k8s.io/minikube/pkg/drivers"
"k8s.io/minikube/pkg/minikube/command"
"k8s.io/minikube/pkg/minikube/cruntime"
@ -94,7 +94,7 @@ func (d *Driver) DriverName() string {
// GetIP returns an IP or hostname that this host is available at
func (d *Driver) GetIP() (string, error) {
ip, err := net.ChooseHostInterface()
ip, err := knet.ChooseHostInterface()
if err != nil {
return "", err
}
@ -123,10 +123,13 @@ func (d *Driver) GetURL() (string, error) {
// GetState returns the state that the host is in (running, stopped, etc)
func (d *Driver) GetState() (state.State, error) {
if err := checkKubelet(d.exec); err != nil {
glog.Infof("kubelet not running: %v", err)
return state.Stopped, nil
_, err := d.GetIP()
if err != nil {
return state.Error, err
}
// If we got this far, it's safe to call the host as running like a VM would.
// Rely on other checks to determine if kubelet and apiserver are healthy.
return state.Running, nil
}
@ -251,13 +254,3 @@ func restartKubelet(cr command.Runner) error {
}
return nil
}
// checkKubelet returns an error if the kubelet is not running.
func checkKubelet(cr command.Runner) error {
glog.Infof("checking for running kubelet ...")
c := exec.Command("systemctl", "is-active", "--quiet", "service", "kubelet")
if _, err := cr.RunCmd(c); err != nil {
return errors.Wrap(err, "check kubelet")
}
return nil
}

82
pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
View File

@ -23,6 +23,8 @@ import (
"net"
"net/http"
"os/exec"
"path"
"strings"
"time"
"github.com/docker/machine/libmachine/state"
@ -101,12 +103,12 @@ func APIServerIsRunning(start time.Time, ip string, port int, timeout time.Durat
return false, fmt.Errorf("cluster wait timed out during healthz check")
}
status, err := APIServerStatus(net.ParseIP(ip), port)
status, err := apiServerHealthz(net.ParseIP(ip), port)
if err != nil {
glog.Warningf("status: %v", err)
return false, nil
}
if status != "Running" {
if status != state.Running {
return false, nil
}
return true, nil
@ -119,9 +121,53 @@ func APIServerIsRunning(start time.Time, ip string, port int, timeout time.Durat
return nil
}
// APIServerStatus hits the /healthz endpoint and returns libmachine style state.State
func APIServerStatus(ip net.IP, apiserverPort int) (string, error) {
url := fmt.Sprintf("https://%s/healthz", net.JoinHostPort(ip.String(), fmt.Sprint(apiserverPort)))
// APIServerStatus returns apiserver status in libmachine style state.State
func APIServerStatus(cr command.Runner, ip net.IP, port int) (state.State, error) {
glog.Infof("Checking apiserver status ...")
// sudo, in case hidepid is set
rr, err := cr.RunCmd(exec.Command("sudo", "pgrep", "kube-apiserver"))
if err != nil {
return state.Stopped, nil
}
pids := strings.Split(strings.TrimSpace(rr.Stdout.String()), "\n")
pid := pids[len(pids)-1]
if len(pids) != 1 {
glog.Errorf("found %d apiserver pids: %v - choosing %s", len(pids), pids, pid)
}
// Get the freezer cgroup entry for this pid
rr, err = cr.RunCmd(exec.Command("sudo", "egrep", "^[0-9]+:freezer:", path.Join("/proc", pid, "cgroup")))
if err != nil {
glog.Warningf("unable to find freezer cgroup: %v", err)
return apiServerHealthz(ip, port)
}
freezer := strings.TrimSpace(rr.Stdout.String())
glog.Infof("apiserver freezer: %q", freezer)
fparts := strings.Split(freezer, ":")
if len(fparts) != 3 {
glog.Warningf("unable to parse freezer - found %d parts: %s", len(fparts), freezer)
return apiServerHealthz(ip, port)
}
rr, err = cr.RunCmd(exec.Command("sudo", "cat", path.Join("/sys/fs/cgroup/freezer", fparts[2], "freezer.state")))
if err != nil {
glog.Errorf("unable to get freezer state: %s", rr.Stderr.String())
return apiServerHealthz(ip, port)
}
fs := strings.TrimSpace(rr.Stdout.String())
glog.Infof("freezer state: %q", fs)
if fs == "FREEZING" || fs == "FROZEN" {
return state.Paused, nil
}
return apiServerHealthz(ip, port)
}
// apiServerHealthz hits the /healthz endpoint and returns libmachine style state.State
func apiServerHealthz(ip net.IP, port int) (state.State, error) {
url := fmt.Sprintf("https://%s/healthz", net.JoinHostPort(ip.String(), fmt.Sprint(port)))
glog.Infof("Checking apiserver healthz at %s ...", url)
// To avoid: x509: certificate signed by unknown authority
tr := &http.Transport{
Proxy: nil, // To avoid connectiv issue if http(s)_proxy is set.
@ -131,11 +177,31 @@ func APIServerStatus(ip net.IP, apiserverPort int) (string, error) {
resp, err := client.Get(url)
// Connection refused, usually.
if err != nil {
return state.Stopped.String(), nil
return state.Stopped, nil
}
if resp.StatusCode != http.StatusOK {
glog.Warningf("%s response: %v %+v", url, err, resp)
return state.Error.String(), nil
return state.Error, nil
}
return state.Running.String(), nil
return state.Running, nil
}
func KubeletStatus(cr command.Runner) (state.State, error) {
glog.Infof("Checking kubelet status ...")
rr, err := cr.RunCmd(exec.Command("sudo", "systemctl", "is-active", "kubelet"))
if err != nil {
// Do not return now, as we still have parsing to do!
glog.Warningf("%s returned error: %v", rr.Command(), err)
}
s := strings.TrimSpace(rr.Stdout.String())
glog.Infof("kubelet is-active: %s", s)
switch s {
case "active":
return state.Running, nil
case "inactive":
return state.Stopped, nil
case "activating":
return state.Starting, nil
}
return state.Error, nil
}

38
pkg/minikube/bootstrapper/kubeadm/kubeadm.go
View File

@ -99,42 +99,12 @@ func (k *Bootstrapper) GetKubeletStatus() (string, error) {
}
// GetAPIServerStatus returns the api-server status
func (k *Bootstrapper) GetAPIServerStatus(ip net.IP, apiserverPort int) (string, error) {
// sudo, in case hidepid is set
rr, err := k.c.RunCmd(exec.Command("sudo", "pgrep", "kube-apiserver"))
func (k *Bootstrapper) GetAPIServerStatus(ip net.IP, port int) (string, error) {
s, err := kverify.APIServerStatus(k.c, ip, port)
if err != nil {
return state.Stopped.String(), nil
return state.Error.String(), err
}
pid := strings.TrimSpace(rr.Stdout.String())
// Get the freezer cgroup entry for this pid
rr, err = k.c.RunCmd(exec.Command("sudo", "egrep", "^[0-9]+:freezer:", path.Join("/proc", pid, "cgroup")))
if err != nil {
glog.Warningf("unable to find freezer cgroup: %v", err)
return kverify.APIServerStatus(ip, apiserverPort)
}
freezer := strings.TrimSpace(rr.Stdout.String())
glog.Infof("apiserver freezer: %q", freezer)
fparts := strings.Split(freezer, ":")
if len(fparts) != 3 {
glog.Warningf("unable to parse freezer - found %d parts: %s", len(fparts), freezer)
return kverify.APIServerStatus(ip, apiserverPort)
}
rr, err = k.c.RunCmd(exec.Command("sudo", "cat", path.Join("/sys/fs/cgroup/freezer", fparts[2], "freezer.state")))
if err != nil {
glog.Errorf("unable to get freezer state: %s", rr.Stderr.String())
return kverify.APIServerStatus(ip, apiserverPort)
}
fs := strings.TrimSpace(rr.Stdout.String())
glog.Infof("freezer state: %q", fs)
if fs == "FREEZING" || fs == "FROZEN" {
return state.Paused.String(), nil
}
return kverify.APIServerStatus(ip, apiserverPort)
return s.String(), nil
}
// LogCommands returns a map of log type to a command which will display that log.