diff --git a/cmd/minikube/cmd/delete.go b/cmd/minikube/cmd/delete.go index 1dd6018ba5..def7952dc7 100644 --- a/cmd/minikube/cmd/delete.go +++ b/cmd/minikube/cmd/delete.go @@ -88,7 +88,7 @@ associated files.`, } exit.WithError("Failed to remove profile", err) } - console.OutStyle("crushed", "The %q cluster is now deleted. I hope you are happy.", profile) + console.OutStyle("crushed", "The %q cluster has been deleted.", profile) }, } diff --git a/cmd/minikube/cmd/logs.go b/cmd/minikube/cmd/logs.go index 5340cd0c50..2df75e4e6a 100644 --- a/cmd/minikube/cmd/logs.go +++ b/cmd/minikube/cmd/logs.go @@ -17,17 +17,28 @@ limitations under the License. package cmd import ( - "os" - "github.com/spf13/cobra" "github.com/spf13/viper" cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config" + "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/cruntime" "k8s.io/minikube/pkg/minikube/exit" + "k8s.io/minikube/pkg/minikube/logs" "k8s.io/minikube/pkg/minikube/machine" ) +const ( + // number of problems per log to output + numberOfProblems = 5 +) + var ( - follow bool + // followLogs triggers tail -f mode + followLogs bool + // numberOfLines is how many lines to output, set via -n + numberOfLines int + // showProblems only shows lines that match known issues + showProblems bool ) // logsCmd represents the logs command @@ -36,17 +47,47 @@ var logsCmd = &cobra.Command{ Short: "Gets the logs of the running instance, used for debugging minikube, not user code", Long: `Gets the logs of the running instance, used for debugging minikube, not user code.`, Run: func(cmd *cobra.Command, args []string) { + cfg, err := config.Load() + if err != nil { + exit.WithError("Error getting config", err) + } + api, err := machine.NewAPIClient() if err != nil { exit.WithError("Error getting client", err) } defer api.Close() - clusterBootstrapper, err := GetClusterBootstrapper(api, viper.GetString(cmdcfg.Bootstrapper)) + + h, err := api.Load(config.GetMachineName()) + if err != nil { + exit.WithError("api load", err) + } + runner, err := machine.CommandRunner(h) + if err != nil { + exit.WithError("command runner", err) + } + bs, err := GetClusterBootstrapper(api, viper.GetString(cmdcfg.Bootstrapper)) if err != nil { exit.WithError("Error getting cluster bootstrapper", err) } - err = clusterBootstrapper.GetClusterLogsTo(follow, os.Stdout) + cr, err := cruntime.New(cruntime.Config{Type: cfg.KubernetesConfig.ContainerRuntime, Runner: runner}) + if err != nil { + exit.WithError("Unable to get runtime", err) + } + if followLogs { + err := logs.Follow(cr, bs, runner) + if err != nil { + exit.WithError("Follow", err) + } + return + } + if showProblems { + problems := logs.FindProblems(cr, bs, runner) + logs.OutputProblems(problems, numberOfProblems) + return + } + err = logs.Output(cr, bs, runner, numberOfLines) if err != nil { exit.WithError("Error getting machine logs", err) } @@ -54,6 +95,8 @@ var logsCmd = &cobra.Command{ } func init() { - logsCmd.Flags().BoolVarP(&follow, "follow", "f", false, "Show only the most recent journal entries, and continuously print new entries as they are appended to the journal.") + logsCmd.Flags().BoolVarP(&followLogs, "follow", "f", false, "Show only the most recent journal entries, and continuously print new entries as they are appended to the journal.") + logsCmd.Flags().BoolVar(&showProblems, "problems", false, "Show only log entries which point to known problems") + logsCmd.Flags().IntVarP(&numberOfLines, "length", "n", 50, "Number of lines back to go within the log") RootCmd.AddCommand(logsCmd) } diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index 1ede977003..395b8b0a3e 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -46,6 +46,7 @@ import ( "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/cruntime" "k8s.io/minikube/pkg/minikube/exit" + "k8s.io/minikube/pkg/minikube/logs" "k8s.io/minikube/pkg/minikube/machine" pkgutil "k8s.io/minikube/pkg/util" "k8s.io/minikube/pkg/util/kubeconfig" @@ -187,15 +188,19 @@ func runStart(cmd *cobra.Command, args []string) { if err := saveConfig(config); err != nil { exit.WithError("Failed to save config", err) } + runner, err := machine.CommandRunner(host) + if err != nil { + exit.WithError("Failed to get command runner", err) + } - configureRuntimes(host) + cr := configureRuntimes(host, runner) bs := prepareHostEnvironment(m, config.KubernetesConfig) waitCacheImages(&cacheGroup) // The kube config must be update must come before bootstrapping, otherwise health checks may use a stale IP kubeconfig := updateKubeConfig(host, &config) - bootstrapCluster(bs, config.KubernetesConfig, preexisting) - validateCluster(bs, ip) + bootstrapCluster(bs, cr, runner, config.KubernetesConfig, preexisting) + validateCluster(bs, cr, runner, ip) configureMounts() if err = LoadCachedImagesInConfigFile(); err != nil { console.Failure("Unable to load cached images from config file.") @@ -446,18 +451,13 @@ func updateKubeConfig(h *host.Host, c *cfg.Config) *kubeconfig.KubeConfigSetup { } // configureRuntimes does what needs to happen to get a runtime going. -func configureRuntimes(h *host.Host) { - runner, err := machine.CommandRunner(h) - if err != nil { - exit.WithError("Failed to get command runner", err) - } - +func configureRuntimes(h *host.Host, runner bootstrapper.CommandRunner) cruntime.Manager { config := cruntime.Config{Type: viper.GetString(containerRuntime), Runner: runner} cr, err := cruntime.New(config) if err != nil { exit.WithError(fmt.Sprintf("Failed runtime for %+v", config), err) } - console.OutStyle(cr.Name(), "Configuring %s as your container runtime ...", cr.Name()) + console.OutStyle(cr.Name(), "Configuring %s as the container runtime ...", cr.Name()) for _, v := range dockerOpt { console.OutStyle("option", "opt %s", v) } @@ -469,7 +469,7 @@ func configureRuntimes(h *host.Host) { if err != nil { exit.WithError("Failed to enable container runtime", err) } - + return cr } // waitCacheImages blocks until the image cache jobs complete @@ -484,8 +484,8 @@ func waitCacheImages(g *errgroup.Group) { } // bootstrapCluster starts Kubernetes using the chosen bootstrapper -func bootstrapCluster(bs bootstrapper.Bootstrapper, kc cfg.KubernetesConfig, preexisting bool) { - console.OutStyle("pulling", "Pulling images used by Kubernetes %s ...", kc.KubernetesVersion) +func bootstrapCluster(bs bootstrapper.Bootstrapper, r cruntime.Manager, runner bootstrapper.CommandRunner, kc cfg.KubernetesConfig, preexisting bool) { + console.OutStyle("pulling", "Pulling images required by Kubernetes %s ...", kc.KubernetesVersion) if err := bs.PullImages(kc); err != nil { console.OutStyle("failure", "Unable to pull images, which may be OK: %v", err) } @@ -495,19 +495,19 @@ func bootstrapCluster(bs bootstrapper.Bootstrapper, kc cfg.KubernetesConfig, pre if preexisting { console.OutStyle("restarting", "Relaunching Kubernetes %s using %s ... ", kc.KubernetesVersion, bsName) if err := bs.RestartCluster(kc); err != nil { - exit.WithError("Error restarting cluster", err) + exit.WithProblems("Error restarting cluster", err, logs.FindProblems(r, bs, runner)) } return } console.OutStyle("launch", "Launching Kubernetes %s using %s ... ", kc.KubernetesVersion, bsName) if err := bs.StartCluster(kc); err != nil { - exit.WithError("Error starting cluster", err) + exit.WithProblems("Error starting cluster", err, logs.FindProblems(r, bs, runner)) } } // validateCluster validates that the cluster is well-configured and healthy -func validateCluster(bs bootstrapper.Bootstrapper, ip string) { +func validateCluster(bs bootstrapper.Bootstrapper, r cruntime.Manager, runner bootstrapper.CommandRunner, ip string) { console.OutStyle("verifying-noline", "Verifying component health ...") kStat := func() (err error) { st, err := bs.GetKubeletStatus() @@ -519,7 +519,7 @@ func validateCluster(bs bootstrapper.Bootstrapper, ip string) { } err := pkgutil.RetryAfter(20, kStat, 3*time.Second) if err != nil { - exit.WithError("kubelet checks failed", err) + exit.WithProblems("kubelet checks failed", err, logs.FindProblems(r, bs, runner)) } aStat := func() (err error) { st, err := bs.GetApiServerStatus(net.ParseIP(ip)) @@ -532,7 +532,7 @@ func validateCluster(bs bootstrapper.Bootstrapper, ip string) { err = pkgutil.RetryAfter(30, aStat, 10*time.Second) if err != nil { - exit.WithError("apiserver checks failed", err) + exit.WithProblems("apiserver checks failed", err, logs.FindProblems(r, bs, runner)) } console.OutLn("") } diff --git a/docs/README.md b/docs/README.md index bdfd18a68e..3c6a2dd1a4 100644 --- a/docs/README.md +++ b/docs/README.md @@ -14,6 +14,8 @@ * **GPUs** ([gpu.md](gpu.md)): Using NVIDIA GPUs on minikube +* **OpenID Connect Authentication** ([openid_connect_auth](openid_connect_auth)): Using OIDC Authentication on minikube + ### Installation and debugging * **Driver installation** ([drivers.md](drivers.md)): In depth instructions for installing the various hypervisor drivers diff --git a/docs/drivers.md b/docs/drivers.md index efaeaebdbd..2cb5acfe9e 100644 --- a/docs/drivers.md +++ b/docs/drivers.md @@ -154,3 +154,9 @@ export LATEST_VERSION=$(curl -L -s -H 'Accept: application/json' https://github. && chmod +x docker-machine-driver-vmware \ && mv docker-machine-driver-vmware /usr/local/bin/ ``` + +To use the driver you would do: + +```shell +minikube start --vm-driver vmware +``` \ No newline at end of file diff --git a/docs/openid_connect_auth.md b/docs/openid_connect_auth.md new file mode 100644 index 0000000000..27345e0a61 --- /dev/null +++ b/docs/openid_connect_auth.md @@ -0,0 +1,34 @@ +# OpenID Connect Authentication + +Minikube `kube-apiserver` can be configured to support OpenID Connect Authentication. + +Read more about OpenID Connect Authentication for Kubernetes here: https://kubernetes.io/docs/reference/access-authn-authz/authentication/#openid-connect-tokens + + +## Configuring the API Server + +Configuration values can be passed to the API server using the `--extra-config` flag on the `minikube start` command. See [configuring_kubernetes.md](https://github.com/kubernetes/minikube/blob/master/docs/configuring_kubernetes.md) for more details. + +The following example configures your Minikube cluster to support RBAC and OIDC: + +```shell +minikube start \ + --extra-config=apiserver.authorization-mode=RBAC \ + --extra-config=apiserver.oidc-issuer-url=https://example.com \ + --extra-config=apiserver.oidc-username-claim=email \ + --extra-config=apiserver.oidc-client-id=kubernetes-local +``` + +## Configuring kubectl + +You can use the kubectl `oidc` authenticator to create a kubeconfig as shown in the Kubernetes docs: https://kubernetes.io/docs/reference/access-authn-authz/authentication/#option-1-oidc-authenticator + +`minikube start` already creates a kubeconfig that includes a `cluster`, in order to use it with your `oidc` authenticator kubeconfig, you can run: + +```shell +kubectl config set-context kubernetes-local-oidc --cluster=minikube --user username@example.com +Context "kubernetes-local-oidc" created. +kubectl config use-context kubernetes-local-oidc +``` + +For the new context to work you will need to create, at the very minimum, a `Role` and a `RoleBinding` in your cluster to grant permissions to the `subjects` included in your `oidc-username-claim`. diff --git a/hack/jenkins/linux_integration_tests_none.sh b/hack/jenkins/linux_integration_tests_none.sh index f3b1b7da9a..83d4348058 100755 --- a/hack/jenkins/linux_integration_tests_none.sh +++ b/hack/jenkins/linux_integration_tests_none.sh @@ -42,6 +42,8 @@ sudo kubeadm reset || sudo kubeadm reset -f || true sudo rm -rf /data/* # Cleanup old Kubernetes configs sudo rm -rf /etc/kubernetes/* +# Cleanup old minikube files +sudo rm -rf /var/lib/minikube/* # Stop any leftover kubelets systemctl is-active --quiet kubelet \ && echo "stopping kubelet" \ diff --git a/pkg/drivers/hyperkit/driver.go b/pkg/drivers/hyperkit/driver.go index 6314f7094c..450620cfaa 100644 --- a/pkg/drivers/hyperkit/driver.go +++ b/pkg/drivers/hyperkit/driver.go @@ -22,6 +22,7 @@ import ( "encoding/json" "fmt" "io/ioutil" + golog "log" "os" "os/user" "path" @@ -79,19 +80,28 @@ func NewDriver(hostName, storePath string) *Driver { // PreCreateCheck is called to enforce pre-creation steps func (d *Driver) PreCreateCheck() error { + return d.verifyRootPermissions() +} + +// verifyRootPermissions is called before any step which needs root access +func (d *Driver) verifyRootPermissions() error { exe, err := os.Executable() if err != nil { return err } - - if syscall.Geteuid() != 0 { + euid := syscall.Geteuid() + log.Debugf("exe=%s uid=%d", exe, euid) + if euid != 0 { return fmt.Errorf(permErr, filepath.Base(exe), exe, exe) } - return nil } func (d *Driver) Create() error { + if err := d.verifyRootPermissions(); err != nil { + return err + } + // TODO: handle different disk types. if err := pkgdrivers.MakeDiskImage(d.BaseDriver, d.Boot2DockerURL, d.DiskSize); err != nil { return errors.Wrap(err, "making disk image") @@ -125,37 +135,55 @@ func (d *Driver) GetURL() (string, error) { return fmt.Sprintf("tcp://%s:2376", ip), nil } -// GetState returns the state that the host is in (running, stopped, etc) -func (d *Driver) GetState() (state.State, error) { - pid := d.getPid() +// Return the state of the hyperkit pid +func pidState(pid int) (state.State, error) { if pid == 0 { return state.Stopped, nil } - p, err := os.FindProcess(pid) + p, err := ps.FindProcess(pid) if err != nil { return state.Error, err } - - // Sending a signal of 0 can be used to check the existence of a process. - if err := p.Signal(syscall.Signal(0)); err != nil { + if p == nil { + log.Debugf("hyperkit pid %d missing from process table", pid) return state.Stopped, nil } - if p == nil { + // hyperkit or com.docker.hyper + if !strings.Contains(p.Executable(), "hyper") { + log.Debugf("pid %d is stale, and is being used by %s", pid, p.Executable()) return state.Stopped, nil } return state.Running, nil } +// GetState returns the state that the host is in (running, stopped, etc) +func (d *Driver) GetState() (state.State, error) { + if err := d.verifyRootPermissions(); err != nil { + return state.Error, err + } + + pid := d.getPid() + log.Debugf("hyperkit pid from json: %d", pid) + return pidState(pid) +} + // Kill stops a host forcefully func (d *Driver) Kill() error { + if err := d.verifyRootPermissions(); err != nil { + return err + } return d.sendSignal(syscall.SIGKILL) } // Remove a host func (d *Driver) Remove() error { + if err := d.verifyRootPermissions(); err != nil { + return err + } + s, err := d.GetState() if err != nil || s == state.Error { - log.Infof("Error checking machine status: %v, assuming it has been removed already", err) + log.Debugf("Error checking machine status: %v, assuming it has been removed already", err) } if s == state.Running { if err := d.Stop(); err != nil { @@ -171,6 +199,10 @@ func (d *Driver) Restart() error { // Start a host func (d *Driver) Start() error { + if err := d.verifyRootPermissions(); err != nil { + return err + } + stateDir := filepath.Join(d.StorePath, "machines", d.MachineName) if err := d.recoverFromUncleanShutdown(); err != nil { return err @@ -189,6 +221,9 @@ func (d *Driver) Start() error { h.CPUs = d.CPU h.Memory = d.Memory h.UUID = d.UUID + // This should stream logs from hyperkit, but doesn't seem to work. + logger := golog.New(os.Stderr, "hyperkit", golog.LstdFlags) + h.SetLogger(logger) if vsockPorts, err := d.extractVSockPorts(); err != nil { return err @@ -197,7 +232,7 @@ func (d *Driver) Start() error { h.VSockPorts = vsockPorts } - log.Infof("Using UUID %s", h.UUID) + log.Debugf("Using UUID %s", h.UUID) mac, err := GetMACAddressFromUUID(h.UUID) if err != nil { return errors.Wrap(err, "getting MAC address from UUID") @@ -205,7 +240,7 @@ func (d *Driver) Start() error { // Need to strip 0's mac = trimMacAddress(mac) - log.Infof("Generated MAC %s", mac) + log.Debugf("Generated MAC %s", mac) h.Disks = []hyperkit.DiskConfig{ { Path: pkgdrivers.GetDiskPath(d.BaseDriver), @@ -213,13 +248,20 @@ func (d *Driver) Start() error { Driver: "virtio-blk", }, } - log.Infof("Starting with cmdline: %s", d.Cmdline) + log.Debugf("Starting with cmdline: %s", d.Cmdline) if err := h.Start(d.Cmdline); err != nil { return errors.Wrapf(err, "starting with cmd line: %s", d.Cmdline) } getIP := func() error { - var err error + st, err := d.GetState() + if err != nil { + return errors.Wrap(err, "get state") + } + if st == state.Error || st == state.Stopped { + return fmt.Errorf("hyperkit crashed! command line:\n hyperkit %s", d.Cmdline) + } + d.IPAddress, err = GetIPAddressByMACAddress(mac) if err != nil { return &commonutil.RetriableError{Err: err} @@ -230,6 +272,7 @@ func (d *Driver) Start() error { if err := commonutil.RetryAfter(30, getIP, 2*time.Second); err != nil { return fmt.Errorf("IP address never found in dhcp leases file %v", err) } + log.Debugf("IP: %s", d.IPAddress) if len(d.NFSShares) > 0 { log.Info("Setting up NFS mounts") @@ -257,47 +300,46 @@ func (d *Driver) recoverFromUncleanShutdown() error { stateDir := filepath.Join(d.StorePath, "machines", d.MachineName) pidFile := filepath.Join(stateDir, pidFileName) - _, err := os.Stat(pidFile) - - if os.IsNotExist(err) { - log.Infof("clean start, hyperkit pid file doesn't exist: %s", pidFile) - return nil - } - - if err != nil { - return errors.Wrap(err, "checking hyperkit pid file existence") + if _, err := os.Stat(pidFile); err != nil { + if os.IsNotExist(err) { + log.Debugf("clean start, hyperkit pid file doesn't exist: %s", pidFile) + return nil + } + return errors.Wrap(err, "stat") } log.Warnf("minikube might have been shutdown in an unclean way, the hyperkit pid file still exists: %s", pidFile) - - content, err := ioutil.ReadFile(pidFile) + bs, err := ioutil.ReadFile(pidFile) if err != nil { return errors.Wrapf(err, "reading pidfile %s", pidFile) } - pid, err := strconv.Atoi(string(content)) + content := strings.TrimSpace(string(bs)) + pid, err := strconv.Atoi(content) if err != nil { return errors.Wrapf(err, "parsing pidfile %s", pidFile) } - p, err := ps.FindProcess(pid) + st, err := pidState(pid) if err != nil { - return errors.Wrapf(err, "trying to find process for PID %d", pid) + return errors.Wrap(err, "pidState") } - if p != nil && !strings.Contains(p.Executable(), "hyperkit") { - return fmt.Errorf("something is not right...please stop all minikube instances, seemingly a hyperkit server is already running with pid %d, executable: %s", pid, p.Executable()) + log.Debugf("pid %d is in state %q", pid, st) + if st == state.Running { + return nil } - - log.Infof("No running hyperkit process found with PID %d, removing %s...", pid, pidFile) + log.Debugf("Removing stale pid file %s...", pidFile) if err := os.Remove(pidFile); err != nil { return errors.Wrap(err, fmt.Sprintf("removing pidFile %s", pidFile)) } - return nil } // Stop a host gracefully func (d *Driver) Stop() error { + if err := d.verifyRootPermissions(); err != nil { + return err + } d.cleanupNfsExports() return d.sendSignal(syscall.SIGTERM) } @@ -334,9 +376,7 @@ func (d *Driver) extractVSockPorts() ([]int, error) { for _, port := range d.VSockPorts { p, err := strconv.Atoi(port) if err != nil { - var err InvalidPortNumberError - err = InvalidPortNumberError(port) - return nil, err + return nil, InvalidPortNumberError(port) } vsockPorts = append(vsockPorts, p) } diff --git a/pkg/drivers/hyperkit/iso.go b/pkg/drivers/hyperkit/iso.go index 9139073db7..14503e1b2f 100644 --- a/pkg/drivers/hyperkit/iso.go +++ b/pkg/drivers/hyperkit/iso.go @@ -21,7 +21,6 @@ import ( "io" "io/ioutil" "os" - "strings" "github.com/hooklift/iso9660" diff --git a/pkg/drivers/hyperkit/network.go b/pkg/drivers/hyperkit/network.go index 61ff3b07a1..a699e071ab 100644 --- a/pkg/drivers/hyperkit/network.go +++ b/pkg/drivers/hyperkit/network.go @@ -25,12 +25,14 @@ import ( "os/exec" "regexp" "strings" + + "github.com/docker/machine/libmachine/log" ) const ( - DHCPLeasesFile = "/var/db/dhcpd_leases" - CONFIG_PLIST = "/Library/Preferences/SystemConfiguration/com.apple.vmnet" - NET_ADDR_KEY = "Shared_Net_Address" + LeasesPath = "/var/db/dhcpd_leases" + VMNetDomain = "/Library/Preferences/SystemConfiguration/com.apple.vmnet" + SharedNetAddrKey = "Shared_Net_Address" ) type DHCPEntry struct { @@ -42,10 +44,11 @@ type DHCPEntry struct { } func GetIPAddressByMACAddress(mac string) (string, error) { - return getIpAddressFromFile(mac, DHCPLeasesFile) + return getIPAddressFromFile(mac, LeasesPath) } -func getIpAddressFromFile(mac, path string) (string, error) { +func getIPAddressFromFile(mac, path string) (string, error) { + log.Debugf("Searching for %s in %s ...", mac, path) file, err := os.Open(path) if err != nil { return "", err @@ -56,12 +59,15 @@ func getIpAddressFromFile(mac, path string) (string, error) { if err != nil { return "", err } + log.Debugf("Found %d entries in %s!", len(dhcpEntries), path) for _, dhcpEntry := range dhcpEntries { + log.Debugf("dhcp entry: %+v", dhcpEntry) if dhcpEntry.HWAddress == mac { + log.Debugf("Found match: %s", mac) return dhcpEntry.IPAddress, nil } } - return "", fmt.Errorf("Could not find an IP address for %s", mac) + return "", fmt.Errorf("could not find an IP address for %s", mac) } func parseDHCPdLeasesFile(file io.Reader) ([]DHCPEntry, error) { @@ -99,7 +105,7 @@ func parseDHCPdLeasesFile(file io.Reader) ([]DHCPEntry, error) { case "lease": dhcpEntry.Lease = val default: - return dhcpEntries, fmt.Errorf("Unable to parse line: %s", line) + return dhcpEntries, fmt.Errorf("unable to parse line: %s", line) } } return dhcpEntries, scanner.Err() @@ -114,18 +120,17 @@ func trimMacAddress(rawUUID string) string { } func GetNetAddr() (net.IP, error) { - _, err := os.Stat(CONFIG_PLIST + ".plist") - if err != nil { - return nil, fmt.Errorf("Does not exist %s", CONFIG_PLIST+".plist") + plistPath := VMNetDomain + ".plist" + if _, err := os.Stat(plistPath); err != nil { + return nil, fmt.Errorf("stat: %v", err) } - - out, err := exec.Command("defaults", "read", CONFIG_PLIST, NET_ADDR_KEY).Output() + out, err := exec.Command("defaults", "read", VMNetDomain, SharedNetAddrKey).Output() if err != nil { return nil, err } ip := net.ParseIP(strings.TrimSpace(string(out))) if ip == nil { - return nil, fmt.Errorf("Could not get the network address for vmnet") + return nil, fmt.Errorf("could not get the network address for vmnet") } return ip, nil } diff --git a/pkg/drivers/hyperkit/network_test.go b/pkg/drivers/hyperkit/network_test.go index d8b831b823..45f72ec183 100644 --- a/pkg/drivers/hyperkit/network_test.go +++ b/pkg/drivers/hyperkit/network_test.go @@ -88,13 +88,13 @@ func Test_getIpAddressFromFile(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := getIpAddressFromFile(tt.args.mac, tt.args.path) + got, err := getIPAddressFromFile(tt.args.mac, tt.args.path) if (err != nil) != tt.wantErr { - t.Errorf("getIpAddressFromFile() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("getIPAddressFromFile() error = %v, wantErr %v", err, tt.wantErr) return } if got != tt.want { - t.Errorf("getIpAddressFromFile() = %v, want %v", got, tt.want) + t.Errorf("getIPAddressFromFile() = %v, want %v", got, tt.want) } }) } diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go index 604b0109ae..b6db690e76 100644 --- a/pkg/minikube/assets/addons.go +++ b/pkg/minikube/assets/addons.go @@ -302,14 +302,15 @@ func addMinikubeDirToAssets(basedir, vmpath string, assets *[]CopyableFile) erro return errors.Wrapf(err, "checking if %s is directory", hostpath) } if !isDir { - if vmpath == "" { + vmdir := vmpath + if vmdir == "" { rPath, err := filepath.Rel(basedir, hostpath) if err != nil { return errors.Wrap(err, "generating relative path") } rPath = filepath.Dir(rPath) rPath = filepath.ToSlash(rPath) - vmpath = path.Join("/", rPath) + vmdir = path.Join("/", rPath) } permString := fmt.Sprintf("%o", info.Mode().Perm()) // The conversion will strip the leading 0 if present, so add it back @@ -318,7 +319,7 @@ func addMinikubeDirToAssets(basedir, vmpath string, assets *[]CopyableFile) erro permString = fmt.Sprintf("0%s", permString) } - f, err := NewFileAsset(hostpath, vmpath, filepath.Base(hostpath), permString) + f, err := NewFileAsset(hostpath, vmdir, filepath.Base(hostpath), permString) if err != nil { return errors.Wrapf(err, "creating file asset for %s", hostpath) } diff --git a/pkg/minikube/assets/addons_test.go b/pkg/minikube/assets/addons_test.go new file mode 100644 index 0000000000..bfd8bfeeea --- /dev/null +++ b/pkg/minikube/assets/addons_test.go @@ -0,0 +1,159 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package assets + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/google/go-cmp/cmp" + "k8s.io/minikube/pkg/minikube/constants" +) + +func setupTestDir() (string, error) { + path, err := ioutil.TempDir("", "minipath") + if err != nil { + return "", err + } + + os.Setenv(constants.MinikubeHome, path) + return path, err +} + +func TestAddMinikubeDirAssets(t *testing.T) { + + tests := []struct { + description string + baseDir string + files []struct { + relativePath string + expectedPath string + } + vmPath string + expectedCfg string + }{ + { + description: "relative path assets", + baseDir: "/files", + files: []struct { + relativePath string + expectedPath string + }{ + { + relativePath: "/dir1/file1.txt", + expectedPath: constants.AddonsPath, + }, + { + relativePath: "/dir1/file2.txt", + expectedPath: constants.AddonsPath, + }, + { + relativePath: "/dir2/file1.txt", + expectedPath: constants.AddonsPath, + }, + }, + vmPath: constants.AddonsPath, + }, + { + description: "absolute path assets", + baseDir: "/files", + files: []struct { + relativePath string + expectedPath string + }{ + { + relativePath: "/dir1/file1.txt", + expectedPath: "/dir1", + }, + { + relativePath: "/dir1/file2.txt", + expectedPath: "/dir1", + }, + { + relativePath: "/dir2/file1.txt", + expectedPath: "/dir2", + }, + }, + vmPath: "", + }, + } + var testDirs = make([]string, 0) + defer func() { + for _, testDir := range testDirs { + err := os.RemoveAll(testDir) + if err != nil { + t.Logf("got unexpected error removing test dir: %v", err) + } + } + }() + + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + testDir, err := setupTestDir() + if err != nil { + t.Errorf("got unexpected error creating test dir: %v", err) + return + } + + testDirs = append(testDirs, testDir) + testFileBaseDir := filepath.Join(testDir, test.baseDir) + want := make(map[string]string, 0) + for _, fileDef := range test.files { + err := func() error { + path := filepath.Join(testFileBaseDir, fileDef.relativePath) + err := os.MkdirAll(filepath.Dir(path), 0755) + want[path] = fileDef.expectedPath + if err != nil { + return err + } + + file, err := os.Create(path) + if err != nil { + return err + } + + defer file.Close() + + _, err = file.WriteString("test") + return err + }() + if err != nil { + t.Errorf("unable to create file on fs: %v", err) + return + } + } + + var actualFiles []CopyableFile + err = addMinikubeDirToAssets(testFileBaseDir, test.vmPath, &actualFiles) + if err != nil { + t.Errorf("got unexpected error adding minikube dir assets: %v", err) + return + } + + got := make(map[string]string, 0) + for _, actualFile := range actualFiles { + got[actualFile.GetAssetName()] = actualFile.GetTargetDir() + } + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("files differ: (-want +got)\n%s", diff) + } + }) + } + +} diff --git a/pkg/minikube/bootstrapper/bootstrapper.go b/pkg/minikube/bootstrapper/bootstrapper.go index 6cd961b18d..4140f58548 100644 --- a/pkg/minikube/bootstrapper/bootstrapper.go +++ b/pkg/minikube/bootstrapper/bootstrapper.go @@ -17,13 +17,20 @@ limitations under the License. package bootstrapper import ( - "io" "net" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/constants" ) +// LogOptions are options to be passed to LogCommands +type LogOptions struct { + // Lines is the number of recent log lines to include, as in tail -n. + Lines int + // Follow is whether or not to actively follow the logs, as in tail -f. + Follow bool +} + // Bootstrapper contains all the methods needed to bootstrap a kubernetes cluster type Bootstrapper interface { // PullImages pulls images necessary for a cluster. Success should not be required. @@ -32,7 +39,8 @@ type Bootstrapper interface { UpdateCluster(config.KubernetesConfig) error RestartCluster(config.KubernetesConfig) error DeleteCluster(config.KubernetesConfig) error - GetClusterLogsTo(follow bool, out io.Writer) error + // LogCommands returns a map of log type to a command which will display that log. + LogCommands(LogOptions) map[string]string SetupCerts(cfg config.KubernetesConfig) error GetKubeletStatus() (string, error) GetApiServerStatus(net.IP) (string, error) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 1268eb108b..aa68d68c0f 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -21,7 +21,6 @@ import ( "crypto" "crypto/tls" "fmt" - "io" "net" "net/http" "os" @@ -121,28 +120,17 @@ func (k *KubeadmBootstrapper) GetApiServerStatus(ip net.IP) (string, error) { return state.Running.String(), nil } -// TODO(r2d4): Should this aggregate all the logs from the control plane? -// Maybe subcommands for each component? minikube logs apiserver? -func (k *KubeadmBootstrapper) GetClusterLogsTo(follow bool, out io.Writer) error { - var flags []string - if follow { - flags = append(flags, "-f") +// LogCommands returns a map of log type to a command which will display that log. +func (k *KubeadmBootstrapper) LogCommands(o bootstrapper.LogOptions) map[string]string { + var kcmd strings.Builder + kcmd.WriteString("journalctl -u kubelet") + if o.Lines > 0 { + kcmd.WriteString(fmt.Sprintf(" -n %d", o.Lines)) } - logsCommand := fmt.Sprintf("sudo journalctl %s -u kubelet", strings.Join(flags, " ")) - - if follow { - if err := k.c.CombinedOutputTo(logsCommand, out); err != nil { - return errors.Wrap(err, "getting cluster logs") - } - } else { - - logs, err := k.c.CombinedOutput(logsCommand) - if err != nil { - return errors.Wrap(err, "getting cluster logs") - } - fmt.Fprint(out, logs) + if o.Follow { + kcmd.WriteString(" -f") } - return nil + return map[string]string{"kubelet": kcmd.String()} } func (k *KubeadmBootstrapper) StartCluster(k8s config.KubernetesConfig) error { @@ -245,7 +233,7 @@ func (k *KubeadmBootstrapper) RestartCluster(k8s config.KubernetesConfig) error } // NOTE: Perhaps now would be a good time to check apiserver health? - console.OutStyle("waiting", "Restarting kube-proxy ...") + console.OutStyle("waiting", "Waiting for kube-proxy to come back up ...") if err := restartKubeProxy(k8s); err != nil { return errors.Wrap(err, "restarting kube-proxy") } diff --git a/pkg/minikube/cluster/cluster.go b/pkg/minikube/cluster/cluster.go index d4a689decb..052535fad4 100644 --- a/pkg/minikube/cluster/cluster.go +++ b/pkg/minikube/cluster/cluster.go @@ -87,7 +87,7 @@ func StartHost(api libmachine.API, config cfg.MachineConfig) (*host.Host, error) console.Warning("Alternatively, you may delete the existing VM using `minikube delete -p %s`", cfg.GetMachineName()) console.Out("\n") } else if exists && cfg.GetMachineName() == constants.DefaultMachineName { - console.OutStyle("tip", "Tip: To create a new cluster, use 'minikube start -p ' or use 'minikube delete' to delete this one.") + console.OutStyle("tip", "Tip: Use 'minikube start -p ' to create a new cluster, or 'minikube delete' to delete this one.") } s, err := h.Driver.GetState() diff --git a/pkg/minikube/console/console_test.go b/pkg/minikube/console/console_test.go index cde2b86f89..7e0af8e1da 100644 --- a/pkg/minikube/console/console_test.go +++ b/pkg/minikube/console/console_test.go @@ -46,17 +46,34 @@ func (f *fakeFile) String() string { } func TestOutStyle(t *testing.T) { - os.Setenv(OverrideEnv, "1") - f := newFakeFile() - SetOutFile(f) - if err := OutStyle("happy", "This is a happy message."); err != nil { - t.Errorf("unexpected error: %q", err) - } - got := f.String() - want := "πŸ˜„ This is a happy message.\n" - if got != want { - t.Errorf("OutStyle() = %q, want %q", got, want) + var tests = []struct { + style string + envValue string + message string + want string + }{ + {"happy", "true", "This is happy.", "πŸ˜„ This is happy.\n"}, + {"Docker", "true", "This is Docker.", "🐳 This is Docker.\n"}, + {"option", "true", "This is option.", " β–ͺ This is option.\n"}, + + {"happy", "false", "This is happy.", "o This is happy.\n"}, + {"Docker", "false", "This is Docker.", "- This is Docker.\n"}, + {"option", "false", "This is option.", " - This is option.\n"}, + } + for _, tc := range tests { + t.Run(tc.style+"-"+tc.envValue, func(t *testing.T) { + os.Setenv(OverrideEnv, tc.envValue) + f := newFakeFile() + SetOutFile(f) + if err := OutStyle(tc.style, tc.message); err != nil { + t.Errorf("unexpected error: %q", err) + } + got := f.String() + if got != tc.want { + t.Errorf("OutStyle() = %q, want %q", got, tc.want) + } + }) } } diff --git a/pkg/minikube/console/style.go b/pkg/minikube/console/style.go index 1265a65580..e7b3b541f9 100644 --- a/pkg/minikube/console/style.go +++ b/pkg/minikube/console/style.go @@ -18,14 +18,22 @@ package console import ( "fmt" + "strings" "golang.org/x/text/message" ) +var ( + defaultLowPrefix = "- " + defautlLowIndentPrefix = " - " +) + // style describes how to stylize a message. type style struct { // Prefix is a string to place in the beginning of a message Prefix string + // LowPrefix is the 7-bit compatible prefix we fallback to for less-awesome terminals + LowPrefix string // OmitNewline omits a newline at the end of a message. OmitNewline bool } @@ -33,39 +41,40 @@ type style struct { // styles is a map of style name to style struct // For consistency, ensure that emojis added render with the same width across platforms. var styles = map[string]style{ - "happy": {Prefix: "πŸ˜„ "}, + "happy": {Prefix: "πŸ˜„ ", LowPrefix: "o "}, "success": {Prefix: "βœ… "}, - "failure": {Prefix: "❌ "}, - "conflict": {Prefix: "πŸ’₯ "}, - "fatal": {Prefix: "πŸ’£ "}, - "notice": {Prefix: "πŸ“Œ "}, - "ready": {Prefix: "πŸ„ "}, - "running": {Prefix: "πŸƒ "}, - "provisioning": {Prefix: "🌱 "}, - "restarting": {Prefix: "πŸ”„ "}, - "stopping": {Prefix: "βœ‹ "}, + "failure": {Prefix: "❌ ", LowPrefix: "X "}, + "conflict": {Prefix: "πŸ’₯ ", LowPrefix: "x "}, + "fatal": {Prefix: "πŸ’£ ", LowPrefix: "! "}, + "notice": {Prefix: "πŸ“Œ ", LowPrefix: "* "}, + "ready": {Prefix: "πŸ„ ", LowPrefix: "= "}, + "running": {Prefix: "πŸƒ ", LowPrefix: ": "}, + "provisioning": {Prefix: "🌱 ", LowPrefix: "> "}, + "restarting": {Prefix: "πŸ”„ ", LowPrefix: ": "}, + "stopping": {Prefix: "βœ‹ ", LowPrefix: ": "}, "stopped": {Prefix: "πŸ›‘ "}, - "warning": {Prefix: "⚠️ "}, - "waiting": {Prefix: "βŒ› "}, + "warning": {Prefix: "⚠️ ", LowPrefix: "! "}, + "waiting": {Prefix: "βŒ› ", LowPrefix: ": "}, "usage": {Prefix: "πŸ’‘ "}, "launch": {Prefix: "πŸš€ "}, - "sad": {Prefix: "😿 "}, + "sad": {Prefix: "😿 ", LowPrefix: "* "}, "thumbs-up": {Prefix: "πŸ‘ "}, "option": {Prefix: " β–ͺ "}, // Indented bullet - "url": {Prefix: "πŸ‘‰ "}, + "log-entry": {Prefix: " "}, // Indent "crushed": {Prefix: "πŸ’” "}, + "url": {Prefix: "πŸ‘‰ "}, // Specialized purpose styles - "iso-download": {Prefix: "πŸ’Ώ "}, - "file-download": {Prefix: "πŸ’Ύ "}, - "caching": {Prefix: "🀹 "}, - "starting-vm": {Prefix: "πŸ”₯ "}, - "starting-none": {Prefix: "🀹 "}, - "resetting": {Prefix: "πŸ”„ "}, - "deleting-host": {Prefix: "πŸ”₯ "}, + "iso-download": {Prefix: "πŸ’Ώ ", LowPrefix: "@ "}, + "file-download": {Prefix: "πŸ’Ύ ", LowPrefix: "@ "}, + "caching": {Prefix: "🀹 ", LowPrefix: "$ "}, + "starting-vm": {Prefix: "πŸ”₯ ", LowPrefix: "> "}, + "starting-none": {Prefix: "🀹 ", LowPrefix: "> "}, + "resetting": {Prefix: "πŸ”„ ", LowPrefix: "# "}, + "deleting-host": {Prefix: "πŸ”₯ ", LowPrefix: "x "}, "copying": {Prefix: "✨ "}, "connectivity": {Prefix: "πŸ“Ά "}, - "internet": {Prefix: "🌐 "}, + "internet": {Prefix: "🌐 ", LowPrefix: "o "}, "mounting": {Prefix: "πŸ“ "}, "celebrate": {Prefix: "πŸŽ‰ "}, "container-runtime": {Prefix: "🎁 "}, @@ -78,10 +87,10 @@ var styles = map[string]style{ "pulling": {Prefix: "🚜 "}, "verifying": {Prefix: "πŸ€” "}, "verifying-noline": {Prefix: "πŸ€” ", OmitNewline: true}, - "kubectl": {Prefix: "πŸ’— "}, - "meh": {Prefix: "πŸ™„ "}, - "embarassed": {Prefix: "🀦 "}, - "tip": {Prefix: "πŸ’‘ "}, + "kubectl": {Prefix: "πŸ’— ", LowPrefix: "+ "}, + "meh": {Prefix: "πŸ™„ ", LowPrefix: "? "}, + "embarassed": {Prefix: "🀦 ", LowPrefix: "* "}, + "tip": {Prefix: "πŸ’‘ ", LowPrefix: "i "}, } // Add a prefix to a string @@ -98,6 +107,17 @@ func hasStyle(style string) bool { return exists } +// lowPrefix returns a 7-bit compatible prefix for a style +func lowPrefix(s style) string { + if s.LowPrefix != "" { + return s.LowPrefix + } + if strings.HasPrefix(s.Prefix, " ") { + return defautlLowIndentPrefix + } + return defaultLowPrefix +} + // Apply styling to a format string func applyStyle(style string, useColor bool, format string, a ...interface{}) (string, error) { p := message.NewPrinter(preferredLanguage) @@ -113,10 +133,8 @@ func applyStyle(style string, useColor bool, format string, a ...interface{}) (s return p.Sprintf(format, a...), fmt.Errorf("unknown style: %q", style) } - prefix := s.Prefix - if !useColor && prefix != "" { - prefix = "-" + if !useColor { + return applyPrefix(lowPrefix(s), out), nil } - out = applyPrefix(prefix, out) - return out, nil + return applyPrefix(s.Prefix, out), nil } diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go index 611197a3b5..e46a30e5da 100644 --- a/pkg/minikube/cruntime/containerd.go +++ b/pkg/minikube/cruntime/containerd.go @@ -107,3 +107,8 @@ func (r *Containerd) KillContainers(ids []string) error { func (r *Containerd) StopContainers(ids []string) error { return stopCRIContainers(r.Runner, ids) } + +// ContainerLogCmd returns the command to retrieve the log for a container based on ID +func (r *Containerd) ContainerLogCmd(id string, len int, follow bool) string { + return criContainerLogCmd(id, len, follow) +} diff --git a/pkg/minikube/cruntime/cri.go b/pkg/minikube/cruntime/cri.go index cbf4a1cd00..b346b42803 100644 --- a/pkg/minikube/cruntime/cri.go +++ b/pkg/minikube/cruntime/cri.go @@ -75,3 +75,18 @@ image-endpoint: unix://{{.Socket}} } return cr.Run(fmt.Sprintf("sudo mkdir -p %s && printf %%s \"%s\" | sudo tee %s", path.Dir(cPath), b.String(), cPath)) } + +// criContainerLogCmd returns the command to retrieve the log for a container based on ID +func criContainerLogCmd(id string, len int, follow bool) string { + var cmd strings.Builder + cmd.WriteString("crictl logs ") + if len > 0 { + cmd.WriteString(fmt.Sprintf("--tail %d ", len)) + } + if follow { + cmd.WriteString("--follow ") + } + + cmd.WriteString(id) + return cmd.String() +} diff --git a/pkg/minikube/cruntime/crio.go b/pkg/minikube/cruntime/crio.go index 2897dd2c06..93aa426ddf 100644 --- a/pkg/minikube/cruntime/crio.go +++ b/pkg/minikube/cruntime/crio.go @@ -106,3 +106,8 @@ func (r *CRIO) KillContainers(ids []string) error { func (r *CRIO) StopContainers(ids []string) error { return stopCRIContainers(r.Runner, ids) } + +// ContainerLogCmd returns the command to retrieve the log for a container based on ID +func (r *CRIO) ContainerLogCmd(id string, len int, follow bool) string { + return criContainerLogCmd(id, len, follow) +} diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go index 9f47808616..fe670c29e5 100644 --- a/pkg/minikube/cruntime/cruntime.go +++ b/pkg/minikube/cruntime/cruntime.go @@ -61,6 +61,8 @@ type Manager interface { KillContainers([]string) error // StopContainers stops containers based on ID StopContainers([]string) error + // ContainerLogCmd returns the command to retrieve the log for a container based on ID + ContainerLogCmd(string, int, bool) string } // Config is runtime configuration diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go index 51f0df5243..5f80e723a0 100644 --- a/pkg/minikube/cruntime/docker.go +++ b/pkg/minikube/cruntime/docker.go @@ -116,3 +116,18 @@ func (r *Docker) StopContainers(ids []string) error { glog.Infof("Killing containers: %s", ids) return r.Runner.Run(fmt.Sprintf("docker stop %s", strings.Join(ids, " "))) } + +// ContainerLogCmd returns the command to retrieve the log for a container based on ID +func (r *Docker) ContainerLogCmd(id string, len int, follow bool) string { + var cmd strings.Builder + cmd.WriteString("docker logs ") + if len > 0 { + cmd.WriteString(fmt.Sprintf("--tail %d ", len)) + } + if follow { + cmd.WriteString("--follow ") + } + + cmd.WriteString(id) + return cmd.String() +} diff --git a/pkg/minikube/exit/exit.go b/pkg/minikube/exit/exit.go index f5a1d9ea19..fad5b18df3 100644 --- a/pkg/minikube/exit/exit.go +++ b/pkg/minikube/exit/exit.go @@ -18,6 +18,7 @@ limitations under the License. package exit import ( + "fmt" "os" "github.com/golang/glog" @@ -35,6 +36,9 @@ const ( IO = 74 // IO represents an I/O error Config = 78 // Config represents an unconfigured or misconΒ­figured state Permissions = 77 // Permissions represents a permissions error + + // MaxProblems controls the number of problems to show for each source + MaxProblems = 3 ) // Usage outputs a usage error and exits with error code 64 @@ -53,14 +57,34 @@ func WithCode(code int, format string, a ...interface{}) { // WithError outputs an error and exits. func WithError(msg string, err error) { - console.Fatal(msg+": %v", err) - console.Err("\n") - console.ErrStyle("sad", "Sorry that minikube crashed. If this was unexpected, we would love to hear from you:") - console.ErrStyle("url", "https://github.com/kubernetes/minikube/issues/new") - // use Warning because Error will display a duplicate message to stderr - glog.Warningf(msg) + displayError(msg, err) // Here is where we would insert code to optionally upload a stack trace. // We can be smarter about guessing exit codes, but EX_SOFTWARE should suffice. os.Exit(Software) } + +// WithProblems outputs an error along with any autodetected problems, and exits. +func WithProblems(msg string, err error, problems map[string][]string) { + displayError(msg, err) + + for name, lines := range problems { + console.OutStyle("failure", "Problems detected in %q:", name) + if len(lines) > MaxProblems { + lines = lines[:MaxProblems] + } + for _, l := range lines { + console.OutStyle("log-entry", l) + } + } + os.Exit(Software) +} + +func displayError(msg string, err error) { + // use Warning because Error will display a duplicate message to stderr + glog.Warningf(fmt.Sprintf("%s: %v", msg, err)) + console.Fatal(msg+": %v", err) + console.Err("\n") + console.ErrStyle("sad", "Sorry that minikube crashed. If this was unexpected, we would love to hear from you:") + console.ErrStyle("url", "https://github.com/kubernetes/minikube/issues/new") +} diff --git a/pkg/minikube/logs/logs.go b/pkg/minikube/logs/logs.go new file mode 100644 index 0000000000..11f63e9013 --- /dev/null +++ b/pkg/minikube/logs/logs.go @@ -0,0 +1,152 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// package logs are convenience methods for fetching logs from a minikube cluster +package logs + +import ( + "bufio" + "bytes" + "fmt" + "os" + "regexp" + "sort" + "strings" + + "github.com/golang/glog" + "k8s.io/minikube/pkg/minikube/bootstrapper" + "k8s.io/minikube/pkg/minikube/console" + "k8s.io/minikube/pkg/minikube/cruntime" +) + +// rootCauseRe is a regular expression that matches known failure root causes +var rootCauseRe = regexp.MustCompile(`^error: |eviction manager: pods.* evicted|unknown flag: --`) + +// importantPods are a list of pods to retrieve logs for, in addition to the bootstrapper logs. +var importantPods = []string{ + "k8s_kube-apiserver", + "k8s_coredns_coredns", + "k8s_kube-scheduler", +} + +// lookbackwardsCount is how far back to look in a log for problems. This should be large enough to +// include usage messages from a failed binary, but small enough to not include irrelevant problems. +const lookBackwardsCount = 200 + +// Follow follows logs from multiple files in tail(1) format +func Follow(r cruntime.Manager, bs bootstrapper.Bootstrapper, runner bootstrapper.CommandRunner) error { + cs := []string{} + for _, v := range logCommands(r, bs, 0, true) { + cs = append(cs, v+" &") + } + cs = append(cs, "wait") + return runner.CombinedOutputTo(strings.Join(cs, " "), os.Stdout) +} + +// IsProblem returns whether this line matches a known problem +func IsProblem(line string) bool { + return rootCauseRe.MatchString(line) +} + +// FindProblems finds possible root causes among the logs +func FindProblems(r cruntime.Manager, bs bootstrapper.Bootstrapper, runner bootstrapper.CommandRunner) map[string][]string { + pMap := map[string][]string{} + cmds := logCommands(r, bs, lookBackwardsCount, false) + for name, cmd := range cmds { + glog.Infof("Gathering logs for %s ...", name) + var b bytes.Buffer + err := runner.CombinedOutputTo(cmds[name], &b) + if err != nil { + glog.Warningf("failed %s: %s: %v", name, cmd, err) + continue + } + scanner := bufio.NewScanner(&b) + problems := []string{} + for scanner.Scan() { + l := scanner.Text() + if IsProblem(l) { + glog.Warningf("Found %s problem: %s", name, l) + problems = append(problems, l) + } + } + if len(problems) > 0 { + pMap[name] = problems + } + } + return pMap +} + +// OutputProblems outputs discovered problems. +func OutputProblems(problems map[string][]string, maxLines int) { + for name, lines := range problems { + console.OutStyle("failure", "Problems detected in %q:", name) + if len(lines) > maxLines { + lines = lines[len(lines)-maxLines:] + } + for _, l := range lines { + console.OutStyle("log-entry", l) + } + } +} + +// Output displays logs from multiple sources in tail(1) format +func Output(r cruntime.Manager, bs bootstrapper.Bootstrapper, runner bootstrapper.CommandRunner, lines int) error { + cmds := logCommands(r, bs, lines, false) + names := []string{} + for k := range cmds { + names = append(names, k) + } + sort.Strings(names) + + failed := []string{} + for _, name := range names { + console.OutLn("==> %s <==", name) + var b bytes.Buffer + err := runner.CombinedOutputTo(cmds[name], &b) + if err != nil { + glog.Errorf("failed: %v", err) + failed = append(failed, name) + continue + } + scanner := bufio.NewScanner(&b) + for scanner.Scan() { + console.OutLn(scanner.Text()) + } + } + if len(failed) > 0 { + return fmt.Errorf("unable to fetch logs for: %s", strings.Join(failed, ", ")) + } + return nil +} + +// logCommands returns a list of commands that would be run to receive the anticipated logs +func logCommands(r cruntime.Manager, bs bootstrapper.Bootstrapper, length int, follow bool) map[string]string { + cmds := bs.LogCommands(bootstrapper.LogOptions{Lines: length, Follow: follow}) + for _, pod := range importantPods { + ids, err := r.ListContainers(pod) + if err != nil { + glog.Errorf("Failed to list containers for %q: %v", pod, err) + continue + } + glog.Infof("%d containers: %s", len(ids), ids) + if len(ids) == 0 { + cmds[pod] = fmt.Sprintf("No container was found matching %q", pod) + continue + } + cmds[pod] = r.ContainerLogCmd(ids[0], length, follow) + } + return cmds +} diff --git a/pkg/minikube/logs/logs_test.go b/pkg/minikube/logs/logs_test.go new file mode 100644 index 0000000000..245f5aa4d3 --- /dev/null +++ b/pkg/minikube/logs/logs_test.go @@ -0,0 +1,44 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package logs + +import ( + "testing" +) + +func TestIsProblem(t *testing.T) { + var tests = []struct { + name string + want bool + input string + }{ + {"almost", false, "F2350 I would love to be an unknown flag, but I am not -- :( --"}, + {"apiserver-required-flag #1962", true, "error: [service-account-issuer is a required flag when BoundServiceAccountTokenVolume is enabled, --service-account-signing-key-file and --service-account-issuer are required flags"}, + {"kubelet-eviction #", true, "I0213 07:16:44.041623 2410 eviction_manager.go:187] eviction manager: pods kube-apiserver-minikube_kube-system(87f41e2e0629c3deb5c2239e08d8045d) evicted, waiting for pod to be cleaned up"}, + {"kubelet-unknown-flag #3655", true, "F0212 14:55:46.443031 2693 server.go:148] unknown flag: --AllowedUnsafeSysctls"}, + {"apiserver-auth-mode #2852", true, `{"log":"Error: unknown flag: --Authorization.Mode\n","stream":"stderr","time":"2018-06-17T22:16:35.134161966Z"}`}, + {"apiserver-admission #3524", true, "error: unknown flag: --GenericServerRunOptions.AdmissionControl"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := IsProblem(tc.input) + if got != tc.want { + t.Fatalf("IsProblem(%s)=%v, want %v", tc.input, got, tc.want) + } + }) + } +} diff --git a/pkg/util/kubernetes.go b/pkg/util/kubernetes.go index 927cf1dd78..0b8df0d0f7 100644 --- a/pkg/util/kubernetes.go +++ b/pkg/util/kubernetes.go @@ -20,8 +20,11 @@ import ( "fmt" "time" + "github.com/golang/glog" "github.com/pkg/errors" - + appsv1 "k8s.io/api/apps/v1" + "k8s.io/api/core/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" @@ -29,16 +32,15 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/watch" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - - appsv1 "k8s.io/api/apps/v1" - "k8s.io/api/core/v1" - apierrs "k8s.io/apimachinery/pkg/api/errors" "k8s.io/client-go/kubernetes" - - "github.com/golang/glog" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/clientcmd" + "k8s.io/kubernetes/cmd/kubeadm/app/constants" +) + +var ( + ReasonableMutateTime = time.Minute * 1 + ReasonableStartTime = time.Minute * 5 ) type PodStore struct { @@ -108,11 +110,11 @@ func StartPods(c kubernetes.Interface, namespace string, pod v1.Pod, waitForRunn return nil } -// Wait up to 10 minutes for all matching pods to become Running and at least one -// matching pod exists. +// WaitForPodsWithLabelRunning waits for all matching pods to become Running and at least one matching pod exists. func WaitForPodsWithLabelRunning(c kubernetes.Interface, ns string, label labels.Selector) error { + glog.Infof("Waiting for pod with label %q in ns %q ...", ns, label) lastKnownPodNumber := -1 - return wait.PollImmediate(constants.APICallRetryInterval, time.Minute*10, func() (bool, error) { + return wait.PollImmediate(constants.APICallRetryInterval, ReasonableStartTime, func() (bool, error) { listOpts := metav1.ListOptions{LabelSelector: label.String()} pods, err := c.CoreV1().Pods(ns).List(listOpts) if err != nil { @@ -139,9 +141,9 @@ func WaitForPodsWithLabelRunning(c kubernetes.Interface, ns string, label labels }) } -// Wait up to 10 minutes for a pod to be deleted +// WaitForPodDelete waits for a pod to be deleted func WaitForPodDelete(c kubernetes.Interface, ns string, label labels.Selector) error { - return wait.PollImmediate(constants.APICallRetryInterval, time.Minute*10, func() (bool, error) { + return wait.PollImmediate(constants.APICallRetryInterval, ReasonableMutateTime, func() (bool, error) { listOpts := metav1.ListOptions{LabelSelector: label.String()} pods, err := c.CoreV1().Pods(ns).List(listOpts) if err != nil { @@ -152,9 +154,9 @@ func WaitForPodDelete(c kubernetes.Interface, ns string, label labels.Selector) }) } -// Wait up to 10 minutes for the given event to appear +// WaitForEvent waits for the given event to appear func WaitForEvent(c kubernetes.Interface, ns string, reason string) error { - return wait.PollImmediate(constants.APICallRetryInterval, time.Minute*10, func() (bool, error) { + return wait.PollImmediate(constants.APICallRetryInterval, ReasonableMutateTime, func() (bool, error) { events, err := c.Events().Events("default").List(metav1.ListOptions{}) if err != nil { glog.Infof("error getting events: %v", err) diff --git a/test/integration/start_stop_delete_test.go b/test/integration/start_stop_delete_test.go index 9eb7dcf294..693611044c 100644 --- a/test/integration/start_stop_delete_test.go +++ b/test/integration/start_stop_delete_test.go @@ -40,7 +40,7 @@ func TestStartStop(t *testing.T) { for _, test := range tests { t.Run(test.runtime, func(t *testing.T) { runner := NewMinikubeRunner(t) - if test.runtime != "" && usingNoneDriver(runner) { + if test.runtime != "docker" && usingNoneDriver(runner) { t.Skipf("skipping, can't use %s with none driver", test.runtime) }