From 0c1d3f2e335cf3bc2d634ed6ac7ea2fe5f907d3f Mon Sep 17 00:00:00 2001
From: Zhongcheng Lao <laozhongcheng@gmail.com>
Date: Fri, 28 Dec 2018 19:32:38 +0800
Subject: [PATCH 01/24] Fixes multiple files behavior in files rootfs

Some files may be left uncopied to the VM when there are multiple
files in the .minikube/files directory as the code misinterprets
the last parent dir as parent dirs for all files, which results
some files with the same name were not copied.

This patch fixes the behavior.
---
 pkg/minikube/assets/addons.go | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go
index 604b0109ae..b6db690e76 100644
--- a/pkg/minikube/assets/addons.go
+++ b/pkg/minikube/assets/addons.go
@@ -302,14 +302,15 @@ func addMinikubeDirToAssets(basedir, vmpath string, assets *[]CopyableFile) erro
 			return errors.Wrapf(err, "checking if %s is directory", hostpath)
 		}
 		if !isDir {
-			if vmpath == "" {
+			vmdir := vmpath
+			if vmdir == "" {
 				rPath, err := filepath.Rel(basedir, hostpath)
 				if err != nil {
 					return errors.Wrap(err, "generating relative path")
 				}
 				rPath = filepath.Dir(rPath)
 				rPath = filepath.ToSlash(rPath)
-				vmpath = path.Join("/", rPath)
+				vmdir = path.Join("/", rPath)
 			}
 			permString := fmt.Sprintf("%o", info.Mode().Perm())
 			// The conversion will strip the leading 0 if present, so add it back
@@ -318,7 +319,7 @@ func addMinikubeDirToAssets(basedir, vmpath string, assets *[]CopyableFile) erro
 				permString = fmt.Sprintf("0%s", permString)
 			}
 
-			f, err := NewFileAsset(hostpath, vmpath, filepath.Base(hostpath), permString)
+			f, err := NewFileAsset(hostpath, vmdir, filepath.Base(hostpath), permString)
 			if err != nil {
 				return errors.Wrapf(err, "creating file asset for %s", hostpath)
 			}

From a6332e3c6e16da443974d18e9aa3e379ac7c639a Mon Sep 17 00:00:00 2001
From: Zhongcheng Lao <laozhongcheng@gmail.com>
Date: Wed, 16 Jan 2019 23:33:30 +0800
Subject: [PATCH 02/24] Add unit testcase for addon file patterns

afero introduced for mocking a filesystem.
---
 pkg/minikube/assets/addons_test.go | 159 +++++++++++++++++++++++++++++
 1 file changed, 159 insertions(+)
 create mode 100644 pkg/minikube/assets/addons_test.go

diff --git a/pkg/minikube/assets/addons_test.go b/pkg/minikube/assets/addons_test.go
new file mode 100644
index 0000000000..bfd8bfeeea
--- /dev/null
+++ b/pkg/minikube/assets/addons_test.go
@@ -0,0 +1,159 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package assets
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"k8s.io/minikube/pkg/minikube/constants"
+)
+
+func setupTestDir() (string, error) {
+	path, err := ioutil.TempDir("", "minipath")
+	if err != nil {
+		return "", err
+	}
+
+	os.Setenv(constants.MinikubeHome, path)
+	return path, err
+}
+
+func TestAddMinikubeDirAssets(t *testing.T) {
+
+	tests := []struct {
+		description string
+		baseDir     string
+		files       []struct {
+			relativePath string
+			expectedPath string
+		}
+		vmPath      string
+		expectedCfg string
+	}{
+		{
+			description: "relative path assets",
+			baseDir:     "/files",
+			files: []struct {
+				relativePath string
+				expectedPath string
+			}{
+				{
+					relativePath: "/dir1/file1.txt",
+					expectedPath: constants.AddonsPath,
+				},
+				{
+					relativePath: "/dir1/file2.txt",
+					expectedPath: constants.AddonsPath,
+				},
+				{
+					relativePath: "/dir2/file1.txt",
+					expectedPath: constants.AddonsPath,
+				},
+			},
+			vmPath: constants.AddonsPath,
+		},
+		{
+			description: "absolute path assets",
+			baseDir:     "/files",
+			files: []struct {
+				relativePath string
+				expectedPath string
+			}{
+				{
+					relativePath: "/dir1/file1.txt",
+					expectedPath: "/dir1",
+				},
+				{
+					relativePath: "/dir1/file2.txt",
+					expectedPath: "/dir1",
+				},
+				{
+					relativePath: "/dir2/file1.txt",
+					expectedPath: "/dir2",
+				},
+			},
+			vmPath: "",
+		},
+	}
+	var testDirs = make([]string, 0)
+	defer func() {
+		for _, testDir := range testDirs {
+			err := os.RemoveAll(testDir)
+			if err != nil {
+				t.Logf("got unexpected error removing test dir: %v", err)
+			}
+		}
+	}()
+
+	for _, test := range tests {
+		t.Run(test.description, func(t *testing.T) {
+			testDir, err := setupTestDir()
+			if err != nil {
+				t.Errorf("got unexpected error creating test dir: %v", err)
+				return
+			}
+
+			testDirs = append(testDirs, testDir)
+			testFileBaseDir := filepath.Join(testDir, test.baseDir)
+			want := make(map[string]string, 0)
+			for _, fileDef := range test.files {
+				err := func() error {
+					path := filepath.Join(testFileBaseDir, fileDef.relativePath)
+					err := os.MkdirAll(filepath.Dir(path), 0755)
+					want[path] = fileDef.expectedPath
+					if err != nil {
+						return err
+					}
+
+					file, err := os.Create(path)
+					if err != nil {
+						return err
+					}
+
+					defer file.Close()
+
+					_, err = file.WriteString("test")
+					return err
+				}()
+				if err != nil {
+					t.Errorf("unable to create file on fs: %v", err)
+					return
+				}
+			}
+
+			var actualFiles []CopyableFile
+			err = addMinikubeDirToAssets(testFileBaseDir, test.vmPath, &actualFiles)
+			if err != nil {
+				t.Errorf("got unexpected error adding minikube dir assets: %v", err)
+				return
+			}
+
+			got := make(map[string]string, 0)
+			for _, actualFile := range actualFiles {
+				got[actualFile.GetAssetName()] = actualFile.GetTargetDir()
+			}
+			if diff := cmp.Diff(want, got); diff != "" {
+				t.Errorf("files differ: (-want +got)\n%s", diff)
+			}
+		})
+	}
+
+}

From f346e5628b12a503100f58af35cd2d7fb8093235 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Tue, 12 Feb 2019 13:57:02 -0800
Subject: [PATCH 03/24] Add logging around the DHCP discovery process

---
 pkg/drivers/hyperkit/network.go | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/pkg/drivers/hyperkit/network.go b/pkg/drivers/hyperkit/network.go
index 61ff3b07a1..8c453ec057 100644
--- a/pkg/drivers/hyperkit/network.go
+++ b/pkg/drivers/hyperkit/network.go
@@ -25,6 +25,8 @@ import (
 	"os/exec"
 	"regexp"
 	"strings"
+
+	"github.com/docker/machine/libmachine/log"
 )
 
 const (
@@ -46,6 +48,7 @@ func GetIPAddressByMACAddress(mac string) (string, error) {
 }
 
 func getIpAddressFromFile(mac, path string) (string, error) {
+	log.Infof("Searching for %s in %s ...", mac, path)
 	file, err := os.Open(path)
 	if err != nil {
 		return "", err
@@ -56,8 +59,11 @@ func getIpAddressFromFile(mac, path string) (string, error) {
 	if err != nil {
 		return "", err
 	}
+	log.Infof("Found %d entries in %s!", len(dhcpEntries), path)
 	for _, dhcpEntry := range dhcpEntries {
+		log.Infof("dhcp entry: %+v", dhcpEntry)
 		if dhcpEntry.HWAddress == mac {
+			log.Infof("Found match: %s", mac)
 			return dhcpEntry.IPAddress, nil
 		}
 	}

From 888978c3a9aafc0325de03305b6f30a333a2d651 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Tue, 12 Feb 2019 13:58:18 -0800
Subject: [PATCH 04/24] Improve VM re-use reliability: detect crashing and/or
 badly installed hyperkit

---
 pkg/drivers/hyperkit/driver.go | 62 +++++++++++++++++++++++++++-------
 1 file changed, 50 insertions(+), 12 deletions(-)

diff --git a/pkg/drivers/hyperkit/driver.go b/pkg/drivers/hyperkit/driver.go
index 6314f7094c..fe4d2cc9bc 100644
--- a/pkg/drivers/hyperkit/driver.go
+++ b/pkg/drivers/hyperkit/driver.go
@@ -79,19 +79,28 @@ func NewDriver(hostName, storePath string) *Driver {
 
 // PreCreateCheck is called to enforce pre-creation steps
 func (d *Driver) PreCreateCheck() error {
+	return d.verifyRootPermissions()
+}
+
+// verifyRootPermissions is called before any step which needs root access
+func (d *Driver) verifyRootPermissions() error {
 	exe, err := os.Executable()
 	if err != nil {
 		return err
 	}
-
-	if syscall.Geteuid() != 0 {
+	euid := syscall.Geteuid()
+	log.Infof("exe=%s uid=%d", exe, euid)
+	if euid != 0 {
 		return fmt.Errorf(permErr, filepath.Base(exe), exe, exe)
 	}
-
 	return nil
 }
 
 func (d *Driver) Create() error {
+	if err := d.verifyRootPermissions(); err != nil {
+		return err
+	}
+
 	// TODO: handle different disk types.
 	if err := pkgdrivers.MakeDiskImage(d.BaseDriver, d.Boot2DockerURL, d.DiskSize); err != nil {
 		return errors.Wrap(err, "making disk image")
@@ -127,20 +136,27 @@ func (d *Driver) GetURL() (string, error) {
 
 // GetState returns the state that the host is in (running, stopped, etc)
 func (d *Driver) GetState() (state.State, error) {
-	pid := d.getPid()
-	if pid == 0 {
-		return state.Stopped, nil
-	}
-	p, err := os.FindProcess(pid)
-	if err != nil {
+	if err := d.verifyRootPermissions(); err != nil {
 		return state.Error, err
 	}
 
-	// Sending a signal of 0 can be used to check the existence of a process.
-	if err := p.Signal(syscall.Signal(0)); err != nil {
+	pid := d.getPid()
+	log.Infof("hyperkit pid from json: %d", pid)
+	if pid == 0 {
 		return state.Stopped, nil
 	}
+
+	p, err := ps.FindProcess(pid)
+	if err != nil {
+		log.Errorf("findprocess %d failed: %v", pid, err)
+		return state.Error, err
+	}
 	if p == nil {
+		log.Infof("hyperkit pid not running: %v", err)
+		return state.Stopped, nil
+	}
+	if !strings.Contains(p.Executable(), "hyperkit") {
+		log.Infof("pid %d is stale -- executable is %s, not hyperkit", pid, p.Executable())
 		return state.Stopped, nil
 	}
 	return state.Running, nil
@@ -148,11 +164,18 @@ func (d *Driver) GetState() (state.State, error) {
 
 // Kill stops a host forcefully
 func (d *Driver) Kill() error {
+	if err := d.verifyRootPermissions(); err != nil {
+		return err
+	}
 	return d.sendSignal(syscall.SIGKILL)
 }
 
 // Remove a host
 func (d *Driver) Remove() error {
+	if err := d.verifyRootPermissions(); err != nil {
+		return err
+	}
+
 	s, err := d.GetState()
 	if err != nil || s == state.Error {
 		log.Infof("Error checking machine status: %v, assuming it has been removed already", err)
@@ -171,6 +194,10 @@ func (d *Driver) Restart() error {
 
 // Start a host
 func (d *Driver) Start() error {
+	if err := d.verifyRootPermissions(); err != nil {
+		return err
+	}
+
 	stateDir := filepath.Join(d.StorePath, "machines", d.MachineName)
 	if err := d.recoverFromUncleanShutdown(); err != nil {
 		return err
@@ -219,7 +246,14 @@ func (d *Driver) Start() error {
 	}
 
 	getIP := func() error {
-		var err error
+		st, err := d.GetState()
+		if err != nil {
+			return errors.Wrap(err, "get state")
+		}
+		if st == state.Error || st == state.Stopped {
+			return fmt.Errorf("hyperkit crashed! command line:\n  hyperkit %s", d.Cmdline)
+		}
+
 		d.IPAddress, err = GetIPAddressByMACAddress(mac)
 		if err != nil {
 			return &commonutil.RetriableError{Err: err}
@@ -230,6 +264,7 @@ func (d *Driver) Start() error {
 	if err := commonutil.RetryAfter(30, getIP, 2*time.Second); err != nil {
 		return fmt.Errorf("IP address never found in dhcp leases file %v", err)
 	}
+	log.Infof("IP: %s", d.IPAddress)
 
 	if len(d.NFSShares) > 0 {
 		log.Info("Setting up NFS mounts")
@@ -298,6 +333,9 @@ func (d *Driver) recoverFromUncleanShutdown() error {
 
 // Stop a host gracefully
 func (d *Driver) Stop() error {
+	if err := d.verifyRootPermissions(); err != nil {
+		return err
+	}
 	d.cleanupNfsExports()
 	return d.sendSignal(syscall.SIGTERM)
 }

From 2045dc9ba419592d4cd70676b13ecb7573ee1c68 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Wed, 13 Feb 2019 13:35:08 -0800
Subject: [PATCH 05/24] Lint cleanup

---
 pkg/drivers/hyperkit/network.go      | 19 +++++++++----------
 pkg/drivers/hyperkit/network_test.go |  6 +++---
 2 files changed, 12 insertions(+), 13 deletions(-)

diff --git a/pkg/drivers/hyperkit/network.go b/pkg/drivers/hyperkit/network.go
index 8c453ec057..b801d50ebf 100644
--- a/pkg/drivers/hyperkit/network.go
+++ b/pkg/drivers/hyperkit/network.go
@@ -30,9 +30,9 @@ import (
 )
 
 const (
-	DHCPLeasesFile = "/var/db/dhcpd_leases"
-	CONFIG_PLIST   = "/Library/Preferences/SystemConfiguration/com.apple.vmnet"
-	NET_ADDR_KEY   = "Shared_Net_Address"
+	LeasesPath       = "/var/db/dhcpd_leases"
+	VMNetDomain      = "/Library/Preferences/SystemConfiguration/com.apple.vmnet"
+	SharedNetAddrKey = "Shared_Net_Address"
 )
 
 type DHCPEntry struct {
@@ -44,10 +44,10 @@ type DHCPEntry struct {
 }
 
 func GetIPAddressByMACAddress(mac string) (string, error) {
-	return getIpAddressFromFile(mac, DHCPLeasesFile)
+	return getIPAddressFromFile(mac, LeasesPath)
 }
 
-func getIpAddressFromFile(mac, path string) (string, error) {
+func getIPAddressFromFile(mac, path string) (string, error) {
 	log.Infof("Searching for %s in %s ...", mac, path)
 	file, err := os.Open(path)
 	if err != nil {
@@ -120,12 +120,11 @@ func trimMacAddress(rawUUID string) string {
 }
 
 func GetNetAddr() (net.IP, error) {
-	_, err := os.Stat(CONFIG_PLIST + ".plist")
-	if err != nil {
-		return nil, fmt.Errorf("Does not exist %s", CONFIG_PLIST+".plist")
+	plistPath := VMNetDomain + ".plist"
+	if _, err := os.Stat(plistPath); err != nil {
+		return nil, fmt.Errorf("stat: %v", err)
 	}
-
-	out, err := exec.Command("defaults", "read", CONFIG_PLIST, NET_ADDR_KEY).Output()
+	out, err := exec.Command("defaults", "read", VMNetDomain, SharedNetAddrKey).Output()
 	if err != nil {
 		return nil, err
 	}
diff --git a/pkg/drivers/hyperkit/network_test.go b/pkg/drivers/hyperkit/network_test.go
index d8b831b823..45f72ec183 100644
--- a/pkg/drivers/hyperkit/network_test.go
+++ b/pkg/drivers/hyperkit/network_test.go
@@ -88,13 +88,13 @@ func Test_getIpAddressFromFile(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := getIpAddressFromFile(tt.args.mac, tt.args.path)
+			got, err := getIPAddressFromFile(tt.args.mac, tt.args.path)
 			if (err != nil) != tt.wantErr {
-				t.Errorf("getIpAddressFromFile() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("getIPAddressFromFile() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
 			if got != tt.want {
-				t.Errorf("getIpAddressFromFile() = %v, want %v", got, tt.want)
+				t.Errorf("getIPAddressFromFile() = %v, want %v", got, tt.want)
 			}
 		})
 	}

From 4b875ddd7f5f9577fd9baa38586072ea3ff75bca Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Wed, 13 Feb 2019 13:35:28 -0800
Subject: [PATCH 06/24] Run goimports

---
 pkg/drivers/hyperkit/iso.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/pkg/drivers/hyperkit/iso.go b/pkg/drivers/hyperkit/iso.go
index 9139073db7..14503e1b2f 100644
--- a/pkg/drivers/hyperkit/iso.go
+++ b/pkg/drivers/hyperkit/iso.go
@@ -21,7 +21,6 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
-
 	"strings"
 
 	"github.com/hooklift/iso9660"

From 473f5983cb8b7e61a15290fc12bbbf87432708fe Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Wed, 13 Feb 2019 13:36:45 -0800
Subject: [PATCH 07/24] Allow driver name to be com.docker.hyper, as seen in
 integration testing

---
 pkg/drivers/hyperkit/driver.go | 78 +++++++++++++++++-----------------
 1 file changed, 38 insertions(+), 40 deletions(-)

diff --git a/pkg/drivers/hyperkit/driver.go b/pkg/drivers/hyperkit/driver.go
index fe4d2cc9bc..2da6910fd2 100644
--- a/pkg/drivers/hyperkit/driver.go
+++ b/pkg/drivers/hyperkit/driver.go
@@ -134,6 +134,27 @@ func (d *Driver) GetURL() (string, error) {
 	return fmt.Sprintf("tcp://%s:2376", ip), nil
 }
 
+// Return the state of the hyperkit pid
+func pidState(pid int) (state.State, error) {
+	if pid == 0 {
+		return state.Stopped, nil
+	}
+	p, err := ps.FindProcess(pid)
+	if err != nil {
+		return state.Error, err
+	}
+	if p == nil {
+		log.Infof("hyperkit pid missing from process table, err=%v", err)
+		return state.Stopped, nil
+	}
+	// hyperkit or com.docker.hyper
+	if !strings.Contains(p.Executable(), "hyper") {
+		log.Infof("pid %d is stale, and is being used by %s", pid, p.Executable())
+		return state.Stopped, nil
+	}
+	return state.Running, nil
+}
+
 // GetState returns the state that the host is in (running, stopped, etc)
 func (d *Driver) GetState() (state.State, error) {
 	if err := d.verifyRootPermissions(); err != nil {
@@ -142,24 +163,7 @@ func (d *Driver) GetState() (state.State, error) {
 
 	pid := d.getPid()
 	log.Infof("hyperkit pid from json: %d", pid)
-	if pid == 0 {
-		return state.Stopped, nil
-	}
-
-	p, err := ps.FindProcess(pid)
-	if err != nil {
-		log.Errorf("findprocess %d failed: %v", pid, err)
-		return state.Error, err
-	}
-	if p == nil {
-		log.Infof("hyperkit pid not running: %v", err)
-		return state.Stopped, nil
-	}
-	if !strings.Contains(p.Executable(), "hyperkit") {
-		log.Infof("pid %d is stale -- executable is %s, not hyperkit", pid, p.Executable())
-		return state.Stopped, nil
-	}
-	return state.Running, nil
+	return pidState(pid)
 }
 
 // Kill stops a host forcefully
@@ -292,42 +296,38 @@ func (d *Driver) recoverFromUncleanShutdown() error {
 	stateDir := filepath.Join(d.StorePath, "machines", d.MachineName)
 	pidFile := filepath.Join(stateDir, pidFileName)
 
-	_, err := os.Stat(pidFile)
-
-	if os.IsNotExist(err) {
-		log.Infof("clean start, hyperkit pid file doesn't exist: %s", pidFile)
-		return nil
-	}
-
-	if err != nil {
-		return errors.Wrap(err, "checking hyperkit pid file existence")
+	if _, err := os.Stat(pidFile); err != nil {
+		if os.IsNotExist(err) {
+			log.Infof("clean start, hyperkit pid file doesn't exist: %s", pidFile)
+			return nil
+		}
+		return errors.Wrap(err, "stat")
 	}
 
 	log.Warnf("minikube might have been shutdown in an unclean way, the hyperkit pid file still exists: %s", pidFile)
-
-	content, err := ioutil.ReadFile(pidFile)
+	bs, err := ioutil.ReadFile(pidFile)
 	if err != nil {
 		return errors.Wrapf(err, "reading pidfile %s", pidFile)
 	}
-	pid, err := strconv.Atoi(string(content))
+	content := strings.TrimSpace(string(bs))
+	pid, err := strconv.Atoi(content)
 	if err != nil {
 		return errors.Wrapf(err, "parsing pidfile %s", pidFile)
 	}
 
-	p, err := ps.FindProcess(pid)
+	st, err := pidState(pid)
 	if err != nil {
-		return errors.Wrapf(err, "trying to find process for PID %d", pid)
+		return errors.Wrap(err, "pidState")
 	}
 
-	if p != nil && !strings.Contains(p.Executable(), "hyperkit") {
-		return fmt.Errorf("something is not right...please stop all minikube instances, seemingly a hyperkit server is already running with pid %d, executable: %s", pid, p.Executable())
+	log.Infof("pid %d is in state %q", pid, st)
+	if st == state.Running {
+		return nil
 	}
-
-	log.Infof("No running hyperkit process found with PID %d, removing %s...", pid, pidFile)
+	log.Infof("Removing stale pid file %s...", pidFile)
 	if err := os.Remove(pidFile); err != nil {
 		return errors.Wrap(err, fmt.Sprintf("removing pidFile %s", pidFile))
 	}
-
 	return nil
 }
 
@@ -372,9 +372,7 @@ func (d *Driver) extractVSockPorts() ([]int, error) {
 	for _, port := range d.VSockPorts {
 		p, err := strconv.Atoi(port)
 		if err != nil {
-			var err InvalidPortNumberError
-			err = InvalidPortNumberError(port)
-			return nil, err
+			return nil, InvalidPortNumberError(port)
 		}
 		vsockPorts = append(vsockPorts, p)
 	}

From 6a18ad9bc509132786e5969f225447b5c432dab8 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Wed, 13 Feb 2019 21:15:46 -0800
Subject: [PATCH 08/24] Reduce wait times, and wait more consistently

---
 pkg/util/kubernetes.go | 32 +++++++++++++++++---------------
 1 file changed, 17 insertions(+), 15 deletions(-)

diff --git a/pkg/util/kubernetes.go b/pkg/util/kubernetes.go
index 927cf1dd78..0b8df0d0f7 100644
--- a/pkg/util/kubernetes.go
+++ b/pkg/util/kubernetes.go
@@ -20,8 +20,11 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/golang/glog"
 	"github.com/pkg/errors"
-
+	appsv1 "k8s.io/api/apps/v1"
+	"k8s.io/api/core/v1"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/labels"
@@ -29,16 +32,15 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apimachinery/pkg/watch"
-	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
-
-	appsv1 "k8s.io/api/apps/v1"
-	"k8s.io/api/core/v1"
-	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/client-go/kubernetes"
-
-	"github.com/golang/glog"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
+)
+
+var (
+	ReasonableMutateTime = time.Minute * 1
+	ReasonableStartTime  = time.Minute * 5
 )
 
 type PodStore struct {
@@ -108,11 +110,11 @@ func StartPods(c kubernetes.Interface, namespace string, pod v1.Pod, waitForRunn
 	return nil
 }
 
-// Wait up to 10 minutes for all matching pods to become Running and at least one
-// matching pod exists.
+// WaitForPodsWithLabelRunning waits for all matching pods to become Running and at least one matching pod exists.
 func WaitForPodsWithLabelRunning(c kubernetes.Interface, ns string, label labels.Selector) error {
+	glog.Infof("Waiting for pod with label %q in ns %q ...", ns, label)
 	lastKnownPodNumber := -1
-	return wait.PollImmediate(constants.APICallRetryInterval, time.Minute*10, func() (bool, error) {
+	return wait.PollImmediate(constants.APICallRetryInterval, ReasonableStartTime, func() (bool, error) {
 		listOpts := metav1.ListOptions{LabelSelector: label.String()}
 		pods, err := c.CoreV1().Pods(ns).List(listOpts)
 		if err != nil {
@@ -139,9 +141,9 @@ func WaitForPodsWithLabelRunning(c kubernetes.Interface, ns string, label labels
 	})
 }
 
-// Wait up to 10 minutes for a pod to be deleted
+// WaitForPodDelete waits for a pod to be deleted
 func WaitForPodDelete(c kubernetes.Interface, ns string, label labels.Selector) error {
-	return wait.PollImmediate(constants.APICallRetryInterval, time.Minute*10, func() (bool, error) {
+	return wait.PollImmediate(constants.APICallRetryInterval, ReasonableMutateTime, func() (bool, error) {
 		listOpts := metav1.ListOptions{LabelSelector: label.String()}
 		pods, err := c.CoreV1().Pods(ns).List(listOpts)
 		if err != nil {
@@ -152,9 +154,9 @@ func WaitForPodDelete(c kubernetes.Interface, ns string, label labels.Selector)
 	})
 }
 
-// Wait up to 10 minutes for the given event to appear
+// WaitForEvent waits for the given event to appear
 func WaitForEvent(c kubernetes.Interface, ns string, reason string) error {
-	return wait.PollImmediate(constants.APICallRetryInterval, time.Minute*10, func() (bool, error) {
+	return wait.PollImmediate(constants.APICallRetryInterval, ReasonableMutateTime, func() (bool, error) {
 		events, err := c.Events().Events("default").List(metav1.ListOptions{})
 		if err != nil {
 			glog.Infof("error getting events: %v", err)

From 08da5fe3b5fa575dc1df6dd6afd8ced33a7b76a7 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Wed, 13 Feb 2019 21:29:20 -0800
Subject: [PATCH 09/24] Add pod logs to 'logs' command, show detected problems
 during minikube start

---
 cmd/minikube/cmd/logs.go                     |  57 +++++++-
 cmd/minikube/cmd/start.go                    |  42 +++---
 pkg/minikube/bootstrapper/bootstrapper.go    |   4 +-
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go |  28 ++--
 pkg/minikube/console/style.go                |   1 +
 pkg/minikube/cruntime/containerd.go          |   5 +
 pkg/minikube/cruntime/cri.go                 |  15 ++
 pkg/minikube/cruntime/crio.go                |   5 +
 pkg/minikube/cruntime/cruntime.go            |   2 +
 pkg/minikube/cruntime/docker.go              |  15 ++
 pkg/minikube/logs/logs.go                    | 141 +++++++++++++++++++
 pkg/minikube/logs/logs_test.go               |  44 ++++++
 12 files changed, 313 insertions(+), 46 deletions(-)
 create mode 100644 pkg/minikube/logs/logs.go
 create mode 100644 pkg/minikube/logs/logs_test.go

diff --git a/cmd/minikube/cmd/logs.go b/cmd/minikube/cmd/logs.go
index 4a632d772a..fd2b05f567 100644
--- a/cmd/minikube/cmd/logs.go
+++ b/cmd/minikube/cmd/logs.go
@@ -23,13 +23,25 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config"
-	cmdUtil "k8s.io/minikube/cmd/util"
+	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/console"
+	"k8s.io/minikube/pkg/minikube/cruntime"
+	"k8s.io/minikube/pkg/minikube/logs"
 	"k8s.io/minikube/pkg/minikube/machine"
 )
 
+const (
+	// number of problems per log to output
+	numberOfProblems = 5
+)
+
 var (
-	follow bool
+	// followLogs triggers tail -f mode
+	followLogs bool
+	// numberOfLines is how many lines to output, set via -n
+	numberOfLines int
+	// showProblems only shows lines that match known issues
+	showProblems bool
 )
 
 // logsCmd represents the logs command
@@ -38,26 +50,57 @@ var logsCmd = &cobra.Command{
 	Short: "Gets the logs of the running instance, used for debugging minikube, not user code",
 	Long:  `Gets the logs of the running instance, used for debugging minikube, not user code.`,
 	Run: func(cmd *cobra.Command, args []string) {
+		cfg, err := config.Load()
+		if err != nil {
+			console.Fatal("Error getting config: %v", err)
+		}
+
 		api, err := machine.NewAPIClient()
 		if err != nil {
 			console.Fatal("Error getting client: %v", err)
 			os.Exit(1)
 		}
 		defer api.Close()
-		clusterBootstrapper, err := GetClusterBootstrapper(api, viper.GetString(cmdcfg.Bootstrapper))
+
+		h, err := api.Load(config.GetMachineName())
+		if err != nil {
+			glog.Exitf("api load: %v", err)
+		}
+		runner, err := machine.CommandRunner(h)
+		if err != nil {
+			glog.Exitf("command runner: %v", err)
+		}
+		bs, err := GetClusterBootstrapper(api, viper.GetString(cmdcfg.Bootstrapper))
 		if err != nil {
 			glog.Exitf("Error getting cluster bootstrapper: %v", err)
 		}
 
-		err = clusterBootstrapper.GetClusterLogsTo(follow, os.Stdout)
+		cr, err := cruntime.New(cruntime.Config{Type: cfg.KubernetesConfig.ContainerRuntime, Runner: runner})
 		if err != nil {
-			console.Fatal("Error getting machine logs:", err)
-			cmdUtil.MaybeReportErrorAndExit(err)
+			glog.Exitf("Unable to get runtime: %v", err)
+		}
+		if followLogs {
+			err := logs.Follow(cr, bs, runner)
+			if err != nil {
+				console.Failure("output: %v", err)
+			}
+			return
+		}
+		if showProblems {
+			problems := logs.FindProblems(cr, bs, runner)
+			logs.OutputProblems(problems, numberOfProblems)
+			return
+		}
+		err = logs.Output(cr, bs, runner, numberOfLines)
+		if err != nil {
+			console.Failure("output: %v", err)
 		}
 	},
 }
 
 func init() {
-	logsCmd.Flags().BoolVarP(&follow, "follow", "f", false, "Show only the most recent journal entries, and continuously print new entries as they are appended to the journal.")
+	logsCmd.Flags().BoolVarP(&followLogs, "follow", "f", false, "Show only the most recent journal entries, and continuously print new entries as they are appended to the journal.")
+	logsCmd.Flags().BoolVar(&showProblems, "problems", false, "Show only log entries which point to known problems")
+	logsCmd.Flags().IntVarP(&numberOfLines, "length", "n", 50, "Number of lines back to go within the log")
 	RootCmd.AddCommand(logsCmd)
 }
diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index dba7186966..854ed84a5e 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -45,6 +45,7 @@ import (
 	"k8s.io/minikube/pkg/minikube/console"
 	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/cruntime"
+	"k8s.io/minikube/pkg/minikube/logs"
 	"k8s.io/minikube/pkg/minikube/machine"
 	pkgutil "k8s.io/minikube/pkg/util"
 	"k8s.io/minikube/pkg/util/kubeconfig"
@@ -186,15 +187,19 @@ func runStart(cmd *cobra.Command, args []string) {
 	if err := saveConfig(config); err != nil {
 		reportErrAndExit("Failed to save config", err)
 	}
+	runner, err := machine.CommandRunner(host)
+	if err != nil {
+		reportErrAndExit("Failed to get command runner", err)
+	}
 
-	configureRuntimes(host)
+	cr := configureRuntimes(host, runner)
 	bs := prepareHostEnvironment(m, config.KubernetesConfig)
 	waitCacheImages(&cacheGroup)
 
 	// The kube config must be update must come before bootstrapping, otherwise health checks may use a stale IP
 	kubeconfig := updateKubeConfig(host, &config)
-	bootstrapCluster(bs, config.KubernetesConfig, preexisting)
-	validateCluster(bs, ip)
+	bootstrapCluster(bs, cr, runner, config.KubernetesConfig, preexisting)
+	validateCluster(bs, cr, runner, ip)
 	configureMounts()
 	if err = LoadCachedImagesInConfigFile(); err != nil {
 		console.Failure("Unable to load cached images from config file.")
@@ -451,12 +456,7 @@ func updateKubeConfig(h *host.Host, c *cfg.Config) *kubeconfig.KubeConfigSetup {
 }
 
 // configureRuntimes does what needs to happen to get a runtime going.
-func configureRuntimes(h *host.Host) {
-	runner, err := machine.CommandRunner(h)
-	if err != nil {
-		reportErrAndExit("Failed to get command runner", err)
-	}
-
+func configureRuntimes(h *host.Host, runner bootstrapper.CommandRunner) cruntime.Manager {
 	config := cruntime.Config{Type: viper.GetString(containerRuntime), Runner: runner}
 	cr, err := cruntime.New(config)
 	if err != nil {
@@ -475,7 +475,7 @@ func configureRuntimes(h *host.Host) {
 	if err != nil {
 		cmdutil.MaybeReportErrorAndExit(err)
 	}
-
+	return cr
 }
 
 // waitCacheImages blocks until the image cache jobs complete
@@ -490,7 +490,7 @@ func waitCacheImages(g *errgroup.Group) {
 }
 
 // bootstrapCluster starts Kubernetes using the chosen bootstrapper
-func bootstrapCluster(bs bootstrapper.Bootstrapper, kc cfg.KubernetesConfig, preexisting bool) {
+func bootstrapCluster(bs bootstrapper.Bootstrapper, r cruntime.Manager, runner bootstrapper.CommandRunner, kc cfg.KubernetesConfig, preexisting bool) {
 	console.OutStyle("pulling", "Pulling images used by Kubernetes %s ...", kc.KubernetesVersion)
 	if err := bs.PullImages(kc); err != nil {
 		fmt.Printf("Unable to pull images, which may be OK: %v", err)
@@ -501,19 +501,19 @@ func bootstrapCluster(bs bootstrapper.Bootstrapper, kc cfg.KubernetesConfig, pre
 	if preexisting {
 		console.OutStyle("restarting", "Relaunching Kubernetes %s using %s ... ", kc.KubernetesVersion, bsName)
 		if err := bs.RestartCluster(kc); err != nil {
-			reportErrAndExit("Error restarting cluster", err)
+			reportProblemsAndExit("Error restarting cluster", err, logs.FindProblems(r, bs, runner))
 		}
 		return
 	}
 
 	console.OutStyle("launch", "Launching Kubernetes %s using %s ... ", kc.KubernetesVersion, bsName)
 	if err := bs.StartCluster(kc); err != nil {
-		reportErrAndExit("Error starting cluster", err)
+		reportProblemsAndExit("Error starting cluster", err, logs.FindProblems(r, bs, runner))
 	}
 }
 
 // validateCluster validates that the cluster is well-configured and healthy
-func validateCluster(bs bootstrapper.Bootstrapper, ip string) {
+func validateCluster(bs bootstrapper.Bootstrapper, r cruntime.Manager, runner bootstrapper.CommandRunner, ip string) {
 	console.OutStyle("verifying-noline", "Verifying component health ...")
 	kStat := func() (err error) {
 		st, err := bs.GetKubeletStatus()
@@ -525,7 +525,7 @@ func validateCluster(bs bootstrapper.Bootstrapper, ip string) {
 	}
 	err := pkgutil.RetryAfter(20, kStat, 3*time.Second)
 	if err != nil {
-		reportErrAndExit("kubelet checks failed", err)
+		reportProblemsAndExit("kubelet checks failed", err, logs.FindProblems(r, bs, runner))
 	}
 	aStat := func() (err error) {
 		st, err := bs.GetApiServerStatus(net.ParseIP(ip))
@@ -538,7 +538,7 @@ func validateCluster(bs bootstrapper.Bootstrapper, ip string) {
 
 	err = pkgutil.RetryAfter(30, aStat, 10*time.Second)
 	if err != nil {
-		reportErrAndExit("apiserver checks failed", err)
+		reportProblemsAndExit("apiserver checks failed", err, logs.FindProblems(r, bs, runner))
 	}
 	console.OutLn("")
 }
@@ -623,9 +623,17 @@ func fatalExit(format string, a ...interface{}) {
 	os.Exit(1)
 }
 
-// reportFatalExit is a shortcut for outputting an error, reporting it, and exiting.
+// reportErrAndExit is a shortcut for outputting an error, reporting it, and exiting.
 func reportErrAndExit(msg string, err error) {
 	console.Fatal(msg+": %v", err)
 	cmdutil.MaybeReportErrorAndExit(err)
 	os.Exit(1)
 }
+
+// reportFatalExit is a shortcut for outputting an error, reporting it, and exiting.
+func reportProblemsAndExit(msg string, err error, problems map[string][]string) {
+	console.Fatal(msg+": %v", err)
+	logs.OutputProblems(problems, 3)
+	cmdutil.MaybeReportErrorAndExit(err)
+	os.Exit(1)
+}
diff --git a/pkg/minikube/bootstrapper/bootstrapper.go b/pkg/minikube/bootstrapper/bootstrapper.go
index 6cd961b18d..35cb89288a 100644
--- a/pkg/minikube/bootstrapper/bootstrapper.go
+++ b/pkg/minikube/bootstrapper/bootstrapper.go
@@ -17,7 +17,6 @@ limitations under the License.
 package bootstrapper
 
 import (
-	"io"
 	"net"
 
 	"k8s.io/minikube/pkg/minikube/config"
@@ -32,7 +31,8 @@ type Bootstrapper interface {
 	UpdateCluster(config.KubernetesConfig) error
 	RestartCluster(config.KubernetesConfig) error
 	DeleteCluster(config.KubernetesConfig) error
-	GetClusterLogsTo(follow bool, out io.Writer) error
+	// LogCommands returns a map of log type to a command which will display that log.
+	LogCommands(int, bool) map[string]string
 	SetupCerts(cfg config.KubernetesConfig) error
 	GetKubeletStatus() (string, error)
 	GetApiServerStatus(net.IP) (string, error)
diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index 1eb4d0fe8f..23eb762222 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -21,7 +21,6 @@ import (
 	"crypto"
 	"crypto/tls"
 	"fmt"
-	"io"
 	"net"
 	"net/http"
 	"os"
@@ -121,28 +120,17 @@ func (k *KubeadmBootstrapper) GetApiServerStatus(ip net.IP) (string, error) {
 	return state.Running.String(), nil
 }
 
-// TODO(r2d4): Should this aggregate all the logs from the control plane?
-// Maybe subcommands for each component? minikube logs apiserver?
-func (k *KubeadmBootstrapper) GetClusterLogsTo(follow bool, out io.Writer) error {
-	var flags []string
-	if follow {
-		flags = append(flags, "-f")
+// LogCommands returns a map of log type to a command which will display that log.
+func (k *KubeadmBootstrapper) LogCommands(len int, follow bool) map[string]string {
+	var kcmd strings.Builder
+	kcmd.WriteString("journalctl -u kubelet")
+	if len > 0 {
+		kcmd.WriteString(fmt.Sprintf(" -n %d", len))
 	}
-	logsCommand := fmt.Sprintf("sudo journalctl %s -u kubelet", strings.Join(flags, " "))
-
 	if follow {
-		if err := k.c.CombinedOutputTo(logsCommand, out); err != nil {
-			return errors.Wrap(err, "getting cluster logs")
+		kcmd.WriteString(" -f")
 		}
-	} else {
-
-		logs, err := k.c.CombinedOutput(logsCommand)
-		if err != nil {
-			return errors.Wrap(err, "getting cluster logs")
-		}
-		fmt.Fprint(out, logs)
-	}
-	return nil
+	return map[string]string{"kubelet": kcmd.String()}
 }
 
 func (k *KubeadmBootstrapper) StartCluster(k8s config.KubernetesConfig) error {
diff --git a/pkg/minikube/console/style.go b/pkg/minikube/console/style.go
index 1380769995..1e6a18cff7 100644
--- a/pkg/minikube/console/style.go
+++ b/pkg/minikube/console/style.go
@@ -49,6 +49,7 @@ var styles = map[string]style{
 	"launch":     {Prefix: "🚀  "},
 	"thumbs-up":  {Prefix: "👍  "},
 	"option":     {Prefix: "    ▪ "}, // Indented bullet
+	"bullet":     {Prefix: "    ▪ "}, // Indented bullet
 	"crushed":    {Prefix: "💔  "},
 
 	// Specialized purpose styles
diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go
index 611197a3b5..e46a30e5da 100644
--- a/pkg/minikube/cruntime/containerd.go
+++ b/pkg/minikube/cruntime/containerd.go
@@ -107,3 +107,8 @@ func (r *Containerd) KillContainers(ids []string) error {
 func (r *Containerd) StopContainers(ids []string) error {
 	return stopCRIContainers(r.Runner, ids)
 }
+
+// ContainerLogCmd returns the command to retrieve the log for a container based on ID
+func (r *Containerd) ContainerLogCmd(id string, len int, follow bool) string {
+	return criContainerLogCmd(id, len, follow)
+}
diff --git a/pkg/minikube/cruntime/cri.go b/pkg/minikube/cruntime/cri.go
index 67f4595ef0..0ccc7bfa57 100644
--- a/pkg/minikube/cruntime/cri.go
+++ b/pkg/minikube/cruntime/cri.go
@@ -60,3 +60,18 @@ image-endpoint: unix://{{.Socket}}
 	}
 	return cr.Run(fmt.Sprintf("sudo mkdir -p %s && printf %%s \"%s\" | sudo tee %s", path.Dir(cPath), b.String(), cPath))
 }
+
+// criContainerLogCmd returns the command to retrieve the log for a container based on ID
+func criContainerLogCmd(id string, len int, follow bool) string {
+	var cmd strings.Builder
+	cmd.WriteString("crictl logs ")
+	if len > 0 {
+		cmd.WriteString(fmt.Sprintf("--tail %d ", len))
+	}
+	if follow {
+		cmd.WriteString("--follow ")
+	}
+
+	cmd.WriteString(id)
+	return cmd.String()
+}
diff --git a/pkg/minikube/cruntime/crio.go b/pkg/minikube/cruntime/crio.go
index 2897dd2c06..93aa426ddf 100644
--- a/pkg/minikube/cruntime/crio.go
+++ b/pkg/minikube/cruntime/crio.go
@@ -106,3 +106,8 @@ func (r *CRIO) KillContainers(ids []string) error {
 func (r *CRIO) StopContainers(ids []string) error {
 	return stopCRIContainers(r.Runner, ids)
 }
+
+// ContainerLogCmd returns the command to retrieve the log for a container based on ID
+func (r *CRIO) ContainerLogCmd(id string, len int, follow bool) string {
+	return criContainerLogCmd(id, len, follow)
+}
diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go
index 9f47808616..fe670c29e5 100644
--- a/pkg/minikube/cruntime/cruntime.go
+++ b/pkg/minikube/cruntime/cruntime.go
@@ -61,6 +61,8 @@ type Manager interface {
 	KillContainers([]string) error
 	// StopContainers stops containers based on ID
 	StopContainers([]string) error
+	// ContainerLogCmd returns the command to retrieve the log for a container based on ID
+	ContainerLogCmd(string, int, bool) string
 }
 
 // Config is runtime configuration
diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go
index 92f4c0bb54..f70a513603 100644
--- a/pkg/minikube/cruntime/docker.go
+++ b/pkg/minikube/cruntime/docker.go
@@ -101,3 +101,18 @@ func (r *Docker) KillContainers(ids []string) error {
 func (r *Docker) StopContainers(ids []string) error {
 	return r.Runner.Run(fmt.Sprintf("docker stop %s", strings.Join(ids, " ")))
 }
+
+// ContainerLogCmd returns the command to retrieve the log for a container based on ID
+func (r *Docker) ContainerLogCmd(id string, len int, follow bool) string {
+	var cmd strings.Builder
+	cmd.WriteString("docker logs ")
+	if len > 0 {
+		cmd.WriteString(fmt.Sprintf("--tail %d ", len))
+	}
+	if follow {
+		cmd.WriteString("--follow ")
+	}
+
+	cmd.WriteString(id)
+	return cmd.String()
+}
diff --git a/pkg/minikube/logs/logs.go b/pkg/minikube/logs/logs.go
new file mode 100644
index 0000000000..95c6c4d891
--- /dev/null
+++ b/pkg/minikube/logs/logs.go
@@ -0,0 +1,141 @@
+/*
+Copyright 2019 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// package logs are convenience methods for fetching logs from a minikube cluster
+package logs
+
+import (
+	"fmt"
+	"os"
+	"regexp"
+	"sort"
+	"strings"
+
+	"github.com/docker/machine/libmachine/log"
+	"github.com/golang/glog"
+	"k8s.io/minikube/pkg/minikube/bootstrapper"
+	"k8s.io/minikube/pkg/minikube/console"
+	"k8s.io/minikube/pkg/minikube/cruntime"
+)
+
+// rootCauseRe is a regular expression that matches known failure root causes
+var rootCauseRe = regexp.MustCompile(`^error: |eviction manager: pods.* evicted|unknown flag: --`)
+
+// importantPods are a list of pods to retrieve logs for, in addition to the bootstrapper logs.
+var importantPods = []string{
+	"k8s_kube-apiserver",
+	"k8s_coredns_coredns",
+	"k8s_kube-scheduler",
+}
+
+// Follow follows logs from multiple files in tail(1) format
+func Follow(r cruntime.Manager, bs bootstrapper.Bootstrapper, runner bootstrapper.CommandRunner) error {
+	cs := []string{}
+	for _, v := range logCommands(r, bs, 0, true) {
+		cs = append(cs, v+" &")
+	}
+	cs = append(cs, "wait")
+	return runner.CombinedOutputTo(strings.Join(cs, " "), os.Stdout)
+}
+
+// IsProblem returns whether this line matches a known problem
+func IsProblem(line string) bool {
+	return rootCauseRe.MatchString(line)
+}
+
+// FindProblems finds possible root causes among the logs
+func FindProblems(r cruntime.Manager, bs bootstrapper.Bootstrapper, runner bootstrapper.CommandRunner) map[string][]string {
+	pMap := map[string][]string{}
+	cmds := logCommands(r, bs, 200, false)
+	for name, cmd := range cmds {
+		log.Infof("Gathering logs for %s ...", name)
+		out, err := runner.CombinedOutput(cmds[name])
+		if err != nil {
+			glog.Warningf("failed %s: %s: %v", name, cmd, err)
+			continue
+		}
+		log.Infof("log length: %d", len(out))
+
+		problems := []string{}
+		for _, l := range strings.Split(out, "\n") {
+			if IsProblem(l) {
+				glog.Warningf("Found %s problem: %s", name, l)
+				problems = append(problems, l)
+			}
+		}
+		if len(problems) > 0 {
+			pMap[name] = problems
+		}
+	}
+	return pMap
+}
+
+// OutputProblems outputs discovered problems.
+func OutputProblems(problems map[string][]string, maxLines int) {
+	for name, lines := range problems {
+		console.OutStyle("failure", "Problem detected in %q", name)
+		if len(lines) > maxLines {
+			lines = lines[len(lines)-maxLines:]
+		}
+		for _, l := range lines {
+			console.OutStyle("bullet", l)
+		}
+	}
+}
+
+// Output displays logs from multiple sources in tail(1) format
+func Output(r cruntime.Manager, bs bootstrapper.Bootstrapper, runner bootstrapper.CommandRunner, lines int) error {
+	cmds := logCommands(r, bs, lines, false)
+	names := []string{}
+	for k := range cmds {
+		names = append(names, k)
+	}
+	sort.Strings(names)
+
+	failed := []string{}
+	for _, name := range names {
+		console.OutLn("==> %s <==", name)
+		out, err := runner.CombinedOutput(cmds[name])
+		if err != nil {
+			glog.Errorf("failed: %v", err)
+			failed = append(failed, name)
+			continue
+		}
+		console.OutLn(out)
+	}
+	if len(failed) > 0 {
+		return fmt.Errorf("unable to fetch logs for: %s", strings.Join(failed, ", "))
+	}
+	return nil
+}
+
+// logCommands returns a list of commands that would be run to receive the anticipated logs
+func logCommands(r cruntime.Manager, bs bootstrapper.Bootstrapper, length int, follow bool) map[string]string {
+	cmds := bs.LogCommands(length, follow)
+	for _, pod := range importantPods {
+		ids, err := r.ListContainers(pod)
+		if err != nil {
+			glog.Errorf("Failed to list containers for %q: %v", pod, err)
+			continue
+		}
+		if len(ids) == 0 {
+			glog.Errorf("No containers found matching %q", pod)
+			continue
+		}
+		cmds[pod] = r.ContainerLogCmd(ids[0], length, follow)
+	}
+	return cmds
+}
diff --git a/pkg/minikube/logs/logs_test.go b/pkg/minikube/logs/logs_test.go
new file mode 100644
index 0000000000..5e1405acec
--- /dev/null
+++ b/pkg/minikube/logs/logs_test.go
@@ -0,0 +1,44 @@
+/*
+Copyright 2019 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package logs
+
+import (
+	"testing"
+)
+
+func TestIsProblem(t *testing.T) {
+	var tests = []struct {
+		name  string
+		want  bool
+		input string
+	}{
+		{"almost", false, "F2350 I would love to be an unknown flag, but I am not -- :( --"},
+		{"apiserver-required-flag #1962", true, "error: [service-account-issuer is a required flag when BoundServiceAccountTokenVolume is enabled, --service-account-signing-key-file and --service-account-issuer are required flags"},
+		{"kubelet-eviction #", true, "I0213 07:16:44.041623    2410 eviction_manager.go:187] eviction manager: pods kube-apiserver-minikube_kube-system(87f41e2e0629c3deb5c2239e08d8045d) evicted, waiting for pod to be cleaned up"},
+		{"kubelet-unknown-flag #3655", true, "F0212 14:55:46.443031    2693 server.go:148] unknown flag: --AllowedUnsafeSysctls"},
+		{"apiserver-auth-mode #2852", true, `{"log":"Error: unknown flag: --Authorization.Mode\n","stream":"stderr","time":"2018-06-17T22:16:35.134161966Z"}`},
+		{"apiserver-admmission #3524", true, "error: unknown flag: --GenericServerRunOptions.AdmissionControl"},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			got := IsProblem(tc.input)
+			if got != tc.want {
+				t.Fatalf("IsProblem(%s)=%v, want %v", tc.input, got, tc.want)
+			}
+		})
+	}
+}

From 496bbe00bba09ac8e4c31c9b68dfdc62bcc20380 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Wed, 13 Feb 2019 21:35:42 -0800
Subject: [PATCH 10/24] Pluralize problem

---
 pkg/minikube/logs/logs.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/minikube/logs/logs.go b/pkg/minikube/logs/logs.go
index 95c6c4d891..64a12ed3da 100644
--- a/pkg/minikube/logs/logs.go
+++ b/pkg/minikube/logs/logs.go
@@ -86,7 +86,7 @@ func FindProblems(r cruntime.Manager, bs bootstrapper.Bootstrapper, runner boots
 // OutputProblems outputs discovered problems.
 func OutputProblems(problems map[string][]string, maxLines int) {
 	for name, lines := range problems {
-		console.OutStyle("failure", "Problem detected in %q", name)
+		console.OutStyle("failure", "Problems detected in %q:", name)
 		if len(lines) > maxLines {
 			lines = lines[len(lines)-maxLines:]
 		}

From 247e957e8ff30b1ed0213613a4dad2297e198923 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Wed, 13 Feb 2019 21:47:54 -0800
Subject: [PATCH 11/24] Revert accidental indentation

---
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index 76872a125b..60ca50c622 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -129,7 +129,7 @@ func (k *KubeadmBootstrapper) LogCommands(len int, follow bool) map[string]strin
 	}
 	if follow {
 		kcmd.WriteString(" -f")
-		}
+	}
 	return map[string]string{"kubelet": kcmd.String()}
 }
 

From 5c1a72f428daddda30550e7d75aa575aaa60635e Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Thu, 14 Feb 2019 10:14:41 -0800
Subject: [PATCH 12/24] Switch Infof statements to Debugf/Warningf, as they go
 to stderr

---
 pkg/drivers/hyperkit/driver.go  | 24 ++++++++++++------------
 pkg/drivers/hyperkit/network.go |  8 ++++----
 2 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/pkg/drivers/hyperkit/driver.go b/pkg/drivers/hyperkit/driver.go
index 2da6910fd2..24b45778be 100644
--- a/pkg/drivers/hyperkit/driver.go
+++ b/pkg/drivers/hyperkit/driver.go
@@ -89,7 +89,7 @@ func (d *Driver) verifyRootPermissions() error {
 		return err
 	}
 	euid := syscall.Geteuid()
-	log.Infof("exe=%s uid=%d", exe, euid)
+	log.Debugf("exe=%s uid=%d", exe, euid)
 	if euid != 0 {
 		return fmt.Errorf(permErr, filepath.Base(exe), exe, exe)
 	}
@@ -144,12 +144,12 @@ func pidState(pid int) (state.State, error) {
 		return state.Error, err
 	}
 	if p == nil {
-		log.Infof("hyperkit pid missing from process table, err=%v", err)
+		log.Warnf("hyperkit pid missing from process table, err=%v", err)
 		return state.Stopped, nil
 	}
 	// hyperkit or com.docker.hyper
 	if !strings.Contains(p.Executable(), "hyper") {
-		log.Infof("pid %d is stale, and is being used by %s", pid, p.Executable())
+		log.Warnf("pid %d is stale, and is being used by %s", pid, p.Executable())
 		return state.Stopped, nil
 	}
 	return state.Running, nil
@@ -162,7 +162,7 @@ func (d *Driver) GetState() (state.State, error) {
 	}
 
 	pid := d.getPid()
-	log.Infof("hyperkit pid from json: %d", pid)
+	log.Debugf("hyperkit pid from json: %d", pid)
 	return pidState(pid)
 }
 
@@ -182,7 +182,7 @@ func (d *Driver) Remove() error {
 
 	s, err := d.GetState()
 	if err != nil || s == state.Error {
-		log.Infof("Error checking machine status: %v, assuming it has been removed already", err)
+		log.Debugf("Error checking machine status: %v, assuming it has been removed already", err)
 	}
 	if s == state.Running {
 		if err := d.Stop(); err != nil {
@@ -228,7 +228,7 @@ func (d *Driver) Start() error {
 		h.VSockPorts = vsockPorts
 	}
 
-	log.Infof("Using UUID %s", h.UUID)
+	log.Debugf("Using UUID %s", h.UUID)
 	mac, err := GetMACAddressFromUUID(h.UUID)
 	if err != nil {
 		return errors.Wrap(err, "getting MAC address from UUID")
@@ -236,7 +236,7 @@ func (d *Driver) Start() error {
 
 	// Need to strip 0's
 	mac = trimMacAddress(mac)
-	log.Infof("Generated MAC %s", mac)
+	log.Debugf("Generated MAC %s", mac)
 	h.Disks = []hyperkit.DiskConfig{
 		{
 			Path:   pkgdrivers.GetDiskPath(d.BaseDriver),
@@ -244,7 +244,7 @@ func (d *Driver) Start() error {
 			Driver: "virtio-blk",
 		},
 	}
-	log.Infof("Starting with cmdline: %s", d.Cmdline)
+	log.Debugf("Starting with cmdline: %s", d.Cmdline)
 	if err := h.Start(d.Cmdline); err != nil {
 		return errors.Wrapf(err, "starting with cmd line: %s", d.Cmdline)
 	}
@@ -268,7 +268,7 @@ func (d *Driver) Start() error {
 	if err := commonutil.RetryAfter(30, getIP, 2*time.Second); err != nil {
 		return fmt.Errorf("IP address never found in dhcp leases file %v", err)
 	}
-	log.Infof("IP: %s", d.IPAddress)
+	log.Debugf("IP: %s", d.IPAddress)
 
 	if len(d.NFSShares) > 0 {
 		log.Info("Setting up NFS mounts")
@@ -298,7 +298,7 @@ func (d *Driver) recoverFromUncleanShutdown() error {
 
 	if _, err := os.Stat(pidFile); err != nil {
 		if os.IsNotExist(err) {
-			log.Infof("clean start, hyperkit pid file doesn't exist: %s", pidFile)
+			log.Debugf("clean start, hyperkit pid file doesn't exist: %s", pidFile)
 			return nil
 		}
 		return errors.Wrap(err, "stat")
@@ -320,11 +320,11 @@ func (d *Driver) recoverFromUncleanShutdown() error {
 		return errors.Wrap(err, "pidState")
 	}
 
-	log.Infof("pid %d is in state %q", pid, st)
+	log.Debugf("pid %d is in state %q", pid, st)
 	if st == state.Running {
 		return nil
 	}
-	log.Infof("Removing stale pid file %s...", pidFile)
+	log.Debugf("Removing stale pid file %s...", pidFile)
 	if err := os.Remove(pidFile); err != nil {
 		return errors.Wrap(err, fmt.Sprintf("removing pidFile %s", pidFile))
 	}
diff --git a/pkg/drivers/hyperkit/network.go b/pkg/drivers/hyperkit/network.go
index b801d50ebf..07c49e4450 100644
--- a/pkg/drivers/hyperkit/network.go
+++ b/pkg/drivers/hyperkit/network.go
@@ -48,7 +48,7 @@ func GetIPAddressByMACAddress(mac string) (string, error) {
 }
 
 func getIPAddressFromFile(mac, path string) (string, error) {
-	log.Infof("Searching for %s in %s ...", mac, path)
+	log.Debugf("Searching for %s in %s ...", mac, path)
 	file, err := os.Open(path)
 	if err != nil {
 		return "", err
@@ -59,11 +59,11 @@ func getIPAddressFromFile(mac, path string) (string, error) {
 	if err != nil {
 		return "", err
 	}
-	log.Infof("Found %d entries in %s!", len(dhcpEntries), path)
+	log.Debugf("Found %d entries in %s!", len(dhcpEntries), path)
 	for _, dhcpEntry := range dhcpEntries {
-		log.Infof("dhcp entry: %+v", dhcpEntry)
+		log.Debugf("dhcp entry: %+v", dhcpEntry)
 		if dhcpEntry.HWAddress == mac {
-			log.Infof("Found match: %s", mac)
+			log.Debugf("Found match: %s", mac)
 			return dhcpEntry.IPAddress, nil
 		}
 	}

From 0b10e33d10fa23275b257d8fae1a428f3ea52e30 Mon Sep 17 00:00:00 2001
From: Abhilash Pallerlamudi <stp.abhi@gmail.com>
Date: Thu, 14 Feb 2019 13:11:27 -0800
Subject: [PATCH 13/24] Add docs to use new vmware unified driver

Signed-off-by: Abhilash Pallerlamudi <stp.abhi@gmail.com>
---
 docs/drivers.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/docs/drivers.md b/docs/drivers.md
index efaeaebdbd..2cb5acfe9e 100644
--- a/docs/drivers.md
+++ b/docs/drivers.md
@@ -154,3 +154,9 @@ export LATEST_VERSION=$(curl -L -s -H 'Accept: application/json' https://github.
 && chmod +x docker-machine-driver-vmware \
 && mv docker-machine-driver-vmware /usr/local/bin/
 ```
+
+To use the driver you would do:
+
+```shell
+minikube start --vm-driver vmware
+```
\ No newline at end of file

From 4c2ba194558c9fddb71010922005d93ebc811ac2 Mon Sep 17 00:00:00 2001
From: Balint Pato <balopat@users.noreply.github.com>
Date: Thu, 14 Feb 2019 13:16:57 -0800
Subject: [PATCH 14/24] Update pkg/minikube/logs/logs_test.go

Co-Authored-By: tstromberg <tstromberg@google.com>
---
 pkg/minikube/logs/logs_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/minikube/logs/logs_test.go b/pkg/minikube/logs/logs_test.go
index 5e1405acec..245f5aa4d3 100644
--- a/pkg/minikube/logs/logs_test.go
+++ b/pkg/minikube/logs/logs_test.go
@@ -31,7 +31,7 @@ func TestIsProblem(t *testing.T) {
 		{"kubelet-eviction #", true, "I0213 07:16:44.041623    2410 eviction_manager.go:187] eviction manager: pods kube-apiserver-minikube_kube-system(87f41e2e0629c3deb5c2239e08d8045d) evicted, waiting for pod to be cleaned up"},
 		{"kubelet-unknown-flag #3655", true, "F0212 14:55:46.443031    2693 server.go:148] unknown flag: --AllowedUnsafeSysctls"},
 		{"apiserver-auth-mode #2852", true, `{"log":"Error: unknown flag: --Authorization.Mode\n","stream":"stderr","time":"2018-06-17T22:16:35.134161966Z"}`},
-		{"apiserver-admmission #3524", true, "error: unknown flag: --GenericServerRunOptions.AdmissionControl"},
+		{"apiserver-admission #3524", true, "error: unknown flag: --GenericServerRunOptions.AdmissionControl"},
 	}
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {

From 5029f8a66e61022b36ebe4fe7629cb4441d139a5 Mon Sep 17 00:00:00 2001
From: Balint Pato <balopat@users.noreply.github.com>
Date: Thu, 14 Feb 2019 13:17:32 -0800
Subject: [PATCH 15/24] Update pkg/minikube/exit/exit.go

Co-Authored-By: tstromberg <tstromberg@google.com>
---
 pkg/minikube/exit/exit.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/minikube/exit/exit.go b/pkg/minikube/exit/exit.go
index f599832703..971426ad37 100644
--- a/pkg/minikube/exit/exit.go
+++ b/pkg/minikube/exit/exit.go
@@ -63,7 +63,7 @@ func WithError(msg string, err error) {
 	os.Exit(Software)
 }
 
-// WithError outputs an error along with any autodetected problems, and exits.
+// WithProblems outputs an error along with any autodetected problems, and exits.
 func WithProblems(msg string, err error, problems map[string][]string) {
 	displayError(msg, err)
 

From 45303bac07a24b1faa4ac20216504981637f4e6b Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Thu, 14 Feb 2019 13:59:08 -0800
Subject: [PATCH 16/24] Code review comments

---
 pkg/minikube/bootstrapper/bootstrapper.go    | 10 +++++-
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go |  8 ++---
 pkg/minikube/exit/exit.go                    |  5 +--
 pkg/minikube/logs/logs.go                    | 33 +++++++++++++-------
 4 files changed, 38 insertions(+), 18 deletions(-)

diff --git a/pkg/minikube/bootstrapper/bootstrapper.go b/pkg/minikube/bootstrapper/bootstrapper.go
index 35cb89288a..4140f58548 100644
--- a/pkg/minikube/bootstrapper/bootstrapper.go
+++ b/pkg/minikube/bootstrapper/bootstrapper.go
@@ -23,6 +23,14 @@ import (
 	"k8s.io/minikube/pkg/minikube/constants"
 )
 
+// LogOptions are options to be passed to LogCommands
+type LogOptions struct {
+	// Lines is the number of recent log lines to include, as in tail -n.
+	Lines int
+	// Follow is whether or not to actively follow the logs, as in tail -f.
+	Follow bool
+}
+
 // Bootstrapper contains all the methods needed to bootstrap a kubernetes cluster
 type Bootstrapper interface {
 	// PullImages pulls images necessary for a cluster. Success should not be required.
@@ -32,7 +40,7 @@ type Bootstrapper interface {
 	RestartCluster(config.KubernetesConfig) error
 	DeleteCluster(config.KubernetesConfig) error
 	// LogCommands returns a map of log type to a command which will display that log.
-	LogCommands(int, bool) map[string]string
+	LogCommands(LogOptions) map[string]string
 	SetupCerts(cfg config.KubernetesConfig) error
 	GetKubeletStatus() (string, error)
 	GetApiServerStatus(net.IP) (string, error)
diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index 60ca50c622..f5f4a8e3fc 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -121,13 +121,13 @@ func (k *KubeadmBootstrapper) GetApiServerStatus(ip net.IP) (string, error) {
 }
 
 // LogCommands returns a map of log type to a command which will display that log.
-func (k *KubeadmBootstrapper) LogCommands(len int, follow bool) map[string]string {
+func (k *KubeadmBootstrapper) LogCommands(o bootstrapper.LogOptions) map[string]string {
 	var kcmd strings.Builder
 	kcmd.WriteString("journalctl -u kubelet")
-	if len > 0 {
-		kcmd.WriteString(fmt.Sprintf(" -n %d", len))
+	if o.Lines > 0 {
+		kcmd.WriteString(fmt.Sprintf(" -n %d", o.Lines))
 	}
-	if follow {
+	if o.Follow {
 		kcmd.WriteString(" -f")
 	}
 	return map[string]string{"kubelet": kcmd.String()}
diff --git a/pkg/minikube/exit/exit.go b/pkg/minikube/exit/exit.go
index 971426ad37..fad5b18df3 100644
--- a/pkg/minikube/exit/exit.go
+++ b/pkg/minikube/exit/exit.go
@@ -18,6 +18,7 @@ limitations under the License.
 package exit
 
 import (
+	"fmt"
 	"os"
 
 	"github.com/golang/glog"
@@ -80,10 +81,10 @@ func WithProblems(msg string, err error, problems map[string][]string) {
 }
 
 func displayError(msg string, err error) {
+	// use Warning because Error will display a duplicate message to stderr
+	glog.Warningf(fmt.Sprintf("%s: %v", msg, err))
 	console.Fatal(msg+": %v", err)
 	console.Err("\n")
 	console.ErrStyle("sad", "Sorry that minikube crashed. If this was unexpected, we would love to hear from you:")
 	console.ErrStyle("url", "https://github.com/kubernetes/minikube/issues/new")
-	// use Warning because Error will display a duplicate message to stderr
-	glog.Warningf(msg+"%s: %v", msg)
 }
diff --git a/pkg/minikube/logs/logs.go b/pkg/minikube/logs/logs.go
index 7cca818d6c..11f63e9013 100644
--- a/pkg/minikube/logs/logs.go
+++ b/pkg/minikube/logs/logs.go
@@ -18,13 +18,14 @@ limitations under the License.
 package logs
 
 import (
+	"bufio"
+	"bytes"
 	"fmt"
 	"os"
 	"regexp"
 	"sort"
 	"strings"
 
-	"github.com/docker/machine/libmachine/log"
 	"github.com/golang/glog"
 	"k8s.io/minikube/pkg/minikube/bootstrapper"
 	"k8s.io/minikube/pkg/minikube/console"
@@ -41,6 +42,10 @@ var importantPods = []string{
 	"k8s_kube-scheduler",
 }
 
+// lookbackwardsCount is how far back to look in a log for problems. This should be large enough to
+// include usage messages from a failed binary, but small enough to not include irrelevant problems.
+const lookBackwardsCount = 200
+
 // Follow follows logs from multiple files in tail(1) format
 func Follow(r cruntime.Manager, bs bootstrapper.Bootstrapper, runner bootstrapper.CommandRunner) error {
 	cs := []string{}
@@ -59,18 +64,19 @@ func IsProblem(line string) bool {
 // FindProblems finds possible root causes among the logs
 func FindProblems(r cruntime.Manager, bs bootstrapper.Bootstrapper, runner bootstrapper.CommandRunner) map[string][]string {
 	pMap := map[string][]string{}
-	cmds := logCommands(r, bs, 200, false)
+	cmds := logCommands(r, bs, lookBackwardsCount, false)
 	for name, cmd := range cmds {
-		log.Infof("Gathering logs for %s ...", name)
-		out, err := runner.CombinedOutput(cmds[name])
+		glog.Infof("Gathering logs for %s ...", name)
+		var b bytes.Buffer
+		err := runner.CombinedOutputTo(cmds[name], &b)
 		if err != nil {
 			glog.Warningf("failed %s: %s: %v", name, cmd, err)
 			continue
 		}
-		log.Infof("log length: %d", len(out))
-
+		scanner := bufio.NewScanner(&b)
 		problems := []string{}
-		for _, l := range strings.Split(out, "\n") {
+		for scanner.Scan() {
+			l := scanner.Text()
 			if IsProblem(l) {
 				glog.Warningf("Found %s problem: %s", name, l)
 				problems = append(problems, l)
@@ -108,13 +114,17 @@ func Output(r cruntime.Manager, bs bootstrapper.Bootstrapper, runner bootstrappe
 	failed := []string{}
 	for _, name := range names {
 		console.OutLn("==> %s <==", name)
-		out, err := runner.CombinedOutput(cmds[name])
+		var b bytes.Buffer
+		err := runner.CombinedOutputTo(cmds[name], &b)
 		if err != nil {
 			glog.Errorf("failed: %v", err)
 			failed = append(failed, name)
 			continue
 		}
-		console.OutLn(out)
+		scanner := bufio.NewScanner(&b)
+		for scanner.Scan() {
+			console.OutLn(scanner.Text())
+		}
 	}
 	if len(failed) > 0 {
 		return fmt.Errorf("unable to fetch logs for: %s", strings.Join(failed, ", "))
@@ -124,15 +134,16 @@ func Output(r cruntime.Manager, bs bootstrapper.Bootstrapper, runner bootstrappe
 
 // logCommands returns a list of commands that would be run to receive the anticipated logs
 func logCommands(r cruntime.Manager, bs bootstrapper.Bootstrapper, length int, follow bool) map[string]string {
-	cmds := bs.LogCommands(length, follow)
+	cmds := bs.LogCommands(bootstrapper.LogOptions{Lines: length, Follow: follow})
 	for _, pod := range importantPods {
 		ids, err := r.ListContainers(pod)
 		if err != nil {
 			glog.Errorf("Failed to list containers for %q: %v", pod, err)
 			continue
 		}
+		glog.Infof("%d containers: %s", len(ids), ids)
 		if len(ids) == 0 {
-			glog.Errorf("No containers found matching %q", pod)
+			cmds[pod] = fmt.Sprintf("No container was found matching %q", pod)
 			continue
 		}
 		cmds[pod] = r.ContainerLogCmd(ids[0], length, follow)

From 0c8a452e977d21c5216b4debbfc4d7790291c188 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Thu, 14 Feb 2019 14:40:16 -0800
Subject: [PATCH 17/24] lowercase error messages

---
 pkg/drivers/hyperkit/network.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/pkg/drivers/hyperkit/network.go b/pkg/drivers/hyperkit/network.go
index 07c49e4450..a699e071ab 100644
--- a/pkg/drivers/hyperkit/network.go
+++ b/pkg/drivers/hyperkit/network.go
@@ -67,7 +67,7 @@ func getIPAddressFromFile(mac, path string) (string, error) {
 			return dhcpEntry.IPAddress, nil
 		}
 	}
-	return "", fmt.Errorf("Could not find an IP address for %s", mac)
+	return "", fmt.Errorf("could not find an IP address for %s", mac)
 }
 
 func parseDHCPdLeasesFile(file io.Reader) ([]DHCPEntry, error) {
@@ -105,7 +105,7 @@ func parseDHCPdLeasesFile(file io.Reader) ([]DHCPEntry, error) {
 		case "lease":
 			dhcpEntry.Lease = val
 		default:
-			return dhcpEntries, fmt.Errorf("Unable to parse line: %s", line)
+			return dhcpEntries, fmt.Errorf("unable to parse line: %s", line)
 		}
 	}
 	return dhcpEntries, scanner.Err()
@@ -130,7 +130,7 @@ func GetNetAddr() (net.IP, error) {
 	}
 	ip := net.ParseIP(strings.TrimSpace(string(out)))
 	if ip == nil {
-		return nil, fmt.Errorf("Could not get the network address for vmnet")
+		return nil, fmt.Errorf("could not get the network address for vmnet")
 	}
 	return ip, nil
 }

From 531e6b0f726da2db061394eaaa21e99439f4078b Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Thu, 14 Feb 2019 14:40:32 -0800
Subject: [PATCH 18/24] Use h.SetLogger

---
 pkg/drivers/hyperkit/driver.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/pkg/drivers/hyperkit/driver.go b/pkg/drivers/hyperkit/driver.go
index 24b45778be..c07fd7a9b3 100644
--- a/pkg/drivers/hyperkit/driver.go
+++ b/pkg/drivers/hyperkit/driver.go
@@ -22,6 +22,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	golog "log"
 	"os"
 	"os/user"
 	"path"
@@ -220,6 +221,9 @@ func (d *Driver) Start() error {
 	h.CPUs = d.CPU
 	h.Memory = d.Memory
 	h.UUID = d.UUID
+	// This should stream logs from hyperkit, but doesn't seem to work.
+	logger := glog.New(os.Stderr, "hyperkit", golog.LstdFlags)
+	h.SetLogger(logger)
 
 	if vsockPorts, err := d.extractVSockPorts(); err != nil {
 		return err

From 7dd7e5417dfb75b8a6e469cf297809795e0b0bd6 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Thu, 14 Feb 2019 14:43:22 -0800
Subject: [PATCH 19/24] glog, not golog

---
 pkg/drivers/hyperkit/driver.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/drivers/hyperkit/driver.go b/pkg/drivers/hyperkit/driver.go
index c07fd7a9b3..f345a4534a 100644
--- a/pkg/drivers/hyperkit/driver.go
+++ b/pkg/drivers/hyperkit/driver.go
@@ -222,7 +222,7 @@ func (d *Driver) Start() error {
 	h.Memory = d.Memory
 	h.UUID = d.UUID
 	// This should stream logs from hyperkit, but doesn't seem to work.
-	logger := glog.New(os.Stderr, "hyperkit", golog.LstdFlags)
+	logger := golog.New(os.Stderr, "hyperkit", golog.LstdFlags)
 	h.SetLogger(logger)
 
 	if vsockPorts, err := d.extractVSockPorts(); err != nil {

From 07fc64fbe169ea4e13bd4730ef66769e93494c1e Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Thu, 14 Feb 2019 20:12:34 -0800
Subject: [PATCH 20/24] Warnf -> Debugf so that it doesn't interfere with
 output

---
 pkg/drivers/hyperkit/driver.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkg/drivers/hyperkit/driver.go b/pkg/drivers/hyperkit/driver.go
index f345a4534a..450620cfaa 100644
--- a/pkg/drivers/hyperkit/driver.go
+++ b/pkg/drivers/hyperkit/driver.go
@@ -145,12 +145,12 @@ func pidState(pid int) (state.State, error) {
 		return state.Error, err
 	}
 	if p == nil {
-		log.Warnf("hyperkit pid missing from process table, err=%v", err)
+		log.Debugf("hyperkit pid %d missing from process table", pid)
 		return state.Stopped, nil
 	}
 	// hyperkit or com.docker.hyper
 	if !strings.Contains(p.Executable(), "hyper") {
-		log.Warnf("pid %d is stale, and is being used by %s", pid, p.Executable())
+		log.Debugf("pid %d is stale, and is being used by %s", pid, p.Executable())
 		return state.Stopped, nil
 	}
 	return state.Running, nil

From 8b93331ff32235b71b7026692e4f1721c935c563 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Thu, 14 Feb 2019 20:20:49 -0800
Subject: [PATCH 21/24] none tests: Cleanup /var/lib/minikube

---
 hack/jenkins/linux_integration_tests_none.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/hack/jenkins/linux_integration_tests_none.sh b/hack/jenkins/linux_integration_tests_none.sh
index f3b1b7da9a..83d4348058 100755
--- a/hack/jenkins/linux_integration_tests_none.sh
+++ b/hack/jenkins/linux_integration_tests_none.sh
@@ -42,6 +42,8 @@ sudo kubeadm reset || sudo kubeadm reset -f || true
 sudo rm -rf /data/*
 # Cleanup old Kubernetes configs
 sudo rm -rf /etc/kubernetes/*
+# Cleanup old minikube files
+sudo rm -rf /var/lib/minikube/*
 # Stop any leftover kubelets
 systemctl is-active --quiet kubelet \
   && echo "stopping kubelet" \

From 61d0516bbab3ee4a271347a22e529dbd95713806 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Thu, 14 Feb 2019 20:26:24 -0800
Subject: [PATCH 22/24] Fix TestStartStop so that none tests are executed

---
 test/integration/start_stop_delete_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/integration/start_stop_delete_test.go b/test/integration/start_stop_delete_test.go
index 9eb7dcf294..693611044c 100644
--- a/test/integration/start_stop_delete_test.go
+++ b/test/integration/start_stop_delete_test.go
@@ -40,7 +40,7 @@ func TestStartStop(t *testing.T) {
 	for _, test := range tests {
 		t.Run(test.runtime, func(t *testing.T) {
 			runner := NewMinikubeRunner(t)
-			if test.runtime != "" && usingNoneDriver(runner) {
+			if test.runtime != "docker" && usingNoneDriver(runner) {
 				t.Skipf("skipping, can't use %s with none driver", test.runtime)
 			}
 

From 3f20dcd38039cc66a0dc5a0c1d341716632a8c71 Mon Sep 17 00:00:00 2001
From: Alberto Alvarez <me@albertoal.com>
Date: Thu, 14 Feb 2019 21:35:05 -0800
Subject: [PATCH 23/24] Adding docs for OpenID Connect Authentication

---
 docs/contributors/openid_connect_auth.md | 34 ++++++++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100644 docs/contributors/openid_connect_auth.md

diff --git a/docs/contributors/openid_connect_auth.md b/docs/contributors/openid_connect_auth.md
new file mode 100644
index 0000000000..b2020b13aa
--- /dev/null
+++ b/docs/contributors/openid_connect_auth.md
@@ -0,0 +1,34 @@
+# OpenID Connect Authentication
+
+Minikube `kube-apiserver` can be configured to support OpenID Connect Authentication.
+
+Read more about OpenID Connect Authentication for Kubernetes here: https://kubernetes.io/docs/reference/access-authn-authz/authentication/#openid-connect-tokens
+
+
+## Configuring the API Server
+
+Configuration values can be passed to the API server using the `--extra-config` flag on the `minikube start` command. See [configuring_kubernetes.md](configuring_kubernetes.md) for more details.
+
+The following example configures your Minikube cluster to support RBAC and OIDC:
+
+```shell
+minikube start \
+  --extra-config=apiserver.authorization-mode=RBAC \
+  --extra-config=apiserver.oidc-issuer-url=https://example.com \
+  --extra-config=apiserver.oidc-username-claim=email \
+  --extra-config=apiserver.oidc-client-id="kubernetes-local"
+```
+
+## Configuring kubectl
+
+You can use the kubectl `oidc` authenticator to create a kubeconfig as shown in the Kubernetes docs: https://kubernetes.io/docs/reference/access-authn-authz/authentication/#option-1-oidc-authenticator
+
+`minikube start` already creates a kubeconfig that includes a `cluster`, in order to use it with your `oidc` authenticator kubeconfig, you can run:
+
+```shell
+kubectl config set-context kubernetes-local-oidc --cluster=minikube --user username@example.com
+Context "kubernetes-local-oidc" created.
+kubectl config use-context kubernetes-local-oidc
+```
+
+For the new context to work you will need to create, at the very minimum, a `Role` and a `RoleBinding` in your cluster to grant permissions to the `subjects` included in your `oidc-username-claim`.

From 2f8d4b1d503f3495afded2b6bc0993394d68928d Mon Sep 17 00:00:00 2001
From: Alberto Alvarez <me@albertoal.com>
Date: Thu, 14 Feb 2019 22:01:05 -0800
Subject: [PATCH 24/24] Update docs/README.md, fix configuring_kubernetes URL
 and move the doc to the right dir

---
 docs/README.md                                 | 2 ++
 docs/{contributors => }/openid_connect_auth.md | 4 ++--
 2 files changed, 4 insertions(+), 2 deletions(-)
 rename docs/{contributors => }/openid_connect_auth.md (89%)

diff --git a/docs/README.md b/docs/README.md
index bdfd18a68e..3c6a2dd1a4 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -14,6 +14,8 @@
 
 * **GPUs** ([gpu.md](gpu.md)): Using NVIDIA GPUs on minikube
 
+* **OpenID Connect Authentication** ([openid_connect_auth](openid_connect_auth)): Using OIDC Authentication on minikube
+
 ### Installation and debugging
 
 * **Driver installation** ([drivers.md](drivers.md)): In depth instructions for installing the various hypervisor drivers
diff --git a/docs/contributors/openid_connect_auth.md b/docs/openid_connect_auth.md
similarity index 89%
rename from docs/contributors/openid_connect_auth.md
rename to docs/openid_connect_auth.md
index b2020b13aa..27345e0a61 100644
--- a/docs/contributors/openid_connect_auth.md
+++ b/docs/openid_connect_auth.md
@@ -7,7 +7,7 @@ Read more about OpenID Connect Authentication for Kubernetes here: https://kuber
 
 ## Configuring the API Server
 
-Configuration values can be passed to the API server using the `--extra-config` flag on the `minikube start` command. See [configuring_kubernetes.md](configuring_kubernetes.md) for more details.
+Configuration values can be passed to the API server using the `--extra-config` flag on the `minikube start` command. See [configuring_kubernetes.md](https://github.com/kubernetes/minikube/blob/master/docs/configuring_kubernetes.md) for more details.
 
 The following example configures your Minikube cluster to support RBAC and OIDC:
 
@@ -16,7 +16,7 @@ minikube start \
   --extra-config=apiserver.authorization-mode=RBAC \
   --extra-config=apiserver.oidc-issuer-url=https://example.com \
   --extra-config=apiserver.oidc-username-claim=email \
-  --extra-config=apiserver.oidc-client-id="kubernetes-local"
+  --extra-config=apiserver.oidc-client-id=kubernetes-local
 ```
 
 ## Configuring kubectl