From 68f973c193111bb79cbf79cb8578333ab0c77d71 Mon Sep 17 00:00:00 2001
From: Prasad Katti <prasadmkatti@gmail.com>
Date: Wed, 18 Mar 2020 16:26:42 -0700
Subject: [PATCH 01/19] Add describe nodes to minikube logs

---
 pkg/minikube/logs/logs.go | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/pkg/minikube/logs/logs.go b/pkg/minikube/logs/logs.go
index ef26e9d7d7..350eacc8f7 100644
--- a/pkg/minikube/logs/logs.go
+++ b/pkg/minikube/logs/logs.go
@@ -23,16 +23,20 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
+	"path"
 	"regexp"
 	"sort"
 	"strings"
 
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
+	"github.com/spf13/viper"
 	"k8s.io/minikube/pkg/minikube/bootstrapper"
 	"k8s.io/minikube/pkg/minikube/command"
+	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/cruntime"
 	"k8s.io/minikube/pkg/minikube/out"
+	"k8s.io/minikube/pkg/minikube/vmpath"
 )
 
 // rootCauseRe is a regular expression that matches known failure root causes
@@ -186,5 +190,15 @@ func logCommands(r cruntime.Manager, bs bootstrapper.Bootstrapper, length int, f
 	}
 	cmds[r.Name()] = r.SystemLogCmd(length)
 	cmds["container status"] = cruntime.ContainerStatusCommand()
+
+	cfg, err := config.Load(viper.GetString(config.ProfileName))
+	if err != nil && !config.IsNotExist(err) {
+		out.ErrLn("Error loading profile config: %v", err)
+	}
+
+	cmds["describe nodes"] = fmt.Sprintf("sudo %s describe node -A --kubeconfig=%s",
+		path.Join(vmpath.GuestPersistentDir, "binaries", cfg.KubernetesConfig.KubernetesVersion, "kubectl"),
+		path.Join(vmpath.GuestPersistentDir, "kubeconfig"))
+
 	return cmds
 }

From 8d48c68aece27b6bef66ce989bcb3fe8e3fb63b3 Mon Sep 17 00:00:00 2001
From: Prasad Katti <prasadmkatti@gmail.com>
Date: Thu, 19 Mar 2020 14:52:52 -0700
Subject: [PATCH 02/19] minor update based on review feedback

---
 pkg/minikube/logs/logs.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/minikube/logs/logs.go b/pkg/minikube/logs/logs.go
index 350eacc8f7..a44a9b6073 100644
--- a/pkg/minikube/logs/logs.go
+++ b/pkg/minikube/logs/logs.go
@@ -192,7 +192,7 @@ func logCommands(r cruntime.Manager, bs bootstrapper.Bootstrapper, length int, f
 	cmds["container status"] = cruntime.ContainerStatusCommand()
 
 	cfg, err := config.Load(viper.GetString(config.ProfileName))
-	if err != nil && !config.IsNotExist(err) {
+	if err != nil {
 		out.ErrLn("Error loading profile config: %v", err)
 	}
 

From 5cd7660d1e49089155bb3c2b59ae25430e7eb7b5 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Thu, 19 Mar 2020 17:44:21 -0700
Subject: [PATCH 03/19] fixHost: only reprovision if necessary, and only once

---
 cmd/minikube/cmd/node_add.go         |  2 +-
 cmd/minikube/cmd/node_start.go       |  2 +-
 cmd/minikube/cmd/start.go            | 18 +++++++++++-------
 go.sum                               |  3 +++
 pkg/minikube/machine/cluster_test.go | 12 ++++++------
 pkg/minikube/machine/fix.go          | 23 ++++++++++-------------
 pkg/minikube/machine/start.go        |  4 ++--
 pkg/minikube/node/config.go          |  8 ++++++++
 pkg/minikube/node/machine.go         |  8 ++++----
 pkg/minikube/node/node.go            |  2 +-
 pkg/minikube/node/start.go           |  4 ++--
 11 files changed, 49 insertions(+), 37 deletions(-)

diff --git a/cmd/minikube/cmd/node_add.go b/cmd/minikube/cmd/node_add.go
index 9ee9e39f1e..f450d20540 100644
--- a/cmd/minikube/cmd/node_add.go
+++ b/cmd/minikube/cmd/node_add.go
@@ -53,7 +53,7 @@ var nodeAddCmd = &cobra.Command{
 			exit.WithError("Error adding node to cluster", err)
 		}
 
-		_, err = node.Start(*cc, *n, false, nil)
+		_, err = node.Start(*cc, config.ClusterConfig{}, *n, false, nil)
 		if err != nil {
 			exit.WithError("Error starting node", err)
 		}
diff --git a/cmd/minikube/cmd/node_start.go b/cmd/minikube/cmd/node_start.go
index c0090b6287..ede4130c87 100644
--- a/cmd/minikube/cmd/node_start.go
+++ b/cmd/minikube/cmd/node_start.go
@@ -61,7 +61,7 @@ var nodeStartCmd = &cobra.Command{
 		}
 
 		// Start it up baby
-		_, err = node.Start(*cc, *n, false, nil)
+		_, err = node.Start(*cc, *cc, *n, false, nil)
 		if err != nil {
 			out.FatalT("Failed to start node {{.name}}", out.V{"name": name})
 		}
diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 6227dfa16a..35019c76ec 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -313,7 +313,7 @@ func runStart(cmd *cobra.Command, args []string) {
 	}
 
 	k8sVersion := getKubernetesVersion(existing)
-	mc, n, err := generateCfgFromFlags(cmd, k8sVersion, driverName)
+	cc, n, err := generateCfgFromFlags(cmd, k8sVersion, driverName)
 	if err != nil {
 		exit.WithError("Failed to generate config", err)
 	}
@@ -329,7 +329,7 @@ func runStart(cmd *cobra.Command, args []string) {
 		if err != nil {
 			exit.WithError("Failed to cache ISO", err)
 		}
-		mc.MinikubeISO = url
+		cc.MinikubeISO = url
 	}
 
 	if viper.GetBool(nativeSSH) {
@@ -338,12 +338,12 @@ func runStart(cmd *cobra.Command, args []string) {
 		ssh.SetDefaultClient(ssh.External)
 	}
 
-	kubeconfig, err := startNode(existing, mc, n)
+	kubeconfig, err := startNode(existing, cc, n)
 	if err != nil {
 		exit.WithError("Starting node", err)
 	}
 
-	if err := showKubectlInfo(kubeconfig, k8sVersion, mc.Name); err != nil {
+	if err := showKubectlInfo(kubeconfig, k8sVersion, cc.Name); err != nil {
 		glog.Errorf("kubectl info: %v", err)
 	}
 }
@@ -383,7 +383,7 @@ func displayEnviron(env []string) {
 	}
 }
 
-func startNode(existing *config.ClusterConfig, mc config.ClusterConfig, n config.Node) (*kubeconfig.Settings, error) {
+func startNode(existing *config.ClusterConfig, cc config.ClusterConfig, n config.Node) (*kubeconfig.Settings, error) {
 	var existingAddons map[string]bool
 	if viper.GetBool(installAddons) {
 		existingAddons = map[string]bool{}
@@ -391,7 +391,11 @@ func startNode(existing *config.ClusterConfig, mc config.ClusterConfig, n config
 			existingAddons = existing.Addons
 		}
 	}
-	return node.Start(mc, n, true, existingAddons)
+
+	if existing == nil {
+		existing = &config.ClusterConfig{}
+	}
+	return node.Start(cc, *existing, n, true, existingAddons)
 }
 
 func showKubectlInfo(kcs *kubeconfig.Settings, k8sVersion string, machineName string) error {
@@ -814,7 +818,7 @@ func validateRegistryMirror() {
 	}
 }
 
-// generateCfgFromFlags generates config.Config based on flags and supplied arguments
+// generateCfgFromFlags generates config.ClusterConfig based on flags and supplied arguments
 func generateCfgFromFlags(cmd *cobra.Command, k8sVersion string, drvName string) (config.ClusterConfig, config.Node, error) {
 	r, err := cruntime.New(cruntime.Config{Type: viper.GetString(containerRuntime)})
 	if err != nil {
diff --git a/go.sum b/go.sum
index d84ef83e6e..8a4fe67c98 100644
--- a/go.sum
+++ b/go.sum
@@ -61,7 +61,9 @@ github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmx
 github.com/afbjorklund/go-getter v1.4.1-0.20190910175809-eb9f6c26742c h1:18gEt7qzn7CW7qMkfPTFyyotlPbvPQo9o4IDV8jZqP4=
 github.com/afbjorklund/go-getter v1.4.1-0.20190910175809-eb9f6c26742c/go.mod h1:7qxyCd8rBfcShwsvxgIguu4KbS3l8bUCwg2Umn7RjeY=
 github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
@@ -972,6 +974,7 @@ google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg=
 google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 gopkg.in/airbrake/gobrake.v2 v2.0.9 h1:7z2uVWwn7oVeeugY1DtlPAy5H+KYgB1KeKTnqjNatLo=
 gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
diff --git a/pkg/minikube/machine/cluster_test.go b/pkg/minikube/machine/cluster_test.go
index 0c32c8642a..fbb0066691 100644
--- a/pkg/minikube/machine/cluster_test.go
+++ b/pkg/minikube/machine/cluster_test.go
@@ -129,7 +129,7 @@ func TestStartHostExists(t *testing.T) {
 	n := config.Node{Name: ih.Name}
 
 	// This should pass without calling Create because the host exists already.
-	h, err := StartHost(api, mc, n)
+	h, err := StartHost(api, mc, mc, n)
 	if err != nil {
 		t.Fatalf("Error starting host: %v", err)
 	}
@@ -163,7 +163,7 @@ func TestStartHostErrMachineNotExist(t *testing.T) {
 	n := config.Node{Name: h.Name}
 
 	// This should pass with creating host, while machine does not exist.
-	h, err = StartHost(api, mc, n)
+	h, err = StartHost(api, mc, mc, n)
 	if err != nil {
 		if err != ErrorMachineNotExist {
 			t.Fatalf("Error starting host: %v", err)
@@ -174,7 +174,7 @@ func TestStartHostErrMachineNotExist(t *testing.T) {
 	n.Name = h.Name
 
 	// Second call. This should pass without calling Create because the host exists already.
-	h, err = StartHost(api, mc, n)
+	h, err = StartHost(api, mc, mc, n)
 	if err != nil {
 		t.Fatalf("Error starting host: %v", err)
 	}
@@ -207,7 +207,7 @@ func TestStartStoppedHost(t *testing.T) {
 	mc := defaultClusterConfig
 	mc.Name = h.Name
 	n := config.Node{Name: h.Name}
-	h, err = StartHost(api, mc, n)
+	h, err = StartHost(api, mc, mc, n)
 	if err != nil {
 		t.Fatal("Error starting host.")
 	}
@@ -235,7 +235,7 @@ func TestStartHost(t *testing.T) {
 	md := &tests.MockDetector{Provisioner: &tests.MockProvisioner{}}
 	provision.SetDetector(md)
 
-	h, err := StartHost(api, defaultClusterConfig, config.Node{Name: "minikube"})
+	h, err := StartHost(api, defaultClusterConfig, defaultClusterConfig, config.Node{Name: "minikube"})
 	if err != nil {
 		t.Fatal("Error starting host.")
 	}
@@ -269,7 +269,7 @@ func TestStartHostConfig(t *testing.T) {
 		DockerOpt: []string{"param=value"},
 	}
 
-	h, err := StartHost(api, cfg, config.Node{Name: "minikube"})
+	h, err := StartHost(api, cfg, cfg, config.Node{Name: "minikube"})
 	if err != nil {
 		t.Fatal("Error starting host.")
 	}
diff --git a/pkg/minikube/machine/fix.go b/pkg/minikube/machine/fix.go
index 25f921cd5a..c8fcee801e 100644
--- a/pkg/minikube/machine/fix.go
+++ b/pkg/minikube/machine/fix.go
@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"math"
 	"os"
+	"reflect"
 	"strconv"
 	"strings"
 	"time"
@@ -28,7 +29,6 @@ import (
 
 	"github.com/docker/machine/libmachine"
 	"github.com/docker/machine/libmachine/host"
-	"github.com/docker/machine/libmachine/provision"
 	"github.com/docker/machine/libmachine/state"
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
@@ -36,7 +36,7 @@ import (
 	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/driver"
 	"k8s.io/minikube/pkg/minikube/out"
-	"k8s.io/minikube/pkg/util/retry"
+	"k8s.io/minikube/pkg/provision"
 )
 
 // hostRunner is a minimal host.Host based interface for running commands
@@ -56,7 +56,7 @@ var (
 )
 
 // fixHost fixes up a previously configured VM so that it is ready to run Kubernetes
-func fixHost(api libmachine.API, cc config.ClusterConfig, n config.Node) (*host.Host, error) {
+func fixHost(api libmachine.API, cc config.ClusterConfig, existing config.ClusterConfig, n config.Node) (*host.Host, error) {
 	out.T(out.Waiting, "Reconfiguring existing host ...")
 
 	start := time.Now()
@@ -78,15 +78,16 @@ func fixHost(api libmachine.API, cc config.ClusterConfig, n config.Node) (*host.
 		return h, err
 	}
 
+	old := engineOptions(existing)
 	e := engineOptions(cc)
-	if len(e.Env) > 0 {
+	if !reflect.DeepEqual(old, e) {
+		glog.Infof("docker config changed, updating provisioner: %+v", e)
 		h.HostOptions.EngineOptions.Env = e.Env
-		glog.Infof("Detecting provisioner ...")
-		provisioner, err := provision.DetectProvisioner(h.Driver)
-		if err != nil {
-			return h, errors.Wrap(err, "detecting provisioner")
+		p := provision.NewBuildrootProvisioner(h.Driver)
+		if driver.IsKIC(h.Driver.DriverName()) {
+			p = provision.NewUbuntuProvisioner(h.Driver)
 		}
-		if err := provisioner.Provision(*h.HostOptions.SwarmOptions, *h.HostOptions.AuthOptions, *h.HostOptions.EngineOptions); err != nil {
+		if err := p.Provision(*h.HostOptions.SwarmOptions, *h.HostOptions.AuthOptions, *h.HostOptions.EngineOptions); err != nil {
 			return h, errors.Wrap(err, "provision")
 		}
 	}
@@ -104,10 +105,6 @@ func fixHost(api libmachine.API, cc config.ClusterConfig, n config.Node) (*host.
 		return h, nil
 	}
 
-	glog.Infof("Configuring auth for driver %s ...", h.Driver.DriverName())
-	if err := h.ConfigureAuth(); err != nil {
-		return h, &retry.RetriableError{Err: errors.Wrap(err, "Error configuring auth on host")}
-	}
 	return h, ensureSyncedGuestClock(h, cc.Driver)
 }
 
diff --git a/pkg/minikube/machine/start.go b/pkg/minikube/machine/start.go
index 73c982fa80..a885b0476e 100644
--- a/pkg/minikube/machine/start.go
+++ b/pkg/minikube/machine/start.go
@@ -62,7 +62,7 @@ var (
 )
 
 // StartHost starts a host VM.
-func StartHost(api libmachine.API, cfg config.ClusterConfig, n config.Node) (*host.Host, error) {
+func StartHost(api libmachine.API, cfg config.ClusterConfig, existing config.ClusterConfig, n config.Node) (*host.Host, error) {
 	// Prevent machine-driver boot races, as well as our own certificate race
 	releaser, err := acquireMachinesLock(cfg.Name)
 	if err != nil {
@@ -83,7 +83,7 @@ func StartHost(api libmachine.API, cfg config.ClusterConfig, n config.Node) (*ho
 		return createHost(api, cfg, n)
 	}
 	glog.Infoln("Skipping create...Using existing machine configuration")
-	return fixHost(api, cfg, n)
+	return fixHost(api, cfg, existing, n)
 }
 
 func engineOptions(cfg config.ClusterConfig) *engine.Options {
diff --git a/pkg/minikube/node/config.go b/pkg/minikube/node/config.go
index b29867f1b6..df7eed881a 100644
--- a/pkg/minikube/node/config.go
+++ b/pkg/minikube/node/config.go
@@ -23,6 +23,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"strconv"
+	"time"
 
 	"github.com/blang/semver"
 	"github.com/docker/machine/libmachine"
@@ -59,6 +60,12 @@ var (
 
 // configureRuntimes does what needs to happen to get a runtime going.
 func configureRuntimes(runner cruntime.CommandRunner, drvName string, k8s config.KubernetesConfig, kv semver.Version) cruntime.Manager {
+	start := time.Now()
+	glog.Infof("configureRuntimes start")
+	defer func() {
+		glog.Infof("configureRuntimes took %s", time.Since(start))
+	}()
+
 	co := cruntime.Config{
 		Type:   viper.GetString(containerRuntime),
 		Runner: runner, ImageRepository: k8s.ImageRepository,
@@ -90,6 +97,7 @@ func configureRuntimes(runner cruntime.CommandRunner, drvName string, k8s config
 		}
 	}
 
+	glog.Infof("enabling %s ...", cr.Name())
 	err = cr.Enable(disableOthers)
 	if err != nil {
 		exit.WithError("Failed to enable container runtime", err)
diff --git a/pkg/minikube/node/machine.go b/pkg/minikube/node/machine.go
index 483131515a..ffe4a792d7 100644
--- a/pkg/minikube/node/machine.go
+++ b/pkg/minikube/node/machine.go
@@ -39,12 +39,12 @@ import (
 	"k8s.io/minikube/pkg/util/retry"
 )
 
-func startMachine(cfg *config.ClusterConfig, node *config.Node) (runner command.Runner, preExists bool, machineAPI libmachine.API, host *host.Host) {
+func startMachine(cfg *config.ClusterConfig, existing *config.ClusterConfig, node *config.Node) (runner command.Runner, preExists bool, machineAPI libmachine.API, host *host.Host) {
 	m, err := machine.NewAPIClient()
 	if err != nil {
 		exit.WithError("Failed to get machine client", err)
 	}
-	host, preExists = startHost(m, *cfg, *node)
+	host, preExists = startHost(m, *cfg, *existing, *node)
 	runner, err = machine.CommandRunner(host)
 	if err != nil {
 		exit.WithError("Failed to get command runner", err)
@@ -68,13 +68,13 @@ func startMachine(cfg *config.ClusterConfig, node *config.Node) (runner command.
 }
 
 // startHost starts a new minikube host using a VM or None
-func startHost(api libmachine.API, mc config.ClusterConfig, n config.Node) (*host.Host, bool) {
+func startHost(api libmachine.API, mc config.ClusterConfig, existing config.ClusterConfig, n config.Node) (*host.Host, bool) {
 	exists, err := api.Exists(mc.Name)
 	if err != nil {
 		exit.WithError("Failed to check if machine exists", err)
 	}
 
-	host, err := machine.StartHost(api, mc, n)
+	host, err := machine.StartHost(api, mc, existing, n)
 	if err != nil {
 		exit.WithError("Unable to start VM. Please investigate and run 'minikube delete' if possible", err)
 	}
diff --git a/pkg/minikube/node/node.go b/pkg/minikube/node/node.go
index e92bad65b5..c25a9c95bd 100644
--- a/pkg/minikube/node/node.go
+++ b/pkg/minikube/node/node.go
@@ -65,7 +65,7 @@ func Add(cc *config.ClusterConfig, name string, controlPlane bool, worker bool,
 		return nil, err
 	}
 
-	_, err = Start(*cc, n, false, nil)
+	_, err = Start(*cc, *cc, n, false, nil)
 	return &n, err
 }
 
diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go
index a3c5eee92b..8b75831f58 100644
--- a/pkg/minikube/node/start.go
+++ b/pkg/minikube/node/start.go
@@ -33,7 +33,7 @@ import (
 )
 
 // Start spins up a guest and starts the kubernetes node.
-func Start(mc config.ClusterConfig, n config.Node, primary bool, existingAddons map[string]bool) (*kubeconfig.Settings, error) {
+func Start(mc config.ClusterConfig, existing config.ClusterConfig, n config.Node, primary bool, existingAddons map[string]bool) (*kubeconfig.Settings, error) {
 	k8sVersion := mc.KubernetesConfig.KubernetesVersion
 	driverName := mc.Driver
 
@@ -59,7 +59,7 @@ func Start(mc config.ClusterConfig, n config.Node, primary bool, existingAddons
 	handleDownloadOnly(&cacheGroup, &kicGroup, k8sVersion)
 	waitDownloadKicArtifacts(&kicGroup)
 
-	mRunner, preExists, machineAPI, host := startMachine(&mc, &n)
+	mRunner, preExists, machineAPI, host := startMachine(&mc, &existing, &n)
 	defer machineAPI.Close()
 
 	// wait for preloaded tarball to finish downloading before configuring runtimes

From e1a766e2c82e254644843c477502db03b6cfe846 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Fri, 20 Mar 2020 08:10:01 -0700
Subject: [PATCH 04/19] Improve code structuring

---
 pkg/minikube/machine/client.go       | 19 ++++++++++---------
 pkg/minikube/machine/cluster_test.go |  9 ---------
 pkg/minikube/machine/fix.go          |  8 ++------
 pkg/minikube/machine/machine.go      | 25 +++++++++++++++++++++++++
 pkg/provision/provision.go           | 13 +++++++++++++
 5 files changed, 50 insertions(+), 24 deletions(-)

diff --git a/pkg/minikube/machine/client.go b/pkg/minikube/machine/client.go
index 0121c92358..e9052ef448 100644
--- a/pkg/minikube/machine/client.go
+++ b/pkg/minikube/machine/client.go
@@ -36,11 +36,11 @@ import (
 	"github.com/docker/machine/libmachine/host"
 	"github.com/docker/machine/libmachine/mcnutils"
 	"github.com/docker/machine/libmachine/persist"
-	lib_provision "github.com/docker/machine/libmachine/provision"
 	"github.com/docker/machine/libmachine/ssh"
 	"github.com/docker/machine/libmachine/state"
 	"github.com/docker/machine/libmachine/swarm"
 	"github.com/docker/machine/libmachine/version"
+	"github.com/golang/glog"
 	"github.com/pkg/errors"
 	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/minikube/driver"
@@ -49,7 +49,6 @@ import (
 	"k8s.io/minikube/pkg/minikube/out"
 	"k8s.io/minikube/pkg/minikube/registry"
 	"k8s.io/minikube/pkg/minikube/sshutil"
-	"k8s.io/minikube/pkg/provision"
 )
 
 // NewRPCClient gets a new client.
@@ -167,6 +166,12 @@ func CommandRunner(h *host.Host) (command.Runner, error) {
 
 // Create creates the host
 func (api *LocalClient) Create(h *host.Host) error {
+	glog.Infof("LocalClient.Create starting")
+	start := time.Now()
+	defer func() {
+		glog.Infof("LocalClient.Create took %s", time.Since(start))
+	}()
+
 	def := registry.Driver(h.DriverName)
 	if def.Empty() {
 		return fmt.Errorf("driver %q does not exist", h.DriverName)
@@ -209,21 +214,17 @@ func (api *LocalClient) Create(h *host.Host) error {
 		{
 			"provisioning",
 			func() error {
+				// Skippable because we don't reconfigure Docker?
 				if driver.BareMetal(h.Driver.DriverName()) {
 					return nil
 				}
-				var pv lib_provision.Provisioner
-				if driver.IsKIC(h.Driver.DriverName()) {
-					pv = provision.NewUbuntuProvisioner(h.Driver)
-				} else {
-					pv = provision.NewBuildrootProvisioner(h.Driver)
-				}
-				return pv.Provision(*h.HostOptions.SwarmOptions, *h.HostOptions.AuthOptions, *h.HostOptions.EngineOptions)
+				return provisionDockerMachine(h)
 			},
 		},
 	}
 
 	for _, step := range steps {
+
 		if err := step.f(); err != nil {
 			return errors.Wrap(err, step.name)
 		}
diff --git a/pkg/minikube/machine/cluster_test.go b/pkg/minikube/machine/cluster_test.go
index fbb0066691..a660b3448e 100644
--- a/pkg/minikube/machine/cluster_test.go
+++ b/pkg/minikube/machine/cluster_test.go
@@ -139,9 +139,6 @@ func TestStartHostExists(t *testing.T) {
 	if s, _ := h.Driver.GetState(); s != state.Running {
 		t.Fatalf("Machine not started.")
 	}
-	if !md.Provisioner.Provisioned {
-		t.Fatalf("Expected provision to be called")
-	}
 }
 
 func TestStartHostErrMachineNotExist(t *testing.T) {
@@ -185,9 +182,6 @@ func TestStartHostErrMachineNotExist(t *testing.T) {
 	if s, _ := h.Driver.GetState(); s != state.Running {
 		t.Fatalf("Machine not started.")
 	}
-	if !md.Provisioner.Provisioned {
-		t.Fatalf("Expected provision to be called")
-	}
 }
 
 func TestStartStoppedHost(t *testing.T) {
@@ -223,9 +217,6 @@ func TestStartStoppedHost(t *testing.T) {
 		t.Fatalf("Machine must be saved after starting.")
 	}
 
-	if !md.Provisioner.Provisioned {
-		t.Fatalf("Expected provision to be called")
-	}
 }
 
 func TestStartHost(t *testing.T) {
diff --git a/pkg/minikube/machine/fix.go b/pkg/minikube/machine/fix.go
index c8fcee801e..83e57c4174 100644
--- a/pkg/minikube/machine/fix.go
+++ b/pkg/minikube/machine/fix.go
@@ -36,7 +36,6 @@ import (
 	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/driver"
 	"k8s.io/minikube/pkg/minikube/out"
-	"k8s.io/minikube/pkg/provision"
 )
 
 // hostRunner is a minimal host.Host based interface for running commands
@@ -83,11 +82,8 @@ func fixHost(api libmachine.API, cc config.ClusterConfig, existing config.Cluste
 	if !reflect.DeepEqual(old, e) {
 		glog.Infof("docker config changed, updating provisioner: %+v", e)
 		h.HostOptions.EngineOptions.Env = e.Env
-		p := provision.NewBuildrootProvisioner(h.Driver)
-		if driver.IsKIC(h.Driver.DriverName()) {
-			p = provision.NewUbuntuProvisioner(h.Driver)
-		}
-		if err := p.Provision(*h.HostOptions.SwarmOptions, *h.HostOptions.AuthOptions, *h.HostOptions.EngineOptions); err != nil {
+		err := provisionDockerMachine(h)
+		if err != nil {
 			return h, errors.Wrap(err, "provision")
 		}
 	}
diff --git a/pkg/minikube/machine/machine.go b/pkg/minikube/machine/machine.go
index 215f240753..a9cba1e413 100644
--- a/pkg/minikube/machine/machine.go
+++ b/pkg/minikube/machine/machine.go
@@ -18,7 +18,10 @@ package machine
 
 import (
 	"github.com/docker/machine/libmachine/host"
+	libprovision "github.com/docker/machine/libmachine/provision"
 	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/minikube/driver"
+	"k8s.io/minikube/pkg/provision"
 )
 
 // Machine contains information about a machine
@@ -74,3 +77,25 @@ func LoadMachine(name string) (*Machine, error) {
 	}
 	return &mm, nil
 }
+
+// provisionDockerMachine provides fast provisioning of a docker machine
+func provisionDockerMachine(h *host.Host) error {
+	p, err := fastDetectProvisioner(h)
+	if err != nil {
+		return errors.Wrap(err, "fast detect")
+	}
+	return p.Provision(*h.HostOptions.SwarmOptions, *h.HostOptions.AuthOptions, *h.HostOptions.EngineOptions)
+}
+
+// fastDetectProvisioner provides a shortcut for provisioner detection
+func fastDetectProvisioner(h *host.Host) (libprovision.Provisioner, error) {
+	d := h.Driver.DriverName()
+	switch {
+	case driver.IsKIC(d):
+		return provision.NewUbuntuProvisioner(h.Driver), nil
+	case driver.BareMetal(d):
+		return libprovision.DetectProvisioner(h.Driver)
+	default:
+		return provision.NewBuildrootProvisioner(h.Driver), nil
+	}
+}
diff --git a/pkg/provision/provision.go b/pkg/provision/provision.go
index 52fb131960..b5dd4651fb 100644
--- a/pkg/provision/provision.go
+++ b/pkg/provision/provision.go
@@ -66,6 +66,19 @@ func init() {
 
 }
 
+// NewSystemdProvisioner is our fork of the same name in the upstream provision library, without the packages
+func NewSystemdProvisioner(osReleaseID string, d drivers.Driver) provision.SystemdProvisioner {
+	return provision.SystemdProvisioner{
+		GenericProvisioner: provision.GenericProvisioner{
+			SSHCommander:      provision.GenericSSHCommander{Driver: d},
+			DockerOptionsDir:  "/etc/docker",
+			DaemonOptionsFile: "/etc/systemd/system/docker.service.d/10-machine.conf",
+			OsReleaseID:       osReleaseID,
+			Driver:            d,
+		},
+	}
+}
+
 func configureAuth(p miniProvisioner) error {
 	log.Infof("configureAuth start")
 	start := time.Now()

From 4faa31ebcf7651220e6fe0527c517707e2a836f3 Mon Sep 17 00:00:00 2001
From: Prasad Katti <prasadmkatti@gmail.com>
Date: Fri, 20 Mar 2020 14:56:04 -0700
Subject: [PATCH 05/19] Add logic to get desc node output in bootstrapper
 LogCommands func

---
 cmd/minikube/cmd/logs.go                     |  6 ++---
 pkg/minikube/bootstrapper/bootstrapper.go    |  2 +-
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 12 ++++++---
 pkg/minikube/logs/logs.go                    | 28 ++++++--------------
 pkg/minikube/node/start.go                   |  2 +-
 5 files changed, 22 insertions(+), 28 deletions(-)

diff --git a/cmd/minikube/cmd/logs.go b/cmd/minikube/cmd/logs.go
index cf36b52cb2..66ed68d675 100644
--- a/cmd/minikube/cmd/logs.go
+++ b/cmd/minikube/cmd/logs.go
@@ -95,18 +95,18 @@ var logsCmd = &cobra.Command{
 			exit.WithError("Unable to get runtime", err)
 		}
 		if followLogs {
-			err := logs.Follow(cr, bs, runner)
+			err := logs.Follow(cr, bs, *cfg, runner)
 			if err != nil {
 				exit.WithError("Follow", err)
 			}
 			return
 		}
 		if showProblems {
-			problems := logs.FindProblems(cr, bs, runner)
+			problems := logs.FindProblems(cr, bs, *cfg, runner)
 			logs.OutputProblems(problems, numberOfProblems)
 			return
 		}
-		err = logs.Output(cr, bs, runner, numberOfLines)
+		err = logs.Output(cr, bs, *cfg, runner, numberOfLines)
 		if err != nil {
 			exit.WithError("Error getting machine logs", err)
 		}
diff --git a/pkg/minikube/bootstrapper/bootstrapper.go b/pkg/minikube/bootstrapper/bootstrapper.go
index 712446285d..6c1e48442a 100644
--- a/pkg/minikube/bootstrapper/bootstrapper.go
+++ b/pkg/minikube/bootstrapper/bootstrapper.go
@@ -40,7 +40,7 @@ type Bootstrapper interface {
 	DeleteCluster(config.KubernetesConfig) error
 	WaitForCluster(config.ClusterConfig, time.Duration) error
 	// LogCommands returns a map of log type to a command which will display that log.
-	LogCommands(LogOptions) map[string]string
+	LogCommands(config.ClusterConfig, LogOptions) map[string]string
 	SetupCerts(config.KubernetesConfig, config.Node) error
 	GetKubeletStatus() (string, error)
 	GetAPIServerStatus(net.IP, int) (string, error)
diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index 92f896e4ef..f2f4d47c73 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -110,7 +110,7 @@ func (k *Bootstrapper) GetAPIServerStatus(ip net.IP, port int) (string, error) {
 }
 
 // LogCommands returns a map of log type to a command which will display that log.
-func (k *Bootstrapper) LogCommands(o bootstrapper.LogOptions) map[string]string {
+func (k *Bootstrapper) LogCommands(cfg config.ClusterConfig, o bootstrapper.LogOptions) map[string]string {
 	var kubelet strings.Builder
 	kubelet.WriteString("sudo journalctl -u kubelet")
 	if o.Lines > 0 {
@@ -128,9 +128,15 @@ func (k *Bootstrapper) LogCommands(o bootstrapper.LogOptions) map[string]string
 	if o.Lines > 0 {
 		dmesg.WriteString(fmt.Sprintf(" | tail -n %d", o.Lines))
 	}
+
+	describe_nodes := fmt.Sprintf("sudo %s describe node -A --kubeconfig=%s",
+		path.Join(vmpath.GuestPersistentDir, "binaries", cfg.KubernetesConfig.KubernetesVersion, "kubectl"),
+		path.Join(vmpath.GuestPersistentDir, "kubeconfig"))
+
 	return map[string]string{
-		"kubelet": kubelet.String(),
-		"dmesg":   dmesg.String(),
+		"kubelet":        kubelet.String(),
+		"dmesg":          dmesg.String(),
+		"describe nodes": describe_nodes,
 	}
 }
 
diff --git a/pkg/minikube/logs/logs.go b/pkg/minikube/logs/logs.go
index a44a9b6073..f284f0e98c 100644
--- a/pkg/minikube/logs/logs.go
+++ b/pkg/minikube/logs/logs.go
@@ -23,20 +23,17 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
-	"path"
 	"regexp"
 	"sort"
 	"strings"
 
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
-	"github.com/spf13/viper"
 	"k8s.io/minikube/pkg/minikube/bootstrapper"
 	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/cruntime"
 	"k8s.io/minikube/pkg/minikube/out"
-	"k8s.io/minikube/pkg/minikube/vmpath"
 )
 
 // rootCauseRe is a regular expression that matches known failure root causes
@@ -66,9 +63,9 @@ type logRunner interface {
 const lookBackwardsCount = 400
 
 // Follow follows logs from multiple files in tail(1) format
-func Follow(r cruntime.Manager, bs bootstrapper.Bootstrapper, cr logRunner) error {
+func Follow(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg config.ClusterConfig, cr logRunner) error {
 	cs := []string{}
-	for _, v := range logCommands(r, bs, 0, true) {
+	for _, v := range logCommands(r, bs, cfg, 0, true) {
 		cs = append(cs, v+" &")
 	}
 	cs = append(cs, "wait")
@@ -88,9 +85,9 @@ func IsProblem(line string) bool {
 }
 
 // FindProblems finds possible root causes among the logs
-func FindProblems(r cruntime.Manager, bs bootstrapper.Bootstrapper, cr logRunner) map[string][]string {
+func FindProblems(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg config.ClusterConfig, cr logRunner) map[string][]string {
 	pMap := map[string][]string{}
-	cmds := logCommands(r, bs, lookBackwardsCount, false)
+	cmds := logCommands(r, bs, cfg, lookBackwardsCount, false)
 	for name := range cmds {
 		glog.Infof("Gathering logs for %s ...", name)
 		var b bytes.Buffer
@@ -132,8 +129,8 @@ func OutputProblems(problems map[string][]string, maxLines int) {
 }
 
 // Output displays logs from multiple sources in tail(1) format
-func Output(r cruntime.Manager, bs bootstrapper.Bootstrapper, runner command.Runner, lines int) error {
-	cmds := logCommands(r, bs, lines, false)
+func Output(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg config.ClusterConfig, runner command.Runner, lines int) error {
+	cmds := logCommands(r, bs, cfg, lines, false)
 	cmds["kernel"] = "uptime && uname -a && grep PRETTY /etc/os-release"
 
 	names := []string{}
@@ -170,8 +167,8 @@ func Output(r cruntime.Manager, bs bootstrapper.Bootstrapper, runner command.Run
 }
 
 // logCommands returns a list of commands that would be run to receive the anticipated logs
-func logCommands(r cruntime.Manager, bs bootstrapper.Bootstrapper, length int, follow bool) map[string]string {
-	cmds := bs.LogCommands(bootstrapper.LogOptions{Lines: length, Follow: follow})
+func logCommands(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg config.ClusterConfig, length int, follow bool) map[string]string {
+	cmds := bs.LogCommands(cfg, bootstrapper.LogOptions{Lines: length, Follow: follow})
 	for _, pod := range importantPods {
 		ids, err := r.ListContainers(cruntime.ListOptions{Name: pod})
 		if err != nil {
@@ -191,14 +188,5 @@ func logCommands(r cruntime.Manager, bs bootstrapper.Bootstrapper, length int, f
 	cmds[r.Name()] = r.SystemLogCmd(length)
 	cmds["container status"] = cruntime.ContainerStatusCommand()
 
-	cfg, err := config.Load(viper.GetString(config.ProfileName))
-	if err != nil {
-		out.ErrLn("Error loading profile config: %v", err)
-	}
-
-	cmds["describe nodes"] = fmt.Sprintf("sudo %s describe node -A --kubeconfig=%s",
-		path.Join(vmpath.GuestPersistentDir, "binaries", cfg.KubernetesConfig.KubernetesVersion, "kubectl"),
-		path.Join(vmpath.GuestPersistentDir, "kubeconfig"))
-
 	return cmds
 }
diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go
index a3c5eee92b..0916134f6d 100644
--- a/pkg/minikube/node/start.go
+++ b/pkg/minikube/node/start.go
@@ -88,7 +88,7 @@ func Start(mc config.ClusterConfig, n config.Node, primary bool, existingAddons
 	// pull images or restart cluster
 	out.T(out.Launch, "Launching Kubernetes ... ")
 	if err := bs.StartCluster(mc); err != nil {
-		exit.WithLogEntries("Error starting cluster", err, logs.FindProblems(cr, bs, mRunner))
+		exit.WithLogEntries("Error starting cluster", err, logs.FindProblems(cr, bs, mc, mRunner))
 	}
 	configureMounts()
 

From c3f1f5e04bc1e7cd3ce45e63538e114f5f1af09f Mon Sep 17 00:00:00 2001
From: Prasad Katti <prasadmkatti@gmail.com>
Date: Fri, 20 Mar 2020 18:10:06 -0700
Subject: [PATCH 06/19] some more plumbing work to pass cfg in kverify.go

---
 .../bootstrapper/bsutil/kverify/kverify.go      | 17 +++++++++--------
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go    | 14 +++++++-------
 2 files changed, 16 insertions(+), 15 deletions(-)

diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
index 0b4c8d887c..47af70f492 100644
--- a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
+++ b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
@@ -36,6 +36,7 @@ import (
 	kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/minikube/pkg/minikube/bootstrapper"
 	"k8s.io/minikube/pkg/minikube/command"
+	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/cruntime"
 	"k8s.io/minikube/pkg/minikube/logs"
 )
@@ -44,7 +45,7 @@ import (
 const minLogCheckTime = 30 * time.Second
 
 // WaitForAPIServerProcess waits for api server to be healthy returns error if it doesn't
-func WaitForAPIServerProcess(r cruntime.Manager, bs bootstrapper.Bootstrapper, cr command.Runner, start time.Time, timeout time.Duration) error {
+func WaitForAPIServerProcess(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg config.ClusterConfig, cr command.Runner, start time.Time, timeout time.Duration) error {
 	glog.Infof("waiting for apiserver process to appear ...")
 	err := wait.PollImmediate(time.Millisecond*500, timeout, func() (bool, error) {
 		if time.Since(start) > timeout {
@@ -52,7 +53,7 @@ func WaitForAPIServerProcess(r cruntime.Manager, bs bootstrapper.Bootstrapper, c
 		}
 
 		if time.Since(start) > minLogCheckTime {
-			announceProblems(r, bs, cr)
+			announceProblems(r, bs, cfg, cr)
 			time.Sleep(kconst.APICallRetryInterval * 5)
 		}
 
@@ -79,7 +80,7 @@ func apiServerPID(cr command.Runner) (int, error) {
 }
 
 // WaitForSystemPods verifies essential pods for running kurnetes is running
-func WaitForSystemPods(r cruntime.Manager, bs bootstrapper.Bootstrapper, cr command.Runner, client *kubernetes.Clientset, start time.Time, timeout time.Duration) error {
+func WaitForSystemPods(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg config.ClusterConfig, cr command.Runner, client *kubernetes.Clientset, start time.Time, timeout time.Duration) error {
 	glog.Info("waiting for kube-system pods to appear ...")
 	pStart := time.Now()
 
@@ -88,7 +89,7 @@ func WaitForSystemPods(r cruntime.Manager, bs bootstrapper.Bootstrapper, cr comm
 			return false, fmt.Errorf("cluster wait timed out during pod check")
 		}
 		if time.Since(start) > minLogCheckTime {
-			announceProblems(r, bs, cr)
+			announceProblems(r, bs, cfg, cr)
 			time.Sleep(kconst.APICallRetryInterval * 5)
 		}
 
@@ -112,7 +113,7 @@ func WaitForSystemPods(r cruntime.Manager, bs bootstrapper.Bootstrapper, cr comm
 }
 
 // WaitForHealthyAPIServer waits for api server status to be running
-func WaitForHealthyAPIServer(r cruntime.Manager, bs bootstrapper.Bootstrapper, cr command.Runner, start time.Time, ip string, port int, timeout time.Duration) error {
+func WaitForHealthyAPIServer(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg config.ClusterConfig, cr command.Runner, start time.Time, ip string, port int, timeout time.Duration) error {
 	glog.Infof("waiting for apiserver healthz status ...")
 	hStart := time.Now()
 
@@ -122,7 +123,7 @@ func WaitForHealthyAPIServer(r cruntime.Manager, bs bootstrapper.Bootstrapper, c
 		}
 
 		if time.Since(start) > minLogCheckTime {
-			announceProblems(r, bs, cr)
+			announceProblems(r, bs, cfg, cr)
 			time.Sleep(kconst.APICallRetryInterval * 5)
 		}
 
@@ -145,8 +146,8 @@ func WaitForHealthyAPIServer(r cruntime.Manager, bs bootstrapper.Bootstrapper, c
 }
 
 // announceProblems checks for problems, and slows polling down if any are found
-func announceProblems(r cruntime.Manager, bs bootstrapper.Bootstrapper, cr command.Runner) {
-	problems := logs.FindProblems(r, bs, cr)
+func announceProblems(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg config.ClusterConfig, cr command.Runner) {
+	problems := logs.FindProblems(r, bs, cfg, cr)
 	if len(problems) > 0 {
 		logs.OutputProblems(problems, 5)
 		time.Sleep(kconst.APICallRetryInterval * 15)
diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index e669621000..4c6a9f9f9e 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -129,14 +129,14 @@ func (k *Bootstrapper) LogCommands(cfg config.ClusterConfig, o bootstrapper.LogO
 		dmesg.WriteString(fmt.Sprintf(" | tail -n %d", o.Lines))
 	}
 
-	describe_nodes := fmt.Sprintf("sudo %s describe node -A --kubeconfig=%s",
+	describeNodes := fmt.Sprintf("sudo %s describe node -A --kubeconfig=%s",
 		path.Join(vmpath.GuestPersistentDir, "binaries", cfg.KubernetesConfig.KubernetesVersion, "kubectl"),
 		path.Join(vmpath.GuestPersistentDir, "kubeconfig"))
 
 	return map[string]string{
 		"kubelet":        kubelet.String(),
 		"dmesg":          dmesg.String(),
-		"describe nodes": describe_nodes,
+		"describe nodes": describeNodes,
 	}
 }
 
@@ -271,7 +271,7 @@ func (k *Bootstrapper) WaitForCluster(cfg config.ClusterConfig, timeout time.Dur
 		return err
 	}
 
-	if err := kverify.WaitForAPIServerProcess(cr, k, k.c, start, timeout); err != nil {
+	if err := kverify.WaitForAPIServerProcess(cr, k, cfg, k.c, start, timeout); err != nil {
 		return err
 	}
 
@@ -285,7 +285,7 @@ func (k *Bootstrapper) WaitForCluster(cfg config.ClusterConfig, timeout time.Dur
 		}
 	}
 
-	if err := kverify.WaitForHealthyAPIServer(cr, k, k.c, start, ip, port, timeout); err != nil {
+	if err := kverify.WaitForHealthyAPIServer(cr, k, cfg, k.c, start, ip, port, timeout); err != nil {
 		return err
 	}
 
@@ -294,7 +294,7 @@ func (k *Bootstrapper) WaitForCluster(cfg config.ClusterConfig, timeout time.Dur
 		return errors.Wrap(err, "get k8s client")
 	}
 
-	if err := kverify.WaitForSystemPods(cr, k, k.c, c, start, timeout); err != nil {
+	if err := kverify.WaitForSystemPods(cr, k, cfg, k.c, c, start, timeout); err != nil {
 		return errors.Wrap(err, "waiting for system pods")
 	}
 	return nil
@@ -347,7 +347,7 @@ func (k *Bootstrapper) restartCluster(cfg config.ClusterConfig) error {
 	}
 
 	// We must ensure that the apiserver is healthy before proceeding
-	if err := kverify.WaitForAPIServerProcess(cr, k, k.c, time.Now(), kconst.DefaultControlPlaneTimeout); err != nil {
+	if err := kverify.WaitForAPIServerProcess(cr, k, cfg, k.c, time.Now(), kconst.DefaultControlPlaneTimeout); err != nil {
 		return errors.Wrap(err, "apiserver healthz")
 	}
 
@@ -366,7 +366,7 @@ func (k *Bootstrapper) restartCluster(cfg config.ClusterConfig) error {
 			return errors.Wrap(err, "getting k8s client")
 		}
 
-		if err := kverify.WaitForSystemPods(cr, k, k.c, client, time.Now(), kconst.DefaultControlPlaneTimeout); err != nil {
+		if err := kverify.WaitForSystemPods(cr, k, cfg, k.c, client, time.Now(), kconst.DefaultControlPlaneTimeout); err != nil {
 			return errors.Wrap(err, "system pods")
 		}
 

From 90a6eb3da0f27a87cbd7f4fc934f8d380f331d67 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Fri, 20 Mar 2020 19:12:30 -0700
Subject: [PATCH 07/19] Recover from ssh restart

---
 pkg/minikube/machine/start.go | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/pkg/minikube/machine/start.go b/pkg/minikube/machine/start.go
index a885b0476e..fe94b92432 100644
--- a/pkg/minikube/machine/start.go
+++ b/pkg/minikube/machine/start.go
@@ -32,6 +32,7 @@ import (
 	"github.com/juju/mutex"
 	"github.com/pkg/errors"
 	"github.com/spf13/viper"
+	"golang.org/x/crypto/ssh"
 	"k8s.io/minikube/pkg/drivers/kic/oci"
 	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/minikube/config"
@@ -43,6 +44,7 @@ import (
 	"k8s.io/minikube/pkg/minikube/sshutil"
 	"k8s.io/minikube/pkg/minikube/vmpath"
 	"k8s.io/minikube/pkg/util/lock"
+	"k8s.io/minikube/pkg/util/retry"
 )
 
 var (
@@ -191,6 +193,7 @@ func postStartSetup(h *host.Host, mc config.ClusterConfig) error {
 	}
 
 	glog.Infof("creating required directories: %v", requiredDirectories)
+
 	r, err := commandRunner(h)
 	if err != nil {
 		return errors.Wrap(err, "command runner")
@@ -229,11 +232,19 @@ func commandRunner(h *host.Host) (command.Runner, error) {
 	}
 
 	glog.Infof("Creating SSH client and returning SSHRunner for %q driver", d)
-	client, err := sshutil.NewSSHClient(h.Driver)
-	if err != nil {
-		return nil, errors.Wrap(err, "ssh client")
+
+	// Retry in order to survive an ssh restart, which sometimes happens due to provisioning
+	var sc *ssh.Client
+	getSSH := func() (err error) {
+		sc, err = sshutil.NewSSHClient(h.Driver)
+		return err
 	}
-	return command.NewSSHRunner(client), nil
+
+	if err := retry.Expo(getSSH, 250*time.Millisecond, 2*time.Second); err != nil {
+		return nil, err
+	}
+
+	return command.NewSSHRunner(sc), nil
 }
 
 // acquireMachinesLock protects against code that is not parallel-safe (libmachine, cert setup)

From f90e59756d8d2cd5a64aa38545b838473b1b1551 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Fri, 20 Mar 2020 19:12:32 -0700
Subject: [PATCH 08/19] Ensure that provisionDockerMachine is always called:
 for old images, and for corrupted ones

---
 pkg/minikube/machine/fix.go     | 15 ++++++---------
 pkg/minikube/machine/machine.go |  9 +++++++++
 2 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/pkg/minikube/machine/fix.go b/pkg/minikube/machine/fix.go
index 83e57c4174..8bf997b6f6 100644
--- a/pkg/minikube/machine/fix.go
+++ b/pkg/minikube/machine/fix.go
@@ -20,7 +20,6 @@ import (
 	"fmt"
 	"math"
 	"os"
-	"reflect"
 	"strconv"
 	"strings"
 	"time"
@@ -77,15 +76,13 @@ func fixHost(api libmachine.API, cc config.ClusterConfig, existing config.Cluste
 		return h, err
 	}
 
-	old := engineOptions(existing)
+	// Technically, we should only have to call provision if Docker has changed,
+	// but who can predict what shape the existing VM is in.
 	e := engineOptions(cc)
-	if !reflect.DeepEqual(old, e) {
-		glog.Infof("docker config changed, updating provisioner: %+v", e)
-		h.HostOptions.EngineOptions.Env = e.Env
-		err := provisionDockerMachine(h)
-		if err != nil {
-			return h, errors.Wrap(err, "provision")
-		}
+	h.HostOptions.EngineOptions.Env = e.Env
+	err = provisionDockerMachine(h)
+	if err != nil {
+		return h, errors.Wrap(err, "provision")
 	}
 
 	if driver.IsMock(h.DriverName) {
diff --git a/pkg/minikube/machine/machine.go b/pkg/minikube/machine/machine.go
index a9cba1e413..26470c3e7a 100644
--- a/pkg/minikube/machine/machine.go
+++ b/pkg/minikube/machine/machine.go
@@ -17,8 +17,11 @@ limitations under the License.
 package machine
 
 import (
+	"time"
+
 	"github.com/docker/machine/libmachine/host"
 	libprovision "github.com/docker/machine/libmachine/provision"
+	"github.com/golang/glog"
 	"github.com/pkg/errors"
 	"k8s.io/minikube/pkg/minikube/driver"
 	"k8s.io/minikube/pkg/provision"
@@ -80,6 +83,12 @@ func LoadMachine(name string) (*Machine, error) {
 
 // provisionDockerMachine provides fast provisioning of a docker machine
 func provisionDockerMachine(h *host.Host) error {
+	glog.Infof("provisioning docker machine ...")
+	start := time.Now()
+	defer func() {
+		glog.Infof("provisioned docker machine in %s", time.Since(start))
+	}()
+
 	p, err := fastDetectProvisioner(h)
 	if err != nil {
 		return errors.Wrap(err, "fast detect")

From 32d32dcc47e083113f41fc8b5001de6ba39427e7 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Fri, 20 Mar 2020 21:20:22 -0700
Subject: [PATCH 09/19] Also clear admin.conf on update

---
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 21 ++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index 1c0268d779..cfa110d46a 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -165,13 +165,6 @@ func (k *Bootstrapper) StartCluster(cfg config.ClusterConfig) error {
 		glog.Infof("StartCluster complete in %s", time.Since(start))
 	}()
 
-	// Remove admin.conf from any previous run
-	c := exec.Command("/bin/bash", "-c", "sudo rm -f /etc/kubernetes/admin.conf")
-	_, err = k.c.RunCmd(c)
-	if err != nil {
-		return errors.Wrap(err, "deleting admin.conf")
-	}
-
 	version, err := util.ParseKubernetesVersion(cfg.KubernetesConfig.KubernetesVersion)
 	if err != nil {
 		return errors.Wrap(err, "parsing kubernetes version")
@@ -209,7 +202,13 @@ func (k *Bootstrapper) StartCluster(cfg config.ClusterConfig) error {
 
 	}
 
-	c = exec.Command("/bin/bash", "-c", fmt.Sprintf("%s init --config %s %s --ignore-preflight-errors=%s", bsutil.InvokeKubeadm(cfg.KubernetesConfig.KubernetesVersion), bsutil.KubeadmYamlPath, extraFlags, strings.Join(ignore, ",")))
+	// Remove the previous kubeadm kubeconfig as the IP may have changed
+	_, err = k.c.RunCmd(exec.Command("sudo", "rm", "-f", "/etc/kubernetes/admin.conf"))
+	if err != nil {
+		return errors.Wrap(err, "deleting admin.conf")
+	}
+
+	c := exec.Command("/bin/bash", "-c", fmt.Sprintf("%s init --config %s %s --ignore-preflight-errors=%s", bsutil.InvokeKubeadm(cfg.KubernetesConfig.KubernetesVersion), bsutil.KubeadmYamlPath, extraFlags, strings.Join(ignore, ",")))
 	rr, err := k.c.RunCmd(c)
 	if err != nil {
 		return errors.Wrapf(err, "init failed. output: %q", rr.Output())
@@ -328,6 +327,12 @@ func (k *Bootstrapper) restartCluster(cfg config.ClusterConfig) error {
 		glog.Errorf("failed to create compat symlinks: %v", err)
 	}
 
+	// Remove the previous kubeadm kubeconfig as the IP may have changed
+	_, err = k.c.RunCmd(exec.Command("sudo", "rm", "-f", "/etc/kubernetes/admin.conf"))
+	if err != nil {
+		return errors.Wrap(err, "deleting admin.conf")
+	}
+
 	baseCmd := fmt.Sprintf("%s %s", bsutil.InvokeKubeadm(cfg.KubernetesConfig.KubernetesVersion), phase)
 	cmds := []string{
 		fmt.Sprintf("%s phase certs all --config %s", baseCmd, bsutil.KubeadmYamlPath),

From c42ecf95e0c198fdd8de7fa7c95e1d1f448da7d6 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Sat, 21 Mar 2020 07:53:03 -0700
Subject: [PATCH 10/19] Download dependencies for non-Docker runtimes

---
 test/integration/aaa_download_only_test.go | 227 +++++++++++----------
 1 file changed, 115 insertions(+), 112 deletions(-)

diff --git a/test/integration/aaa_download_only_test.go b/test/integration/aaa_download_only_test.go
index 3eda60e8cf..300c9323c1 100644
--- a/test/integration/aaa_download_only_test.go
+++ b/test/integration/aaa_download_only_test.go
@@ -40,138 +40,141 @@ import (
 )
 
 func TestDownloadOnly(t *testing.T) {
-	profile := UniqueProfileName("download")
-	ctx, cancel := context.WithTimeout(context.Background(), Minutes(15))
-	defer Cleanup(t, profile, cancel)
+	for _, r := range []string{"crio", "docker", "containerd"} {
+		t.Run(r, func(t *testing.T) {
+			// Stores the startup run result for later error messages
+			var rrr *RunResult
+			var err error
 
-	// Stores the startup run result for later error messages
-	var rrr *RunResult
-	var err error
+			profile := UniqueProfileName(r)
+			ctx, cancel := context.WithTimeout(context.Background(), Minutes(30))
+			defer Cleanup(t, profile, cancel)
 
-	t.Run("group", func(t *testing.T) {
-		versions := []string{
-			constants.OldestKubernetesVersion,
-			constants.DefaultKubernetesVersion,
-			constants.NewestKubernetesVersion,
-		}
-		for _, v := range versions {
-			t.Run(v, func(t *testing.T) {
-				// Explicitly does not pass StartArgs() to test driver default
-				// --force to avoid uid check
-				args := append([]string{"start", "--download-only", "-p", profile, "--force", "--alsologtostderr", fmt.Sprintf("--kubernetes-version=%s", v)}, StartArgs()...)
+			versions := []string{
+				constants.OldestKubernetesVersion,
+				constants.DefaultKubernetesVersion,
+				constants.NewestKubernetesVersion,
+			}
 
-				// Preserve the initial run-result for debugging
-				if rrr == nil {
-					rrr, err = Run(t, exec.CommandContext(ctx, Target(), args...))
-				} else {
-					_, err = Run(t, exec.CommandContext(ctx, Target(), args...))
-				}
+			for _, v := range versions {
+				t.Run(v, func(t *testing.T) {
+					// Explicitly does not pass StartArgs() to test driver default
+					// --force to avoid uid check
+					args := append([]string{"start", "--download-only", "-p", profile, "--force", "--alsologtostderr", fmt.Sprintf("--kubernetes-version=%s", v), fmt.Sprintf("--container-runtime=%s", r)}, StartArgs()...)
 
-				if err != nil {
-					t.Errorf("%s failed: %v", args, err)
-				}
-
-				if download.PreloadExists(v, "docker") {
-					// Just make sure the tarball path exists
-					if _, err := os.Stat(download.TarballPath(v)); err != nil {
-						t.Errorf("preloaded tarball path doesn't exist: %v", err)
+					// Preserve the initial run-result for debugging
+					if rrr == nil {
+						rrr, err = Run(t, exec.CommandContext(ctx, Target(), args...))
+					} else {
+						_, err = Run(t, exec.CommandContext(ctx, Target(), args...))
 					}
-					return
-				}
 
-				imgs, err := images.Kubeadm("", v)
-				if err != nil {
-					t.Errorf("kubeadm images: %v %+v", v, err)
-				}
+					if err != nil {
+						t.Errorf("%s failed: %v", args, err)
+					}
 
-				// skip verify for cache images if --driver=none
-				if !NoneDriver() {
-					for _, img := range imgs {
-						img = strings.Replace(img, ":", "_", 1) // for example kube-scheduler:v1.15.2 --> kube-scheduler_v1.15.2
-						fp := filepath.Join(localpath.MiniPath(), "cache", "images", img)
-						_, err := os.Stat(fp)
-						if err != nil {
-							t.Errorf("expected image file exist at %q but got error: %v", fp, err)
+					if download.PreloadExists(v, "docker") {
+						// Just make sure the tarball path exists
+						if _, err := os.Stat(download.TarballPath(v)); err != nil {
+							t.Errorf("preloaded tarball path doesn't exist: %v", err)
+						}
+						return
+					}
+
+					imgs, err := images.Kubeadm("", v)
+					if err != nil {
+						t.Errorf("kubeadm images: %v %+v", v, err)
+					}
+
+					// skip verify for cache images if --driver=none
+					if !NoneDriver() {
+						for _, img := range imgs {
+							img = strings.Replace(img, ":", "_", 1) // for example kube-scheduler:v1.15.2 --> kube-scheduler_v1.15.2
+							fp := filepath.Join(localpath.MiniPath(), "cache", "images", img)
+							_, err := os.Stat(fp)
+							if err != nil {
+								t.Errorf("expected image file exist at %q but got error: %v", fp, err)
+							}
 						}
 					}
+
+					// checking binaries downloaded (kubelet,kubeadm)
+					for _, bin := range constants.KubernetesReleaseBinaries {
+						fp := filepath.Join(localpath.MiniPath(), "cache", "linux", v, bin)
+						_, err := os.Stat(fp)
+						if err != nil {
+							t.Errorf("expected the file for binary exist at %q but got error %v", fp, err)
+						}
+					}
+
+					// If we are on darwin/windows, check to make sure OS specific kubectl has been downloaded
+					// as well for the `minikube kubectl` command
+					if runtime.GOOS == "linux" {
+						return
+					}
+					binary := "kubectl"
+					if runtime.GOOS == "windows" {
+						binary = "kubectl.exe"
+					}
+					fp := filepath.Join(localpath.MiniPath(), "cache", runtime.GOOS, v, binary)
+					if _, err := os.Stat(fp); err != nil {
+						t.Errorf("expected the file for binary exist at %q but got error %v", fp, err)
+					}
+				})
+			}
+
+			// Check that the profile we've created has the expected driver
+			t.Run("ExpectedDefaultDriver", func(t *testing.T) {
+				if ExpectedDefaultDriver() == "" {
+					t.Skipf("--expected-default-driver is unset, skipping test")
+					return
+				}
+				rr, err := Run(t, exec.CommandContext(ctx, Target(), "profile", "list", "--output", "json"))
+				if err != nil {
+					t.Errorf("%s failed: %v", rr.Args, err)
+				}
+				var ps map[string][]config.Profile
+				err = json.Unmarshal(rr.Stdout.Bytes(), &ps)
+				if err != nil {
+					t.Errorf("%s failed: %v", rr.Args, err)
 				}
 
-				// checking binaries downloaded (kubelet,kubeadm)
-				for _, bin := range constants.KubernetesReleaseBinaries {
-					fp := filepath.Join(localpath.MiniPath(), "cache", "linux", v, bin)
-					_, err := os.Stat(fp)
-					if err != nil {
-						t.Errorf("expected the file for binary exist at %q but got error %v", fp, err)
+				got := ""
+				for _, p := range ps["valid"] {
+					if p.Name == profile {
+						got = p.Config.Driver
 					}
 				}
 
-				// If we are on darwin/windows, check to make sure OS specific kubectl has been downloaded
-				// as well for the `minikube kubectl` command
-				if runtime.GOOS == "linux" {
-					return
-				}
-				binary := "kubectl"
-				if runtime.GOOS == "windows" {
-					binary = "kubectl.exe"
-				}
-				fp := filepath.Join(localpath.MiniPath(), "cache", runtime.GOOS, v, binary)
-				if _, err := os.Stat(fp); err != nil {
-					t.Errorf("expected the file for binary exist at %q but got error %v", fp, err)
+				if got != ExpectedDefaultDriver() {
+					t.Errorf("got driver %q, expected %q\nstart output: %s", got, ExpectedDefaultDriver(), rrr.Output())
 				}
 			})
-		}
 
-		// Check that the profile we've created has the expected driver
-		t.Run("ExpectedDefaultDriver", func(t *testing.T) {
-			if ExpectedDefaultDriver() == "" {
-				t.Skipf("--expected-default-driver is unset, skipping test")
-				return
-			}
-			rr, err := Run(t, exec.CommandContext(ctx, Target(), "profile", "list", "--output", "json"))
-			if err != nil {
-				t.Errorf("%s failed: %v", rr.Args, err)
-			}
-			var ps map[string][]config.Profile
-			err = json.Unmarshal(rr.Stdout.Bytes(), &ps)
-			if err != nil {
-				t.Errorf("%s failed: %v", rr.Args, err)
-			}
-
-			got := ""
-			for _, p := range ps["valid"] {
-				if p.Name == profile {
-					got = p.Config.Driver
+			// This is a weird place to test profile deletion, but this test is serial, and we have a profile to delete!
+			t.Run("DeleteAll", func(t *testing.T) {
+				if !CanCleanup() {
+					t.Skip("skipping, as cleanup is disabled")
 				}
-			}
-
-			if got != ExpectedDefaultDriver() {
-				t.Errorf("got driver %q, expected %q\nstart output: %s", got, ExpectedDefaultDriver(), rrr.Output())
-			}
+				rr, err := Run(t, exec.CommandContext(ctx, Target(), "delete", "--all"))
+				if err != nil {
+					t.Errorf("%s failed: %v", rr.Args, err)
+				}
+			})
+			// Delete should always succeed, even if previously partially or fully deleted.
+			t.Run("DeleteAlwaysSucceeds", func(t *testing.T) {
+				if !CanCleanup() {
+					t.Skip("skipping, as cleanup is disabled")
+				}
+				rr, err := Run(t, exec.CommandContext(ctx, Target(), "delete", "-p", profile))
+				if err != nil {
+					t.Errorf("%s failed: %v", rr.Args, err)
+				}
+			})
 		})
-
-		// This is a weird place to test profile deletion, but this test is serial, and we have a profile to delete!
-		t.Run("DeleteAll", func(t *testing.T) {
-			if !CanCleanup() {
-				t.Skip("skipping, as cleanup is disabled")
-			}
-			rr, err := Run(t, exec.CommandContext(ctx, Target(), "delete", "--all"))
-			if err != nil {
-				t.Errorf("%s failed: %v", rr.Args, err)
-			}
-		})
-		// Delete should always succeed, even if previously partially or fully deleted.
-		t.Run("DeleteAlwaysSucceeds", func(t *testing.T) {
-			if !CanCleanup() {
-				t.Skip("skipping, as cleanup is disabled")
-			}
-			rr, err := Run(t, exec.CommandContext(ctx, Target(), "delete", "-p", profile))
-			if err != nil {
-				t.Errorf("%s failed: %v", rr.Args, err)
-			}
-		})
-	})
-
+	}
 }
+
 func TestDownloadOnlyDocker(t *testing.T) {
 	if !runningDockerDriver(StartArgs()) {
 		t.Skip("this test only runs with the docker driver")

From d75dc6a2294853be85cc6e78411704ab4acb9c6b Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Sat, 21 Mar 2020 10:22:45 -0700
Subject: [PATCH 11/19] Remove broken ExpectedDefaultDriver test, fix call to
 PreloadExists

---
 hack/jenkins/common.sh                        |  1 -
 .../windows_integration_test_hyperv.ps1       |  2 +-
 .../windows_integration_test_virtualbox.ps1   |  2 +-
 test/integration/aaa_download_only_test.go    | 32 +------------------
 test/integration/main.go                      |  1 -
 5 files changed, 3 insertions(+), 35 deletions(-)

diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh
index 404b7aa4f9..8b908d3d73 100755
--- a/hack/jenkins/common.sh
+++ b/hack/jenkins/common.sh
@@ -286,7 +286,6 @@ fi
 touch "${TEST_OUT}"
 ${SUDO_PREFIX}${E2E_BIN} \
   -minikube-start-args="--driver=${VM_DRIVER} ${EXTRA_START_ARGS}" \
-  -expected-default-driver="${EXPECTED_DEFAULT_DRIVER}" \
   -test.timeout=70m -test.v \
   ${EXTRA_TEST_ARGS} \
   -binary="${MINIKUBE_BIN}" 2>&1 | tee "${TEST_OUT}"
diff --git a/hack/jenkins/windows_integration_test_hyperv.ps1 b/hack/jenkins/windows_integration_test_hyperv.ps1
index f1c4db8b9d..536c4e35cc 100644
--- a/hack/jenkins/windows_integration_test_hyperv.ps1
+++ b/hack/jenkins/windows_integration_test_hyperv.ps1
@@ -19,7 +19,7 @@ gsutil.cmd -m cp -r gs://minikube-builds/$env:MINIKUBE_LOCATION/testdata .
 
 ./out/minikube-windows-amd64.exe delete
 
-out/e2e-windows-amd64.exe --expected-default-driver=hyperv -minikube-start-args="--driver=hyperv --hyperv-virtual-switch=primary-virtual-switch" -binary=out/minikube-windows-amd64.exe -test.v -test.timeout=65m
+out/e2e-windows-amd64.exe -minikube-start-args="--driver=hyperv --hyperv-virtual-switch=primary-virtual-switch" -binary=out/minikube-windows-amd64.exe -test.v -test.timeout=65m
 $env:result=$lastexitcode
 # If the last exit code was 0->success, x>0->error
 If($env:result -eq 0){$env:status="success"}
diff --git a/hack/jenkins/windows_integration_test_virtualbox.ps1 b/hack/jenkins/windows_integration_test_virtualbox.ps1
index 6d9c7f318b..2f5957301f 100644
--- a/hack/jenkins/windows_integration_test_virtualbox.ps1
+++ b/hack/jenkins/windows_integration_test_virtualbox.ps1
@@ -19,7 +19,7 @@ gsutil.cmd -m cp -r gs://minikube-builds/$env:MINIKUBE_LOCATION/testdata .
 
 ./out/minikube-windows-amd64.exe delete
 
-out/e2e-windows-amd64.exe -minikube-start-args="--driver=virtualbox" -expected-default-driver=hyperv -binary=out/minikube-windows-amd64.exe -test.v -test.timeout=30m
+out/e2e-windows-amd64.exe -minikube-start-args="--driver=virtualbox" -binary=out/minikube-windows-amd64.exe -test.v -test.timeout=30m
 $env:result=$lastexitcode
 # If the last exit code was 0->success, x>0->error
 If($env:result -eq 0){$env:status="success"}
diff --git a/test/integration/aaa_download_only_test.go b/test/integration/aaa_download_only_test.go
index 300c9323c1..9640212800 100644
--- a/test/integration/aaa_download_only_test.go
+++ b/test/integration/aaa_download_only_test.go
@@ -21,7 +21,6 @@ package integration
 import (
 	"context"
 	"crypto/md5"
-	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -33,7 +32,6 @@ import (
 	"time"
 
 	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
-	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/download"
 	"k8s.io/minikube/pkg/minikube/localpath"
@@ -73,7 +71,7 @@ func TestDownloadOnly(t *testing.T) {
 						t.Errorf("%s failed: %v", args, err)
 					}
 
-					if download.PreloadExists(v, "docker") {
+					if download.PreloadExists(v, r) {
 						// Just make sure the tarball path exists
 						if _, err := os.Stat(download.TarballPath(v)); err != nil {
 							t.Errorf("preloaded tarball path doesn't exist: %v", err)
@@ -123,34 +121,6 @@ func TestDownloadOnly(t *testing.T) {
 				})
 			}
 
-			// Check that the profile we've created has the expected driver
-			t.Run("ExpectedDefaultDriver", func(t *testing.T) {
-				if ExpectedDefaultDriver() == "" {
-					t.Skipf("--expected-default-driver is unset, skipping test")
-					return
-				}
-				rr, err := Run(t, exec.CommandContext(ctx, Target(), "profile", "list", "--output", "json"))
-				if err != nil {
-					t.Errorf("%s failed: %v", rr.Args, err)
-				}
-				var ps map[string][]config.Profile
-				err = json.Unmarshal(rr.Stdout.Bytes(), &ps)
-				if err != nil {
-					t.Errorf("%s failed: %v", rr.Args, err)
-				}
-
-				got := ""
-				for _, p := range ps["valid"] {
-					if p.Name == profile {
-						got = p.Config.Driver
-					}
-				}
-
-				if got != ExpectedDefaultDriver() {
-					t.Errorf("got driver %q, expected %q\nstart output: %s", got, ExpectedDefaultDriver(), rrr.Output())
-				}
-			})
-
 			// This is a weird place to test profile deletion, but this test is serial, and we have a profile to delete!
 			t.Run("DeleteAll", func(t *testing.T) {
 				if !CanCleanup() {
diff --git a/test/integration/main.go b/test/integration/main.go
index 33c5e09618..54a644518c 100644
--- a/test/integration/main.go
+++ b/test/integration/main.go
@@ -27,7 +27,6 @@ import (
 
 // General configuration: used to set the VM Driver
 var startArgs = flag.String("minikube-start-args", "", "Arguments to pass to minikube start")
-var defaultDriver = flag.String("expected-default-driver", "", "Expected default driver")
 
 // Flags for faster local integration testing
 var forceProfile = flag.String("profile", "", "force tests to run against a particular profile")

From e6948b90b4aa9a7047cb4be2303d7425c8938c8c Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Sat, 21 Mar 2020 10:47:08 -0700
Subject: [PATCH 12/19] Trim crio prefix, dedup results

---
 test/integration/start_stop_delete_test.go | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/test/integration/start_stop_delete_test.go b/test/integration/start_stop_delete_test.go
index c891309a69..38ca1d9b9a 100644
--- a/test/integration/start_stop_delete_test.go
+++ b/test/integration/start_stop_delete_test.go
@@ -260,12 +260,14 @@ func testPulledImages(ctx context.Context, t *testing.T, profile string, version
 	if err != nil {
 		t.Errorf("images unmarshal: %v", err)
 	}
-	gotImages := []string{}
+	found := map[string]bool{}
 	for _, img := range jv["images"] {
 		for _, i := range img.Tags {
+			// Remove container-specific prefixes for naming consistency
+			i = strings.TrimPrefix(i, "docker.io/")
+			i = strings.TrimPrefix(i, "localhost/")
 			if defaultImage(i) {
-				// Remove docker.io for naming consistency between container runtimes
-				gotImages = append(gotImages, strings.TrimPrefix(i, "docker.io/"))
+				found[i] = true
 			} else {
 				t.Logf("Found non-minikube image: %s", i)
 			}
@@ -275,6 +277,10 @@ func testPulledImages(ctx context.Context, t *testing.T, profile string, version
 	if err != nil {
 		t.Errorf("kubeadm images: %v", version)
 	}
+	gotImages := []string{}
+	for k := range found {
+		gotImages = append(gotImages, k)
+	}
 	sort.Strings(want)
 	sort.Strings(gotImages)
 	if diff := cmp.Diff(want, gotImages); diff != "" {

From 17e6d84c3f5bf34537fc7153d2e5a0d9a0467a7e Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Sat, 21 Mar 2020 10:47:58 -0700
Subject: [PATCH 13/19] Remove unused code

---
 test/integration/main.go | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/test/integration/main.go b/test/integration/main.go
index 54a644518c..3c159a6e42 100644
--- a/test/integration/main.go
+++ b/test/integration/main.go
@@ -68,11 +68,6 @@ func HyperVDriver() bool {
 	return strings.Contains(*startArgs, "--driver=hyperv")
 }
 
-// ExpectedDefaultDriver returns the expected default driver, if any
-func ExpectedDefaultDriver() string {
-	return *defaultDriver
-}
-
 // CanCleanup returns if cleanup is allowed
 func CanCleanup() bool {
 	return *cleanup

From 314692a03825f9eb9e49702ff16f22ef3693d596 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Sat, 21 Mar 2020 11:09:59 -0700
Subject: [PATCH 14/19] Version bump: v1.9.0-beta.2

---
 CHANGELOG.md | 36 ++++++++++++++++++++++++++++++++++++
 Makefile     |  2 +-
 2 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 835ccf9552..baebf87467 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,41 @@
 # Release Notes
 
+## Version 1.9.0-beta.2 - 2020-03-21
+
+New features & improvements
+
+* Add experimental multi-node support 🎉 [#6787](https://github.com/kubernetes/minikube/pull/6787)
+* Skip kubeadm if cluster is running & properly configured [#7124](https://github.com/kubernetes/minikube/pull/7124)
+* Make certificates per-profile and consistent until IP or names change [#7125](https://github.com/kubernetes/minikube/pull/7125)
+* bumpup helm-tiller v2.16.1 → v2.16.3 [#7130](https://github.com/kubernetes/minikube/pull/7130)
+* Update Nvidia GPU plugin [#7132](https://github.com/kubernetes/minikube/pull/7132)
+* bumpup istio & istio-provisoner addon 1.4.0 → 1.5.0 [#7120](https://github.com/kubernetes/minikube/pull/7120)
+* New addon: registry-aliases [#6657](https://github.com/kubernetes/minikube/pull/6657)
+* Upgrade buildroot minor version [#7101](https://github.com/kubernetes/minikube/pull/7101)
+
+Bugfixes
+
+* Prevent crash if namespace or service doesn't exist [#5844](https://github.com/kubernetes/minikube/pull/5844)
+* Only run deleteHosts if running a VM [#7110](https://github.com/kubernetes/minikube/pull/7110)
+* provisioner: only reload docker if necessary, don't install curl [#7115](https://github.com/kubernetes/minikube/pull/7115)
+* Improve error when docker-env is used with non-docker runtime [#7112](https://github.com/kubernetes/minikube/pull/7112)
+* Add warning if both vm-driver and driver are specified [#7109](https://github.com/kubernetes/minikube/pull/7109)
+
+Thank you to our contributors:
+
+- Anders F Björklund
+- Iso Kenta
+- Kamesh Sampath
+- Kenta Iso
+- Priya Wadhwa
+- Sharif Elgamal
+- Tacio Costa
+- Thomas Stromberg
+- Thomas Strömberg
+- Zhongcheng Lao
+- rajula96reddy
+- sayboras
+
 ## Version 1.9.0-beta.1 - 2020-03-18
 
 New features
diff --git a/Makefile b/Makefile
index f1256f8eb5..f7b45a07ed 100755
--- a/Makefile
+++ b/Makefile
@@ -15,7 +15,7 @@
 # Bump these on release - and please check ISO_VERSION for correctness.
 VERSION_MAJOR ?= 1
 VERSION_MINOR ?= 9
-VERSION_BUILD ?= 0-beta.1
+VERSION_BUILD ?= 0-beta.2
 RAW_VERSION=$(VERSION_MAJOR).$(VERSION_MINOR).$(VERSION_BUILD)
 VERSION ?= v$(RAW_VERSION)
 

From ae9fd2a1e6b86926c22223d9773a7145793b35a6 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Sat, 21 Mar 2020 11:10:48 -0700
Subject: [PATCH 15/19] Remove dupe

---
 CHANGELOG.md | 1 -
 1 file changed, 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index baebf87467..e0c8f2efe0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -30,7 +30,6 @@ Thank you to our contributors:
 - Priya Wadhwa
 - Sharif Elgamal
 - Tacio Costa
-- Thomas Stromberg
 - Thomas Strömberg
 - Zhongcheng Lao
 - rajula96reddy

From ba6dbc57c233cfcdba5f18f43dc0d813854c7ccf Mon Sep 17 00:00:00 2001
From: Prasad Katti <prasadmkatti@gmail.com>
Date: Sat, 21 Mar 2020 12:18:50 -0700
Subject: [PATCH 16/19] Update the call to logProblems in node/start.go

---
 pkg/minikube/node/start.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go
index b46469837e..21a65d15ab 100644
--- a/pkg/minikube/node/start.go
+++ b/pkg/minikube/node/start.go
@@ -112,7 +112,7 @@ func Start(cc config.ClusterConfig, n config.Node, existingAddons map[string]boo
 		bs = setupKubeAdm(machineAPI, cc, n)
 		err = bs.StartCluster(cc)
 		if err != nil {
-			exit.WithLogEntries("Error starting cluster", err, logs.FindProblems(cr, bs, mRunner))
+			exit.WithLogEntries("Error starting cluster", err, logs.FindProblems(cr, bs, cc, mRunner))
 		}
 	} else {
 		bs, err = cluster.Bootstrapper(machineAPI, viper.GetString(cmdcfg.Bootstrapper), cc, n)

From a59750627e71a8e7190b245ead5599c2bb2ca8aa Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Sat, 21 Mar 2020 15:51:31 -0700
Subject: [PATCH 17/19] Improve stale endpoint detection and resolution in
 Kubernetes configs

---
 pkg/minikube/bootstrapper/bsutil/kubeadm.go  |  2 -
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 62 ++++++++++++++------
 2 files changed, 44 insertions(+), 20 deletions(-)

diff --git a/pkg/minikube/bootstrapper/bsutil/kubeadm.go b/pkg/minikube/bootstrapper/bsutil/kubeadm.go
index 8b675ae644..67187d6109 100644
--- a/pkg/minikube/bootstrapper/bsutil/kubeadm.go
+++ b/pkg/minikube/bootstrapper/bsutil/kubeadm.go
@@ -102,8 +102,6 @@ func GenerateKubeadmYAML(cc config.ClusterConfig, n config.Node, r cruntime.Mana
 		NoTaintMaster:    false, // That does not work with k8s 1.12+
 		DNSDomain:        k8s.DNSDomain,
 		NodeIP:           n.IP,
-		// NOTE: If set to an specific VM IP, things may break if the IP changes on host restart
-		// For multi-node, we may need to figure out an alternate strategy, like DNS or hosts files
 		ControlPlaneAddress: cp.IP,
 	}
 
diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index e859511099..a6c1bbe3dd 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -110,7 +110,7 @@ func (k *Bootstrapper) GetAPIServerStatus(ip net.IP, port int) (string, error) {
 }
 
 // LogCommands returns a map of log type to a command which will display that log.
-func (k *Bootstrapper) LogCommands(o bootstrapper.LogOptions) map[string]string {
+func (k *Bootstrapper) LogCommands(cfg config.ClusterConfig, o bootstrapper.LogOptions) map[string]string {
 	var kubelet strings.Builder
 	kubelet.WriteString("sudo journalctl -u kubelet")
 	if o.Lines > 0 {
@@ -128,9 +128,15 @@ func (k *Bootstrapper) LogCommands(o bootstrapper.LogOptions) map[string]string
 	if o.Lines > 0 {
 		dmesg.WriteString(fmt.Sprintf(" | tail -n %d", o.Lines))
 	}
+
+	describeNodes := fmt.Sprintf("sudo %s describe node -A --kubeconfig=%s",
+		path.Join(vmpath.GuestPersistentDir, "binaries", cfg.KubernetesConfig.KubernetesVersion, "kubectl"),
+		path.Join(vmpath.GuestPersistentDir, "kubeconfig"))
+
 	return map[string]string{
-		"kubelet": kubelet.String(),
-		"dmesg":   dmesg.String(),
+		"kubelet":        kubelet.String(),
+		"dmesg":          dmesg.String(),
+		"describe nodes": describeNodes,
 	}
 }
 
@@ -151,6 +157,30 @@ func (k *Bootstrapper) createCompatSymlinks() error {
 	return nil
 }
 
+// clearStaleConfigs clears configurations which may have stale IP addresses
+func (k *Bootstrapper) clearStaleConfigs(cfg config.ClusterConfig) error {
+	cp, err := config.PrimaryControlPlane(&cfg)
+	if err != nil {
+		return err
+	}
+
+	paths := []string{
+		"/etc/kubernetes/admin.conf",
+		"/etc/kubernetes/kubelet.conf",
+		"/etc/kubernetes/controller-manager.conf",
+		"/etc/kubernetes/scheduler.conf",
+	}
+
+	endpoint := fmt.Sprintf("https://%s", net.JoinHostPort(cp.IP, strconv.Itoa(cp.Port)))
+	for _, path := range paths {
+		_, err := k.c.RunCmd(exec.Command("sudo", "/bin/bash", "-c", fmt.Sprintf("grep %s %s || sudo rm -f %s", endpoint, path, path)))
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // StartCluster starts the cluster
 func (k *Bootstrapper) StartCluster(cfg config.ClusterConfig) error {
 	err := bsutil.ExistingConfig(k.c)
@@ -202,10 +232,8 @@ func (k *Bootstrapper) StartCluster(cfg config.ClusterConfig) error {
 
 	}
 
-	// Remove the previous kubeadm kubeconfig as the IP may have changed
-	_, err = k.c.RunCmd(exec.Command("sudo", "rm", "-f", "/etc/kubernetes/admin.conf"))
-	if err != nil {
-		return errors.Wrap(err, "deleting admin.conf")
+	if err := k.clearStaleConfigs(cfg); err != nil {
+		return errors.Wrap(err, "clearing stale configs")
 	}
 
 	conf := bsutil.KubeadmYamlPath
@@ -287,7 +315,7 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time
 		return err
 	}
 
-	if err := kverify.WaitForAPIServerProcess(cr, k, k.c, start, timeout); err != nil {
+	if err := kverify.WaitForAPIServerProcess(cr, k, cfg, k.c, start, timeout); err != nil {
 		return err
 	}
 
@@ -296,7 +324,7 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time
 		return err
 	}
 
-	if err := kverify.WaitForHealthyAPIServer(cr, k, k.c, start, ip, port, timeout); err != nil {
+	if err := kverify.WaitForHealthyAPIServer(cr, k, cfg, k.c, start, ip, port, timeout); err != nil {
 		return err
 	}
 
@@ -305,7 +333,7 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time
 		return errors.Wrap(err, "get k8s client")
 	}
 
-	if err := kverify.WaitForSystemPods(cr, k, k.c, c, start, timeout); err != nil {
+	if err := kverify.WaitForSystemPods(cr, k, cfg, k.c, c, start, timeout); err != nil {
 		return errors.Wrap(err, "waiting for system pods")
 	}
 	return nil
@@ -313,8 +341,8 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time
 
 // needsReset returns whether or not the cluster needs to be reconfigured
 func (k *Bootstrapper) needsReset(conf string, ip string, port int, client *kubernetes.Clientset) bool {
-	if _, err := k.c.RunCmd(exec.Command("sudo", "diff", "-u", conf, conf+".new")); err != nil {
-		glog.Infof("needs reset: configs differ")
+	if rr, err := k.c.RunCmd(exec.Command("sudo", "diff", "-u", conf, conf+".new")); err != nil {
+		glog.Infof("needs reset: configs differ:\n%s", rr.Output())
 		return true
 	}
 
@@ -378,10 +406,8 @@ func (k *Bootstrapper) restartCluster(cfg config.ClusterConfig) error {
 		return nil
 	}
 
-	// Remove the previous kubeadm kubeconfig as the IP may have changed
-	_, err = k.c.RunCmd(exec.Command("sudo", "rm", "-f", "/etc/kubernetes/admin.conf"))
-	if err != nil {
-		return errors.Wrap(err, "deleting admin.conf")
+	if err := k.clearStaleConfigs(cfg); err != nil {
+		return errors.Wrap(err, "clearing stale configs")
 	}
 
 	if _, err := k.c.RunCmd(exec.Command("sudo", "mv", conf+".new", conf)); err != nil {
@@ -411,11 +437,11 @@ func (k *Bootstrapper) restartCluster(cfg config.ClusterConfig) error {
 	}
 
 	// We must ensure that the apiserver is healthy before proceeding
-	if err := kverify.WaitForAPIServerProcess(cr, k, k.c, time.Now(), kconst.DefaultControlPlaneTimeout); err != nil {
+	if err := kverify.WaitForAPIServerProcess(cr, k, cfg, k.c, time.Now(), kconst.DefaultControlPlaneTimeout); err != nil {
 		return errors.Wrap(err, "apiserver healthz")
 	}
 
-	if err := kverify.WaitForSystemPods(cr, k, k.c, client, time.Now(), kconst.DefaultControlPlaneTimeout); err != nil {
+	if err := kverify.WaitForSystemPods(cr, k, cfg, k.c, client, time.Now(), kconst.DefaultControlPlaneTimeout); err != nil {
 		return errors.Wrap(err, "system pods")
 	}
 

From f76cdea82a94707c1f8cea24f2b2bd6e3b1380fa Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Sat, 21 Mar 2020 16:05:00 -0700
Subject: [PATCH 18/19] Run gofmt

---
 pkg/minikube/bootstrapper/bsutil/kubeadm.go | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/pkg/minikube/bootstrapper/bsutil/kubeadm.go b/pkg/minikube/bootstrapper/bsutil/kubeadm.go
index 67187d6109..b2af8d8a90 100644
--- a/pkg/minikube/bootstrapper/bsutil/kubeadm.go
+++ b/pkg/minikube/bootstrapper/bsutil/kubeadm.go
@@ -94,14 +94,14 @@ func GenerateKubeadmYAML(cc config.ClusterConfig, n config.Node, r cruntime.Mana
 		EtcdDataDir:       EtcdDataDir(),
 		ClusterName:       cc.Name,
 		//kubeadm uses NodeName as the --hostname-override parameter, so this needs to be the name of the machine
-		NodeName:         driver.MachineName(cc, n),
-		CRISocket:        r.SocketPath(),
-		ImageRepository:  k8s.ImageRepository,
-		ComponentOptions: componentOpts,
-		FeatureArgs:      kubeadmFeatureArgs,
-		NoTaintMaster:    false, // That does not work with k8s 1.12+
-		DNSDomain:        k8s.DNSDomain,
-		NodeIP:           n.IP,
+		NodeName:            driver.MachineName(cc, n),
+		CRISocket:           r.SocketPath(),
+		ImageRepository:     k8s.ImageRepository,
+		ComponentOptions:    componentOpts,
+		FeatureArgs:         kubeadmFeatureArgs,
+		NoTaintMaster:       false, // That does not work with k8s 1.12+
+		DNSDomain:           k8s.DNSDomain,
+		NodeIP:              n.IP,
 		ControlPlaneAddress: cp.IP,
 	}
 

From 373364d6257c0462675eafb074e10037a54a00fc Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Sat, 21 Mar 2020 18:47:26 -0700
Subject: [PATCH 19/19] Update to include last minute changes

---
 CHANGELOG.md | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e0c8f2efe0..fe559157a9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,22 +4,22 @@
 
 New features & improvements
 
-* Add experimental multi-node support 🎉 [#6787](https://github.com/kubernetes/minikube/pull/6787)
-* Skip kubeadm if cluster is running & properly configured [#7124](https://github.com/kubernetes/minikube/pull/7124)
-* Make certificates per-profile and consistent until IP or names change [#7125](https://github.com/kubernetes/minikube/pull/7125)
+* 🎉 Experimental multi-node support 🎊 [#6787](https://github.com/kubernetes/minikube/pull/6787)
+* Add kubectl desc nodes to minikube logs [#7105](https://github.com/kubernetes/minikube/pull/7105)
 * bumpup helm-tiller v2.16.1 → v2.16.3 [#7130](https://github.com/kubernetes/minikube/pull/7130)
 * Update Nvidia GPU plugin [#7132](https://github.com/kubernetes/minikube/pull/7132)
 * bumpup istio & istio-provisoner addon 1.4.0 → 1.5.0 [#7120](https://github.com/kubernetes/minikube/pull/7120)
 * New addon: registry-aliases [#6657](https://github.com/kubernetes/minikube/pull/6657)
 * Upgrade buildroot minor version [#7101](https://github.com/kubernetes/minikube/pull/7101)
+* Skip kubeadm if cluster is running & properly configured [#7124](https://github.com/kubernetes/minikube/pull/7124)
+* Make certificates per-profile and consistent until IP or names change [#7125](https://github.com/kubernetes/minikube/pull/7125)
 
 Bugfixes
 
-* Prevent crash if namespace or service doesn't exist [#5844](https://github.com/kubernetes/minikube/pull/5844)
-* Only run deleteHosts if running a VM [#7110](https://github.com/kubernetes/minikube/pull/7110)
-* provisioner: only reload docker if necessary, don't install curl [#7115](https://github.com/kubernetes/minikube/pull/7115)
-* Improve error when docker-env is used with non-docker runtime [#7112](https://github.com/kubernetes/minikube/pull/7112)
+* Prevent minikube from crashing if namespace or service doesn't exist [#5844](https://github.com/kubernetes/minikube/pull/5844)
 * Add warning if both vm-driver and driver are specified [#7109](https://github.com/kubernetes/minikube/pull/7109)
+* Improve error when docker-env is used with non-docker runtime [#7112](https://github.com/kubernetes/minikube/pull/7112)
+* provisioner: only reload docker if necessary, don't install curl [#7115](https://github.com/kubernetes/minikube/pull/7115)
 
 Thank you to our contributors:
 
@@ -27,6 +27,7 @@ Thank you to our contributors:
 - Iso Kenta
 - Kamesh Sampath
 - Kenta Iso
+- Prasad Katti
 - Priya Wadhwa
 - Sharif Elgamal
 - Tacio Costa