From 2b87e766c6114bb107cee48c0950c87041704b6e Mon Sep 17 00:00:00 2001
From: Tom <tommyp1ckles@users.noreply.github.com>
Date: Sun, 22 Mar 2020 13:00:14 -0700
Subject: [PATCH 01/27] Update nfsexports version

---
 go.mod | 2 +-
 go.sum | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/go.mod b/go.mod
index a703f57861..be48dd84f6 100644
--- a/go.mod
+++ b/go.mod
@@ -35,7 +35,7 @@ require (
 	github.com/hooklift/iso9660 v0.0.0-20170318115843-1cf07e5970d8
 	github.com/imdario/mergo v0.3.8 // indirect
 	github.com/intel-go/cpuid v0.0.0-20181003105527-1a4a6f06a1c6 // indirect
-	github.com/johanneswuerbach/nfsexports v0.0.0-20181204082207-1aa528dcb345
+	github.com/johanneswuerbach/nfsexports v0.0.0-20200318065542-c48c3734757f
 	github.com/juju/clock v0.0.0-20190205081909-9c5c9712527c
 	github.com/juju/errors v0.0.0-20190806202954-0232dcc7464d // indirect
 	github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8 // indirect
diff --git a/go.sum b/go.sum
index 8a4fe67c98..875768f6c7 100644
--- a/go.sum
+++ b/go.sum
@@ -423,6 +423,8 @@ github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht
 github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8=
 github.com/johanneswuerbach/nfsexports v0.0.0-20181204082207-1aa528dcb345 h1:XP1VL9iOZu4yz/rq8zj+yvB23XEY5erXRzp8JYmkWu0=
 github.com/johanneswuerbach/nfsexports v0.0.0-20181204082207-1aa528dcb345/go.mod h1:+c1/kUpg2zlkoWqTOvzDs36Wpbm3Gd1nlmtXAEB0WGU=
+github.com/johanneswuerbach/nfsexports v0.0.0-20200318065542-c48c3734757f h1:tL0xH80QVHQOde6Qqdohv6PewABH8l8N9pywZtuojJ0=
+github.com/johanneswuerbach/nfsexports v0.0.0-20200318065542-c48c3734757f/go.mod h1:+c1/kUpg2zlkoWqTOvzDs36Wpbm3Gd1nlmtXAEB0WGU=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs=

From 82eafa57d4d8c2d2f9addb20d18f4f66437d21c9 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Mon, 23 Mar 2020 17:25:25 -0700
Subject: [PATCH 02/27] Smoothly retry host startup

---
 pkg/minikube/node/start.go | 39 ++++++++++++++++++++++++++++++++++----
 1 file changed, 35 insertions(+), 4 deletions(-)

diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go
index 204ed14b97..7e3e1836b0 100644
--- a/pkg/minikube/node/start.go
+++ b/pkg/minikube/node/start.go
@@ -320,14 +320,45 @@ func startMachine(cfg *config.ClusterConfig, node *config.Node) (runner command.
 }
 
 // startHost starts a new minikube host using a VM or None
-func startHost(api libmachine.API, mc config.ClusterConfig, n config.Node) (*host.Host, bool) {
-	host, exists, err := machine.StartHost(api, mc, n)
-	if err != nil {
-		exit.WithError("Unable to start VM. Please investigate and run 'minikube delete' if possible", err)
+func startHost(api libmachine.API, cc config.ClusterConfig, n config.Node) (*host.Host, bool) {
+	host, exists, err := machine.StartHost(api, cc, n)
+	if err == nil {
+		return host, exists
 	}
+	out.T(out.Embarrassed, "StartHost failed, but will try again: {{.error}}", out.V{"error": err})
+
+	// NOTE: People get very cranky if you delete their prexisting VM. Only delete new ones.
+	if !exists {
+		err := machine.DeleteHost(api, driver.MachineName(cc, n))
+		if err != nil {
+			glog.Warningf("delete host: %v", err)
+		}
+	}
+
+	// Try again, but just once to avoid copious error messages
+	time.Sleep(5 * time.Second)
+
+	host, exists, err = machine.StartHost(api, cc, n)
+	if err == nil {
+		return host, exists
+	}
+
+	out.T(out.FailureType, "StartHost failed again: {{.error}}", out.V{"error": err})
+	out.T(out.Workaround, `Run: "{{.cmd}} delete", then "{{.cmd}} start --alsologtostderr -v=1" to try again with more logs`,
+		out.V{"cmd": minikubeCmd()})
+
+	exit.WithError("Unable to start VM after repeated tries. Please try {{'minikube delete' if possible", err)
 	return host, exists
 }
 
+// Return a minikube command containing the current profile name
+func minikubeCmd() string {
+	if viper.GetString(config.ProfileName) != constants.DefaultClusterName {
+		return fmt.Sprintf("minikube -p %s", config.ProfileName)
+	}
+	return "minikube"
+}
+
 // validateNetwork tries to catch network problems as soon as possible
 func validateNetwork(h *host.Host, r command.Runner) string {
 	ip, err := h.Driver.GetIP()

From 407637b4a2070122229e91cf7930e2c18d6b08bc Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Tue, 24 Mar 2020 06:07:51 -0700
Subject: [PATCH 03/27] download only for both docker and podman

---
 .github/workflows/main.yml                 | 20 +++++++++++++++-----
 test/integration/aaa_download_only_test.go | 22 ++++++----------------
 test/integration/functional_test.go        |  2 +-
 test/integration/main.go                   |  5 +++++
 4 files changed, 27 insertions(+), 22 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 37d1991a0d..79a9f0982e 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -113,7 +113,7 @@ jobs:
         chmod a+x e2e-*
         chmod a+x minikube-*
         START_TIME=$(date -u +%s)
-        KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--vm-driver=docker -test.timeout=80m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
+        KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--vm-driver=docker -test.timeout=80m -test.v -timeout-multiplier=1.3 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
         END_TIME=$(date -u +%s)
         TIME_ELAPSED=$(($END_TIME-$START_TIME))
         min=$((${TIME_ELAPSED}/60))
@@ -145,6 +145,8 @@ jobs:
         echo "----------------${numFail} Failures----------------------------"
         echo $STAT | jq '.FailedTests' || true
         echo "-------------------------------------------------------"
+        numPass=$(echo $STAT | jq '.NumberOfPass')
+        echo "*** $numPass Passed ***"
         if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
   docker_ubuntu_18_04:
     runs-on: ubuntu-18.04
@@ -186,7 +188,7 @@ jobs:
         chmod a+x e2e-*
         chmod a+x minikube-*
         START_TIME=$(date -u +%s)
-        KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--driver=docker -test.timeout=70m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
+        KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--driver=docker -test.timeout=80m -test.v -timeout-multiplier=1.3 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
         END_TIME=$(date -u +%s)
         TIME_ELAPSED=$(($END_TIME-$START_TIME))
         min=$((${TIME_ELAPSED}/60))
@@ -218,6 +220,8 @@ jobs:
         echo "----------------${numFail} Failures----------------------------"
         echo $STAT | jq '.FailedTests' || true
         echo "-------------------------------------------------------"
+        numPass=$(echo $STAT | jq '.NumberOfPass')
+        echo "*** $numPass Passed ***"
         if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
   none_ubuntu16_04:
     needs: [build_minikube]
@@ -254,7 +258,7 @@ jobs:
         chmod a+x e2e-*
         chmod a+x minikube-*
         START_TIME=$(date -u +%s)
-        KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome sudo -E ./e2e-linux-amd64 -minikube-start-args=--driver=none -test.timeout=70m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
+        KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome sudo -E ./e2e-linux-amd64 -minikube-start-args=--driver=none -test.timeout=50m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
         END_TIME=$(date -u +%s)
         TIME_ELAPSED=$(($END_TIME-$START_TIME))
         min=$((${TIME_ELAPSED}/60))
@@ -286,6 +290,8 @@ jobs:
         echo "----------------${numFail} Failures----------------------------"
         echo $STAT | jq '.FailedTests' || true
         echo "-------------------------------------------------------"
+        numPass=$(echo $STAT | jq '.NumberOfPass')
+        echo "*** $numPass Passed ***"
         if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
   none_ubuntu18_04:
     needs: [build_minikube]
@@ -322,7 +328,7 @@ jobs:
         chmod a+x e2e-*
         chmod a+x minikube-*
         START_TIME=$(date -u +%s)
-        KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome sudo -E ./e2e-linux-amd64 -minikube-start-args=--driver=none -test.timeout=70m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
+        KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome sudo -E ./e2e-linux-amd64 -minikube-start-args=--driver=none -test.timeout=50m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
         END_TIME=$(date -u +%s)
         TIME_ELAPSED=$(($END_TIME-$START_TIME))
         min=$((${TIME_ELAPSED}/60))
@@ -354,6 +360,8 @@ jobs:
         echo "----------------${numFail} Failures----------------------------"
         echo $STAT | jq '.FailedTests' || true
         echo "-------------------------------------------------------"
+        numPass=$(echo $STAT | jq '.NumberOfPass')
+        echo "*** $numPass Passed ***"
         if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
   podman_ubuntu_18_04:
       needs: [build_minikube]
@@ -400,7 +408,7 @@ jobs:
           chmod a+x e2e-*
           chmod a+x minikube-*
           START_TIME=$(date -u +%s)
-          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome sudo -E ./e2e-linux-amd64 -minikube-start-args=--driver=podman -test.timeout=70m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
+          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome sudo -E ./e2e-linux-amd64 -minikube-start-args=--driver=podman -test.timeout=70m -test.v -timeout-multiplier=1.3 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
           END_TIME=$(date -u +%s)
           TIME_ELAPSED=$(($END_TIME-$START_TIME))
           min=$((${TIME_ELAPSED}/60))
@@ -432,6 +440,8 @@ jobs:
           echo "----------------${numFail} Failures----------------------------"
           echo $STAT | jq '.FailedTests' || true
           echo "-------------------------------------------------------"
+          numPass=$(echo $STAT | jq '.NumberOfPass')
+          echo "*** $numPass Passed ***"
           if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
   # After all 4 integration tests finished
   # collect all the reports and upload
diff --git a/test/integration/aaa_download_only_test.go b/test/integration/aaa_download_only_test.go
index 7cd32410ab..8be373a5dd 100644
--- a/test/integration/aaa_download_only_test.go
+++ b/test/integration/aaa_download_only_test.go
@@ -29,7 +29,6 @@ import (
 	"runtime"
 	"strings"
 	"testing"
-	"time"
 
 	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
 	"k8s.io/minikube/pkg/minikube/constants"
@@ -147,16 +146,16 @@ func TestDownloadOnly(t *testing.T) {
 	}
 }
 
-func TestDownloadOnlyDocker(t *testing.T) {
-	if !runningDockerDriver(StartArgs()) {
-		t.Skip("this test only runs with the docker driver")
+func TestDownloadOnlyKic(t *testing.T) {
+	if !KicDriver() {
+		t.Skip("skipping, only for docker or podman driver")
 	}
-
 	profile := UniqueProfileName("download-docker")
-	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
+	ctx, cancel := context.WithTimeout(context.Background(), Minutes(15))
 	defer Cleanup(t, profile, cancel)
 
-	args := []string{"start", "--download-only", "-p", profile, "--force", "--alsologtostderr", "--driver=docker"}
+	args := []string{"start", "--download-only", "-p", profile, "--force", "--alsologtostderr"}
+	args = append(args, StartArgs()...)
 	rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
 	if err != nil {
 		t.Errorf("%s failed: %v:\n%s", args, err, rr.Output())
@@ -178,12 +177,3 @@ func TestDownloadOnlyDocker(t *testing.T) {
 		t.Errorf("checksum of %s does not match remote checksum (%s != %s)", tarball, string(remoteChecksum), string(checksum[:]))
 	}
 }
-
-func runningDockerDriver(startArgs []string) bool {
-	for _, s := range startArgs {
-		if s == "--driver=docker" {
-			return true
-		}
-	}
-	return false
-}
diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go
index ec4ce69051..fc53eea6c0 100644
--- a/test/integration/functional_test.go
+++ b/test/integration/functional_test.go
@@ -725,7 +725,7 @@ func validateMySQL(ctx context.Context, t *testing.T, profile string) {
 		rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "exec", names[0], "--", "mysql", "-ppassword", "-e", "show databases;"))
 		return err
 	}
-	if err = retry.Expo(mysql, 5*time.Second, 180*time.Second); err != nil {
+	if err = retry.Expo(mysql, 5*time.Second, Seconds(180)); err != nil {
 		t.Errorf("mysql failing: %v", err)
 	}
 }
diff --git a/test/integration/main.go b/test/integration/main.go
index 04c22da305..b6c5a3916f 100644
--- a/test/integration/main.go
+++ b/test/integration/main.go
@@ -68,6 +68,11 @@ func HyperVDriver() bool {
 	return strings.Contains(*startArgs, "--driver=hyperv") || strings.Contains(*startArgs, "--vm-driver=hyperv")
 }
 
+// KicDriver returns whether or not this test is using the docker or podman driver
+func KicDriver() bool {
+	return strings.Contains(*startArgs, "--driver=docker") || strings.Contains(*startArgs, "--vm-driver=docker") || strings.Contains(*startArgs, "--vm-driver=podman") || strings.Contains(*startArgs, "driver=podman")
+}
+
 // CanCleanup returns if cleanup is allowed
 func CanCleanup() bool {
 	return *cleanup

From 42980d609bc20c2790567296cab5a861055d2e33 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Tue, 24 Mar 2020 06:29:35 -0700
Subject: [PATCH 04/27] remove not needed retry

---
 test/integration/functional_test.go      |  2 +-
 test/integration/version_upgrade_test.go | 12 +++---------
 2 files changed, 4 insertions(+), 10 deletions(-)

diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go
index fc53eea6c0..5111590fe6 100644
--- a/test/integration/functional_test.go
+++ b/test/integration/functional_test.go
@@ -725,7 +725,7 @@ func validateMySQL(ctx context.Context, t *testing.T, profile string) {
 		rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "exec", names[0], "--", "mysql", "-ppassword", "-e", "show databases;"))
 		return err
 	}
-	if err = retry.Expo(mysql, 5*time.Second, Seconds(180)); err != nil {
+	if err = retry.Expo(mysql, 2*time.Second, Seconds(180)); err != nil {
 		t.Errorf("mysql failing: %v", err)
 	}
 }
diff --git a/test/integration/version_upgrade_test.go b/test/integration/version_upgrade_test.go
index e9c655a9b4..0c553b9998 100644
--- a/test/integration/version_upgrade_test.go
+++ b/test/integration/version_upgrade_test.go
@@ -76,7 +76,7 @@ func TestVersionUpgrade(t *testing.T) {
 	}
 
 	// Retry to allow flakiness for the previous release
-	if err := retry.Expo(r, 1*time.Second, Minutes(30), 3); err != nil {
+	if err := retry.Expo(r, 1*time.Second, Minutes(30), 2); err != nil {
 		t.Fatalf("release start failed: %v", err)
 	}
 
@@ -120,14 +120,8 @@ func TestVersionUpgrade(t *testing.T) {
 	}
 
 	args = append([]string{"start", "-p", profile, fmt.Sprintf("--kubernetes-version=%s", constants.OldestKubernetesVersion), "--alsologtostderr", "-v=1"}, StartArgs()...)
-	rr = &RunResult{}
-	r = func() error {
-		rr, err = Run(t, exec.CommandContext(ctx, tf.Name(), args...))
-		return err
-	}
-
-	if err := retry.Expo(r, 1*time.Second, Minutes(30), 3); err == nil {
-		t.Fatalf("downgrading kubernetes should not be allowed: %v", err)
+	if rr, err := Run(t, exec.CommandContext(ctx, tf.Name(), args...)); err == nil {
+		t.Fatalf("downgrading kubernetes should not be allowed. expected to see error but got %v for %q", err, rr.Args())
 	}
 
 	args = append([]string{"start", "-p", profile, fmt.Sprintf("--kubernetes-version=%s", constants.NewestKubernetesVersion), "--alsologtostderr", "-v=1"}, StartArgs()...)

From 0478f2d2048e7b82627351c9afe530156f0843b9 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Tue, 24 Mar 2020 06:37:16 -0700
Subject: [PATCH 05/27] adjuts the retry times

---
 test/integration/addons_test.go          | 4 ++--
 test/integration/fn_mount_cmd.go         | 2 +-
 test/integration/fn_pvc.go               | 2 +-
 test/integration/fn_tunnel_cmd.go        | 2 +-
 test/integration/version_upgrade_test.go | 4 ++--
 5 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/test/integration/addons_test.go b/test/integration/addons_test.go
index 313fa01d15..a784f381bf 100644
--- a/test/integration/addons_test.go
+++ b/test/integration/addons_test.go
@@ -129,7 +129,7 @@ func validateIngressAddon(ctx context.Context, t *testing.T, profile string) {
 		return nil
 	}
 
-	if err := retry.Expo(checkIngress, 500*time.Millisecond, Minutes(1)); err != nil {
+	if err := retry.Expo(checkIngress, 500*time.Millisecond, Seconds(90)); err != nil {
 		t.Errorf("ingress never responded as expected on 127.0.0.1:80: %v", err)
 	}
 
@@ -241,7 +241,7 @@ func validateMetricsServerAddon(ctx context.Context, t *testing.T, profile strin
 	}
 
 	// metrics-server takes some time to be able to collect metrics
-	if err := retry.Expo(checkMetricsServer, Seconds(13), Minutes(6)); err != nil {
+	if err := retry.Expo(checkMetricsServer, time.Second*3, Minutes(6)); err != nil {
 		t.Errorf(err.Error())
 	}
 
diff --git a/test/integration/fn_mount_cmd.go b/test/integration/fn_mount_cmd.go
index eedc9e7983..8a3a9f68ee 100644
--- a/test/integration/fn_mount_cmd.go
+++ b/test/integration/fn_mount_cmd.go
@@ -106,7 +106,7 @@ func validateMountCmd(ctx context.Context, t *testing.T, profile string) {
 	}
 
 	start := time.Now()
-	if err := retry.Expo(checkMount, time.Second, 15*time.Second); err != nil {
+	if err := retry.Expo(checkMount, time.Millisecond*500, Seconds(15)); err != nil {
 		// For local testing, allow macOS users to click prompt. If they don't, skip the test.
 		if runtime.GOOS == "darwin" {
 			t.Skip("skipping: mount did not appear, likely because macOS requires prompt to allow non-codesigned binaries to listen on non-localhost port")
diff --git a/test/integration/fn_pvc.go b/test/integration/fn_pvc.go
index 4f97c830ef..785783fd80 100644
--- a/test/integration/fn_pvc.go
+++ b/test/integration/fn_pvc.go
@@ -57,7 +57,7 @@ func validatePersistentVolumeClaim(ctx context.Context, t *testing.T, profile st
 	}
 
 	// Ensure the addon-manager has created the StorageClass before creating a claim, otherwise it won't be bound
-	if err := retry.Expo(checkStorageClass, time.Second, 90*time.Second); err != nil {
+	if err := retry.Expo(checkStorageClass, time.Millisecond*500, Seconds(100)); err != nil {
 		t.Errorf("no default storage class after retry: %v", err)
 	}
 
diff --git a/test/integration/fn_tunnel_cmd.go b/test/integration/fn_tunnel_cmd.go
index a21b9350c3..e4598b3da7 100644
--- a/test/integration/fn_tunnel_cmd.go
+++ b/test/integration/fn_tunnel_cmd.go
@@ -119,7 +119,7 @@ func validateTunnelCmd(ctx context.Context, t *testing.T, profile string) {
 		}
 		return nil
 	}
-	if err = retry.Expo(fetch, time.Millisecond*500, Minutes(2), 6); err != nil {
+	if err = retry.Expo(fetch, time.Millisecond*500, Minutes(2), 13); err != nil {
 		t.Errorf("failed to contact nginx at %s: %v", nginxIP, err)
 	}
 
diff --git a/test/integration/version_upgrade_test.go b/test/integration/version_upgrade_test.go
index 0c553b9998..a03676a335 100644
--- a/test/integration/version_upgrade_test.go
+++ b/test/integration/version_upgrade_test.go
@@ -75,7 +75,7 @@ func TestVersionUpgrade(t *testing.T) {
 		return err
 	}
 
-	// Retry to allow flakiness for the previous release
+	// Retry up to two times, to allow flakiness for the previous release
 	if err := retry.Expo(r, 1*time.Second, Minutes(30), 2); err != nil {
 		t.Fatalf("release start failed: %v", err)
 	}
@@ -121,7 +121,7 @@ func TestVersionUpgrade(t *testing.T) {
 
 	args = append([]string{"start", "-p", profile, fmt.Sprintf("--kubernetes-version=%s", constants.OldestKubernetesVersion), "--alsologtostderr", "-v=1"}, StartArgs()...)
 	if rr, err := Run(t, exec.CommandContext(ctx, tf.Name(), args...)); err == nil {
-		t.Fatalf("downgrading kubernetes should not be allowed. expected to see error but got %v for %q", err, rr.Args())
+		t.Fatalf("downgrading kubernetes should not be allowed. expected to see error but got %v for %q", err, rr.Args)
 	}
 
 	args = append([]string{"start", "-p", profile, fmt.Sprintf("--kubernetes-version=%s", constants.NewestKubernetesVersion), "--alsologtostderr", "-v=1"}, StartArgs()...)

From 83cc28e0826a8b01109af317485d38c9e5b629a3 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Tue, 24 Mar 2020 07:11:22 -0700
Subject: [PATCH 06/27] add logging for when container status is running

---
 pkg/drivers/kic/oci/oci.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go
index df5117164f..2de446a4cd 100644
--- a/pkg/drivers/kic/oci/oci.go
+++ b/pkg/drivers/kic/oci/oci.go
@@ -156,6 +156,10 @@ func CreateContainerNode(p CreateParams) error {
 		if s != "running" {
 			return fmt.Errorf("temporary error created container %q is not running yet", p.Name)
 		}
+		if s == "running" {
+			glog.Infof("the created container %q has a running status.", p.Name)
+			return nil
+		}
 		return nil
 	}
 

From 41b16b1568d83e3fcc23645d701f5f41502d32fe Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Tue, 24 Mar 2020 07:27:07 -0700
Subject: [PATCH 07/27] fail if container is not running

---
 pkg/drivers/kic/oci/oci.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go
index 2de446a4cd..b0904d586b 100644
--- a/pkg/drivers/kic/oci/oci.go
+++ b/pkg/drivers/kic/oci/oci.go
@@ -164,8 +164,8 @@ func CreateContainerNode(p CreateParams) error {
 	}
 
 	// retry up to up 5 seconds to make sure the created container status is running.
-	if err := retry.Expo(checkRunning, 13*time.Millisecond, time.Second*5); err != nil {
-		glog.Warningf("The created container %q failed to report to be running in 5 seconds.", p.Name)
+	if err := retry.Expo(checkRunning, 13*time.Millisecond, time.Second*10); err != nil {
+		return errors.Wrapf(err, "check container %q running", p.Name)
 	}
 
 	return nil

From 940baa09eae4197074dad120475f54e3576d59b6 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Tue, 24 Mar 2020 07:31:22 -0700
Subject: [PATCH 08/27] fix comment

---
 pkg/drivers/kic/oci/oci.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go
index b0904d586b..295925b8cf 100644
--- a/pkg/drivers/kic/oci/oci.go
+++ b/pkg/drivers/kic/oci/oci.go
@@ -163,8 +163,8 @@ func CreateContainerNode(p CreateParams) error {
 		return nil
 	}
 
-	// retry up to up 5 seconds to make sure the created container status is running.
-	if err := retry.Expo(checkRunning, 13*time.Millisecond, time.Second*10); err != nil {
+	// retry up to up 13 seconds to make sure the created container status is running.
+	if err := retry.Expo(checkRunning, 13*time.Millisecond, time.Second*13); err != nil {
 		return errors.Wrapf(err, "check container %q running", p.Name)
 	}
 

From fed6713a3fbd1568208bc96104c9e8cac243cb89 Mon Sep 17 00:00:00 2001
From: tstromberg <tstromberg@google.com>
Date: Tue, 24 Mar 2020 08:45:29 -0700
Subject: [PATCH 09/27] Enable HW_RANDOM_VIRTIO

---
 deploy/iso/minikube-iso/board/coreos/minikube/linux_defconfig | 1 +
 1 file changed, 1 insertion(+)

diff --git a/deploy/iso/minikube-iso/board/coreos/minikube/linux_defconfig b/deploy/iso/minikube-iso/board/coreos/minikube/linux_defconfig
index ebf694f191..40c43c77ce 100644
--- a/deploy/iso/minikube-iso/board/coreos/minikube/linux_defconfig
+++ b/deploy/iso/minikube-iso/board/coreos/minikube/linux_defconfig
@@ -380,6 +380,7 @@ CONFIG_SERIAL_8250_SHARE_IRQ=y
 CONFIG_SERIAL_8250_DETECT_IRQ=y
 CONFIG_SERIAL_8250_RSA=y
 CONFIG_HW_RANDOM=y
+CONFIG_HW_RANDOM_VIRTIO=y
 # CONFIG_HW_RANDOM_INTEL is not set
 # CONFIG_HW_RANDOM_AMD is not set
 CONFIG_NVRAM=y

From e9e9a41049fca2d5ad8d7f3d0cfbd9e2953b0ac8 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Tue, 24 Mar 2020 08:58:15 -0700
Subject: [PATCH 10/27] Use ExampleCmd now that it is available

---
 pkg/minikube/node/start.go | 15 ++++-----------
 1 file changed, 4 insertions(+), 11 deletions(-)

diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go
index 7e3e1836b0..5f7b131a6e 100644
--- a/pkg/minikube/node/start.go
+++ b/pkg/minikube/node/start.go
@@ -48,6 +48,7 @@ import (
 	"k8s.io/minikube/pkg/minikube/localpath"
 	"k8s.io/minikube/pkg/minikube/logs"
 	"k8s.io/minikube/pkg/minikube/machine"
+	"k8s.io/minikube/pkg/minikube/mustload"
 	"k8s.io/minikube/pkg/minikube/out"
 	"k8s.io/minikube/pkg/minikube/proxy"
 	"k8s.io/minikube/pkg/util"
@@ -335,7 +336,7 @@ func startHost(api libmachine.API, cc config.ClusterConfig, n config.Node) (*hos
 		}
 	}
 
-	// Try again, but just once to avoid copious error messages
+	// Try again, but just once to avoid making the logs overly confusing
 	time.Sleep(5 * time.Second)
 
 	host, exists, err = machine.StartHost(api, cc, n)
@@ -344,21 +345,13 @@ func startHost(api libmachine.API, cc config.ClusterConfig, n config.Node) (*hos
 	}
 
 	out.T(out.FailureType, "StartHost failed again: {{.error}}", out.V{"error": err})
-	out.T(out.Workaround, `Run: "{{.cmd}} delete", then "{{.cmd}} start --alsologtostderr -v=1" to try again with more logs`,
-		out.V{"cmd": minikubeCmd()})
+	out.T(out.Workaround, `Run: "{{.delete}}", then "{{.start}} --alsologtostderr -v=1" to try again with more logging`,
+		out.V{"delete": mustload.ExampleCmd(cc.Name, "delete"), "start": mustload.ExampleCmd(cc.Name, "start")})
 
 	exit.WithError("Unable to start VM after repeated tries. Please try {{'minikube delete' if possible", err)
 	return host, exists
 }
 
-// Return a minikube command containing the current profile name
-func minikubeCmd() string {
-	if viper.GetString(config.ProfileName) != constants.DefaultClusterName {
-		return fmt.Sprintf("minikube -p %s", config.ProfileName)
-	}
-	return "minikube"
-}
-
 // validateNetwork tries to catch network problems as soon as possible
 func validateNetwork(h *host.Host, r command.Runner) string {
 	ip, err := h.Driver.GetIP()

From b5d5aa1d52b2212986219515f6bec70d641f3b7f Mon Sep 17 00:00:00 2001
From: Vincent Link <mail@linkvt.de>
Date: Tue, 24 Mar 2020 16:46:00 +0100
Subject: [PATCH 11/27] Parse --disk-size and --memory sizes with binary
 suffixes

---
 pkg/util/utils.go      | 5 +++--
 pkg/util/utils_test.go | 3 ++-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/pkg/util/utils.go b/pkg/util/utils.go
index fdd38f4f35..37de6085cd 100644
--- a/pkg/util/utils.go
+++ b/pkg/util/utils.go
@@ -38,12 +38,13 @@ func CalculateSizeInMB(humanReadableSize string) (int, error) {
 	if err == nil {
 		humanReadableSize += "mb"
 	}
-	size, err := units.FromHumanSize(humanReadableSize)
+	// parse the size suffix binary instead of decimal so that 1G -> 1024MB instead of 1000MB
+	size, err := units.RAMInBytes(humanReadableSize)
 	if err != nil {
 		return 0, fmt.Errorf("FromHumanSize: %v", err)
 	}
 
-	return int(size / units.MB), nil
+	return int(size / units.MiB), nil
 }
 
 // GetBinaryDownloadURL returns a suitable URL for the platform
diff --git a/pkg/util/utils_test.go b/pkg/util/utils_test.go
index f1fe867c48..55392d7ebc 100644
--- a/pkg/util/utils_test.go
+++ b/pkg/util/utils_test.go
@@ -51,6 +51,7 @@ func TestCalculateSizeInMB(t *testing.T) {
 		{"1024KB", 1},
 		{"1024mb", 1024},
 		{"1024b", 0},
+		{"1g", 1024},
 	}
 
 	for _, tt := range testData {
@@ -59,7 +60,7 @@ func TestCalculateSizeInMB(t *testing.T) {
 			t.Fatalf("unexpected err: %v", err)
 		}
 		if number != tt.expectedNumber {
-			t.Fatalf("Expected '%d'' but got '%d'", tt.expectedNumber, number)
+			t.Fatalf("Expected '%d' but got '%d' from size '%s'", tt.expectedNumber, number, tt.size)
 		}
 	}
 }

From f00f5ff6008c17652d77626192773927ccf349a3 Mon Sep 17 00:00:00 2001
From: Medya Ghazizadeh <medyagh@users.noreply.github.com>
Date: Tue, 24 Mar 2020 10:21:32 -0700
Subject: [PATCH 12/27] remove extra if

---
 pkg/drivers/kic/oci/oci.go | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go
index 295925b8cf..5885b0e1cf 100644
--- a/pkg/drivers/kic/oci/oci.go
+++ b/pkg/drivers/kic/oci/oci.go
@@ -156,10 +156,7 @@ func CreateContainerNode(p CreateParams) error {
 		if s != "running" {
 			return fmt.Errorf("temporary error created container %q is not running yet", p.Name)
 		}
-		if s == "running" {
-			glog.Infof("the created container %q has a running status.", p.Name)
-			return nil
-		}
+		glog.Infof("the created container %q has a running status.", p.Name)
 		return nil
 	}
 

From 5ef83f35e222c8d4d829dc5776837ee2eccff3cc Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Tue, 24 Mar 2020 10:53:02 -0700
Subject: [PATCH 13/27] Add --preload flag to optionally turn off preload

---
 cmd/minikube/cmd/start.go                  | 2 ++
 go.mod                                     | 2 +-
 go.sum                                     | 2 ++
 pkg/minikube/download/preload.go           | 5 +++++
 test/integration/start_stop_delete_test.go | 2 +-
 5 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 01f27f68c4..7d6e92cbf6 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -123,6 +123,7 @@ const (
 	hostOnlyNicType         = "host-only-nic-type"
 	natNicType              = "nat-nic-type"
 	nodes                   = "nodes"
+	preload                 = "preload"
 )
 
 var (
@@ -175,6 +176,7 @@ func initMinikubeFlags() {
 	startCmd.Flags().Bool(autoUpdate, true, "If set, automatically updates drivers to the latest version. Defaults to true.")
 	startCmd.Flags().Bool(installAddons, true, "If set, install addons. Defaults to true.")
 	startCmd.Flags().IntP(nodes, "n", 1, "The number of nodes to spin up. Defaults to 1.")
+	startCmd.Flags().Bool(preload, true, "If true, download tarball of preloaded images if available to improve start time.")
 }
 
 // initKubernetesFlags inits the commandline flags for kubernetes related options
diff --git a/go.mod b/go.mod
index 3f4a02bf9f..29a5ac30d5 100644
--- a/go.mod
+++ b/go.mod
@@ -35,7 +35,7 @@ require (
 	github.com/hooklift/iso9660 v0.0.0-20170318115843-1cf07e5970d8
 	github.com/imdario/mergo v0.3.8 // indirect
 	github.com/intel-go/cpuid v0.0.0-20181003105527-1a4a6f06a1c6 // indirect
-	github.com/johanneswuerbach/nfsexports v0.0.0-20181204082207-1aa528dcb345
+	github.com/johanneswuerbach/nfsexports v0.0.0-20200318065542-c48c3734757f
 	github.com/juju/clock v0.0.0-20190205081909-9c5c9712527c
 	github.com/juju/errors v0.0.0-20190806202954-0232dcc7464d // indirect
 	github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8 // indirect
diff --git a/go.sum b/go.sum
index 70ea938169..d7d0bc3a61 100644
--- a/go.sum
+++ b/go.sum
@@ -423,6 +423,8 @@ github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht
 github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8=
 github.com/johanneswuerbach/nfsexports v0.0.0-20181204082207-1aa528dcb345 h1:XP1VL9iOZu4yz/rq8zj+yvB23XEY5erXRzp8JYmkWu0=
 github.com/johanneswuerbach/nfsexports v0.0.0-20181204082207-1aa528dcb345/go.mod h1:+c1/kUpg2zlkoWqTOvzDs36Wpbm3Gd1nlmtXAEB0WGU=
+github.com/johanneswuerbach/nfsexports v0.0.0-20200318065542-c48c3734757f h1:tL0xH80QVHQOde6Qqdohv6PewABH8l8N9pywZtuojJ0=
+github.com/johanneswuerbach/nfsexports v0.0.0-20200318065542-c48c3734757f/go.mod h1:+c1/kUpg2zlkoWqTOvzDs36Wpbm3Gd1nlmtXAEB0WGU=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs=
diff --git a/pkg/minikube/download/preload.go b/pkg/minikube/download/preload.go
index 09e0d8d35e..0291a15b93 100644
--- a/pkg/minikube/download/preload.go
+++ b/pkg/minikube/download/preload.go
@@ -31,6 +31,7 @@ import (
 	"github.com/golang/glog"
 	"github.com/hashicorp/go-getter"
 	"github.com/pkg/errors"
+	"github.com/spf13/viper"
 	"k8s.io/minikube/pkg/minikube/localpath"
 	"k8s.io/minikube/pkg/minikube/out"
 )
@@ -76,6 +77,10 @@ func remoteTarballURL(k8sVersion string) string {
 
 // PreloadExists returns true if there is a preloaded tarball that can be used
 func PreloadExists(k8sVersion, containerRuntime string) bool {
+	if !viper.GetBool("preload") {
+		return false
+	}
+
 	if containerRuntime != "docker" {
 		return false
 	}
diff --git a/test/integration/start_stop_delete_test.go b/test/integration/start_stop_delete_test.go
index 38ca1d9b9a..6ba37042ef 100644
--- a/test/integration/start_stop_delete_test.go
+++ b/test/integration/start_stop_delete_test.go
@@ -175,7 +175,7 @@ func TestStartStopWithPreload(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), Minutes(40))
 	defer CleanupWithLogs(t, profile, cancel)
 
-	startArgs := []string{"start", "-p", profile, "--memory=2200", "--alsologtostderr", "-v=3", "--wait=true"}
+	startArgs := []string{"start", "-p", profile, "--memory=2200", "--alsologtostderr", "-v=3", "--wait=true", "--preload=false"}
 	startArgs = append(startArgs, StartArgs()...)
 	k8sVersion := "v1.17.0"
 	startArgs = append(startArgs, fmt.Sprintf("--kubernetes-version=%s", k8sVersion))

From 4a7f59f8a8c7da65b4a9ffd737250cf0020548a2 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Tue, 24 Mar 2020 10:59:25 -0700
Subject: [PATCH 14/27] remove preload flag

---
 test/integration/start_stop_delete_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/integration/start_stop_delete_test.go b/test/integration/start_stop_delete_test.go
index 6ba37042ef..38ca1d9b9a 100644
--- a/test/integration/start_stop_delete_test.go
+++ b/test/integration/start_stop_delete_test.go
@@ -175,7 +175,7 @@ func TestStartStopWithPreload(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), Minutes(40))
 	defer CleanupWithLogs(t, profile, cancel)
 
-	startArgs := []string{"start", "-p", profile, "--memory=2200", "--alsologtostderr", "-v=3", "--wait=true", "--preload=false"}
+	startArgs := []string{"start", "-p", profile, "--memory=2200", "--alsologtostderr", "-v=3", "--wait=true"}
 	startArgs = append(startArgs, StartArgs()...)
 	k8sVersion := "v1.17.0"
 	startArgs = append(startArgs, fmt.Sprintf("--kubernetes-version=%s", k8sVersion))

From 8d9e26d9940a7c2e89387661dd15b6e6d76cb217 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Tue, 24 Mar 2020 11:58:41 -0700
Subject: [PATCH 15/27] Test restart with preloaded tarball and without

---
 test/integration/start_stop_delete_test.go | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/test/integration/start_stop_delete_test.go b/test/integration/start_stop_delete_test.go
index 38ca1d9b9a..f0aeb03450 100644
--- a/test/integration/start_stop_delete_test.go
+++ b/test/integration/start_stop_delete_test.go
@@ -175,7 +175,7 @@ func TestStartStopWithPreload(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), Minutes(40))
 	defer CleanupWithLogs(t, profile, cancel)
 
-	startArgs := []string{"start", "-p", profile, "--memory=2200", "--alsologtostderr", "-v=3", "--wait=true"}
+	startArgs := []string{"start", "-p", profile, "--memory=2200", "--alsologtostderr", "-v=3", "--wait=true", "--preload=false"}
 	startArgs = append(startArgs, StartArgs()...)
 	k8sVersion := "v1.17.0"
 	startArgs = append(startArgs, fmt.Sprintf("--kubernetes-version=%s", k8sVersion))
@@ -191,6 +191,18 @@ func TestStartStopWithPreload(t *testing.T) {
 	if err != nil {
 		t.Fatalf("%s failed: %v", rr.Args, err)
 	}
+
+	// Restart again with v1.17.0, this time with the preloaded tarball
+	startArgs = []string{"start", "-p", profile, "--memory=2200", "--alsologtostderr", "-v=3", "--wait=true"}
+	startArgs = append(startArgs, StartArgs()...)
+	startArgs = append(startArgs, fmt.Sprintf("--kubernetes-version=%s", k8sVersion))
+
+	rr, err = Run(t, exec.CommandContext(ctx, Target(), startArgs...))
+	if err != nil {
+		t.Fatalf("%s failed: %v", rr.Args, err)
+	}
+	verifyImageExistsInDaemon(ctx, t, profile, image)
+
 	// Restart minikube with v1.17.3, which has a preloaded tarball
 	startArgs = []string{"start", "-p", profile, "--memory=2200", "--alsologtostderr", "-v=3", "--wait=true"}
 	startArgs = append(startArgs, StartArgs()...)
@@ -200,9 +212,13 @@ func TestStartStopWithPreload(t *testing.T) {
 	if err != nil {
 		t.Fatalf("%s failed: %v", rr.Args, err)
 	}
+	verifyImageExistsInDaemon(ctx, t, profile, image)
 
+}
+
+func verifyImageExistsInDaemon(ctx context.Context, t *testing.T, profile, image string) {
 	// Ensure that busybox still exists in the daemon
-	rr, err = Run(t, exec.CommandContext(ctx, Target(), "ssh", "-p", profile, "--", "docker", "images"))
+	rr, err := Run(t, exec.CommandContext(ctx, Target(), "ssh", "-p", profile, "--", "docker", "images"))
 	if err != nil {
 		t.Fatalf("%s failed: %v", rr.Args, err)
 	}

From 0d76e2a68477c09dc6ba924d9504d972672063f4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= <anders.f.bjorklund@gmail.com>
Date: Tue, 24 Mar 2020 20:19:01 +0100
Subject: [PATCH 16/27] Avoid dereferencing nil, in case of an error

---
 pkg/minikube/assets/vm_assets.go | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/pkg/minikube/assets/vm_assets.go b/pkg/minikube/assets/vm_assets.go
index c3f30a667d..e3b4678544 100644
--- a/pkg/minikube/assets/vm_assets.go
+++ b/pkg/minikube/assets/vm_assets.go
@@ -120,7 +120,10 @@ func (f *FileAsset) GetLength() (flen int) {
 // GetModTime returns modification time of the file
 func (f *FileAsset) GetModTime() (time.Time, error) {
 	fi, err := os.Stat(f.AssetName)
-	return fi.ModTime(), err
+	if err != nil {
+		return time.Time{}, err
+	}
+	return fi.ModTime(), nil
 }
 
 // Read reads the asset

From e3bcdbf42ac35c685e3f658eec768d2efdff69b2 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Tue, 24 Mar 2020 12:55:31 -0700
Subject: [PATCH 17/27] fix validateSpecificDriver error message

---
 cmd/minikube/cmd/start.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 01f27f68c4..302bf72a0a 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -577,7 +577,7 @@ func validateSpecifiedDriver(existing *config.ClusterConfig) {
 
 	out.ErrT(out.Workaround, `To proceed, either:
 
-1) Delete the existing "{{.name}}" cluster using: '{{.command}} delete'
+1) Delete the existing "{{.name}}" cluster using: 'minikube delete -p {{.name}}'
 
 * or *
 

From 2933953cfa2c95b1d135d449b536c3d2e909c38c Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Tue, 24 Mar 2020 13:18:59 -0700
Subject: [PATCH 18/27] use mustload

---
 cmd/minikube/cmd/start.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 302bf72a0a..60d2cce40b 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -577,12 +577,12 @@ func validateSpecifiedDriver(existing *config.ClusterConfig) {
 
 	out.ErrT(out.Workaround, `To proceed, either:
 
-1) Delete the existing "{{.name}}" cluster using: 'minikube delete -p {{.name}}'
+1) Delete the existing "{{.name}}" cluster using: '{{.delcommand}}'
 
 * or *
 
 2) Start the existing "{{.name}}" cluster using: '{{.command}} --driver={{.old}}'
-`, out.V{"command": mustload.ExampleCmd(existing.Name, "start"), "old": old, "name": existing.Name})
+`, out.V{"command": mustload.ExampleCmd(existing.Name, "start"), "delcommand": mustload.ExampleCmd(existing.Name, "delete"), "old": old, "name": existing.Name})
 
 	exit.WithCodeT(exit.Config, "Exiting.")
 }

From b5e088ad59bb4c320d335633bc0d420223932a6b Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Tue, 24 Mar 2020 13:44:51 -0700
Subject: [PATCH 19/27] run preload on hot restarts as well, in case kubernetes
 version has been upgraded

---
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index a6c1bbe3dd..5de297a003 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -524,17 +524,22 @@ func (k *Bootstrapper) UpdateCluster(cfg config.ClusterConfig) error {
 		return errors.Wrap(err, "kubeadm images")
 	}
 
-	if cfg.KubernetesConfig.ShouldLoadCachedImages {
-		if err := machine.LoadImages(&cfg, k.c, images, constants.ImageCacheDir); err != nil {
-			out.FailureT("Unable to load cached images: {{.error}}", out.V{"error": err})
-		}
-	}
 	r, err := cruntime.New(cruntime.Config{Type: cfg.KubernetesConfig.ContainerRuntime,
 		Runner: k.c, Socket: cfg.KubernetesConfig.CRISocket})
 	if err != nil {
 		return errors.Wrap(err, "runtime")
 	}
 
+	if err := r.Preload(cfg.KubernetesConfig); err != nil {
+		return errors.Wrap(err, "preloading")
+	}
+
+	if cfg.KubernetesConfig.ShouldLoadCachedImages {
+		if err := machine.LoadImages(&cfg, k.c, images, constants.ImageCacheDir); err != nil {
+			out.FailureT("Unable to load cached images: {{.error}}", out.V{"error": err})
+		}
+	}
+
 	for _, n := range cfg.Nodes {
 		err := k.UpdateNode(cfg, n, r)
 		if err != nil {

From 3ad1dfb4bb3d8e4a016a3aca1585a3b07b4b78e9 Mon Sep 17 00:00:00 2001
From: Vincent Link <mail@linkvt.de>
Date: Tue, 24 Mar 2020 20:19:13 +0100
Subject: [PATCH 20/27] Add 'stable' and 'latest' as valid kubernetes-version
 values

---
 cmd/minikube/cmd/start.go      | 22 ++++++++++++----------
 cmd/minikube/cmd/start_test.go | 12 +++++++++++-
 2 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 01f27f68c4..595058f4f9 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -179,7 +179,7 @@ func initMinikubeFlags() {
 
 // initKubernetesFlags inits the commandline flags for kubernetes related options
 func initKubernetesFlags() {
-	startCmd.Flags().String(kubernetesVersion, "", "The kubernetes version that the minikube VM will use (ex: v1.2.3)")
+	startCmd.Flags().String(kubernetesVersion, "", fmt.Sprintf("The kubernetes version that the minikube VM will use (ex: v1.2.3, 'stable' for %s, 'latest' for %s). Defaults to 'stable'.", constants.DefaultKubernetesVersion, constants.NewestKubernetesVersion))
 	startCmd.Flags().Var(&config.ExtraOptions, "extra-config",
 		`A set of key=value pairs that describe configuration that may be passed to different components.
 		The key should be '.' separated, and the first part before the dot is the component to apply the configuration to.
@@ -480,10 +480,10 @@ func selectDriver(existing *config.ClusterConfig) registry.DriverState {
 		if vmd := viper.GetString("vm-driver"); vmd != "" {
 			// Output a warning
 			warning := `Both driver={{.driver}} and vm-driver={{.vmd}} have been set.
-			
+
     Since vm-driver is deprecated, minikube will default to driver={{.driver}}.
 
-    If vm-driver is set in the global config, please run "minikube config unset vm-driver" to resolve this warning.			
+    If vm-driver is set in the global config, please run "minikube config unset vm-driver" to resolve this warning.
 			`
 			out.T(out.Warning, warning, out.V{"driver": d, "vmd": vmd})
 		}
@@ -1065,13 +1065,15 @@ func autoSetDriverOptions(cmd *cobra.Command, drvName string) (err error) {
 func getKubernetesVersion(old *config.ClusterConfig) string {
 	paramVersion := viper.GetString(kubernetesVersion)
 
-	if paramVersion == "" { // if the user did not specify any version then ...
-		if old != nil { // .. use the old version from config (if any)
-			paramVersion = old.KubernetesConfig.KubernetesVersion
-		}
-		if paramVersion == "" { // .. otherwise use the default version
-			paramVersion = constants.DefaultKubernetesVersion
-		}
+	// try to load the old version first if the user didn't specify anything
+	if paramVersion == "" && old != nil {
+		paramVersion = old.KubernetesConfig.KubernetesVersion
+	}
+
+	if paramVersion == "" || strings.EqualFold(paramVersion, "stable") {
+		paramVersion = constants.DefaultKubernetesVersion
+	} else if strings.EqualFold(paramVersion, "latest") {
+		paramVersion = constants.NewestKubernetesVersion
 	}
 
 	nvs, err := semver.Make(strings.TrimPrefix(paramVersion, version.VersionPrefix))
diff --git a/cmd/minikube/cmd/start_test.go b/cmd/minikube/cmd/start_test.go
index ef7e4b7403..0a443aa507 100644
--- a/cmd/minikube/cmd/start_test.go
+++ b/cmd/minikube/cmd/start_test.go
@@ -26,7 +26,7 @@ import (
 	"k8s.io/minikube/pkg/minikube/constants"
 )
 
-func TestGetKuberneterVersion(t *testing.T) {
+func TestGetKubernetesVersion(t *testing.T) {
 	var tests = []struct {
 		description     string
 		expectedVersion string
@@ -55,6 +55,16 @@ func TestGetKuberneterVersion(t *testing.T) {
 			paramVersion:    "v1.16.0",
 			cfg:             &cfg.ClusterConfig{KubernetesConfig: cfg.KubernetesConfig{KubernetesVersion: "v1.15.0"}},
 		},
+		{
+			description:     "kubernetes-version given as 'stable', no config",
+			expectedVersion: constants.DefaultKubernetesVersion,
+			paramVersion:    "stable",
+		},
+		{
+			description:     "kubernetes-version given as 'latest', no config",
+			expectedVersion: constants.NewestKubernetesVersion,
+			paramVersion:    "latest",
+		},
 	}
 
 	for _, test := range tests {

From 1d72a1c8263fe1d40a81d5c3faa24ca532cc7e44 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Tue, 24 Mar 2020 15:01:31 -0700
Subject: [PATCH 21/27] don't extract volume if tarball doesn't exist

---
 pkg/drivers/kic/kic.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go
index 2055f91931..6e05e2de05 100644
--- a/pkg/drivers/kic/kic.go
+++ b/pkg/drivers/kic/kic.go
@@ -121,6 +121,10 @@ func (d *Driver) Create() error {
 		return errors.Wrap(err, "prepare kic ssh")
 	}
 
+	// If preload doesn't exist, don't both extracting tarball to volume
+	if !download.PreloadExists(d.NodeConfig.KubernetesVersion, d.NodeConfig.ContainerRuntime) {
+		return nil
+	}
 	t := time.Now()
 	glog.Infof("Starting extracting preloaded images to volume")
 	// Extract preloaded images to container

From 0e897f8448bfb79b8d58b20b7b266b3dd54a808e Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Tue, 24 Mar 2020 16:25:57 -0700
Subject: [PATCH 22/27] Only run preload if it exists

---
 pkg/minikube/cruntime/docker.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go
index 8641092573..843c7529f8 100644
--- a/pkg/minikube/cruntime/docker.go
+++ b/pkg/minikube/cruntime/docker.go
@@ -290,6 +290,9 @@ func (r *Docker) SystemLogCmd(len int) string {
 // 2. Extract the preloaded tarball to the correct directory
 // 3. Remove the tarball within the VM
 func (r *Docker) Preload(cfg config.KubernetesConfig) error {
+	if !download.PreloadExists(cfg.KubernetesVersion, cfg.ContainerRuntime) {
+		return nil
+	}
 	k8sVersion := cfg.KubernetesVersion
 
 	// If images already exist, return

From 455ebeac0130a245e524b385917bd24737cbf7fd Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Tue, 24 Mar 2020 17:28:57 -0700
Subject: [PATCH 23/27] update test, don't need to rerun v1.17.0 since those
 images exist anyway

---
 test/integration/start_stop_delete_test.go | 19 +------------------
 1 file changed, 1 insertion(+), 18 deletions(-)

diff --git a/test/integration/start_stop_delete_test.go b/test/integration/start_stop_delete_test.go
index f0aeb03450..e948f4b6ef 100644
--- a/test/integration/start_stop_delete_test.go
+++ b/test/integration/start_stop_delete_test.go
@@ -192,17 +192,6 @@ func TestStartStopWithPreload(t *testing.T) {
 		t.Fatalf("%s failed: %v", rr.Args, err)
 	}
 
-	// Restart again with v1.17.0, this time with the preloaded tarball
-	startArgs = []string{"start", "-p", profile, "--memory=2200", "--alsologtostderr", "-v=3", "--wait=true"}
-	startArgs = append(startArgs, StartArgs()...)
-	startArgs = append(startArgs, fmt.Sprintf("--kubernetes-version=%s", k8sVersion))
-
-	rr, err = Run(t, exec.CommandContext(ctx, Target(), startArgs...))
-	if err != nil {
-		t.Fatalf("%s failed: %v", rr.Args, err)
-	}
-	verifyImageExistsInDaemon(ctx, t, profile, image)
-
 	// Restart minikube with v1.17.3, which has a preloaded tarball
 	startArgs = []string{"start", "-p", profile, "--memory=2200", "--alsologtostderr", "-v=3", "--wait=true"}
 	startArgs = append(startArgs, StartArgs()...)
@@ -212,13 +201,7 @@ func TestStartStopWithPreload(t *testing.T) {
 	if err != nil {
 		t.Fatalf("%s failed: %v", rr.Args, err)
 	}
-	verifyImageExistsInDaemon(ctx, t, profile, image)
-
-}
-
-func verifyImageExistsInDaemon(ctx context.Context, t *testing.T, profile, image string) {
-	// Ensure that busybox still exists in the daemon
-	rr, err := Run(t, exec.CommandContext(ctx, Target(), "ssh", "-p", profile, "--", "docker", "images"))
+	rr, err = Run(t, exec.CommandContext(ctx, Target(), "ssh", "-p", profile, "--", "docker", "images"))
 	if err != nil {
 		t.Fatalf("%s failed: %v", rr.Args, err)
 	}

From ab4b7d92b380b477801280289aaf2531454e11e9 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <t+github@stromberg.org>
Date: Tue, 24 Mar 2020 17:36:04 -0700
Subject: [PATCH 24/27] Flush logs before writing to stdout

---
 pkg/minikube/out/out.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/pkg/minikube/out/out.go b/pkg/minikube/out/out.go
index 9ecb23c053..38c817339b 100644
--- a/pkg/minikube/out/out.go
+++ b/pkg/minikube/out/out.go
@@ -68,6 +68,9 @@ func T(style StyleEnum, format string, a ...V) {
 
 // String writes a basic formatted string to stdout
 func String(format string, a ...interface{}) {
+	// Flush log buffer so that output order makes sense
+	glog.Flush()
+
 	if outFile == nil {
 		glog.Warningf("[unset outFile]: %s", fmt.Sprintf(format, a...))
 		return

From d3fcd40feba97e809d115839bc45be45831730a0 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <t+github@stromberg.org>
Date: Tue, 24 Mar 2020 17:37:59 -0700
Subject: [PATCH 25/27] Use correct preload paths on Windows, announce base
 image pull

---
 pkg/minikube/download/preload.go | 12 ++++++------
 pkg/minikube/node/cache.go       |  1 +
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/pkg/minikube/download/preload.go b/pkg/minikube/download/preload.go
index 09e0d8d35e..4a4979acc0 100644
--- a/pkg/minikube/download/preload.go
+++ b/pkg/minikube/download/preload.go
@@ -23,7 +23,7 @@ import (
 	"io/ioutil"
 	"net/http"
 	"os"
-	"path"
+	"path/filepath"
 
 	"cloud.google.com/go/storage"
 	"google.golang.org/api/option"
@@ -59,14 +59,14 @@ func targetDir() string {
 	return localpath.MakeMiniPath("cache", "preloaded-tarball")
 }
 
-// PreloadChecksumPath returns path to checksum file
+// PreloadChecksumPath returns the local path to the cached checksum file
 func PreloadChecksumPath(k8sVersion string) string {
-	return path.Join(targetDir(), checksumName(k8sVersion))
+	return filepath.Join(targetDir(), checksumName(k8sVersion))
 }
 
-// TarballPath returns the path to the preloaded tarball
+// TarballPath returns the local path to the cached preload tarball
 func TarballPath(k8sVersion string) string {
-	return path.Join(targetDir(), TarballName(k8sVersion))
+	return filepath.Join(targetDir(), TarballName(k8sVersion))
 }
 
 // remoteTarballURL returns the URL for the remote tarball in GCS
@@ -122,7 +122,7 @@ func Preload(k8sVersion, containerRuntime string) error {
 		return nil
 	}
 
-	out.T(out.FileDownload, "Downloading preloaded images tarball for k8s {{.version}} ...", out.V{"version": k8sVersion})
+	out.T(out.FileDownload, "Downloading Kubernetes {{.version}} preload ...", out.V{"version": k8sVersion})
 	url := remoteTarballURL(k8sVersion)
 
 	tmpDst := targetPath + ".download"
diff --git a/pkg/minikube/node/cache.go b/pkg/minikube/node/cache.go
index f1b3ac8f32..ebeb04fdc0 100644
--- a/pkg/minikube/node/cache.go
+++ b/pkg/minikube/node/cache.go
@@ -100,6 +100,7 @@ func doCacheBinaries(k8sVersion string) error {
 
 // BeginDownloadKicArtifacts downloads the kic image + preload tarball, returns true if preload is available
 func beginDownloadKicArtifacts(g *errgroup.Group) {
+	out.T(out.Pulling, "Pulling base image ...")
 	glog.Info("Beginning downloading kic artifacts")
 	g.Go(func() error {
 		glog.Infof("Downloading %s to local daemon", kic.BaseImage)

From 646609ec30fcd5aa4f0e7c0be029bbad46873e55 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Tue, 24 Mar 2020 21:24:43 -0700
Subject: [PATCH 26/27] review comments

---
 cmd/minikube/cmd/start.go        | 2 +-
 pkg/minikube/download/preload.go | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 7d6e92cbf6..ef568571ba 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -176,7 +176,7 @@ func initMinikubeFlags() {
 	startCmd.Flags().Bool(autoUpdate, true, "If set, automatically updates drivers to the latest version. Defaults to true.")
 	startCmd.Flags().Bool(installAddons, true, "If set, install addons. Defaults to true.")
 	startCmd.Flags().IntP(nodes, "n", 1, "The number of nodes to spin up. Defaults to 1.")
-	startCmd.Flags().Bool(preload, true, "If true, download tarball of preloaded images if available to improve start time.")
+	startCmd.Flags().Bool(preload, true, "If set, download tarball of preloaded images if available to improve start time. Defaults to true.")
 }
 
 // initKubernetesFlags inits the commandline flags for kubernetes related options
diff --git a/pkg/minikube/download/preload.go b/pkg/minikube/download/preload.go
index 0291a15b93..9016371d68 100644
--- a/pkg/minikube/download/preload.go
+++ b/pkg/minikube/download/preload.go
@@ -81,6 +81,9 @@ func PreloadExists(k8sVersion, containerRuntime string) bool {
 		return false
 	}
 
+	// See https://github.com/kubernetes/minikube/issues/6933
+	// and https://github.com/kubernetes/minikube/issues/6934
+	// to track status of adding containerd & crio
 	if containerRuntime != "docker" {
 		return false
 	}

From 1be46bceec192a87b0525039d8d91076389505cc Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Wed, 25 Mar 2020 01:12:27 -0700
Subject: [PATCH 27/27] implement Preload check for crio and containerd

---
 pkg/minikube/cruntime/containerd.go | 4 ++++
 pkg/minikube/cruntime/crio.go       | 4 ++++
 2 files changed, 8 insertions(+)

diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go
index 67beb710a7..89a2e3912a 100644
--- a/pkg/minikube/cruntime/containerd.go
+++ b/pkg/minikube/cruntime/containerd.go
@@ -30,6 +30,7 @@ import (
 	"github.com/pkg/errors"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
 	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/download"
 	"k8s.io/minikube/pkg/minikube/out"
 )
 
@@ -313,5 +314,8 @@ func (r *Containerd) SystemLogCmd(len int) string {
 
 // Preload preloads the container runtime with k8s images
 func (r *Containerd) Preload(cfg config.KubernetesConfig) error {
+	if !download.PreloadExists(cfg.KubernetesVersion, cfg.ContainerRuntime) {
+		return nil
+	}
 	return fmt.Errorf("not yet implemented for %s", r.Name())
 }
diff --git a/pkg/minikube/cruntime/crio.go b/pkg/minikube/cruntime/crio.go
index fff4a8c270..5678a9d6a4 100644
--- a/pkg/minikube/cruntime/crio.go
+++ b/pkg/minikube/cruntime/crio.go
@@ -26,6 +26,7 @@ import (
 	"github.com/pkg/errors"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
 	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/download"
 	"k8s.io/minikube/pkg/minikube/out"
 )
 
@@ -230,5 +231,8 @@ func (r *CRIO) SystemLogCmd(len int) string {
 
 // Preload preloads the container runtime with k8s images
 func (r *CRIO) Preload(cfg config.KubernetesConfig) error {
+	if !download.PreloadExists(cfg.KubernetesVersion, cfg.ContainerRuntime) {
+		return nil
+	}
 	return fmt.Errorf("not yet implemented for %s", r.Name())
 }