From 9ff030942d77a479564ad68c5d19ada0d6ac3a4a Mon Sep 17 00:00:00 2001
From: Michael Ryan Dempsey <bluestealth@bluestealth.pw>
Date: Wed, 5 Aug 2020 01:52:12 -0700
Subject: [PATCH 01/65] use MiB with kvm driver

---
 pkg/drivers/kvm/domain.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkg/drivers/kvm/domain.go b/pkg/drivers/kvm/domain.go
index 2d8a18d5fa..3a69cc0b93 100644
--- a/pkg/drivers/kvm/domain.go
+++ b/pkg/drivers/kvm/domain.go
@@ -31,8 +31,8 @@ import (
 
 const domainTmpl = `
 <domain type='kvm'>
-  <name>{{.MachineName}}</name> 
-  <memory unit='MB'>{{.Memory}}</memory>
+  <name>{{.MachineName}}</name>
+  <memory unit='MiB'>{{.Memory}}</memory>
   <vcpu>{{.CPU}}</vcpu>
   <features>
     <acpi/>

From 6ad47f9560ec6a4dddf0117e02b540f64fee2208 Mon Sep 17 00:00:00 2001
From: Michael Ryan Dempsey <bluestealth@bluestealth.pw>
Date: Wed, 5 Aug 2020 01:53:02 -0700
Subject: [PATCH 02/65] use MiB for raw disk allocations

---
 pkg/drivers/common.go      | 9 ++++++++-
 pkg/drivers/common_test.go | 2 +-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/pkg/drivers/common.go b/pkg/drivers/common.go
index 2fde2efde8..c62484dd95 100644
--- a/pkg/drivers/common.go
+++ b/pkg/drivers/common.go
@@ -23,6 +23,7 @@ import (
 	"path/filepath"
 	"syscall"
 
+	"github.com/docker/go-units"
 	"github.com/docker/machine/libmachine/drivers"
 	"github.com/docker/machine/libmachine/mcnflag"
 	"github.com/docker/machine/libmachine/mcnutils"
@@ -52,6 +53,12 @@ func (d *CommonDriver) SetConfigFromFlags(flags drivers.DriverOptions) error {
 	return nil
 }
 
+func diskSizeInMiB(diskSizeMB int) int64 {
+	result := int64(diskSizeMB)
+	result *= units.MiB
+	return result
+}
+
 func createRawDiskImage(sshKeyPath, diskPath string, diskSizeMb int) error {
 	tarBuf, err := mcnutils.MakeDiskImage(sshKeyPath)
 	if err != nil {
@@ -74,7 +81,7 @@ func createRawDiskImage(sshKeyPath, diskPath string, diskSizeMb int) error {
 		return errors.Wrapf(err, "closing file %s", diskPath)
 	}
 
-	if err := os.Truncate(diskPath, int64(diskSizeMb*1000000)); err != nil {
+	if err := os.Truncate(diskPath, diskSizeInMiB(diskSizeMb)); err != nil {
 		return errors.Wrap(err, "truncate")
 	}
 	return nil
diff --git a/pkg/drivers/common_test.go b/pkg/drivers/common_test.go
index cc250e03de..14cc631082 100644
--- a/pkg/drivers/common_test.go
+++ b/pkg/drivers/common_test.go
@@ -36,7 +36,7 @@ func Test_createDiskImage(t *testing.T) {
 	diskPath := filepath.Join(tmpdir, "disk")
 
 	sizeInMb := 100
-	sizeInBytes := int64(sizeInMb) * 1000000
+	sizeInBytes := int64(104857600)
 	if err := createRawDiskImage(sshPath, diskPath, sizeInMb); err != nil {
 		t.Errorf("createDiskImage() error = %v", err)
 	}

From ad1e2e2de582ee5cce871d5463049b042d81c334 Mon Sep 17 00:00:00 2001
From: Michael Ryan Dempsey <bluestealth@bluestealth.pw>
Date: Tue, 11 Aug 2020 21:38:28 -0700
Subject: [PATCH 03/65] Use same conversion logic for bytes -> MiB everywhere

---
 cmd/minikube/cmd/start.go    | 3 ++-
 pkg/minikube/machine/info.go | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 26f0d8cf05..971261e756 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -29,6 +29,7 @@ import (
 	"strings"
 
 	"github.com/blang/semver"
+	"github.com/docker/go-units"
 	"github.com/docker/machine/libmachine/ssh"
 	"github.com/golang/glog"
 	"github.com/google/go-containerregistry/pkg/authn"
@@ -793,7 +794,7 @@ func memoryLimits(drvName string) (int, int, error) {
 		if err != nil {
 			return -1, -1, err
 		}
-		containerLimit = int(s.TotalMemory / 1024 / 1024)
+		containerLimit = int(s.TotalMemory / units.MiB)
 	}
 
 	return sysLimit, containerLimit, nil
diff --git a/pkg/minikube/machine/info.go b/pkg/minikube/machine/info.go
index 99bf55982d..14343bae9e 100644
--- a/pkg/minikube/machine/info.go
+++ b/pkg/minikube/machine/info.go
@@ -20,6 +20,7 @@ import (
 	"io/ioutil"
 	"os/exec"
 
+	"github.com/docker/go-units"
 	"github.com/docker/machine/libmachine/provision"
 	"github.com/golang/glog"
 	"github.com/shirou/gopsutil/cpu"
@@ -39,7 +40,7 @@ type HostInfo struct {
 }
 
 func megs(bytes uint64) int64 {
-	return int64(bytes / 1024 / 1024)
+	return int64(bytes / units.MiB)
 }
 
 // CachedHostInfo returns system information such as memory,CPU, DiskSize

From 1859274e8a53fcd4332838279713177e429aa63a Mon Sep 17 00:00:00 2001
From: Michael Ryan Dempsey <bluestealth@bluestealth.pw>
Date: Wed, 12 Aug 2020 10:57:23 -0700
Subject: [PATCH 04/65] refactor conversion functions into util

---
 cmd/minikube/cmd/start.go    |  3 +--
 pkg/drivers/common.go        | 10 ++--------
 pkg/minikube/machine/info.go | 10 +++-------
 pkg/util/utils.go            | 14 +++++++++++++-
 4 files changed, 19 insertions(+), 18 deletions(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 971261e756..fbd8631104 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -29,7 +29,6 @@ import (
 	"strings"
 
 	"github.com/blang/semver"
-	"github.com/docker/go-units"
 	"github.com/docker/machine/libmachine/ssh"
 	"github.com/golang/glog"
 	"github.com/google/go-containerregistry/pkg/authn"
@@ -794,7 +793,7 @@ func memoryLimits(drvName string) (int, int, error) {
 		if err != nil {
 			return -1, -1, err
 		}
-		containerLimit = int(s.TotalMemory / units.MiB)
+		containerLimit = util.ConvertBytesToMB(s.TotalMemory)
 	}
 
 	return sysLimit, containerLimit, nil
diff --git a/pkg/drivers/common.go b/pkg/drivers/common.go
index c62484dd95..8c5c98bc73 100644
--- a/pkg/drivers/common.go
+++ b/pkg/drivers/common.go
@@ -23,13 +23,13 @@ import (
 	"path/filepath"
 	"syscall"
 
-	"github.com/docker/go-units"
 	"github.com/docker/machine/libmachine/drivers"
 	"github.com/docker/machine/libmachine/mcnflag"
 	"github.com/docker/machine/libmachine/mcnutils"
 	"github.com/docker/machine/libmachine/ssh"
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/util"
 )
 
 // This file is for common code shared among internal machine drivers
@@ -53,12 +53,6 @@ func (d *CommonDriver) SetConfigFromFlags(flags drivers.DriverOptions) error {
 	return nil
 }
 
-func diskSizeInMiB(diskSizeMB int) int64 {
-	result := int64(diskSizeMB)
-	result *= units.MiB
-	return result
-}
-
 func createRawDiskImage(sshKeyPath, diskPath string, diskSizeMb int) error {
 	tarBuf, err := mcnutils.MakeDiskImage(sshKeyPath)
 	if err != nil {
@@ -81,7 +75,7 @@ func createRawDiskImage(sshKeyPath, diskPath string, diskSizeMb int) error {
 		return errors.Wrapf(err, "closing file %s", diskPath)
 	}
 
-	if err := os.Truncate(diskPath, diskSizeInMiB(diskSizeMb)); err != nil {
+	if err := os.Truncate(diskPath, util.ConvertMBToBytes(diskSizeMb)); err != nil {
 		return errors.Wrap(err, "truncate")
 	}
 	return nil
diff --git a/pkg/minikube/machine/info.go b/pkg/minikube/machine/info.go
index 14343bae9e..4305a5a143 100644
--- a/pkg/minikube/machine/info.go
+++ b/pkg/minikube/machine/info.go
@@ -20,7 +20,6 @@ import (
 	"io/ioutil"
 	"os/exec"
 
-	"github.com/docker/go-units"
 	"github.com/docker/machine/libmachine/provision"
 	"github.com/golang/glog"
 	"github.com/shirou/gopsutil/cpu"
@@ -30,6 +29,7 @@ import (
 	"k8s.io/minikube/pkg/minikube/out"
 	"k8s.io/minikube/pkg/minikube/out/register"
 	"k8s.io/minikube/pkg/minikube/style"
+	"k8s.io/minikube/pkg/util"
 )
 
 // HostInfo holds information on the user's machine
@@ -39,10 +39,6 @@ type HostInfo struct {
 	DiskSize int64
 }
 
-func megs(bytes uint64) int64 {
-	return int64(bytes / units.MiB)
-}
-
 // CachedHostInfo returns system information such as memory,CPU, DiskSize
 func CachedHostInfo() (*HostInfo, error, error, error) {
 	var cpuErr, memErr, diskErr error
@@ -62,8 +58,8 @@ func CachedHostInfo() (*HostInfo, error, error, error) {
 
 	var info HostInfo
 	info.CPUs = len(i)
-	info.Memory = megs(v.Total)
-	info.DiskSize = megs(d.Total)
+	info.Memory = util.ConvertUnsignedBytesToMB(v.Total)
+	info.DiskSize = util.ConvertUnsignedBytesToMB(d.Total)
 	return &info, cpuErr, memErr, diskErr
 }
 
diff --git a/pkg/util/utils.go b/pkg/util/utils.go
index 6a5c3bf0f6..518c1d7966 100644
--- a/pkg/util/utils.go
+++ b/pkg/util/utils.go
@@ -24,7 +24,7 @@ import (
 	"strconv"
 
 	"github.com/blang/semver"
-	"github.com/docker/go-units"
+	units "github.com/docker/go-units"
 	"github.com/pkg/errors"
 )
 
@@ -47,6 +47,18 @@ func CalculateSizeInMB(humanReadableSize string) (int, error) {
 	return int(size / units.MiB), nil
 }
 
+func ConvertMBToBytes(mbSize int) int64 {
+	return int64(mbSize) * units.MiB
+}
+
+func ConvertBytesToMB(byteSize int64) int {
+	return int(ConvertUnsignedBytesToMB(uint64(byteSize)))
+}
+
+func ConvertUnsignedBytesToMB(byteSize uint64) int64 {
+	return int64(byteSize / units.MiB)
+}
+
 // GetBinaryDownloadURL returns a suitable URL for the platform
 func GetBinaryDownloadURL(version, platform string) string {
 	switch platform {

From 4708f6215c1bea0780cb3885f49ebed184488aa4 Mon Sep 17 00:00:00 2001
From: Tacio Costa <taciomcosta@gmail.com>
Date: Sat, 5 Sep 2020 18:37:24 -0300
Subject: [PATCH 05/65]  fix: profile list shows paused clusters as Running

---
 cmd/minikube/cmd/config/profile_list.go | 165 ++++++++++++++----------
 1 file changed, 96 insertions(+), 69 deletions(-)

diff --git a/cmd/minikube/cmd/config/profile_list.go b/cmd/minikube/cmd/config/profile_list.go
index 86e30b9eed..8384408c8a 100644
--- a/cmd/minikube/cmd/config/profile_list.go
+++ b/cmd/minikube/cmd/config/profile_list.go
@@ -23,6 +23,7 @@ import (
 	"strconv"
 	"strings"
 
+	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/kverify"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/driver"
 	"k8s.io/minikube/pkg/minikube/exit"
@@ -31,6 +32,7 @@ import (
 	"k8s.io/minikube/pkg/minikube/reason"
 	"k8s.io/minikube/pkg/minikube/style"
 
+	"github.com/docker/machine/libmachine"
 	"github.com/golang/glog"
 	"github.com/olekukonko/tablewriter"
 	"github.com/spf13/cobra"
@@ -54,96 +56,114 @@ var profileListCmd = &cobra.Command{
 	},
 }
 
-var printProfilesTable = func() {
-	var validData [][]string
+func printProfilesTable() {
+	validProfiles, invalidProfiles, err := config.ListProfiles()
+
+	if err != nil {
+		glog.Warningf("error loading profiles: %v", err)
+	}
+
+	if len(validProfiles) == 0 {
+		exit.Message(reason.Usage, "No minikube profile was found. You can create one using `minikube start`.")
+	}
+
+	updateProfilesStatus(validProfiles)
+	renderProfilesTable(profilesToTableData(validProfiles))
+	warnInvalidProfiles(invalidProfiles)
+}
+
+func updateProfilesStatus(profiles []*config.Profile) {
+	api, err := machine.NewAPIClient()
+	if err != nil {
+		glog.Errorf("failed to get machine api client %v", err)
+	}
+	defer api.Close()
+
+	for _, p := range profiles {
+		p.Status = profileStatus(p, api)
+	}
+}
+
+func profileStatus(p *config.Profile, api libmachine.API) string {
+	cp, err := config.PrimaryControlPlane(p.Config)
+	if err != nil {
+		exit.Error(reason.GuestCpConfig, "error getting primary control plane", err)
+	}
+
+	host, err := machine.LoadHost(api, driver.MachineName(*p.Config, cp))
+	if err != nil {
+		glog.Warningf("error loading profiles: %v", err)
+		return "Unknown"
+	}
+
+	cr, err := machine.CommandRunner(host)
+	if err != nil {
+		glog.Warningf("error loading profiles: %v", err)
+		return "Unknown"
+	}
+
+	hostname, _, port, err := driver.ControlPlaneEndpoint(p.Config, &cp, host.DriverName)
+	if err != nil {
+		glog.Warningf("error loading profiles: %v", err)
+		return "Unknown"
+	}
+
+	status, err := kverify.APIServerStatus(cr, hostname, port)
+	if err != nil {
+		glog.Warningf("error getting apiserver status for %s: %v", p.Name, err)
+		return "Unknown"
+	}
+	return status.String()
+}
+
+func renderProfilesTable(ps [][]string) {
 	table := tablewriter.NewWriter(os.Stdout)
 	table.SetHeader([]string{"Profile", "VM Driver", "Runtime", "IP", "Port", "Version", "Status"})
 	table.SetAutoFormatHeaders(false)
 	table.SetBorders(tablewriter.Border{Left: true, Top: true, Right: true, Bottom: true})
 	table.SetCenterSeparator("|")
-	validProfiles, invalidProfiles, err := config.ListProfiles()
+	table.AppendBulk(ps)
+	table.Render()
+}
 
-	if len(validProfiles) == 0 || err != nil {
-		exit.Message(reason.Usage, "No minikube profile was found. You can create one using `minikube start`.")
-	}
-	api, err := machine.NewAPIClient()
-	if err != nil {
-		glog.Errorf("failed to get machine api client %v", err)
-	}
-	defer api.Close()
-
-	for _, p := range validProfiles {
+func profilesToTableData(profiles []*config.Profile) [][]string {
+	var data [][]string
+	for _, p := range profiles {
 		cp, err := config.PrimaryControlPlane(p.Config)
 		if err != nil {
 			exit.Error(reason.GuestCpConfig, "error getting primary control plane", err)
 		}
-		p.Status, err = machine.Status(api, driver.MachineName(*p.Config, cp))
-		if err != nil {
-			glog.Warningf("error getting host status for %s: %v", p.Name, err)
-		}
-		validData = append(validData, []string{p.Name, p.Config.Driver, p.Config.KubernetesConfig.ContainerRuntime, cp.IP, strconv.Itoa(cp.Port), p.Config.KubernetesConfig.KubernetesVersion, p.Status})
+
+		data = append(data, []string{p.Name, p.Config.Driver, p.Config.KubernetesConfig.ContainerRuntime, cp.IP, strconv.Itoa(cp.Port), p.Config.KubernetesConfig.KubernetesVersion, p.Status})
+	}
+	return data
+}
+
+func warnInvalidProfiles(invalidProfiles []*config.Profile) {
+	if invalidProfiles == nil {
+		return
 	}
 
-	table.AppendBulk(validData)
-	table.Render()
-
-	if invalidProfiles != nil {
-		out.WarningT("Found {{.number}} invalid profile(s) ! ", out.V{"number": len(invalidProfiles)})
-		for _, p := range invalidProfiles {
-			out.ErrT(style.Empty, "\t "+p.Name)
-		}
-		out.ErrT(style.Tip, "You can delete them using the following command(s): ")
-		for _, p := range invalidProfiles {
-			out.Err(fmt.Sprintf("\t $ minikube delete -p %s \n", p.Name))
-		}
-
+	out.WarningT("Found {{.number}} invalid profile(s) ! ", out.V{"number": len(invalidProfiles)})
+	for _, p := range invalidProfiles {
+		out.ErrT(style.Empty, "\t "+p.Name)
 	}
 
-	if err != nil {
-		glog.Warningf("error loading profiles: %v", err)
+	out.ErrT(style.Tip, "You can delete them using the following command(s): ")
+	for _, p := range invalidProfiles {
+		out.Err(fmt.Sprintf("\t $ minikube delete -p %s \n", p.Name))
 	}
 }
 
-var printProfilesJSON = func() {
-	api, err := machine.NewAPIClient()
-	if err != nil {
-		glog.Errorf("failed to get machine api client %v", err)
-	}
-	defer api.Close()
-
+func printProfilesJSON() {
 	validProfiles, invalidProfiles, err := config.ListProfiles()
-	for _, v := range validProfiles {
-		cp, err := config.PrimaryControlPlane(v.Config)
-		if err != nil {
-			exit.Error(reason.GuestCpConfig, "error getting primary control plane", err)
-		}
-		status, err := machine.Status(api, driver.MachineName(*v.Config, cp))
-		if err != nil {
-			glog.Warningf("error getting host status for %s: %v", v.Name, err)
-		}
-		v.Status = status
-	}
 
-	var valid []*config.Profile
-	var invalid []*config.Profile
-
-	if validProfiles != nil {
-		valid = validProfiles
-	} else {
-		valid = []*config.Profile{}
-	}
-
-	if invalidProfiles != nil {
-		invalid = invalidProfiles
-	} else {
-		invalid = []*config.Profile{}
-	}
-
-	body := map[string]interface{}{}
+	updateProfilesStatus(validProfiles)
 
+	var body = map[string]interface{}{}
 	if err == nil || config.IsNotExist(err) {
-		body["valid"] = valid
-		body["invalid"] = invalid
+		body["valid"] = profilesOrDefault(validProfiles)
+		body["invalid"] = profilesOrDefault(invalidProfiles)
 		jsonString, _ := json.Marshal(body)
 		out.String(string(jsonString))
 	} else {
@@ -154,6 +174,13 @@ var printProfilesJSON = func() {
 	}
 }
 
+func profilesOrDefault(profiles []*config.Profile) []*config.Profile {
+	if profiles != nil {
+		return profiles
+	}
+	return []*config.Profile{}
+}
+
 func init() {
 	profileListCmd.Flags().StringVarP(&output, "output", "o", "table", "The output format. One of 'json', 'table'")
 	ProfileCmd.AddCommand(profileListCmd)

From 7e7344da18a44a9469d3cc99005e8bfad25d6cfc Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Fri, 18 Sep 2020 16:20:58 -0700
Subject: [PATCH 06/65] Update Makefile and release notes for v1.13.1

---
 CHANGELOG.md                           | 25 +++++++++++++++++++++++++
 Makefile                               |  4 ++--
 site/content/en/docs/commands/start.md |  2 +-
 3 files changed, 28 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4398a816c4..93406981be 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,30 @@
 # Release Notes
 
+## Version 1.13.1 - 2020-09-18
+* Update Default Kubernetes Version to v1.19.2 [#9265](https://github.com/kubernetes/minikube/pull/9265)
+* fix mounting for docker driver in windows [#9263](https://github.com/kubernetes/minikube/pull/9263)
+* CSI Hostpath Driver & VolumeSnapshots addons [#8461](https://github.com/kubernetes/minikube/pull/8461)
+* docker/podman drivers: Make sure CFS_BANDWIDTH is available for --cpus [#9255](https://github.com/kubernetes/minikube/pull/9255)
+* Fix ForwardedPort for podman version 2.0.1 and up [#9237](https://github.com/kubernetes/minikube/pull/9237)
+* Avoid setting time for memory assets [#9256](https://github.com/kubernetes/minikube/pull/9256)
+* point to newest gcp-auth-webhook version [#9199](https://github.com/kubernetes/minikube/pull/9199)
+* Set preload=false if not using overlay2 as docker storage driver [#8831](https://github.com/kubernetes/minikube/pull/8831)
+* Upgrade crio to 1.17.3 [#8922](https://github.com/kubernetes/minikube/pull/8922)
+* Add Docker Desktop instructions if memory is greater than minimum but less than recommended [#9181](https://github.com/kubernetes/minikube/pull/9181)
+* Update minimum memory constants to use MiB instead of MB [#9180](https://github.com/kubernetes/minikube/pull/9180)
+
+Thank you to our contributors for this release! 
+- Anders F Björklund
+- Dean Coakley
+- Li Zhijian
+- Medya Ghazizadeh
+- Priya Wadhwa
+- Sharif Elgamal
+- Thomas Strömberg
+- Zadjad Rezai
+- jjanik
+
+
 ## Version 1.13.0 - 2020-09-03
 
 ## Features
diff --git a/Makefile b/Makefile
index 1592fe5f0a..4e37b1a575 100644
--- a/Makefile
+++ b/Makefile
@@ -15,7 +15,7 @@
 # Bump these on release - and please check ISO_VERSION for correctness.
 VERSION_MAJOR ?= 1
 VERSION_MINOR ?= 13
-VERSION_BUILD ?= 0
+VERSION_BUILD ?= 1
 RAW_VERSION=$(VERSION_MAJOR).$(VERSION_MINOR).$(VERSION_BUILD)
 VERSION ?= v$(RAW_VERSION)
 
@@ -23,7 +23,7 @@ KUBERNETES_VERSION ?= $(shell egrep "DefaultKubernetesVersion =" pkg/minikube/co
 KIC_VERSION ?= $(shell egrep "Version =" pkg/drivers/kic/types.go | cut -d \" -f2)
 
 # Default to .0 for higher cache hit rates, as build increments typically don't require new ISO versions
-ISO_VERSION ?= v$(VERSION_MAJOR).$(VERSION_MINOR).0
+ISO_VERSION ?= v$(VERSION_MAJOR).$(VERSION_MINOR).1
 # Dashes are valid in semver, but not Linux packaging. Use ~ to delimit alpha/beta
 DEB_VERSION ?= $(subst -,~,$(RAW_VERSION))
 RPM_VERSION ?= $(DEB_VERSION)
diff --git a/site/content/en/docs/commands/start.md b/site/content/en/docs/commands/start.md
index 4897e40f8d..72163a3365 100644
--- a/site/content/en/docs/commands/start.md
+++ b/site/content/en/docs/commands/start.md
@@ -65,7 +65,7 @@ minikube start [flags]
       --insecure-registry strings         Insecure Docker registries to pass to the Docker daemon.  The default service CIDR range will automatically be added.
       --install-addons                    If set, install addons. Defaults to true. (default true)
       --interactive                       Allow user prompts for more information (default true)
-      --iso-url strings                   Locations to fetch the minikube ISO from. (default [https://storage.googleapis.com/minikube/iso/minikube-v1.13.0.iso,https://github.com/kubernetes/minikube/releases/download/v1.13.0/minikube-v1.13.0.iso,https://kubernetes.oss-cn-hangzhou.aliyuncs.com/minikube/iso/minikube-v1.13.0.iso])
+      --iso-url strings                   Locations to fetch the minikube ISO from. (default [https://storage.googleapis.com/minikube/iso/minikube-v1.13.1.iso,https://github.com/kubernetes/minikube/releases/download/v1.13.1/minikube-v1.13.1.iso,https://kubernetes.oss-cn-hangzhou.aliyuncs.com/minikube/iso/minikube-v1.13.1.iso])
       --keep-context                      This will keep the existing kubectl context and will create a minikube context.
       --kubernetes-version string         The Kubernetes version that the minikube VM will use (ex: v1.2.3, 'stable' for v1.19.2, 'latest' for v1.19.2). Defaults to 'stable'.
       --kvm-gpu                           Enable experimental NVIDIA GPU support in minikube

From ba0db2f72a95ee27fcab8cbb5ca0a51f91578a9e Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Fri, 18 Sep 2020 16:23:45 -0700
Subject: [PATCH 07/65] add contributor

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 93406981be..3959a82268 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -16,6 +16,7 @@
 Thank you to our contributors for this release! 
 - Anders F Björklund
 - Dean Coakley
+- Julien Breux
 - Li Zhijian
 - Medya Ghazizadeh
 - Priya Wadhwa

From 9fbd2e94bd54a9b67b47b772c9c8fa056cfb62bc Mon Sep 17 00:00:00 2001
From: Predrag Rogic <predrag.rogic@talkout.com>
Date: Mon, 21 Sep 2020 00:06:15 +0100
Subject: [PATCH 08/65] update-context: Automatically set the context as
 current/active in kubeconfig

---
 cmd/minikube/cmd/update-context.go | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/cmd/minikube/cmd/update-context.go b/cmd/minikube/cmd/update-context.go
index 70f3f2e079..b9659795cd 100644
--- a/cmd/minikube/cmd/update-context.go
+++ b/cmd/minikube/cmd/update-context.go
@@ -45,5 +45,11 @@ var updateContextCmd = &cobra.Command{
 		} else {
 			out.T(style.Meh, `No changes required for the "{{.context}}" context`, out.V{"context": cname})
 		}
+
+		if err := kubeconfig.SetCurrentContext(cname, kubeconfig.PathFromEnv()); err != nil {
+			out.ErrT(style.Sad, `Error while setting kubectl current context:  {{.error}}`, out.V{"error": err})
+		} else {
+			out.T(style.Kubectl, `Current context is "{{.context}}"`, out.V{"context": cname})
+		}
 	},
 }

From 85bf8a5522f6a02c2f91fdce270a0fcd92ad0226 Mon Sep 17 00:00:00 2001
From: minikube-bot <minikube-bot@google.com>
Date: Mon, 21 Sep 2020 10:13:32 -0700
Subject: [PATCH 09/65] Update releases.json to include v1.13.1

---
 deploy/minikube/releases.json | 8 ++++++++
 go.mod                        | 1 +
 2 files changed, 9 insertions(+)

diff --git a/deploy/minikube/releases.json b/deploy/minikube/releases.json
index cfd1fc148b..a37571ded3 100644
--- a/deploy/minikube/releases.json
+++ b/deploy/minikube/releases.json
@@ -1,4 +1,12 @@
 [
+  {
+      "name": "v1.13.1",
+      "checksums": {
+          "darwin": "cc7eaadea2becc48eee78136f8d569df55a28c46d58d1c8bb434895382aced78",
+          "linux": "3564b685f8d797df78ebfa2f5b34c99b3c77b0d1f49eab6aab37500f1ba61d98",
+          "windows": "0d315ec21ca8a34eff5fa4cc478c09ed2d48ae88b3c5d586df9de111ac414d44"
+      }
+  },
   {
       "name": "v1.13.0",
       "checksums": {
diff --git a/go.mod b/go.mod
index 225bdf9537..5a5e6025e8 100644
--- a/go.mod
+++ b/go.mod
@@ -72,6 +72,7 @@ require (
 	github.com/zchee/go-vmnet v0.0.0-20161021174912-97ebf9174097
 	golang.org/x/build v0.0.0-20190927031335-2835ba2e683f
 	golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37
+	golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 // indirect
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
 	golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a
 	golang.org/x/sys v0.0.0-20200523222454-059865788121

From 98591a6656603947a5b26c66749ef9793ecd68bc Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Mon, 21 Sep 2020 16:35:50 -0400
Subject: [PATCH 10/65] Add advice for the DRV_CP_ENDPOINT error

---
 pkg/minikube/reason/reason.go | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/pkg/minikube/reason/reason.go b/pkg/minikube/reason/reason.go
index ee9a3faf6b..712c255109 100644
--- a/pkg/minikube/reason/reason.go
+++ b/pkg/minikube/reason/reason.go
@@ -207,7 +207,13 @@ var (
 	ProviderNotFound    = Kind{ID: "PROVIDER_NOT_FOUND", ExitCode: ExProviderNotFound}
 	ProviderUnavailable = Kind{ID: "PROVIDER_UNAVAILABLE", ExitCode: ExProviderNotFound, Style: style.Shrug}
 
-	DrvCPEndpoint         = Kind{ID: "DRV_CP_ENDPOINT", ExitCode: ExDriverError}
+	DrvCPEndpoint = Kind{ID: "DRV_CP_ENDPOINT",
+		Advice: `Recreate the cluster by running:
+		minikube delete{{.profile}}
+		minikube start{{.profile}}`,
+		ExitCode: ExDriverError,
+		Style:    style.Failure,
+	}
 	DrvPortForward        = Kind{ID: "DRV_PORT_FORWARD", ExitCode: ExDriverError}
 	DrvUnsupportedMulti   = Kind{ID: "DRV_UNSUPPORTED_MULTINODE", ExitCode: ExDriverConflict}
 	DrvUnsupportedOS      = Kind{ID: "DRV_UNSUPPORTED_OS", ExitCode: ExDriverUnsupported}

From f6b524b372163ee1006c6f91ee70aaafae30fa48 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Mon, 21 Sep 2020 19:05:58 -0400
Subject: [PATCH 11/65] add profile arg

---
 pkg/minikube/node/start.go    | 2 +-
 pkg/minikube/reason/reason.go | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go
index 2a40e806f9..6982f10135 100644
--- a/pkg/minikube/node/start.go
+++ b/pkg/minikube/node/start.go
@@ -311,7 +311,7 @@ func setupKubeAdm(mAPI libmachine.API, cfg config.ClusterConfig, n config.Node,
 func setupKubeconfig(h *host.Host, cc *config.ClusterConfig, n *config.Node, clusterName string) *kubeconfig.Settings {
 	addr, err := apiServerURL(*h, *cc, *n)
 	if err != nil {
-		exit.Error(reason.DrvCPEndpoint, "Failed to get API Server URL", err)
+		exit.Message(reason.DrvCPEndpoint, fmt.Sprintf("failed to get API Server URL: %v", err), out.V{"profileArg": fmt.Sprintf("--profile=%s", clusterName)})
 	}
 
 	if cc.KubernetesConfig.APIServerName != constants.APIServerName {
diff --git a/pkg/minikube/reason/reason.go b/pkg/minikube/reason/reason.go
index 712c255109..48af438986 100644
--- a/pkg/minikube/reason/reason.go
+++ b/pkg/minikube/reason/reason.go
@@ -209,8 +209,8 @@ var (
 
 	DrvCPEndpoint = Kind{ID: "DRV_CP_ENDPOINT",
 		Advice: `Recreate the cluster by running:
-		minikube delete{{.profile}}
-		minikube start{{.profile}}`,
+		minikube delete {{.profileArg}}
+		minikube start {{.profileArg}}`,
 		ExitCode: ExDriverError,
 		Style:    style.Failure,
 	}

From 19cc40f420703176d4977abc648ddb39322c0e9b Mon Sep 17 00:00:00 2001
From: Predrag Rogic <predrag.rogic@talkout.com>
Date: Tue, 22 Sep 2020 03:41:27 +0100
Subject: [PATCH 12/65] update_kubernetes_version: implement a standalone
 program that polls GitHub for the latest Kubernetes release

---
 go.mod                                        |  1 +
 go.sum                                        |  2 +
 .../update_kubernetes_version.go              | 71 ++++++++++++++++---
 3 files changed, 64 insertions(+), 10 deletions(-)

diff --git a/go.mod b/go.mod
index 225bdf9537..c4bedc33b9 100644
--- a/go.mod
+++ b/go.mod
@@ -28,6 +28,7 @@ require (
 	github.com/google/go-cmp v0.4.1
 	github.com/google/go-containerregistry v0.0.0-20200601195303-96cf69f03a3c
 	github.com/google/go-github v17.0.0+incompatible
+	github.com/google/go-github/v32 v32.1.0
 	github.com/google/slowjam v0.0.0-20200530021616-df27e642fe7b
 	github.com/google/uuid v1.1.1
 	github.com/googleapis/gnostic v0.3.0 // indirect
diff --git a/go.sum b/go.sum
index 38b4e9aed3..5801ce02c5 100644
--- a/go.sum
+++ b/go.sum
@@ -517,6 +517,8 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
 github.com/google/go-github/v28 v28.1.1/go.mod h1:bsqJWQX05omyWVmc00nEUql9mhQyv38lDZ8kPZcQVoM=
+github.com/google/go-github/v32 v32.1.0 h1:GWkQOdXqviCPx7Q7Fj+KyPoGm4SwHRh8rheoPhd27II=
+github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI=
 github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/go-replayers/grpcreplay v0.1.0/go.mod h1:8Ig2Idjpr6gifRd6pNVggX6TC1Zw6Jx74AKp7QNH2QE=
diff --git a/hack/kubernetes_version/update_kubernetes_version.go b/hack/kubernetes_version/update_kubernetes_version.go
index 6bd5325c21..02c9b08e13 100644
--- a/hack/kubernetes_version/update_kubernetes_version.go
+++ b/hack/kubernetes_version/update_kubernetes_version.go
@@ -17,6 +17,7 @@ limitations under the License.
 package main
 
 import (
+	"context"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -25,17 +26,65 @@ import (
 	"strings"
 
 	"github.com/golang/glog"
+
+	"github.com/google/go-github/v32/github"
 )
 
 func main() {
-	if len(os.Args) == 1 {
-		fmt.Println("Usage: go run update_kubernetes_version.go <kubernetes_version>")
-		os.Exit(1)
+	vDefault := ""
+	vNewest := ""
+
+	client := github.NewClient(nil)
+
+	// walk through the paginated list of all 'kubernetes/kubernetes' repo releases
+	// from latest to older releases, until latest release and pre-release are found
+	// use max value (100) for PerPage to avoid hitting the rate limits (60 per hour, 10 per minute)
+	// see https://godoc.org/github.com/google/go-github/github#hdr-Rate_Limiting
+	opt := &github.ListOptions{PerPage: 100}
+out:
+	for {
+		rels, resp, err := client.Repositories.ListReleases(context.Background(), "kubernetes", "kubernetes", opt)
+		if err != nil {
+			glog.Errorf("GitHub ListReleases failed: %v", err)
+			break
+		}
+
+		for _, r := range rels {
+			// GetName returns the Name field if it's non-nil, zero value otherwise.
+			ver := r.GetName()
+			if ver == "" {
+				continue
+			}
+
+			rel := strings.Split(ver, "-")
+			// check if it is a release channel (ie, 'v1.19.2') or a
+			// pre-release channel (ie, 'v1.19.3-rc.0' or 'v1.19.0-beta.2')
+			if len(rel) == 1 && vDefault == "" {
+				vDefault = ver
+			} else if len(rel) > 1 && vNewest == "" {
+				if strings.HasPrefix(rel[1], "rc") || strings.HasPrefix(rel[1], "beta") {
+					vNewest = ver
+				}
+			}
+
+			if vDefault != "" && vNewest != "" {
+				// make sure that vNewest >= vDefault
+				if vNewest < vDefault {
+					vNewest = vDefault
+				}
+				break out
+			}
+		}
+
+		if resp.NextPage == 0 {
+			break
+		}
+		opt.Page = resp.NextPage
 	}
 
-	v := os.Args[1]
-	if !strings.HasPrefix(v, "v") {
-		v = "v" + v
+	if vDefault == "" || vNewest == "" {
+		glog.Errorf("Cannot determine DefaultKubernetesVersion or NewestKubernetesVersion")
+		os.Exit(1)
 	}
 
 	constantsFile := "../../pkg/minikube/constants/constants.go"
@@ -53,17 +102,19 @@ func main() {
 	mode := info.Mode()
 
 	re := regexp.MustCompile(`DefaultKubernetesVersion = \".*`)
-	f := re.ReplaceAllString(string(cf), "DefaultKubernetesVersion = \""+v+"\"")
+	f := re.ReplaceAllString(string(cf), "DefaultKubernetesVersion = \""+vDefault+"\"")
 
 	re = regexp.MustCompile(`NewestKubernetesVersion = \".*`)
-	f = re.ReplaceAllString(f, "NewestKubernetesVersion = \""+v+"\"")
+	f = re.ReplaceAllString(f, "NewestKubernetesVersion = \""+vNewest+"\"")
 
 	if err := ioutil.WriteFile(constantsFile, []byte(f), mode); err != nil {
 		fmt.Println(err)
 		os.Exit(1)
 	}
 
-	testData := "../../pkg/minikube/bootstrapper/bsutil/testdata"
+	// update testData just for the latest 'v<MAJOR>.<MINOR>' from vDefault
+	vDefaultMM := vDefault[:strings.LastIndex(vDefault, ".")]
+	testData := "../../pkg/minikube/bootstrapper/bsutil/testdata/" + vDefaultMM
 
 	err = filepath.Walk(testData, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
@@ -77,7 +128,7 @@ func main() {
 			return err
 		}
 		re = regexp.MustCompile(`kubernetesVersion: .*`)
-		cf = []byte(re.ReplaceAllString(string(cf), "kubernetesVersion: "+v))
+		cf = []byte(re.ReplaceAllString(string(cf), "kubernetesVersion: "+vDefaultMM+".0")) // TODO: let <PATCH> version to be the latest one instead of "0"
 		return ioutil.WriteFile(path, cf, info.Mode())
 	})
 	if err != nil {

From 6bf7ca3dd25ceefb3870752c2876850e0c8b5b09 Mon Sep 17 00:00:00 2001
From: Ilya Zuyev <izuyev@google.com>
Date: Mon, 21 Sep 2020 23:06:54 -0700
Subject: [PATCH 13/65] Validate --base-image flag and active driver combo

---
 cmd/minikube/cmd/start.go      | 25 +++++++++++++++++
 cmd/minikube/cmd/start_test.go | 49 ++++++++++++++++++++++++++++++++++
 2 files changed, 74 insertions(+)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index fc3acd592d..192d5732f4 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -38,7 +38,9 @@ import (
 	"github.com/shirou/gopsutil/cpu"
 	gopshost "github.com/shirou/gopsutil/host"
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 	"github.com/spf13/viper"
+
 	cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config"
 	"k8s.io/minikube/pkg/drivers/kic/oci"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil"
@@ -167,7 +169,10 @@ func runStart(cmd *cobra.Command, args []string) {
 
 	validateSpecifiedDriver(existing)
 	validateKubernetesVersion(existing)
+
 	ds, alts, specified := selectDriver(existing)
+	validateBaseImage(cmd.Flag(kicBaseImage), ds.Name)
+
 	starter, err := provisionWithDriver(cmd, ds, existing)
 	if err != nil {
 		node.ExitIfFatal(err)
@@ -516,6 +521,7 @@ func kubectlVersion(path string) (string, error) {
 	return cv.ClientVersion.GitVersion, nil
 }
 
+// returns (current_driver, suggested_drivers, "true, if the driver is set by command line arg or in the config file")
 func selectDriver(existing *config.ClusterConfig) (registry.DriverState, []registry.DriverState, bool) {
 	// Technically unrelated, but important to perform before detection
 	driver.SetLibvirtURI(viper.GetString(kvmQemuURI))
@@ -1185,6 +1191,25 @@ func validateKubernetesVersion(old *config.ClusterConfig) {
 	}
 }
 
+// validateBaseImage checks that --base-image is not passed if the drive being in use is KIC (docker/podman)
+// if so, the function exits the process
+func validateBaseImage(baseImage *pflag.Flag, driver string) {
+	if !validBaseImageFlag(baseImage.Changed, driver) {
+		exit.Message(reason.Usage, "TODO:  image {{.image}} with driver {{.driver}}",
+			out.V{
+				"driver": driver,
+				"image":  baseImage.Value,
+			},
+		)
+	}
+}
+func validBaseImageFlag(baseImageFlagSet bool, driver string) bool {
+	if baseImageFlagSet {
+		return registry.IsKIC(driver)
+	}
+	return true
+}
+
 func getKubernetesVersion(old *config.ClusterConfig) string {
 	paramVersion := viper.GetString(kubernetesVersion)
 
diff --git a/cmd/minikube/cmd/start_test.go b/cmd/minikube/cmd/start_test.go
index 8c315b375c..13da5639e4 100644
--- a/cmd/minikube/cmd/start_test.go
+++ b/cmd/minikube/cmd/start_test.go
@@ -23,8 +23,10 @@ import (
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
+
 	cfg "k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
+	"k8s.io/minikube/pkg/minikube/driver"
 	"k8s.io/minikube/pkg/minikube/proxy"
 )
 
@@ -278,3 +280,50 @@ func TestSuggestMemoryAllocation(t *testing.T) {
 		})
 	}
 }
+
+func TestBaseImageFlagDriverCombo(t *testing.T) {
+	type testStruct struct {
+		description string
+		imageSet    bool
+		driver      string
+	}
+	invalidCombos := []testStruct{
+		{"KVM2 with flag", true, driver.KVM2},
+		{"VirtualBox with flag", true, driver.VirtualBox},
+		{"HyperKit with flag", true, driver.HyperKit},
+		{"VMware with flag", true, driver.VMware},
+		{"VMwareFusion with flag", true, driver.VMwareFusion},
+		{"HyperV with flag", true, driver.HyperV},
+		{"Parallels with flag", true, driver.Parallels},
+	}
+	validCombos := []testStruct{
+		{"Docker with flag", true, driver.Docker},
+		{"Podman with flag", true, driver.Podman},
+		{"Docker w/o flag", false, driver.Docker},
+		{"Podman w/o flag", false, driver.Podman},
+		{"KVM2 w/o flag", false, driver.KVM2},
+		{"VirtualBox w/o flag", false, driver.VirtualBox},
+		{"HyperKit w/o flag", false, driver.HyperKit},
+		{"VMware w/o flag", false, driver.VMware},
+		{"VMwareFusion w/o flag", false, driver.VMwareFusion},
+		{"HyperV w/o flag", false, driver.HyperV},
+		{"Parallels w/o flag", false, driver.Parallels},
+	}
+
+	for _, test := range validCombos {
+		t.Run(test.description, func(t *testing.T) {
+			if !validBaseImageFlag(test.imageSet, test.driver) {
+				t.Errorf("invalidBaseImageFlag(base-image-set=%v, driver=%v): got false, expected true",
+					test.imageSet, test.driver)
+			}
+		})
+	}
+	for _, test := range invalidCombos {
+		t.Run(test.description, func(t *testing.T) {
+			if validBaseImageFlag(test.imageSet, test.driver) {
+				t.Errorf("invalidBaseImageFlag(base-image-set=%v, driver=%v): got true, expected false",
+					test.imageSet, test.driver)
+			}
+		})
+	}
+}

From 6a6098367970ce1c210c8e3a07cb7a07c69895e6 Mon Sep 17 00:00:00 2001
From: Ilya Zuyev <izuyev@google.com>
Date: Mon, 21 Sep 2020 23:28:34 -0700
Subject: [PATCH 14/65] Show invalid flag diagnostic message to user

---
 cmd/minikube/cmd/start.go | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 192d5732f4..652c73afbf 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -1193,12 +1193,18 @@ func validateKubernetesVersion(old *config.ClusterConfig) {
 
 // validateBaseImage checks that --base-image is not passed if the drive being in use is KIC (docker/podman)
 // if so, the function exits the process
-func validateBaseImage(baseImage *pflag.Flag, driver string) {
-	if !validBaseImageFlag(baseImage.Changed, driver) {
-		exit.Message(reason.Usage, "TODO:  image {{.image}} with driver {{.driver}}",
+func validateBaseImage(imageFlag *pflag.Flag, driver string) {
+	if !validBaseImageFlag(imageFlag.Changed, driver) {
+		exit.Message(reason.Usage,
+			"flag --{{.imgFlag}} is not available for driver '{{.driver}}'. Did you mean to use '{{.docker}}' or '{{.podman}}' driver instead?\n"+
+				"Please use --{{.isoFlag}} flag to configure VM based drivers",
 			out.V{
-				"driver": driver,
-				"image":  baseImage.Value,
+				"imgFlag": imageFlag.Name,
+				"driver":  driver,
+				"image":   imageFlag.Value,
+				"docker":  registry.Docker,
+				"podman":  registry.Podman,
+				"isoFlag": isoURL,
 			},
 		)
 	}

From 603ae8fc6dcef43adca6d72b4f401879d41aae37 Mon Sep 17 00:00:00 2001
From: Dominik Braun <mail@dominikbraun.io>
Date: Tue, 22 Sep 2020 10:16:27 +0200
Subject: [PATCH 15/65] Return ExHostPermission for insufficient config file
 permissions

---
 cmd/minikube/cmd/start.go     |  7 ++++++-
 pkg/minikube/config/config.go | 21 +++++++++++++++++++++
 pkg/minikube/reason/reason.go |  1 +
 3 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index fc3acd592d..3f9635cc70 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -156,9 +156,14 @@ func runStart(cmd *cobra.Command, args []string) {
 		out.WarningT("Profile name '{{.name}}' is not valid", out.V{"name": ClusterFlagValue()})
 		exit.Message(reason.Usage, "Only alphanumeric and dashes '-' are permitted. Minimum 1 character, starting with alphanumeric.")
 	}
+
 	existing, err := config.Load(ClusterFlagValue())
 	if err != nil && !config.IsNotExist(err) {
-		exit.Message(reason.HostConfigLoad, "Unable to load config: {{.error}}", out.V{"error": err})
+		kind := reason.HostConfigLoad
+		if config.IsPermissionDenied(err) {
+			kind = reason.HostPermissionDenied
+		}
+		exit.Message(kind, "Unable to load config: {{.error}}", out.V{"error": err})
 	}
 
 	if existing != nil {
diff --git a/pkg/minikube/config/config.go b/pkg/minikube/config/config.go
index 93e9332dcc..5fd296630c 100644
--- a/pkg/minikube/config/config.go
+++ b/pkg/minikube/config/config.go
@@ -80,6 +80,24 @@ func IsNotExist(err error) bool {
 	return false
 }
 
+// ErrPermissionDenied is the error returned when the config cannot be read
+// due to insufficient permissions
+type ErrPermissionDenied struct {
+	s string
+}
+
+func (e *ErrPermissionDenied) Error() string {
+	return e.s
+}
+
+// IsPermissionDenied returns whether the error is a ErrPermissionDenied instance
+func IsPermissionDenied(err error) bool {
+	if _, ok := err.(*ErrPermissionDenied); ok {
+		return true
+	}
+	return false
+}
+
 // MinikubeConfig represents minikube config
 type MinikubeConfig map[string]interface{}
 
@@ -184,6 +202,9 @@ func (c *simpleConfigLoader) LoadConfigFromFile(profileName string, miniHome ...
 
 	data, err := ioutil.ReadFile(path)
 	if err != nil {
+		if os.IsPermission(err) {
+			return nil, &ErrPermissionDenied{err.Error()}
+		}
 		return nil, errors.Wrap(err, "read")
 	}
 
diff --git a/pkg/minikube/reason/reason.go b/pkg/minikube/reason/reason.go
index ee9a3faf6b..caeaab1a35 100644
--- a/pkg/minikube/reason/reason.go
+++ b/pkg/minikube/reason/reason.go
@@ -192,6 +192,7 @@ var (
 	HostHomeChown        = Kind{ID: "HOST_HOME_CHOWN", ExitCode: ExHostPermission}
 	HostBrowser          = Kind{ID: "HOST_BROWSER", ExitCode: ExHostError}
 	HostConfigLoad       = Kind{ID: "HOST_CONFIG_LOAD", ExitCode: ExHostConfig}
+	HostPermissionDenied = Kind{ID: "HOST_PERMISSION_DENIED", ExitCode: ExHostPermission}
 	HostCurrentUser      = Kind{ID: "HOST_CURRENT_USER", ExitCode: ExHostConfig}
 	HostDelCache         = Kind{ID: "HOST_DEL_CACHE", ExitCode: ExHostError}
 	HostKillMountProc    = Kind{ID: "HOST_KILL_MOUNT_PROC", ExitCode: ExHostError}

From 8f4127b202d70e0154edd567226ee84d48a560f3 Mon Sep 17 00:00:00 2001
From: Dominik Braun <mail@dominikbraun.io>
Date: Tue, 22 Sep 2020 18:04:24 +0200
Subject: [PATCH 16/65] Use HOST_HOME_PERMISSION kind for insufficient
 permissions

---
 cmd/minikube/cmd/start.go     |  2 +-
 pkg/minikube/reason/reason.go | 16 +++++++++++-----
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 3f9635cc70..17c0207918 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -161,7 +161,7 @@ func runStart(cmd *cobra.Command, args []string) {
 	if err != nil && !config.IsNotExist(err) {
 		kind := reason.HostConfigLoad
 		if config.IsPermissionDenied(err) {
-			kind = reason.HostPermissionDenied
+			kind = reason.HostHomePermission
 		}
 		exit.Message(kind, "Unable to load config: {{.error}}", out.V{"error": err})
 	}
diff --git a/pkg/minikube/reason/reason.go b/pkg/minikube/reason/reason.go
index caeaab1a35..c5bbfb5c6b 100644
--- a/pkg/minikube/reason/reason.go
+++ b/pkg/minikube/reason/reason.go
@@ -188,11 +188,17 @@ var (
 
 	RsrcInsufficientStorage = Kind{ID: "RSRC_INSUFFICIENT_STORAGE", ExitCode: ExInsufficientStorage, Style: style.UnmetRequirement}
 
-	HostHomeMkdir        = Kind{ID: "HOST_HOME_MKDIR", ExitCode: ExHostPermission}
-	HostHomeChown        = Kind{ID: "HOST_HOME_CHOWN", ExitCode: ExHostPermission}
-	HostBrowser          = Kind{ID: "HOST_BROWSER", ExitCode: ExHostError}
-	HostConfigLoad       = Kind{ID: "HOST_CONFIG_LOAD", ExitCode: ExHostConfig}
-	HostPermissionDenied = Kind{ID: "HOST_PERMISSION_DENIED", ExitCode: ExHostPermission}
+	HostHomeMkdir      = Kind{ID: "HOST_HOME_MKDIR", ExitCode: ExHostPermission}
+	HostHomeChown      = Kind{ID: "HOST_HOME_CHOWN", ExitCode: ExHostPermission}
+	HostBrowser        = Kind{ID: "HOST_BROWSER", ExitCode: ExHostError}
+	HostConfigLoad     = Kind{ID: "HOST_CONFIG_LOAD", ExitCode: ExHostConfig}
+	HostHomePermission = Kind{
+		ID:       "HOST_HOME_PERMISSION",
+		ExitCode: ExHostPermission,
+		Advice:   "Your user lacks permissions to the minikube profile directory. Run: 'sudo chown -R $USER $HOME/.minikube; chmod -R u+wrx $HOME/.minikube' to fix",
+		Issues:   []int{9165},
+	}
+
 	HostCurrentUser      = Kind{ID: "HOST_CURRENT_USER", ExitCode: ExHostConfig}
 	HostDelCache         = Kind{ID: "HOST_DEL_CACHE", ExitCode: ExHostError}
 	HostKillMountProc    = Kind{ID: "HOST_KILL_MOUNT_PROC", ExitCode: ExHostError}

From 3f44d470eb30a193aca0c9c9b604cfdf9615b7c2 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Tue, 22 Sep 2020 13:54:23 -0400
Subject: [PATCH 17/65] Add performance monitor code

---
 cmd/performance/pr-bot/bot.go |  38 ++++++++
 pkg/perf/monitor/constants.go |  24 +++++
 pkg/perf/monitor/execute.go   |  63 ++++++++++++
 pkg/perf/monitor/github.go    | 179 ++++++++++++++++++++++++++++++++++
 4 files changed, 304 insertions(+)
 create mode 100644 pkg/perf/monitor/constants.go
 create mode 100644 pkg/perf/monitor/execute.go
 create mode 100644 pkg/perf/monitor/github.go

diff --git a/cmd/performance/pr-bot/bot.go b/cmd/performance/pr-bot/bot.go
index 174c10d9e4..2614ea6884 100644
--- a/cmd/performance/pr-bot/bot.go
+++ b/cmd/performance/pr-bot/bot.go
@@ -18,12 +18,17 @@ package main
 
 import (
 	"context"
+	"fmt"
 	"log"
+	"os"
 	"time"
+
+	"github.com/pkg/errors"
 )
 
 func main() {
 	for {
+		log.Print("~~~~~~~~~ Starting performance analysis ~~~~~~~~~~~~~~")
 		if err := analyzePerformance(context.Background()); err != nil {
 			log.Printf("error executing performance analysis: %v", err)
 		}
@@ -36,5 +41,38 @@ func main() {
 //   2. running mkcmp against those PRs
 //   3. commenting results on those PRs
 func analyzePerformance(ctx context.Context) error {
+	logsFile := "/home/performance-monitor/logs.txt"
+	if _, err := os.Stat(logsFile); err != nil {
+		return err
+	}
+	client := monitor.NewClient(context.Background(), "kubernetes", "minikube")
+	prs, err := client.ListOpenPRsWithLabel("")
+	if err != nil {
+		return errors.Wrap(err, "listing open prs")
+	}
+	log.Print("got prs:", prs)
+	// TODO: priyawadhwa@ for each PR we should comment the error if we get one?
+	for _, pr := range prs {
+		log.Printf("~~~ Analyzing PR %d ~~~", pr)
+		newCommitsExist, err := client.NewCommitsExist(pr, "minikube-pr-bot")
+		if err != nil {
+			return err
+		}
+		if !newCommitsExist {
+			log.Println("New commits don't exist, skipping rerun...")
+			continue
+		}
+		// TODO: priyawadhwa@ we should download mkcmp for each run?
+		var message string
+		message, err = monitor.RunMkcmp(ctx, pr)
+		if err != nil {
+			message = fmt.Sprintf("Error: %v\n%s", err, message)
+		}
+		log.Printf("got message for pr %d:\n%s\n", pr, message)
+		if err := client.CommentOnPR(pr, message); err != nil {
+			return err
+		}
+		log.Print("successfully commented on PR")
+	}
 	return nil
 }
diff --git a/pkg/perf/monitor/constants.go b/pkg/perf/monitor/constants.go
new file mode 100644
index 0000000000..0221374464
--- /dev/null
+++ b/pkg/perf/monitor/constants.go
@@ -0,0 +1,24 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package monitor
+
+const (
+	GithubAccessTokenEnvVar = "GITHUB_ACCESS_TOKEN"
+	OkToTestLabel           = "ok-to-test"
+	GithubOwner             = "kubernetes"
+	GithubRepo              = "minikube"
+)
diff --git a/pkg/perf/monitor/execute.go b/pkg/perf/monitor/execute.go
new file mode 100644
index 0000000000..543ed0114c
--- /dev/null
+++ b/pkg/perf/monitor/execute.go
@@ -0,0 +1,63 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package monitor
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+)
+
+// RunMkcmp runs minikube built at the given pr against minikube at master
+func RunMkcmp(ctx context.Context, pr int) (string, error) {
+	// run 'git pull' so that minikube dir is up to date
+	if _, err := runCmdInMinikube(ctx, []string{"git", "pull", "origin", "master"}); err != nil {
+		return "", errors.Wrap(err, "running git pull")
+	}
+	mkcmpPath := "out/mkcmp"
+	minikubePath := "out/minikube"
+	if _, err := runCmdInMinikube(ctx, []string{"make", mkcmpPath, minikubePath}); err != nil {
+		return "", errors.Wrap(err, "building minikube and mkcmp at head")
+	}
+	return runCmdInMinikube(ctx, []string{mkcmpPath, minikubePath, fmt.Sprintf("pr://%d", pr)})
+}
+
+// runCmdInMinikube runs the cmd and return stdout
+func runCmdInMinikube(ctx context.Context, command []string) (string, error) {
+	cmd := exec.CommandContext(ctx, command[0], command[1:]...)
+	cmd.Dir = minikubeDir()
+	cmd.Env = os.Environ()
+
+	buf := bytes.NewBuffer([]byte{})
+	cmd.Stdout = buf
+
+	log.Printf("Running: %v", cmd.Args)
+	if err := cmd.Run(); err != nil {
+		return "", errors.Wrapf(err, "running %v: %v", cmd.Args, buf.String())
+	}
+	return buf.String(), nil
+}
+
+func minikubeDir() string {
+	return filepath.Join(os.Getenv("HOME"), "minikube")
+}
diff --git a/pkg/perf/monitor/github.go b/pkg/perf/monitor/github.go
new file mode 100644
index 0000000000..18afdbb580
--- /dev/null
+++ b/pkg/perf/monitor/github.go
@@ -0,0 +1,179 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package monitor
+
+import (
+	"context"
+	"log"
+	"os"
+	"time"
+
+	"github.com/google/go-github/github"
+	"github.com/pkg/errors"
+	"golang.org/x/oauth2"
+)
+
+// Client provides the context and client with necessary auth
+// for interacting with the Github API
+type Client struct {
+	ctx context.Context
+	*github.Client
+	owner string
+	repo  string
+}
+
+// NewClient returns a github client with the necessary auth
+func NewClient(ctx context.Context, owner, repo string) *Client {
+	githubToken := os.Getenv(GithubAccessTokenEnvVar)
+	// Setup the token for github authentication
+	ts := oauth2.StaticTokenSource(
+		&oauth2.Token{AccessToken: githubToken},
+	)
+	tc := oauth2.NewClient(context.Background(), ts)
+
+	// Return a client instance from github
+	client := github.NewClient(tc)
+	return &Client{
+		ctx:    ctx,
+		Client: client,
+		owner:  owner,
+		repo:   repo,
+	}
+}
+
+// CommentOnPR comments message on the PR
+func (g *Client) CommentOnPR(pr int, message string) error {
+	comment := &github.IssueComment{
+		Body: &message,
+	}
+
+	log.Printf("Creating comment on PR %d: %s", pr, message)
+	_, _, err := g.Client.Issues.CreateComment(g.ctx, g.owner, g.repo, pr, comment)
+	if err != nil {
+		return errors.Wrap(err, "creating github comment")
+	}
+	log.Printf("Successfully commented on PR %d.", pr)
+	return nil
+}
+
+// ListOpenPRsWithLabel returns all open PRs with the specified label
+func (g *Client) ListOpenPRsWithLabel(label string) ([]int, error) {
+	validPrs := []int{}
+	prs, _, err := g.Client.PullRequests.List(g.ctx, g.owner, g.repo, &github.PullRequestListOptions{})
+	if err != nil {
+		return nil, errors.Wrap(err, "listing pull requests")
+	}
+	for _, pr := range prs {
+		if prContainsLabel(pr.Labels, "ok-to-test") {
+			validPrs = append(validPrs, pr.GetNumber())
+		}
+	}
+	return validPrs, nil
+}
+
+func prContainsLabel(labels []*github.Label, label string) bool {
+	for _, l := range labels {
+		if l == nil {
+			continue
+		}
+		if l.GetName() == label {
+			return true
+		}
+	}
+	return false
+}
+
+// NewCommitsExist checks if new commits exist since minikube-pr-bot
+// commented on the PR. If so, return true.
+func (g *Client) NewCommitsExist(pr int, login string) (bool, error) {
+	lastCommentTime, err := g.timeOfLastComment(pr, login)
+	if err != nil {
+		return false, errors.Wrapf(err, "getting time of last comment by %s on pr %d", login, pr)
+	}
+	lastCommitTime, err := g.timeOfLastCommit(pr)
+	if err != nil {
+		return false, errors.Wrapf(err, "getting time of last commit on pr %d", pr)
+	}
+	return lastCommentTime.Before(lastCommitTime), nil
+}
+
+func (g *Client) timeOfLastCommit(pr int) (time.Time, error) {
+	var commits []*github.RepositoryCommit
+
+	page := 0
+	resultsPerPage := 30
+	for {
+		c, _, err := g.Client.PullRequests.ListCommits(g.ctx, g.owner, g.repo, pr, &github.ListOptions{
+			Page:    page,
+			PerPage: resultsPerPage,
+		})
+		if err != nil {
+			return time.Time{}, nil
+		}
+		commits = append(commits, c...)
+		if len(c) < resultsPerPage {
+			break
+		}
+		page++
+	}
+
+	lastCommitTime := time.Time{}
+	for _, c := range commits {
+		if newCommitTime := c.GetCommit().GetAuthor().GetDate(); newCommitTime.After(lastCommitTime) {
+			lastCommitTime = newCommitTime
+		}
+	}
+	return lastCommitTime, nil
+}
+
+func (g *Client) timeOfLastComment(pr int, login string) (time.Time, error) {
+	var comments []*github.IssueComment
+
+	page := 0
+	resultsPerPage := 30
+	for {
+		c, _, err := g.Client.Issues.ListComments(g.ctx, g.owner, g.repo, pr, &github.IssueListCommentsOptions{
+			ListOptions: github.ListOptions{
+				Page:    page,
+				PerPage: resultsPerPage,
+			},
+		})
+		if err != nil {
+			return time.Time{}, nil
+		}
+		comments = append(comments, c...)
+		if len(c) < resultsPerPage {
+			break
+		}
+		page++
+	}
+
+	// go through comments backwards to find the most recent
+	lastCommentTime := time.Time{}
+
+	for _, c := range comments {
+		if u := c.GetUser(); u != nil {
+			if u.GetLogin() == login {
+				if c.GetCreatedAt().After(lastCommentTime) {
+					lastCommentTime = c.GetCreatedAt()
+				}
+			}
+		}
+	}
+
+	return lastCommentTime, nil
+}

From 3d547a937f91ec825815c6e51a13224ea4b1506e Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Tue, 22 Sep 2020 13:56:53 -0400
Subject: [PATCH 18/65] Fix Makefile rule to build bot

---
 Makefile                      |  6 +++---
 cmd/performance/pr-bot/bot.go | 16 +++++-----------
 pkg/perf/monitor/constants.go |  1 +
 pkg/perf/monitor/github.go    |  2 +-
 4 files changed, 10 insertions(+), 15 deletions(-)

diff --git a/Makefile b/Makefile
index 4e37b1a575..2a87f19f67 100644
--- a/Makefile
+++ b/Makefile
@@ -747,9 +747,9 @@ site: site/themes/docsy/assets/vendor/bootstrap/package.js out/hugo/hugo ## Serv
 out/mkcmp:
 	GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o $@ cmd/performance/mkcmp/main.go
 
-.PHONY: out/performance-monitor
-out/performance-monitor:
-	GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o $@ cmd/performance/monitor/monitor.go
+.PHONY: out/performance-bot
+out/performance-bot:
+	GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o $@ cmd/performance/pr-bot/bot.go
 
 .PHONY: compare
 compare: out/mkcmp out/minikube
diff --git a/cmd/performance/pr-bot/bot.go b/cmd/performance/pr-bot/bot.go
index 2614ea6884..340c4ef8f4 100644
--- a/cmd/performance/pr-bot/bot.go
+++ b/cmd/performance/pr-bot/bot.go
@@ -20,10 +20,10 @@ import (
 	"context"
 	"fmt"
 	"log"
-	"os"
 	"time"
 
 	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/perf/monitor"
 )
 
 func main() {
@@ -41,20 +41,15 @@ func main() {
 //   2. running mkcmp against those PRs
 //   3. commenting results on those PRs
 func analyzePerformance(ctx context.Context) error {
-	logsFile := "/home/performance-monitor/logs.txt"
-	if _, err := os.Stat(logsFile); err != nil {
-		return err
-	}
-	client := monitor.NewClient(context.Background(), "kubernetes", "minikube")
-	prs, err := client.ListOpenPRsWithLabel("")
+	client := monitor.NewClient(ctx, monitor.GithubOwner, monitor.GithubRepo)
+	prs, err := client.ListOpenPRsWithLabel(monitor.OkToTestLabel)
 	if err != nil {
 		return errors.Wrap(err, "listing open prs")
 	}
 	log.Print("got prs:", prs)
-	// TODO: priyawadhwa@ for each PR we should comment the error if we get one?
 	for _, pr := range prs {
 		log.Printf("~~~ Analyzing PR %d ~~~", pr)
-		newCommitsExist, err := client.NewCommitsExist(pr, "minikube-pr-bot")
+		newCommitsExist, err := client.NewCommitsExist(pr, monitor.BotName)
 		if err != nil {
 			return err
 		}
@@ -62,13 +57,12 @@ func analyzePerformance(ctx context.Context) error {
 			log.Println("New commits don't exist, skipping rerun...")
 			continue
 		}
-		// TODO: priyawadhwa@ we should download mkcmp for each run?
 		var message string
 		message, err = monitor.RunMkcmp(ctx, pr)
 		if err != nil {
 			message = fmt.Sprintf("Error: %v\n%s", err, message)
 		}
-		log.Printf("got message for pr %d:\n%s\n", pr, message)
+		log.Printf("message for pr %d:\n%s\n", pr, message)
 		if err := client.CommentOnPR(pr, message); err != nil {
 			return err
 		}
diff --git a/pkg/perf/monitor/constants.go b/pkg/perf/monitor/constants.go
index 0221374464..be4a0011ab 100644
--- a/pkg/perf/monitor/constants.go
+++ b/pkg/perf/monitor/constants.go
@@ -21,4 +21,5 @@ const (
 	OkToTestLabel           = "ok-to-test"
 	GithubOwner             = "kubernetes"
 	GithubRepo              = "minikube"
+	BotName                 = "minikube-pr-bot"
 )
diff --git a/pkg/perf/monitor/github.go b/pkg/perf/monitor/github.go
index 18afdbb580..c50f9f1563 100644
--- a/pkg/perf/monitor/github.go
+++ b/pkg/perf/monitor/github.go
@@ -78,7 +78,7 @@ func (g *Client) ListOpenPRsWithLabel(label string) ([]int, error) {
 		return nil, errors.Wrap(err, "listing pull requests")
 	}
 	for _, pr := range prs {
-		if prContainsLabel(pr.Labels, "ok-to-test") {
+		if prContainsLabel(pr.Labels, label) {
 			validPrs = append(validPrs, pr.GetNumber())
 		}
 	}

From 3f457dbdc1e7ce53479c23db38ab89c842f4867c Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Tue, 22 Sep 2020 14:14:04 -0400
Subject: [PATCH 19/65] fix error handling

---
 pkg/perf/monitor/github.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkg/perf/monitor/github.go b/pkg/perf/monitor/github.go
index c50f9f1563..eb3146136e 100644
--- a/pkg/perf/monitor/github.go
+++ b/pkg/perf/monitor/github.go
@@ -122,7 +122,7 @@ func (g *Client) timeOfLastCommit(pr int) (time.Time, error) {
 			PerPage: resultsPerPage,
 		})
 		if err != nil {
-			return time.Time{}, nil
+			return time.Time{}, err
 		}
 		commits = append(commits, c...)
 		if len(c) < resultsPerPage {
@@ -153,7 +153,7 @@ func (g *Client) timeOfLastComment(pr int, login string) (time.Time, error) {
 			},
 		})
 		if err != nil {
-			return time.Time{}, nil
+			return time.Time{}, err
 		}
 		comments = append(comments, c...)
 		if len(c) < resultsPerPage {

From 02cdb59184a089684085f7cb07f929d2da6a59a8 Mon Sep 17 00:00:00 2001
From: Dominik Braun <mail@dominikbraun.io>
Date: Tue, 22 Sep 2020 21:40:26 +0200
Subject: [PATCH 20/65] Use HostHomePermission variable in known_issues.go

---
 pkg/minikube/reason/known_issues.go | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/pkg/minikube/reason/known_issues.go b/pkg/minikube/reason/known_issues.go
index 49ed0417dd..17347661b4 100644
--- a/pkg/minikube/reason/known_issues.go
+++ b/pkg/minikube/reason/known_issues.go
@@ -202,12 +202,7 @@ var hostIssues = []match{
 		GOOS:   []string{"linux"},
 	},
 	{
-		Kind: Kind{
-			ID:       "HOST_HOME_PERMISSION",
-			ExitCode: ExGuestPermission,
-			Advice:   "Your user lacks permissions to the minikube profile directory. Run: 'sudo chown -R $USER $HOME/.minikube; chmod -R u+wrx $HOME/.minikube' to fix",
-			Issues:   []int{9165},
-		},
+		Kind:   HostHomePermission,
 		Regexp: re(`/.minikube/.*: permission denied`),
 	},
 }

From 76d3c9d7bbd7679a6b4ca368bb3d5721efcd4b19 Mon Sep 17 00:00:00 2001
From: Ilya Zuyev <izuyev@google.com>
Date: Tue, 22 Sep 2020 13:09:58 -0700
Subject: [PATCH 21/65] Fix variable name (avoid collision with imported
 package)

---
 cmd/minikube/cmd/start.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 652c73afbf..30a20e98ab 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -1193,14 +1193,14 @@ func validateKubernetesVersion(old *config.ClusterConfig) {
 
 // validateBaseImage checks that --base-image is not passed if the drive being in use is KIC (docker/podman)
 // if so, the function exits the process
-func validateBaseImage(imageFlag *pflag.Flag, driver string) {
-	if !validBaseImageFlag(imageFlag.Changed, driver) {
+func validateBaseImage(imageFlag *pflag.Flag, drv string) {
+	if !validBaseImageFlag(imageFlag.Changed, drv) {
 		exit.Message(reason.Usage,
 			"flag --{{.imgFlag}} is not available for driver '{{.driver}}'. Did you mean to use '{{.docker}}' or '{{.podman}}' driver instead?\n"+
 				"Please use --{{.isoFlag}} flag to configure VM based drivers",
 			out.V{
 				"imgFlag": imageFlag.Name,
-				"driver":  driver,
+				"driver":  drv,
 				"image":   imageFlag.Value,
 				"docker":  registry.Docker,
 				"podman":  registry.Podman,

From 1444ce37d7aadb3796c9b94345c21b6edfc9d49b Mon Sep 17 00:00:00 2001
From: Predrag Rogic <predrag.rogic@talkout.com>
Date: Tue, 22 Sep 2020 21:46:18 +0100
Subject: [PATCH 22/65] update_kubernetes_version: refactor

- add context.WithTimeout - limit the time spent on quering GitHub
  APIs
- extract Kubernetes GitHub Releases API quering into a separate
  fetchKubernetesReleases func
- extract Kubernetes Version Update logic into a separate
  updateKubernetesVersions func
- extract file patching instructions into a separate replaceAllString
  func
- patch '../../site/content/en/docs/commands/start.md' => remove the
  need to run 'make generate-docs' before 'make test'
- resolve 'ERROR: logging before flag.Parse'
- consolidate and add more verbose logging
- add more descriptive code comments
- removed TODO - will stick to the '0' patch version for all testData
  sets
- ensure 'make test' completes w/o any issues
---
 .../update_kubernetes_version.go              | 127 +++++++++++-------
 1 file changed, 81 insertions(+), 46 deletions(-)

diff --git a/hack/kubernetes_version/update_kubernetes_version.go b/hack/kubernetes_version/update_kubernetes_version.go
index 02c9b08e13..75265150fd 100644
--- a/hack/kubernetes_version/update_kubernetes_version.go
+++ b/hack/kubernetes_version/update_kubernetes_version.go
@@ -18,12 +18,13 @@ package main
 
 import (
 	"context"
-	"fmt"
+	"flag"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"regexp"
 	"strings"
+	"time"
 
 	"github.com/golang/glog"
 
@@ -31,22 +32,49 @@ import (
 )
 
 func main() {
-	vDefault := ""
-	vNewest := ""
+	// init glog: by default, all log statements write to files in a temporary directory, also
+	// flag.Parse must be called before any logging is done
+	flag.Parse()
+	_ = flag.Set("logtostderr", "true")
 
+	// fetch respective current stable (vDefault as DefaultKubernetesVersion) and
+	// latest rc or beta (vDefault as NewestKubernetesVersion) Kubernetes GitHub Releases
+	vDefault, vNewest, err := fetchKubernetesReleases()
+	if err != nil {
+		glog.Errorf("Fetching current GitHub Releases failed: %v", err)
+	}
+	if vDefault == "" || vNewest == "" {
+		glog.Fatalf("Cannot determine current 'DefaultKubernetesVersion' and 'NewestKubernetesVersion'")
+	}
+	glog.Infof("Current Kubernetes GitHub Releases: 'stable' is %s and 'latest' is %s", vDefault, vNewest)
+
+	if err := updateKubernetesVersions(vDefault, vNewest); err != nil {
+		glog.Fatalf("Updating 'DefaultKubernetesVersion' and 'NewestKubernetesVersion' failed: %v", err)
+	}
+	glog.Infof("Update successful: 'DefaultKubernetesVersion' was set to %s and 'NewestKubernetesVersion' was set to %s", vDefault, vNewest)
+
+	// Flush before exiting to guarantee all log output is written
+	glog.Flush()
+}
+
+// fetchKubernetesReleases returns respective current stable (as vDefault) and
+// latest rc or beta (as vNewest) Kubernetes GitHub Releases, and any error
+func fetchKubernetesReleases() (vDefault, vNewest string, err error) {
 	client := github.NewClient(nil)
 
+	// set a context with a deadline - timeout after at most 10 seconds
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
 	// walk through the paginated list of all 'kubernetes/kubernetes' repo releases
 	// from latest to older releases, until latest release and pre-release are found
 	// use max value (100) for PerPage to avoid hitting the rate limits (60 per hour, 10 per minute)
 	// see https://godoc.org/github.com/google/go-github/github#hdr-Rate_Limiting
 	opt := &github.ListOptions{PerPage: 100}
-out:
 	for {
-		rels, resp, err := client.Repositories.ListReleases(context.Background(), "kubernetes", "kubernetes", opt)
+		rels, resp, err := client.Repositories.ListReleases(ctx, "kubernetes", "kubernetes", opt)
 		if err != nil {
-			glog.Errorf("GitHub ListReleases failed: %v", err)
-			break
+			return "", "", err
 		}
 
 		for _, r := range rels {
@@ -72,7 +100,7 @@ out:
 				if vNewest < vDefault {
 					vNewest = vDefault
 				}
-				break out
+				return vDefault, vNewest, nil
 			}
 		}
 
@@ -81,57 +109,64 @@ out:
 		}
 		opt.Page = resp.NextPage
 	}
+	return vDefault, vNewest, nil
+}
 
-	if vDefault == "" || vNewest == "" {
-		glog.Errorf("Cannot determine DefaultKubernetesVersion or NewestKubernetesVersion")
-		os.Exit(1)
+// updateKubernetesVersions updates DefaultKubernetesVersion to vDefault release and
+// NewestKubernetesVersion to vNewest release, and returns any error
+func updateKubernetesVersions(vDefault, vNewest string) error {
+	if err := replaceAllString("../../pkg/minikube/constants/constants.go", map[string]string{
+		`DefaultKubernetesVersion = \".*`: "DefaultKubernetesVersion = \"" + vDefault + "\"",
+		`NewestKubernetesVersion = \".*`:  "NewestKubernetesVersion = \"" + vNewest + "\"",
+	}); err != nil {
+		return err
 	}
 
-	constantsFile := "../../pkg/minikube/constants/constants.go"
-	cf, err := ioutil.ReadFile(constantsFile)
-	if err != nil {
-		fmt.Println(err)
-		os.Exit(1)
+	if err := replaceAllString("../../site/content/en/docs/commands/start.md", map[string]string{
+		`'stable' for .*,`:  "'stable' for " + vDefault + ",",
+		`'latest' for .*\)`: "'latest' for " + vNewest + ")",
+	}); err != nil {
+		return err
 	}
 
-	info, err := os.Stat(constantsFile)
-	if err != nil {
-		fmt.Println(err)
-		os.Exit(1)
-	}
-	mode := info.Mode()
-
-	re := regexp.MustCompile(`DefaultKubernetesVersion = \".*`)
-	f := re.ReplaceAllString(string(cf), "DefaultKubernetesVersion = \""+vDefault+"\"")
-
-	re = regexp.MustCompile(`NewestKubernetesVersion = \".*`)
-	f = re.ReplaceAllString(f, "NewestKubernetesVersion = \""+vNewest+"\"")
-
-	if err := ioutil.WriteFile(constantsFile, []byte(f), mode); err != nil {
-		fmt.Println(err)
-		os.Exit(1)
-	}
-
-	// update testData just for the latest 'v<MAJOR>.<MINOR>' from vDefault
+	// update testData just for the latest 'v<MAJOR>.<MINOR>.0' from vDefault
 	vDefaultMM := vDefault[:strings.LastIndex(vDefault, ".")]
 	testData := "../../pkg/minikube/bootstrapper/bsutil/testdata/" + vDefaultMM
 
-	err = filepath.Walk(testData, func(path string, info os.FileInfo, err error) error {
+	return filepath.Walk(testData, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
 		if !strings.HasSuffix(path, "default.yaml") {
 			return nil
 		}
-		cf, err = ioutil.ReadFile(path)
-		if err != nil {
-			return err
-		}
-		re = regexp.MustCompile(`kubernetesVersion: .*`)
-		cf = []byte(re.ReplaceAllString(string(cf), "kubernetesVersion: "+vDefaultMM+".0")) // TODO: let <PATCH> version to be the latest one instead of "0"
-		return ioutil.WriteFile(path, cf, info.Mode())
+		return replaceAllString(path, map[string]string{
+			`kubernetesVersion: .*`: "kubernetesVersion: " + vDefaultMM + ".0",
+		})
 	})
-	if err != nil {
-		glog.Errorf("Walk failed: %v", err)
-	}
+}
+
+// replaceAllString replaces all occuranes of map's keys with their respective values in the file
+func replaceAllString(path string, pairs map[string]string) error {
+	fb, err := ioutil.ReadFile(path)
+	if err != nil {
+		return err
+	}
+
+	info, err := os.Stat(path)
+	if err != nil {
+		return err
+	}
+	mode := info.Mode()
+
+	f := string(fb)
+	for org, new := range pairs {
+		re := regexp.MustCompile(org)
+		f = re.ReplaceAllString(f, new)
+	}
+	if err := ioutil.WriteFile(path, []byte(f), mode); err != nil {
+		return err
+	}
+
+	return nil
 }

From bf8ec63ca996debddf63fdd7ff9cd298efbee23f Mon Sep 17 00:00:00 2001
From: Ilya Zuyev <izuyev@google.com>
Date: Tue, 22 Sep 2020 14:10:47 -0700
Subject: [PATCH 23/65] Simplify base-image flag validation

---
 cmd/minikube/cmd/start.go      | 42 ++++++++++--------------
 cmd/minikube/cmd/start_test.go | 59 ++++++++++++----------------------
 2 files changed, 37 insertions(+), 64 deletions(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 30a20e98ab..63620892cd 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -38,7 +38,6 @@ import (
 	"github.com/shirou/gopsutil/cpu"
 	gopshost "github.com/shirou/gopsutil/host"
 	"github.com/spf13/cobra"
-	"github.com/spf13/pflag"
 	"github.com/spf13/viper"
 
 	cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config"
@@ -171,7 +170,21 @@ func runStart(cmd *cobra.Command, args []string) {
 	validateKubernetesVersion(existing)
 
 	ds, alts, specified := selectDriver(existing)
-	validateBaseImage(cmd.Flag(kicBaseImage), ds.Name)
+	if cmd.Flag(kicBaseImage).Changed {
+		if !isBaseImageApplicable(ds.Name) {
+			exit.Message(reason.Usage,
+				"flag --{{.imgFlag}} is not available for driver '{{.driver}}'. Did you mean to use '{{.docker}}' or '{{.podman}}' driver instead?\n"+
+					"Please use --{{.isoFlag}} flag to configure VM based drivers",
+				out.V{
+					"imgFlag": kicBaseImage,
+					"driver":  ds.Name,
+					"docker":  registry.Docker,
+					"podman":  registry.Podman,
+					"isoFlag": isoURL,
+				},
+			)
+		}
+	}
 
 	starter, err := provisionWithDriver(cmd, ds, existing)
 	if err != nil {
@@ -1191,29 +1204,8 @@ func validateKubernetesVersion(old *config.ClusterConfig) {
 	}
 }
 
-// validateBaseImage checks that --base-image is not passed if the drive being in use is KIC (docker/podman)
-// if so, the function exits the process
-func validateBaseImage(imageFlag *pflag.Flag, drv string) {
-	if !validBaseImageFlag(imageFlag.Changed, drv) {
-		exit.Message(reason.Usage,
-			"flag --{{.imgFlag}} is not available for driver '{{.driver}}'. Did you mean to use '{{.docker}}' or '{{.podman}}' driver instead?\n"+
-				"Please use --{{.isoFlag}} flag to configure VM based drivers",
-			out.V{
-				"imgFlag": imageFlag.Name,
-				"driver":  drv,
-				"image":   imageFlag.Value,
-				"docker":  registry.Docker,
-				"podman":  registry.Podman,
-				"isoFlag": isoURL,
-			},
-		)
-	}
-}
-func validBaseImageFlag(baseImageFlagSet bool, driver string) bool {
-	if baseImageFlagSet {
-		return registry.IsKIC(driver)
-	}
-	return true
+func isBaseImageApplicable(drv string) bool {
+	return registry.IsKIC(drv)
 }
 
 func getKubernetesVersion(old *config.ClusterConfig) string {
diff --git a/cmd/minikube/cmd/start_test.go b/cmd/minikube/cmd/start_test.go
index 13da5639e4..a0752c1fd6 100644
--- a/cmd/minikube/cmd/start_test.go
+++ b/cmd/minikube/cmd/start_test.go
@@ -282,47 +282,28 @@ func TestSuggestMemoryAllocation(t *testing.T) {
 }
 
 func TestBaseImageFlagDriverCombo(t *testing.T) {
-	type testStruct struct {
-		description string
-		imageSet    bool
-		driver      string
-	}
-	invalidCombos := []testStruct{
-		{"KVM2 with flag", true, driver.KVM2},
-		{"VirtualBox with flag", true, driver.VirtualBox},
-		{"HyperKit with flag", true, driver.HyperKit},
-		{"VMware with flag", true, driver.VMware},
-		{"VMwareFusion with flag", true, driver.VMwareFusion},
-		{"HyperV with flag", true, driver.HyperV},
-		{"Parallels with flag", true, driver.Parallels},
-	}
-	validCombos := []testStruct{
-		{"Docker with flag", true, driver.Docker},
-		{"Podman with flag", true, driver.Podman},
-		{"Docker w/o flag", false, driver.Docker},
-		{"Podman w/o flag", false, driver.Podman},
-		{"KVM2 w/o flag", false, driver.KVM2},
-		{"VirtualBox w/o flag", false, driver.VirtualBox},
-		{"HyperKit w/o flag", false, driver.HyperKit},
-		{"VMware w/o flag", false, driver.VMware},
-		{"VMwareFusion w/o flag", false, driver.VMwareFusion},
-		{"HyperV w/o flag", false, driver.HyperV},
-		{"Parallels w/o flag", false, driver.Parallels},
+	tests := []struct {
+		driver        string
+		canUseBaseImg bool
+	}{
+		{driver.Docker, true},
+		{driver.Podman, true},
+		{driver.None, false},
+		{driver.KVM2, false},
+		{driver.VirtualBox, false},
+		{driver.HyperKit, false},
+		{driver.VMware, false},
+		{driver.VMwareFusion, false},
+		{driver.HyperV, false},
+		{driver.Parallels, false},
 	}
 
-	for _, test := range validCombos {
-		t.Run(test.description, func(t *testing.T) {
-			if !validBaseImageFlag(test.imageSet, test.driver) {
-				t.Errorf("invalidBaseImageFlag(base-image-set=%v, driver=%v): got false, expected true",
-					test.imageSet, test.driver)
-			}
-		})
-	}
-	for _, test := range invalidCombos {
-		t.Run(test.description, func(t *testing.T) {
-			if validBaseImageFlag(test.imageSet, test.driver) {
-				t.Errorf("invalidBaseImageFlag(base-image-set=%v, driver=%v): got true, expected false",
-					test.imageSet, test.driver)
+	for _, test := range tests {
+		t.Run(test.driver, func(t *testing.T) {
+			got := isBaseImageApplicable(test.driver)
+			if got != test.canUseBaseImg {
+				t.Errorf("isBaseImageApplicable(driver=%v): got %v, expected %v",
+					test.driver, got, test.canUseBaseImg)
 			}
 		})
 	}

From 887390194e4d1b55e88b3e2eb5a7333709d802fa Mon Sep 17 00:00:00 2001
From: Ilya Zuyev <izuyev@google.com>
Date: Tue, 22 Sep 2020 14:19:38 -0700
Subject: [PATCH 24/65] Add more unit tests

---
 cmd/minikube/cmd/start_test.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/cmd/minikube/cmd/start_test.go b/cmd/minikube/cmd/start_test.go
index a0752c1fd6..a1071c28f9 100644
--- a/cmd/minikube/cmd/start_test.go
+++ b/cmd/minikube/cmd/start_test.go
@@ -296,6 +296,8 @@ func TestBaseImageFlagDriverCombo(t *testing.T) {
 		{driver.VMwareFusion, false},
 		{driver.HyperV, false},
 		{driver.Parallels, false},
+		{"something_invalid", false},
+		{"", false},
 	}
 
 	for _, test := range tests {

From fd44e0d8fb93e317d4e8c40558e8a5120ea17206 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Wed, 23 Sep 2020 15:00:03 -0700
Subject: [PATCH 25/65] Force dependency on exp to prevent dirty commit release
 binaries (#9309)

* force dependency for release purposes

* revert original idea and add loads of debugging

* hey more debugging cool

* fix debugging

* drilling down debugging

* revert to original solution

* fix comment
---
 cmd/minikube/main.go | 3 +++
 go.mod               | 2 +-
 go.sum               | 1 +
 3 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/cmd/minikube/main.go b/cmd/minikube/main.go
index 1f379790c5..71bee8516f 100644
--- a/cmd/minikube/main.go
+++ b/cmd/minikube/main.go
@@ -31,6 +31,9 @@ import (
 	// Register drivers
 	_ "k8s.io/minikube/pkg/minikube/registry/drvs"
 
+	// Force exp dependency
+	_ "golang.org/x/exp/ebnf"
+
 	mlog "github.com/docker/machine/libmachine/log"
 
 	"github.com/golang/glog"
diff --git a/go.mod b/go.mod
index 7bcdb6e20b..6170d4656e 100644
--- a/go.mod
+++ b/go.mod
@@ -73,7 +73,7 @@ require (
 	github.com/zchee/go-vmnet v0.0.0-20161021174912-97ebf9174097
 	golang.org/x/build v0.0.0-20190927031335-2835ba2e683f
 	golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37
-	golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 // indirect
+	golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
 	golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a
 	golang.org/x/sys v0.0.0-20200523222454-059865788121
diff --git a/go.sum b/go.sum
index 5801ce02c5..854420fd06 100644
--- a/go.sum
+++ b/go.sum
@@ -1206,6 +1206,7 @@ golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
+golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 h1:QE6XYQK6naiK1EPAe1g/ILLxN5RBoH5xkJk3CqlMI/Y=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
 golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=

From 7a2cd0f4592559cbb729b71ba3f3b973d5b938ca Mon Sep 17 00:00:00 2001
From: loftkun <loftkun@gmail.com>
Date: Fri, 25 Sep 2020 17:45:51 +0900
Subject: [PATCH 26/65] Removing Duplicate Command Additions

---
 cmd/minikube/cmd/delete.go | 1 -
 cmd/minikube/cmd/stop.go   | 2 --
 2 files changed, 3 deletions(-)

diff --git a/cmd/minikube/cmd/delete.go b/cmd/minikube/cmd/delete.go
index 33a31488c1..6b50e46316 100644
--- a/cmd/minikube/cmd/delete.go
+++ b/cmd/minikube/cmd/delete.go
@@ -91,7 +91,6 @@ func init() {
 	if err := viper.BindPFlags(deleteCmd.Flags()); err != nil {
 		exit.Error(reason.InternalBindFlags, "unable to bind flags", err)
 	}
-	RootCmd.AddCommand(deleteCmd)
 }
 
 // shotgun cleanup to delete orphaned docker container data
diff --git a/cmd/minikube/cmd/stop.go b/cmd/minikube/cmd/stop.go
index 72dbce79a0..dc595cff91 100644
--- a/cmd/minikube/cmd/stop.go
+++ b/cmd/minikube/cmd/stop.go
@@ -60,8 +60,6 @@ func init() {
 	if err := viper.GetViper().BindPFlags(stopCmd.Flags()); err != nil {
 		exit.Error(reason.InternalFlagsBind, "unable to bind flags", err)
 	}
-
-	RootCmd.AddCommand(stopCmd)
 }
 
 // runStop handles the executes the flow of "minikube stop"

From b4b181b64784b5d0ddc38c32d98a9d907a5924b4 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Fri, 25 Sep 2020 10:06:45 -0700
Subject: [PATCH 27/65] give the logs test a little wiggle room

---
 pkg/minikube/perf/logs_test.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/pkg/minikube/perf/logs_test.go b/pkg/minikube/perf/logs_test.go
index b62e0cb295..0fbdf35bfb 100644
--- a/pkg/minikube/perf/logs_test.go
+++ b/pkg/minikube/perf/logs_test.go
@@ -38,7 +38,8 @@ func TestTimeCommandLogs(t *testing.T) {
 		if !ok {
 			t.Fatalf("expected log %s but didn't find it", log)
 		}
-		if actualTime < time {
+		// Let's give a little wiggle room so we don't fail if time is 3 and actualTime is 2.999
+		if actualTime < time && time-actualTime > 0.001 {
 			t.Fatalf("expected log \"%s\" to take more time than it actually did. got %v, expected > %v", log, actualTime, time)
 		}
 	}

From 54f94000065d186b05fe6363d090fb1923b9b3e5 Mon Sep 17 00:00:00 2001
From: Daniel Weibel <danielmweibel@gmail.com>
Date: Sat, 26 Sep 2020 14:45:59 +0200
Subject: [PATCH 28/65] Fix white version of SVG logo (icon)

---
 images/logo/logo_white.svg | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/images/logo/logo_white.svg b/images/logo/logo_white.svg
index c932459ae6..1937fdc8b1 100644
--- a/images/logo/logo_white.svg
+++ b/images/logo/logo_white.svg
@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<svg width="702px" height="683px" viewBox="0 0 702 683" version="1.1" xmlns="https://www.w3.org/2000/svg" xmlns:xlink="https://www.w3.org/1999/xlink">
-    <!-- Generator: Sketch 39.1 (31720) - https://www.sketchapp.com/ -->
+<svg width="702px" height="683px" viewBox="0 0 702 683" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+    <!-- Generator: Sketch 39.1 (31720) - http://www.bohemiancoding.com/sketch -->
     <title>minikube</title>
     <desc>Created with Sketch.</desc>
     <defs>

From d3dc36e1463106b7323a248441ff39914d1666e0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= <anders.f.bjorklund@gmail.com>
Date: Mon, 31 Aug 2020 20:00:06 +0200
Subject: [PATCH 29/65] Clean up the docker installation

Remove local things from kind, replace with packages.

Use the "clean-install" script for a nicer Dockerfile.
---
 deploy/kicbase/Dockerfile | 46 ++++++++++++++++++++++++---------------
 1 file changed, 28 insertions(+), 18 deletions(-)

diff --git a/deploy/kicbase/Dockerfile b/deploy/kicbase/Dockerfile
index 776a036013..eb5a355d5c 100644
--- a/deploy/kicbase/Dockerfile
+++ b/deploy/kicbase/Dockerfile
@@ -4,39 +4,54 @@ ARG COMMIT_SHA
 # could be changed to any debian that can run systemd
 FROM kindest/base:v20200430-2c0eee40 as base
 USER root
-# specify version of everything explicitly using 'apt-cache policy'
-RUN apt-get update && apt-get install -y --no-install-recommends \
+
+# remove files that were installed by kind, replaced by packages
+RUN rm \
+    /etc/crictl.yaml \
+    /etc/systemd/system/multi-user.target.wants/containerd.service \
+    /opt/cni/bin/host-local \
+    /opt/cni/bin/loopback \
+    /opt/cni/bin/portmap \
+    /opt/cni/bin/ptp \
+    /usr/local/bin/containerd \
+    /usr/local/bin/containerd-shim \
+    /usr/local/bin/containerd-shim-runc-v2 \
+    /usr/local/bin/crictl \
+    /usr/local/bin/ctr \
+    /usr/local/sbin/runc
+
+# install system requirements from the regular distro repositories
+RUN clean-install \
     lz4 \
     gnupg \ 
     sudo \
     docker.io \
+    containerd \
     openssh-server \
     dnsutils \
     runc \
     # libglib2.0-0 is required for conmon, which is required for podman
-    libglib2.0-0 \
-    # removing kind's crictl config
-    && rm /etc/crictl.yaml
+    libglib2.0-0
 
 # Install cri-o/podman dependencies:
 RUN sh -c "echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_20.04/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list" && \ 
     curl -LO https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_20.04/Release.key && \
-    apt-key add - < Release.key && apt-get update && \
-    apt-get install -y --no-install-recommends containers-common catatonit conmon containernetworking-plugins podman-plugins varlink
+    apt-key add - < Release.key && \
+    clean-install containers-common catatonit conmon containernetworking-plugins cri-tools podman-plugins varlink
 
 # install cri-o based on https://github.com/cri-o/cri-o/commit/96b0c34b31a9fc181e46d7d8e34fb8ee6c4dc4e1#diff-04c6e90faac2675aa89e2176d2eec7d8R128
 RUN sh -c "echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/1.18:/1.18.3/xUbuntu_20.04/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list" && \
     curl -LO https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/1.18:/1.18.3/xUbuntu_20.04/Release.key && \
-    apt-key add - < Release.key && apt-get update && \
-    apt-get install -y --no-install-recommends cri-o=1.18.3~3
+    apt-key add - < Release.key && \
+    clean-install cri-o=1.18.3~3
 
 # install podman
 RUN sh -c "echo 'deb https://dl.bintray.com/afbjorklund/podman focal main' > /etc/apt/sources.list.d/podman.list" && \
     curl -L https://bintray.com/user/downloadSubjectPublicKey?username=afbjorklund -o afbjorklund-public.key.asc && \
-    apt-key add - < afbjorklund-public.key.asc && apt-get update && \
-    apt-get install -y --no-install-recommends podman=1.9.3~1
+    apt-key add - < afbjorklund-public.key.asc && \
+    clean-install podman=1.9.3~1
 
-RUN mkdir -p /usr/lib/cri-o-runc/sbin && cp /usr/local/sbin/runc /usr/lib/cri-o-runc/sbin/runc
+RUN mkdir -p /usr/lib/cri-o-runc/sbin && cp /usr/sbin/runc /usr/lib/cri-o-runc/sbin/runc
 
 COPY entrypoint /usr/local/bin/entrypoint
 # automount service
@@ -71,12 +86,7 @@ USER root
 # https://github.com/kubernetes-sigs/kind/blob/master/images/base/files/usr/local/bin/entrypoint
 RUN mkdir -p /kind
 # Deleting leftovers
-RUN apt-get clean -y && rm -rf \
-  /var/cache/debconf/* \
-  /var/lib/apt/lists/* \
-  /var/log/* \
-  /tmp/* \
-  /var/tmp/* \
+RUN rm -rf \
   /usr/share/doc/* \
   /usr/share/man/* \
   /usr/share/local/* \

From 90566878443ddcd90930638325228b3e5978ac37 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= <anders.f.bjorklund@gmail.com>
Date: Sun, 27 Sep 2020 15:14:01 +0200
Subject: [PATCH 30/65] Also remove the extra containerd configuration

---
 deploy/kicbase/Dockerfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/deploy/kicbase/Dockerfile b/deploy/kicbase/Dockerfile
index eb5a355d5c..f7045f1d7a 100644
--- a/deploy/kicbase/Dockerfile
+++ b/deploy/kicbase/Dockerfile
@@ -8,6 +8,7 @@ USER root
 # remove files that were installed by kind, replaced by packages
 RUN rm \
     /etc/crictl.yaml \
+    /etc/systemd/system/containerd.service \
     /etc/systemd/system/multi-user.target.wants/containerd.service \
     /opt/cni/bin/host-local \
     /opt/cni/bin/loopback \

From 85f8d057570eac829767615407f6dd9ba7209c31 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= <anders.f.bjorklund@gmail.com>
Date: Sat, 12 Sep 2020 12:34:51 +0200
Subject: [PATCH 31/65] Fork all files needed for kindbase from upstream

This is for reference, to be able to combine them
---
 Makefile                                      |  11 +-
 deploy/kicbase/Dockerfile                     |   2 +-
 deploy/kindbase/Dockerfile                    | 122 +++++++++
 deploy/kindbase/files/etc/crictl.yaml         |   1 +
 .../etc/sysctl.d/10-network-security.conf     |   4 +
 .../etc/systemd/system/containerd.service     |  29 +++
 .../files/usr/local/bin/clean-install         |  39 +++
 .../kindbase/files/usr/local/bin/entrypoint   | 235 ++++++++++++++++++
 8 files changed, 441 insertions(+), 2 deletions(-)
 create mode 100644 deploy/kindbase/Dockerfile
 create mode 100644 deploy/kindbase/files/etc/crictl.yaml
 create mode 100644 deploy/kindbase/files/etc/sysctl.d/10-network-security.conf
 create mode 100644 deploy/kindbase/files/etc/systemd/system/containerd.service
 create mode 100755 deploy/kindbase/files/usr/local/bin/clean-install
 create mode 100755 deploy/kindbase/files/usr/local/bin/entrypoint

diff --git a/Makefile b/Makefile
index 2a87f19f67..98e4b3c1e0 100644
--- a/Makefile
+++ b/Makefile
@@ -20,6 +20,7 @@ RAW_VERSION=$(VERSION_MAJOR).$(VERSION_MINOR).$(VERSION_BUILD)
 VERSION ?= v$(RAW_VERSION)
 
 KUBERNETES_VERSION ?= $(shell egrep "DefaultKubernetesVersion =" pkg/minikube/constants/constants.go | cut -d \" -f2)
+KIND_VERSION ?= v20200430-2c0eee40
 KIC_VERSION ?= $(shell egrep "Version =" pkg/drivers/kic/types.go | cut -d \" -f2)
 
 # Default to .0 for higher cache hit rates, as build increments typically don't require new ISO versions
@@ -47,6 +48,7 @@ BUILD_IMAGE 	?= us.gcr.io/k8s-artifacts-prod/build-image/kube-cross:v$(GO_VERSIO
 ISO_BUILD_IMAGE ?= $(REGISTRY)/buildroot-image
 KVM_BUILD_IMAGE ?= $(REGISTRY)/kvm-build-image:$(GO_VERSION)
 
+KIND_BASE_IMAGE_GCR ?= $(REGISTRY)/kindbase:$(KIND_VERSION)
 KIC_BASE_IMAGE_GCR ?= $(REGISTRY)/kicbase:$(KIC_VERSION)
 KIC_BASE_IMAGE_GH ?= $(REGISTRY_GH)/kicbase:$(KIC_VERSION)
 KIC_BASE_IMAGE_HUB ?= kicbase/stable:$(KIC_VERSION)
@@ -576,8 +578,15 @@ endif
 storage-provisioner-image: out/storage-provisioner-$(GOARCH) ## Build storage-provisioner docker image
 	docker build -t $(STORAGE_PROVISIONER_IMAGE) -f deploy/storage-provisioner/Dockerfile  --build-arg arch=$(GOARCH) .
 
+.PHONY: kind-base-image
+kind-base-image: ## builds the base image used for kind.
+	docker rmi -f $(KIND_BASE_IMAGE_GCR)-snapshot || true
+	docker build -f ./deploy/kindbase/Dockerfile -t local/kindbase:$(KIND_VERSION)-snapshot ./deploy/kindbase
+	docker tag local/kindbase:$(KIND_VERSION)-snapshot $(KIND_BASE_IMAGE_GCR)-snapshot
+	docker tag local/kindbase:$(KIND_VERSION)-snapshot $(KIND_BASE_IMAGE_GCR)
+
 .PHONY: kic-base-image
-kic-base-image: ## builds the base image used for kic.
+kic-base-image: kind-base-image ## builds the base image used for kic.
 	docker rmi -f $(KIC_BASE_IMAGE_GCR)-snapshot || true
 	docker build -f ./deploy/kicbase/Dockerfile -t local/kicbase:$(KIC_VERSION)-snapshot  --build-arg COMMIT_SHA=${VERSION}-$(COMMIT) --cache-from $(KIC_BASE_IMAGE_GCR) --target base ./deploy/kicbase
 	docker tag local/kicbase:$(KIC_VERSION)-snapshot $(KIC_BASE_IMAGE_GCR)-snapshot
diff --git a/deploy/kicbase/Dockerfile b/deploy/kicbase/Dockerfile
index f7045f1d7a..00ee69d6af 100644
--- a/deploy/kicbase/Dockerfile
+++ b/deploy/kicbase/Dockerfile
@@ -2,7 +2,7 @@ ARG COMMIT_SHA
 # using base image created by kind https://github.com/kubernetes-sigs/kind/blob/v0.8.1/images/base/Dockerfile
 # which is an ubuntu 20.04 with an entry-point that helps running systemd
 # could be changed to any debian that can run systemd
-FROM kindest/base:v20200430-2c0eee40 as base
+FROM gcr.io/k8s-minikube/kindbase:v20200430-2c0eee40 as base
 USER root
 
 # remove files that were installed by kind, replaced by packages
diff --git a/deploy/kindbase/Dockerfile b/deploy/kindbase/Dockerfile
new file mode 100644
index 0000000000..d03bb0a62e
--- /dev/null
+++ b/deploy/kindbase/Dockerfile
@@ -0,0 +1,122 @@
+# Copyright 2018 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# kind node base image
+#
+# For systemd + docker configuration used below, see the following references:
+# https://www.freedesktop.org/wiki/Software/systemd/ContainerInterface/
+
+# start from ubuntu 19.10, this image is reasonably small as a starting point
+# for a kubernetes node image, it doesn't contain much we don't need
+FROM ubuntu:20.04
+
+# Configure containerd and runc binaries from kind-ci/containerd-nightlies repository
+# The repository contains latest stable releases and nightlies built for multiple architectures
+ARG CONTAINERD_VERSION="v1.3.3-14-g449e9269"
+# Configure CNI binaries from upstream
+ARG CNI_VERSION="v0.8.5"
+# Configure crictl binary from upstream
+ARG CRICTL_VERSION="v1.18.0"
+
+# copy in static files (configs, scripts)
+COPY files/ /
+
+# Install dependencies, first from apt, then from release tarballs.
+# NOTE: we use one RUN to minimize layers.
+#
+# First we must ensure that our util scripts are executable.
+#
+# The base image already has: ssh, apt, snapd, but we need to install more packages.
+# Packages installed are broken down into (each on a line):
+# - packages needed to run services (systemd)
+# - packages needed for kubernetes components
+# - packages needed by the container runtime
+# - misc packages kind uses itself
+# After installing packages we cleanup by:
+# - removing unwanted systemd services
+# - disabling kmsg in journald (these log entries would be confusing)
+#
+# Then we install containerd from our nightly build infrastructure, as this
+# build for multiple architectures and allows us to upgrade to patched releases
+# more quickly.
+#
+# Next we download and extract crictl and CNI plugin binaries from upstream.
+#
+# Next we ensure the /etc/kubernetes/manifests directory exists. Normally
+# a kubeadm debain / rpm package would ensure that this exists but we install
+# freshly built binaries directly when we build the node image.
+#
+# Finally we adjust tempfiles cleanup to be 1 minute after "boot" instead of 15m
+# This is plenty after we've done initial setup for a node, but before we are
+# likely to try to export logs etc.
+RUN echo "Ensuring scripts are executable ..." \
+    && chmod +x /usr/local/bin/clean-install /usr/local/bin/entrypoint \
+ && echo "Installing Packages ..." \
+    && DEBIAN_FRONTEND=noninteractive clean-install \
+      systemd \
+      conntrack iptables iproute2 ethtool socat util-linux mount ebtables udev kmod \
+      libseccomp2 \
+      bash ca-certificates curl rsync \
+    && find /lib/systemd/system/sysinit.target.wants/ -name "systemd-tmpfiles-setup.service" -delete \
+    && rm -f /lib/systemd/system/multi-user.target.wants/* \
+    && rm -f /etc/systemd/system/*.wants/* \
+    && rm -f /lib/systemd/system/local-fs.target.wants/* \
+    && rm -f /lib/systemd/system/sockets.target.wants/*udev* \
+    && rm -f /lib/systemd/system/sockets.target.wants/*initctl* \
+    && rm -f /lib/systemd/system/basic.target.wants/* \
+    && echo "ReadKMsg=no" >> /etc/systemd/journald.conf \
+    && ln -s "$(which systemd)" /sbin/init \
+ && echo "Installing containerd ..." \
+    && export ARCH=$(dpkg --print-architecture | sed 's/ppc64el/ppc64le/' | sed 's/armhf/arm/') \
+    && export CONTAINERD_BASE_URL="https://github.com/kind-ci/containerd-nightlies/releases/download/containerd-${CONTAINERD_VERSION#v}" \
+    && curl -sSL --retry 5 --output /tmp/containerd.tgz "${CONTAINERD_BASE_URL}/containerd-${CONTAINERD_VERSION#v}.linux-${ARCH}.tar.gz" \
+    && tar -C /usr/local -xzvf /tmp/containerd.tgz \
+    && rm -rf /tmp/containerd.tgz \
+    && rm -f /usr/local/bin/containerd-stress /usr/local/bin/containerd-shim-runc-v1 \
+    && curl -sSL --retry 5 --output /usr/local/sbin/runc "${CONTAINERD_BASE_URL}/runc.${ARCH}" \
+    && chmod 755 /usr/local/sbin/runc \
+    && containerd --version \
+    && systemctl enable containerd \
+ && echo "Installing crictl ..." \
+    && curl -fSL "https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-${ARCH}.tar.gz" | tar xzC /usr/local/bin \
+ && echo "Installing CNI binaries ..." \
+    && export ARCH=$(dpkg --print-architecture | sed 's/ppc64el/ppc64le/' | sed 's/armhf/arm/') \
+    && export CNI_TARBALL="${CNI_VERSION}/cni-plugins-linux-${ARCH}-${CNI_VERSION}.tgz" \
+    && export CNI_URL="https://github.com/containernetworking/plugins/releases/download/${CNI_TARBALL}" \
+    && curl -sSL --retry 5 --output /tmp/cni.tgz "${CNI_URL}" \
+    && mkdir -p /opt/cni/bin \
+    && tar -C /opt/cni/bin -xzf /tmp/cni.tgz \
+    && rm -rf /tmp/cni.tgz \
+    && find /opt/cni/bin -type f -not \( \
+         -iname host-local \
+         -o -iname ptp \
+         -o -iname portmap \
+         -o -iname loopback \
+      \) \
+      -delete \
+ && echo "Ensuring /etc/kubernetes/manifests" \
+    && mkdir -p /etc/kubernetes/manifests \
+ && echo "Adjusting systemd-tmpfiles timer" \
+    && sed -i /usr/lib/systemd/system/systemd-tmpfiles-clean.timer -e 's#OnBootSec=.*#OnBootSec=1min#' \
+ && echo "Modifying /etc/nsswitch.conf to prefer hosts" \
+    && sed -i /etc/nsswitch.conf -re 's#^(hosts:\s*).*#\1dns files#'
+
+# tell systemd that it is in docker (it will check for the container env)
+# https://www.freedesktop.org/wiki/Software/systemd/ContainerInterface/
+ENV container docker
+# systemd exits on SIGRTMIN+3, not SIGTERM (which re-executes it)
+# https://bugzilla.redhat.com/show_bug.cgi?id=1201657
+STOPSIGNAL SIGRTMIN+3
+# NOTE: this is *only* for documentation, the entrypoint is overridden later
+ENTRYPOINT [ "/usr/local/bin/entrypoint", "/sbin/init" ]
diff --git a/deploy/kindbase/files/etc/crictl.yaml b/deploy/kindbase/files/etc/crictl.yaml
new file mode 100644
index 0000000000..1a5daba3b9
--- /dev/null
+++ b/deploy/kindbase/files/etc/crictl.yaml
@@ -0,0 +1 @@
+runtime-endpoint: unix:///var/run/containerd/containerd.sock
\ No newline at end of file
diff --git a/deploy/kindbase/files/etc/sysctl.d/10-network-security.conf b/deploy/kindbase/files/etc/sysctl.d/10-network-security.conf
new file mode 100644
index 0000000000..3d9c71c3e7
--- /dev/null
+++ b/deploy/kindbase/files/etc/sysctl.d/10-network-security.conf
@@ -0,0 +1,4 @@
+# Turn on Source Address Verification in all interfaces to
+# prevent some spoofing attacks.
+net.ipv4.conf.default.rp_filter=1
+net.ipv4.conf.all.rp_filter=1
diff --git a/deploy/kindbase/files/etc/systemd/system/containerd.service b/deploy/kindbase/files/etc/systemd/system/containerd.service
new file mode 100644
index 0000000000..fb97bf814c
--- /dev/null
+++ b/deploy/kindbase/files/etc/systemd/system/containerd.service
@@ -0,0 +1,29 @@
+# derived containerd systemd service file from the official:
+# https://github.com/containerd/containerd/blob/master/containerd.service
+[Unit]
+Description=containerd container runtime
+Documentation=https://containerd.io
+After=network.target
+# disable rate limiting
+StartLimitIntervalSec=0
+
+[Service]
+ExecStartPre=-/sbin/modprobe overlay
+ExecStart=/usr/local/bin/containerd
+Restart=always
+RestartSec=1
+
+Delegate=yes
+KillMode=process
+Restart=always
+# Having non-zero Limit*s causes performance problems due to accounting overhead
+# in the kernel. We recommend using cgroups to do container-local accounting.
+LimitNPROC=infinity
+LimitCORE=infinity
+LimitNOFILE=1048576
+# Comment TasksMax if your systemd version does not supports it.
+# Only systemd 226 and above support this version.
+TasksMax=infinity
+
+[Install]
+WantedBy=multi-user.target
diff --git a/deploy/kindbase/files/usr/local/bin/clean-install b/deploy/kindbase/files/usr/local/bin/clean-install
new file mode 100755
index 0000000000..33b3238b75
--- /dev/null
+++ b/deploy/kindbase/files/usr/local/bin/clean-install
@@ -0,0 +1,39 @@
+#!/bin/sh
+
+# Copyright 2017 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# A script encapsulating a common Dockerimage pattern for installing packages
+# and then cleaning up the unnecessary install artifacts.
+# e.g. clean-install iptables ebtables conntrack
+
+set -o errexit
+
+if [ $# = 0 ]; then
+  echo >&2 "No packages specified"
+  exit 1
+fi
+
+apt-get update
+apt-get install -y --no-install-recommends "$@"
+apt-get clean -y
+rm -rf \
+   /var/cache/debconf/* \
+   /var/lib/apt/lists/* \
+   /var/log/* \
+   /tmp/* \
+   /var/tmp/* \
+   /usr/share/doc/* \
+   /usr/share/man/* \
+   /usr/share/local/*
diff --git a/deploy/kindbase/files/usr/local/bin/entrypoint b/deploy/kindbase/files/usr/local/bin/entrypoint
new file mode 100755
index 0000000000..98dc77022a
--- /dev/null
+++ b/deploy/kindbase/files/usr/local/bin/entrypoint
@@ -0,0 +1,235 @@
+#!/bin/bash
+
+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+fix_mount() {
+  echo 'INFO: ensuring we can execute /bin/mount even with userns-remap' 
+  # necessary only when userns-remap is enabled on the host, but harmless
+  # The binary /bin/mount should be owned by root and have the setuid bit
+  chown root:root /bin/mount
+  chmod -s /bin/mount
+
+  # This is a workaround to an AUFS bug that might cause `Text file
+  # busy` on `mount` command below. See more details in
+  # https://github.com/moby/moby/issues/9547
+  if [[ "$(stat -f -c %T /bin/mount)" == 'aufs' ]]; then
+    echo 'INFO: detected aufs, calling sync' >&2
+    sync
+  fi
+
+  echo 'INFO: remounting /sys read-only'
+  # systemd-in-a-container should have read only /sys
+  # https://www.freedesktop.org/wiki/Software/systemd/ContainerInterface/
+  # however, we need other things from `docker run --privileged` ...
+  # and this flag also happens to make /sys rw, amongst other things
+  mount -o remount,ro /sys
+
+  echo 'INFO: making mounts shared' >&2
+  # for mount propagation
+  mount --make-rshared /
+}
+
+fix_cgroup() {
+  echo 'INFO: fix cgroup mounts for all subsystems'
+  # For each cgroup subsystem, Docker does a bind mount from the current
+  # cgroup to the root of the cgroup subsystem. For instance:
+  #   /sys/fs/cgroup/memory/docker/<cid> -> /sys/fs/cgroup/memory
+  #
+  # This will confuse Kubelet and cadvisor and will dump the following error
+  # messages in kubelet log:
+  #   `summary_sys_containers.go:47] Failed to get system container stats for ".../kubelet.service"`
+  #
+  # This is because `/proc/<pid>/cgroup` is not affected by the bind mount.
+  # The following is a workaround to recreate the original cgroup
+  # environment by doing another bind mount for each subsystem.
+  local docker_cgroup_mounts
+  docker_cgroup_mounts=$(grep /sys/fs/cgroup /proc/self/mountinfo | grep docker || true)
+  if [[ -n "${docker_cgroup_mounts}" ]]; then
+    local docker_cgroup cgroup_subsystems subsystem
+    docker_cgroup=$(echo "${docker_cgroup_mounts}" | head -n 1 | cut -d' ' -f 4)
+    cgroup_subsystems=$(echo "${docker_cgroup_mounts}" | cut -d' ' -f 5)
+    echo "${cgroup_subsystems}" |
+    while IFS= read -r subsystem; do
+      mkdir -p "${subsystem}${docker_cgroup}"
+      mount --bind "${subsystem}" "${subsystem}${docker_cgroup}"
+    done
+  fi
+}
+
+fix_machine_id() {
+  # Deletes the machine-id embedded in the node image and generates a new one.
+  # This is necessary because both kubelet and other components like weave net
+  # use machine-id internally to distinguish nodes.
+  echo 'INFO: clearing and regenerating /etc/machine-id' >&2
+  rm -f /etc/machine-id
+  systemd-machine-id-setup
+}
+
+fix_product_name() {
+  # this is a small fix to hide the underlying hardware and fix issue #426
+  # https://github.com/kubernetes-sigs/kind/issues/426
+  if [[ -f /sys/class/dmi/id/product_name ]]; then
+    echo 'INFO: faking /sys/class/dmi/id/product_name to be "kind"' >&2
+    echo 'kind' > /kind/product_name
+    mount -o ro,bind /kind/product_name /sys/class/dmi/id/product_name
+  fi
+}
+
+fix_product_uuid() {
+  # The system UUID is usually read from DMI via sysfs, the problem is that
+  # in the kind case this means that all (container) nodes share the same
+  # system/product uuid, as they share the same DMI.
+  # Note: The UUID is read from DMI, this tool is overwriting the sysfs files
+  # which should fix the attached issue, but this workaround does not address
+  # the issue if a tool is reading directly from DMI.
+  # https://github.com/kubernetes-sigs/kind/issues/1027
+  [[ ! -f /kind/product_uuid ]] && cat /proc/sys/kernel/random/uuid > /kind/product_uuid
+  if [[ -f /sys/class/dmi/id/product_uuid ]]; then
+    echo 'INFO: faking /sys/class/dmi/id/product_uuid to be random' >&2
+    mount -o ro,bind /kind/product_uuid /sys/class/dmi/id/product_uuid
+  fi
+  if [[ -f /sys/devices/virtual/dmi/id/product_uuid ]]; then
+    echo 'INFO: faking /sys/devices/virtual/dmi/id/product_uuid as well' >&2
+    mount -o ro,bind /kind/product_uuid /sys/devices/virtual/dmi/id/product_uuid
+  fi
+}
+
+fix_kmsg() {
+  # In environments where /dev/kmsg is not available, the kubelet (1.15+) won't
+  # start because it cannot open /dev/kmsg when starting the kmsgparser in the
+  # OOM parser.
+  # To support those environments, we link /dev/kmsg to /dev/console.
+  # https://github.com/kubernetes-sigs/kind/issues/662
+  if [[ ! -e /dev/kmsg ]]; then
+    if [[ -e /dev/console ]]; then
+      echo 'WARN: /dev/kmsg does not exist, symlinking /dev/console' >&2
+      ln -s /dev/console /dev/kmsg
+    else
+      echo 'WARN: /dev/kmsg does not exist, nor does /dev/console!' >&2
+    fi
+  fi
+}
+
+configure_proxy() {
+  # ensure all processes receive the proxy settings by default
+  # https://www.freedesktop.org/software/systemd/man/systemd-system.conf.html
+  mkdir -p /etc/systemd/system.conf.d/
+  cat <<EOF >/etc/systemd/system.conf.d/proxy-default-environment.conf
+[Manager]
+DefaultEnvironment="HTTP_PROXY=${HTTP_PROXY:-}" "HTTPS_PROXY=${HTTPS_PROXY:-}" "NO_PROXY=${NO_PROXY:-}"
+EOF
+}
+
+select_iptables() {
+  # based on: https://github.com/kubernetes/kubernetes/blob/ffe93b3979486feb41a0f85191bdd189cbd56ccc/build/debian-iptables/iptables-wrapper
+  local mode=nft
+  num_legacy_lines=$( (iptables-legacy-save || true; ip6tables-legacy-save || true) 2>/dev/null | grep '^-' | wc -l || true)
+  if [ "${num_legacy_lines}" -ge 10 ]; then
+    mode=legacy
+  else
+    num_nft_lines=$( (timeout 5 sh -c "iptables-nft-save; ip6tables-nft-save" || true) 2>/dev/null | grep '^-' | wc -l || true)
+    if [ "${num_legacy_lines}" -ge "${num_nft_lines}" ]; then
+      mode=legacy
+    fi
+  fi
+
+  echo "INFO: setting iptables to detected mode: ${mode}" >&2
+  update-alternatives --set iptables "/usr/sbin/iptables-${mode}" > /dev/null
+  update-alternatives --set ip6tables "/usr/sbin/ip6tables-${mode}" > /dev/null
+}
+
+enable_network_magic(){
+  # well-known docker embedded DNS is at 127.0.0.11:53
+  local docker_embedded_dns_ip='127.0.0.11'
+
+  # first we need to detect an IP to use for reaching the docker host
+  local docker_host_ip
+  docker_host_ip="$( (getent ahostsv4 'host.docker.internal' | head -n1 | cut -d' ' -f1) || true)"
+  if [[ -z "${docker_host_ip}" ]]; then
+    docker_host_ip=$(ip -4 route show default | cut -d' ' -f3)
+  fi
+
+  # patch docker's iptables rules to switch out the DNS IP
+  iptables-save \
+    | sed \
+      `# switch docker DNS DNAT rules to our chosen IP` \
+      -e "s/-d ${docker_embedded_dns_ip}/-d ${docker_host_ip}/g" \
+      `# we need to also apply these rules to non-local traffic (from pods)` \
+      -e 's/-A OUTPUT \(.*\) -j DOCKER_OUTPUT/\0\n-A PREROUTING \1 -j DOCKER_OUTPUT/' \
+      `# switch docker DNS SNAT rules rules to our chosen IP` \
+      -e "s/--to-source :53/--to-source ${docker_host_ip}:53/g"\
+    | iptables-restore
+
+  # now we can ensure that DNS is configured to use our IP
+  cp /etc/resolv.conf /etc/resolv.conf.original
+  sed -e "s/${docker_embedded_dns_ip}/${docker_host_ip}/g" /etc/resolv.conf.original >/etc/resolv.conf
+
+  # fixup IPs in manifests ...
+  curr_ipv4="$( (getent ahostsv4 $(hostname) | head -n1 | cut -d' ' -f1) || true)"
+  echo "INFO: Detected IPv4 address: ${curr_ipv4}" >&2
+  if [ -f /kind/old-ipv4 ]; then
+      old_ipv4=$(cat /kind/old-ipv4)
+      echo "INFO: Detected old IPv4 address: ${old_ipv4}" >&2
+      # sanity check that we have a current address
+      if [[ -z $curr_ipv4 ]]; then
+        echo "ERROR: Have an old IPv4 address but no current IPv4 address (!)" >&2
+        exit 1
+      fi
+      # kubernetes manifests are only present on control-plane nodes
+      sed -i "s#${old_ipv4}#${curr_ipv4}#" /etc/kubernetes/manifests/*.yaml || true
+      # this is no longer required with autodiscovery
+      sed -i "s#${old_ipv4}#${curr_ipv4}#" /var/lib/kubelet/kubeadm-flags.env || true
+  fi
+  if [[ -n $curr_ipv4 ]]; then
+    echo -n "${curr_ipv4}" >/kind/old-ipv4
+  fi
+
+  # do IPv6
+  curr_ipv6="$( (getent ahostsv6 $(hostname) | head -n1 | cut -d' ' -f1) || true)"
+  echo "INFO: Detected IPv6 address: ${curr_ipv6}" >&2
+  if [ -f /kind/old-ipv6 ]; then
+      old_ipv6=$(cat /kind/old-ipv6)
+      echo "INFO: Detected old IPv6 address: ${old_ipv6}" >&2
+      # sanity check that we have a current address
+      if [[ -z $curr_ipv6 ]]; then
+        echo "ERROR: Have an old IPv6 address but no current IPv6 address (!)" >&2
+      fi
+      # kubernetes manifests are only present on control-plane nodes
+      sed -i "s#${old_ipv6}#${curr_ipv6}#" /etc/kubernetes/manifests/*.yaml || true
+      # this is no longer required with autodiscovery
+      sed -i "s#${old_ipv6}#${curr_ipv6}#" /var/lib/kubelet/kubeadm-flags.env || true
+  fi
+  if [[ -n $curr_ipv6 ]]; then
+    echo -n "${curr_ipv6}" >/kind/old-ipv6
+  fi
+}
+
+# run pre-init fixups
+fix_kmsg
+fix_mount
+fix_cgroup
+fix_machine_id
+fix_product_name
+fix_product_uuid
+configure_proxy
+select_iptables
+enable_network_magic
+
+# we want the command (expected to be systemd) to be PID1, so exec to it
+exec "$@"

From a4e4a719ba3f775ba0563d7a12913ce3af367463 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= <anders.f.bjorklund@gmail.com>
Date: Sat, 12 Sep 2020 12:40:16 +0200
Subject: [PATCH 32/65] Specify snapshot version of ubuntu used for kind

Also fix the 19.10 comment, it hadn't been updated
---
 deploy/kindbase/Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/deploy/kindbase/Dockerfile b/deploy/kindbase/Dockerfile
index d03bb0a62e..bdd3b38c84 100644
--- a/deploy/kindbase/Dockerfile
+++ b/deploy/kindbase/Dockerfile
@@ -17,9 +17,9 @@
 # For systemd + docker configuration used below, see the following references:
 # https://www.freedesktop.org/wiki/Software/systemd/ContainerInterface/
 
-# start from ubuntu 19.10, this image is reasonably small as a starting point
+# start from ubuntu 20.04, this image is reasonably small as a starting point
 # for a kubernetes node image, it doesn't contain much we don't need
-FROM ubuntu:20.04
+FROM ubuntu:focal-20200423
 
 # Configure containerd and runc binaries from kind-ci/containerd-nightlies repository
 # The repository contains latest stable releases and nightlies built for multiple architectures

From 8a18334cba52db1b960060dd0020f54b6e7c352d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= <anders.f.bjorklund@gmail.com>
Date: Sun, 27 Sep 2020 15:00:33 +0200
Subject: [PATCH 33/65] Complete the fork of kindbase by copy/paste

Base kicbase directly on ubuntu, without kindbase

Move all the files that are still copied and used

Avoid installing software only to be deleted later

The entrypoint had already been forked since earlier
---
 Makefile                                      |  11 +-
 .../10-network-security.conf                  |   0
 deploy/kicbase/Dockerfile                     | 103 ++++++--
 .../usr/local/bin => kicbase}/clean-install   |   0
 deploy/kindbase/Dockerfile                    | 122 ---------
 deploy/kindbase/files/etc/crictl.yaml         |   1 -
 .../etc/systemd/system/containerd.service     |  29 ---
 .../kindbase/files/usr/local/bin/entrypoint   | 235 ------------------
 8 files changed, 85 insertions(+), 416 deletions(-)
 rename deploy/{kindbase/files/etc/sysctl.d => kicbase}/10-network-security.conf (100%)
 rename deploy/{kindbase/files/usr/local/bin => kicbase}/clean-install (100%)
 delete mode 100644 deploy/kindbase/Dockerfile
 delete mode 100644 deploy/kindbase/files/etc/crictl.yaml
 delete mode 100644 deploy/kindbase/files/etc/systemd/system/containerd.service
 delete mode 100755 deploy/kindbase/files/usr/local/bin/entrypoint

diff --git a/Makefile b/Makefile
index 98e4b3c1e0..2a87f19f67 100644
--- a/Makefile
+++ b/Makefile
@@ -20,7 +20,6 @@ RAW_VERSION=$(VERSION_MAJOR).$(VERSION_MINOR).$(VERSION_BUILD)
 VERSION ?= v$(RAW_VERSION)
 
 KUBERNETES_VERSION ?= $(shell egrep "DefaultKubernetesVersion =" pkg/minikube/constants/constants.go | cut -d \" -f2)
-KIND_VERSION ?= v20200430-2c0eee40
 KIC_VERSION ?= $(shell egrep "Version =" pkg/drivers/kic/types.go | cut -d \" -f2)
 
 # Default to .0 for higher cache hit rates, as build increments typically don't require new ISO versions
@@ -48,7 +47,6 @@ BUILD_IMAGE 	?= us.gcr.io/k8s-artifacts-prod/build-image/kube-cross:v$(GO_VERSIO
 ISO_BUILD_IMAGE ?= $(REGISTRY)/buildroot-image
 KVM_BUILD_IMAGE ?= $(REGISTRY)/kvm-build-image:$(GO_VERSION)
 
-KIND_BASE_IMAGE_GCR ?= $(REGISTRY)/kindbase:$(KIND_VERSION)
 KIC_BASE_IMAGE_GCR ?= $(REGISTRY)/kicbase:$(KIC_VERSION)
 KIC_BASE_IMAGE_GH ?= $(REGISTRY_GH)/kicbase:$(KIC_VERSION)
 KIC_BASE_IMAGE_HUB ?= kicbase/stable:$(KIC_VERSION)
@@ -578,15 +576,8 @@ endif
 storage-provisioner-image: out/storage-provisioner-$(GOARCH) ## Build storage-provisioner docker image
 	docker build -t $(STORAGE_PROVISIONER_IMAGE) -f deploy/storage-provisioner/Dockerfile  --build-arg arch=$(GOARCH) .
 
-.PHONY: kind-base-image
-kind-base-image: ## builds the base image used for kind.
-	docker rmi -f $(KIND_BASE_IMAGE_GCR)-snapshot || true
-	docker build -f ./deploy/kindbase/Dockerfile -t local/kindbase:$(KIND_VERSION)-snapshot ./deploy/kindbase
-	docker tag local/kindbase:$(KIND_VERSION)-snapshot $(KIND_BASE_IMAGE_GCR)-snapshot
-	docker tag local/kindbase:$(KIND_VERSION)-snapshot $(KIND_BASE_IMAGE_GCR)
-
 .PHONY: kic-base-image
-kic-base-image: kind-base-image ## builds the base image used for kic.
+kic-base-image: ## builds the base image used for kic.
 	docker rmi -f $(KIC_BASE_IMAGE_GCR)-snapshot || true
 	docker build -f ./deploy/kicbase/Dockerfile -t local/kicbase:$(KIC_VERSION)-snapshot  --build-arg COMMIT_SHA=${VERSION}-$(COMMIT) --cache-from $(KIC_BASE_IMAGE_GCR) --target base ./deploy/kicbase
 	docker tag local/kicbase:$(KIC_VERSION)-snapshot $(KIC_BASE_IMAGE_GCR)-snapshot
diff --git a/deploy/kindbase/files/etc/sysctl.d/10-network-security.conf b/deploy/kicbase/10-network-security.conf
similarity index 100%
rename from deploy/kindbase/files/etc/sysctl.d/10-network-security.conf
rename to deploy/kicbase/10-network-security.conf
diff --git a/deploy/kicbase/Dockerfile b/deploy/kicbase/Dockerfile
index 00ee69d6af..13b32a3629 100644
--- a/deploy/kicbase/Dockerfile
+++ b/deploy/kicbase/Dockerfile
@@ -1,26 +1,92 @@
+# Copyright 2018 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# kind node base image
+#
+# For systemd + docker configuration used below, see the following references:
+# https://www.freedesktop.org/wiki/Software/systemd/ContainerInterface/
+
+# start from ubuntu 20.04, this image is reasonably small as a starting point
+# for a kubernetes node image, it doesn't contain much we don't need
+FROM ubuntu:focal-20200423
+
+# copy in static files (configs, scripts)
+COPY 10-network-security.conf /etc/sysctl.d/10-network-security.conf
+COPY clean-install /usr/local/bin/clean-install
+COPY entrypoint /usr/local/bin/entrypoint
+
+# Install dependencies, first from apt, then from release tarballs.
+# NOTE: we use one RUN to minimize layers.
+#
+# First we must ensure that our util scripts are executable.
+#
+# The base image already has: ssh, apt, snapd, but we need to install more packages.
+# Packages installed are broken down into (each on a line):
+# - packages needed to run services (systemd)
+# - packages needed for kubernetes components
+# - packages needed by the container runtime
+# - misc packages kind uses itself
+# After installing packages we cleanup by:
+# - removing unwanted systemd services
+# - disabling kmsg in journald (these log entries would be confusing)
+#
+# Next we ensure the /etc/kubernetes/manifests directory exists. Normally
+# a kubeadm debain / rpm package would ensure that this exists but we install
+# freshly built binaries directly when we build the node image.
+#
+# Finally we adjust tempfiles cleanup to be 1 minute after "boot" instead of 15m
+# This is plenty after we've done initial setup for a node, but before we are
+# likely to try to export logs etc.
+RUN echo "Ensuring scripts are executable ..." \
+    && chmod +x /usr/local/bin/clean-install /usr/local/bin/entrypoint \
+ && echo "Installing Packages ..." \
+    && DEBIAN_FRONTEND=noninteractive clean-install \
+      systemd \
+      conntrack iptables iproute2 ethtool socat util-linux mount ebtables udev kmod \
+      libseccomp2 \
+      bash ca-certificates curl rsync \
+    && find /lib/systemd/system/sysinit.target.wants/ -name "systemd-tmpfiles-setup.service" -delete \
+    && rm -f /lib/systemd/system/multi-user.target.wants/* \
+    && rm -f /etc/systemd/system/*.wants/* \
+    && rm -f /lib/systemd/system/local-fs.target.wants/* \
+    && rm -f /lib/systemd/system/sockets.target.wants/*udev* \
+    && rm -f /lib/systemd/system/sockets.target.wants/*initctl* \
+    && rm -f /lib/systemd/system/basic.target.wants/* \
+    && echo "ReadKMsg=no" >> /etc/systemd/journald.conf \
+    && ln -s "$(which systemd)" /sbin/init \
+ && echo "Ensuring /etc/kubernetes/manifests" \
+    && mkdir -p /etc/kubernetes/manifests \
+ && echo "Adjusting systemd-tmpfiles timer" \
+    && sed -i /usr/lib/systemd/system/systemd-tmpfiles-clean.timer -e 's#OnBootSec=.*#OnBootSec=1min#' \
+ && echo "Modifying /etc/nsswitch.conf to prefer hosts" \
+    && sed -i /etc/nsswitch.conf -re 's#^(hosts:\s*).*#\1dns files#'
+
+# tell systemd that it is in docker (it will check for the container env)
+# https://www.freedesktop.org/wiki/Software/systemd/ContainerInterface/
+ENV container docker
+# systemd exits on SIGRTMIN+3, not SIGTERM (which re-executes it)
+# https://bugzilla.redhat.com/show_bug.cgi?id=1201657
+STOPSIGNAL SIGRTMIN+3
+# NOTE: this is *only* for documentation, the entrypoint is overridden later
+ENTRYPOINT [ "/usr/local/bin/entrypoint", "/sbin/init" ]
+
 ARG COMMIT_SHA
-# using base image created by kind https://github.com/kubernetes-sigs/kind/blob/v0.8.1/images/base/Dockerfile
+# using base image created by kind https://github.com/kubernetes-sigs/kind/blob/2c0eee40/images/base/Dockerfile
 # which is an ubuntu 20.04 with an entry-point that helps running systemd
 # could be changed to any debian that can run systemd
-FROM gcr.io/k8s-minikube/kindbase:v20200430-2c0eee40 as base
 USER root
 
-# remove files that were installed by kind, replaced by packages
-RUN rm \
-    /etc/crictl.yaml \
-    /etc/systemd/system/containerd.service \
-    /etc/systemd/system/multi-user.target.wants/containerd.service \
-    /opt/cni/bin/host-local \
-    /opt/cni/bin/loopback \
-    /opt/cni/bin/portmap \
-    /opt/cni/bin/ptp \
-    /usr/local/bin/containerd \
-    /usr/local/bin/containerd-shim \
-    /usr/local/bin/containerd-shim-runc-v2 \
-    /usr/local/bin/crictl \
-    /usr/local/bin/ctr \
-    /usr/local/sbin/runc
-
 # install system requirements from the regular distro repositories
 RUN clean-install \
     lz4 \
@@ -54,7 +120,6 @@ RUN sh -c "echo 'deb https://dl.bintray.com/afbjorklund/podman focal main' > /et
 
 RUN mkdir -p /usr/lib/cri-o-runc/sbin && cp /usr/sbin/runc /usr/lib/cri-o-runc/sbin/runc
 
-COPY entrypoint /usr/local/bin/entrypoint
 # automount service
 COPY automount/minikube-automount /usr/sbin/minikube-automount
 COPY automount/minikube-automount.service /usr/lib/systemd/system/minikube-automount.service
diff --git a/deploy/kindbase/files/usr/local/bin/clean-install b/deploy/kicbase/clean-install
similarity index 100%
rename from deploy/kindbase/files/usr/local/bin/clean-install
rename to deploy/kicbase/clean-install
diff --git a/deploy/kindbase/Dockerfile b/deploy/kindbase/Dockerfile
deleted file mode 100644
index bdd3b38c84..0000000000
--- a/deploy/kindbase/Dockerfile
+++ /dev/null
@@ -1,122 +0,0 @@
-# Copyright 2018 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# kind node base image
-#
-# For systemd + docker configuration used below, see the following references:
-# https://www.freedesktop.org/wiki/Software/systemd/ContainerInterface/
-
-# start from ubuntu 20.04, this image is reasonably small as a starting point
-# for a kubernetes node image, it doesn't contain much we don't need
-FROM ubuntu:focal-20200423
-
-# Configure containerd and runc binaries from kind-ci/containerd-nightlies repository
-# The repository contains latest stable releases and nightlies built for multiple architectures
-ARG CONTAINERD_VERSION="v1.3.3-14-g449e9269"
-# Configure CNI binaries from upstream
-ARG CNI_VERSION="v0.8.5"
-# Configure crictl binary from upstream
-ARG CRICTL_VERSION="v1.18.0"
-
-# copy in static files (configs, scripts)
-COPY files/ /
-
-# Install dependencies, first from apt, then from release tarballs.
-# NOTE: we use one RUN to minimize layers.
-#
-# First we must ensure that our util scripts are executable.
-#
-# The base image already has: ssh, apt, snapd, but we need to install more packages.
-# Packages installed are broken down into (each on a line):
-# - packages needed to run services (systemd)
-# - packages needed for kubernetes components
-# - packages needed by the container runtime
-# - misc packages kind uses itself
-# After installing packages we cleanup by:
-# - removing unwanted systemd services
-# - disabling kmsg in journald (these log entries would be confusing)
-#
-# Then we install containerd from our nightly build infrastructure, as this
-# build for multiple architectures and allows us to upgrade to patched releases
-# more quickly.
-#
-# Next we download and extract crictl and CNI plugin binaries from upstream.
-#
-# Next we ensure the /etc/kubernetes/manifests directory exists. Normally
-# a kubeadm debain / rpm package would ensure that this exists but we install
-# freshly built binaries directly when we build the node image.
-#
-# Finally we adjust tempfiles cleanup to be 1 minute after "boot" instead of 15m
-# This is plenty after we've done initial setup for a node, but before we are
-# likely to try to export logs etc.
-RUN echo "Ensuring scripts are executable ..." \
-    && chmod +x /usr/local/bin/clean-install /usr/local/bin/entrypoint \
- && echo "Installing Packages ..." \
-    && DEBIAN_FRONTEND=noninteractive clean-install \
-      systemd \
-      conntrack iptables iproute2 ethtool socat util-linux mount ebtables udev kmod \
-      libseccomp2 \
-      bash ca-certificates curl rsync \
-    && find /lib/systemd/system/sysinit.target.wants/ -name "systemd-tmpfiles-setup.service" -delete \
-    && rm -f /lib/systemd/system/multi-user.target.wants/* \
-    && rm -f /etc/systemd/system/*.wants/* \
-    && rm -f /lib/systemd/system/local-fs.target.wants/* \
-    && rm -f /lib/systemd/system/sockets.target.wants/*udev* \
-    && rm -f /lib/systemd/system/sockets.target.wants/*initctl* \
-    && rm -f /lib/systemd/system/basic.target.wants/* \
-    && echo "ReadKMsg=no" >> /etc/systemd/journald.conf \
-    && ln -s "$(which systemd)" /sbin/init \
- && echo "Installing containerd ..." \
-    && export ARCH=$(dpkg --print-architecture | sed 's/ppc64el/ppc64le/' | sed 's/armhf/arm/') \
-    && export CONTAINERD_BASE_URL="https://github.com/kind-ci/containerd-nightlies/releases/download/containerd-${CONTAINERD_VERSION#v}" \
-    && curl -sSL --retry 5 --output /tmp/containerd.tgz "${CONTAINERD_BASE_URL}/containerd-${CONTAINERD_VERSION#v}.linux-${ARCH}.tar.gz" \
-    && tar -C /usr/local -xzvf /tmp/containerd.tgz \
-    && rm -rf /tmp/containerd.tgz \
-    && rm -f /usr/local/bin/containerd-stress /usr/local/bin/containerd-shim-runc-v1 \
-    && curl -sSL --retry 5 --output /usr/local/sbin/runc "${CONTAINERD_BASE_URL}/runc.${ARCH}" \
-    && chmod 755 /usr/local/sbin/runc \
-    && containerd --version \
-    && systemctl enable containerd \
- && echo "Installing crictl ..." \
-    && curl -fSL "https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-${ARCH}.tar.gz" | tar xzC /usr/local/bin \
- && echo "Installing CNI binaries ..." \
-    && export ARCH=$(dpkg --print-architecture | sed 's/ppc64el/ppc64le/' | sed 's/armhf/arm/') \
-    && export CNI_TARBALL="${CNI_VERSION}/cni-plugins-linux-${ARCH}-${CNI_VERSION}.tgz" \
-    && export CNI_URL="https://github.com/containernetworking/plugins/releases/download/${CNI_TARBALL}" \
-    && curl -sSL --retry 5 --output /tmp/cni.tgz "${CNI_URL}" \
-    && mkdir -p /opt/cni/bin \
-    && tar -C /opt/cni/bin -xzf /tmp/cni.tgz \
-    && rm -rf /tmp/cni.tgz \
-    && find /opt/cni/bin -type f -not \( \
-         -iname host-local \
-         -o -iname ptp \
-         -o -iname portmap \
-         -o -iname loopback \
-      \) \
-      -delete \
- && echo "Ensuring /etc/kubernetes/manifests" \
-    && mkdir -p /etc/kubernetes/manifests \
- && echo "Adjusting systemd-tmpfiles timer" \
-    && sed -i /usr/lib/systemd/system/systemd-tmpfiles-clean.timer -e 's#OnBootSec=.*#OnBootSec=1min#' \
- && echo "Modifying /etc/nsswitch.conf to prefer hosts" \
-    && sed -i /etc/nsswitch.conf -re 's#^(hosts:\s*).*#\1dns files#'
-
-# tell systemd that it is in docker (it will check for the container env)
-# https://www.freedesktop.org/wiki/Software/systemd/ContainerInterface/
-ENV container docker
-# systemd exits on SIGRTMIN+3, not SIGTERM (which re-executes it)
-# https://bugzilla.redhat.com/show_bug.cgi?id=1201657
-STOPSIGNAL SIGRTMIN+3
-# NOTE: this is *only* for documentation, the entrypoint is overridden later
-ENTRYPOINT [ "/usr/local/bin/entrypoint", "/sbin/init" ]
diff --git a/deploy/kindbase/files/etc/crictl.yaml b/deploy/kindbase/files/etc/crictl.yaml
deleted file mode 100644
index 1a5daba3b9..0000000000
--- a/deploy/kindbase/files/etc/crictl.yaml
+++ /dev/null
@@ -1 +0,0 @@
-runtime-endpoint: unix:///var/run/containerd/containerd.sock
\ No newline at end of file
diff --git a/deploy/kindbase/files/etc/systemd/system/containerd.service b/deploy/kindbase/files/etc/systemd/system/containerd.service
deleted file mode 100644
index fb97bf814c..0000000000
--- a/deploy/kindbase/files/etc/systemd/system/containerd.service
+++ /dev/null
@@ -1,29 +0,0 @@
-# derived containerd systemd service file from the official:
-# https://github.com/containerd/containerd/blob/master/containerd.service
-[Unit]
-Description=containerd container runtime
-Documentation=https://containerd.io
-After=network.target
-# disable rate limiting
-StartLimitIntervalSec=0
-
-[Service]
-ExecStartPre=-/sbin/modprobe overlay
-ExecStart=/usr/local/bin/containerd
-Restart=always
-RestartSec=1
-
-Delegate=yes
-KillMode=process
-Restart=always
-# Having non-zero Limit*s causes performance problems due to accounting overhead
-# in the kernel. We recommend using cgroups to do container-local accounting.
-LimitNPROC=infinity
-LimitCORE=infinity
-LimitNOFILE=1048576
-# Comment TasksMax if your systemd version does not supports it.
-# Only systemd 226 and above support this version.
-TasksMax=infinity
-
-[Install]
-WantedBy=multi-user.target
diff --git a/deploy/kindbase/files/usr/local/bin/entrypoint b/deploy/kindbase/files/usr/local/bin/entrypoint
deleted file mode 100755
index 98dc77022a..0000000000
--- a/deploy/kindbase/files/usr/local/bin/entrypoint
+++ /dev/null
@@ -1,235 +0,0 @@
-#!/bin/bash
-
-# Copyright 2019 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -o errexit
-set -o nounset
-set -o pipefail
-
-fix_mount() {
-  echo 'INFO: ensuring we can execute /bin/mount even with userns-remap' 
-  # necessary only when userns-remap is enabled on the host, but harmless
-  # The binary /bin/mount should be owned by root and have the setuid bit
-  chown root:root /bin/mount
-  chmod -s /bin/mount
-
-  # This is a workaround to an AUFS bug that might cause `Text file
-  # busy` on `mount` command below. See more details in
-  # https://github.com/moby/moby/issues/9547
-  if [[ "$(stat -f -c %T /bin/mount)" == 'aufs' ]]; then
-    echo 'INFO: detected aufs, calling sync' >&2
-    sync
-  fi
-
-  echo 'INFO: remounting /sys read-only'
-  # systemd-in-a-container should have read only /sys
-  # https://www.freedesktop.org/wiki/Software/systemd/ContainerInterface/
-  # however, we need other things from `docker run --privileged` ...
-  # and this flag also happens to make /sys rw, amongst other things
-  mount -o remount,ro /sys
-
-  echo 'INFO: making mounts shared' >&2
-  # for mount propagation
-  mount --make-rshared /
-}
-
-fix_cgroup() {
-  echo 'INFO: fix cgroup mounts for all subsystems'
-  # For each cgroup subsystem, Docker does a bind mount from the current
-  # cgroup to the root of the cgroup subsystem. For instance:
-  #   /sys/fs/cgroup/memory/docker/<cid> -> /sys/fs/cgroup/memory
-  #
-  # This will confuse Kubelet and cadvisor and will dump the following error
-  # messages in kubelet log:
-  #   `summary_sys_containers.go:47] Failed to get system container stats for ".../kubelet.service"`
-  #
-  # This is because `/proc/<pid>/cgroup` is not affected by the bind mount.
-  # The following is a workaround to recreate the original cgroup
-  # environment by doing another bind mount for each subsystem.
-  local docker_cgroup_mounts
-  docker_cgroup_mounts=$(grep /sys/fs/cgroup /proc/self/mountinfo | grep docker || true)
-  if [[ -n "${docker_cgroup_mounts}" ]]; then
-    local docker_cgroup cgroup_subsystems subsystem
-    docker_cgroup=$(echo "${docker_cgroup_mounts}" | head -n 1 | cut -d' ' -f 4)
-    cgroup_subsystems=$(echo "${docker_cgroup_mounts}" | cut -d' ' -f 5)
-    echo "${cgroup_subsystems}" |
-    while IFS= read -r subsystem; do
-      mkdir -p "${subsystem}${docker_cgroup}"
-      mount --bind "${subsystem}" "${subsystem}${docker_cgroup}"
-    done
-  fi
-}
-
-fix_machine_id() {
-  # Deletes the machine-id embedded in the node image and generates a new one.
-  # This is necessary because both kubelet and other components like weave net
-  # use machine-id internally to distinguish nodes.
-  echo 'INFO: clearing and regenerating /etc/machine-id' >&2
-  rm -f /etc/machine-id
-  systemd-machine-id-setup
-}
-
-fix_product_name() {
-  # this is a small fix to hide the underlying hardware and fix issue #426
-  # https://github.com/kubernetes-sigs/kind/issues/426
-  if [[ -f /sys/class/dmi/id/product_name ]]; then
-    echo 'INFO: faking /sys/class/dmi/id/product_name to be "kind"' >&2
-    echo 'kind' > /kind/product_name
-    mount -o ro,bind /kind/product_name /sys/class/dmi/id/product_name
-  fi
-}
-
-fix_product_uuid() {
-  # The system UUID is usually read from DMI via sysfs, the problem is that
-  # in the kind case this means that all (container) nodes share the same
-  # system/product uuid, as they share the same DMI.
-  # Note: The UUID is read from DMI, this tool is overwriting the sysfs files
-  # which should fix the attached issue, but this workaround does not address
-  # the issue if a tool is reading directly from DMI.
-  # https://github.com/kubernetes-sigs/kind/issues/1027
-  [[ ! -f /kind/product_uuid ]] && cat /proc/sys/kernel/random/uuid > /kind/product_uuid
-  if [[ -f /sys/class/dmi/id/product_uuid ]]; then
-    echo 'INFO: faking /sys/class/dmi/id/product_uuid to be random' >&2
-    mount -o ro,bind /kind/product_uuid /sys/class/dmi/id/product_uuid
-  fi
-  if [[ -f /sys/devices/virtual/dmi/id/product_uuid ]]; then
-    echo 'INFO: faking /sys/devices/virtual/dmi/id/product_uuid as well' >&2
-    mount -o ro,bind /kind/product_uuid /sys/devices/virtual/dmi/id/product_uuid
-  fi
-}
-
-fix_kmsg() {
-  # In environments where /dev/kmsg is not available, the kubelet (1.15+) won't
-  # start because it cannot open /dev/kmsg when starting the kmsgparser in the
-  # OOM parser.
-  # To support those environments, we link /dev/kmsg to /dev/console.
-  # https://github.com/kubernetes-sigs/kind/issues/662
-  if [[ ! -e /dev/kmsg ]]; then
-    if [[ -e /dev/console ]]; then
-      echo 'WARN: /dev/kmsg does not exist, symlinking /dev/console' >&2
-      ln -s /dev/console /dev/kmsg
-    else
-      echo 'WARN: /dev/kmsg does not exist, nor does /dev/console!' >&2
-    fi
-  fi
-}
-
-configure_proxy() {
-  # ensure all processes receive the proxy settings by default
-  # https://www.freedesktop.org/software/systemd/man/systemd-system.conf.html
-  mkdir -p /etc/systemd/system.conf.d/
-  cat <<EOF >/etc/systemd/system.conf.d/proxy-default-environment.conf
-[Manager]
-DefaultEnvironment="HTTP_PROXY=${HTTP_PROXY:-}" "HTTPS_PROXY=${HTTPS_PROXY:-}" "NO_PROXY=${NO_PROXY:-}"
-EOF
-}
-
-select_iptables() {
-  # based on: https://github.com/kubernetes/kubernetes/blob/ffe93b3979486feb41a0f85191bdd189cbd56ccc/build/debian-iptables/iptables-wrapper
-  local mode=nft
-  num_legacy_lines=$( (iptables-legacy-save || true; ip6tables-legacy-save || true) 2>/dev/null | grep '^-' | wc -l || true)
-  if [ "${num_legacy_lines}" -ge 10 ]; then
-    mode=legacy
-  else
-    num_nft_lines=$( (timeout 5 sh -c "iptables-nft-save; ip6tables-nft-save" || true) 2>/dev/null | grep '^-' | wc -l || true)
-    if [ "${num_legacy_lines}" -ge "${num_nft_lines}" ]; then
-      mode=legacy
-    fi
-  fi
-
-  echo "INFO: setting iptables to detected mode: ${mode}" >&2
-  update-alternatives --set iptables "/usr/sbin/iptables-${mode}" > /dev/null
-  update-alternatives --set ip6tables "/usr/sbin/ip6tables-${mode}" > /dev/null
-}
-
-enable_network_magic(){
-  # well-known docker embedded DNS is at 127.0.0.11:53
-  local docker_embedded_dns_ip='127.0.0.11'
-
-  # first we need to detect an IP to use for reaching the docker host
-  local docker_host_ip
-  docker_host_ip="$( (getent ahostsv4 'host.docker.internal' | head -n1 | cut -d' ' -f1) || true)"
-  if [[ -z "${docker_host_ip}" ]]; then
-    docker_host_ip=$(ip -4 route show default | cut -d' ' -f3)
-  fi
-
-  # patch docker's iptables rules to switch out the DNS IP
-  iptables-save \
-    | sed \
-      `# switch docker DNS DNAT rules to our chosen IP` \
-      -e "s/-d ${docker_embedded_dns_ip}/-d ${docker_host_ip}/g" \
-      `# we need to also apply these rules to non-local traffic (from pods)` \
-      -e 's/-A OUTPUT \(.*\) -j DOCKER_OUTPUT/\0\n-A PREROUTING \1 -j DOCKER_OUTPUT/' \
-      `# switch docker DNS SNAT rules rules to our chosen IP` \
-      -e "s/--to-source :53/--to-source ${docker_host_ip}:53/g"\
-    | iptables-restore
-
-  # now we can ensure that DNS is configured to use our IP
-  cp /etc/resolv.conf /etc/resolv.conf.original
-  sed -e "s/${docker_embedded_dns_ip}/${docker_host_ip}/g" /etc/resolv.conf.original >/etc/resolv.conf
-
-  # fixup IPs in manifests ...
-  curr_ipv4="$( (getent ahostsv4 $(hostname) | head -n1 | cut -d' ' -f1) || true)"
-  echo "INFO: Detected IPv4 address: ${curr_ipv4}" >&2
-  if [ -f /kind/old-ipv4 ]; then
-      old_ipv4=$(cat /kind/old-ipv4)
-      echo "INFO: Detected old IPv4 address: ${old_ipv4}" >&2
-      # sanity check that we have a current address
-      if [[ -z $curr_ipv4 ]]; then
-        echo "ERROR: Have an old IPv4 address but no current IPv4 address (!)" >&2
-        exit 1
-      fi
-      # kubernetes manifests are only present on control-plane nodes
-      sed -i "s#${old_ipv4}#${curr_ipv4}#" /etc/kubernetes/manifests/*.yaml || true
-      # this is no longer required with autodiscovery
-      sed -i "s#${old_ipv4}#${curr_ipv4}#" /var/lib/kubelet/kubeadm-flags.env || true
-  fi
-  if [[ -n $curr_ipv4 ]]; then
-    echo -n "${curr_ipv4}" >/kind/old-ipv4
-  fi
-
-  # do IPv6
-  curr_ipv6="$( (getent ahostsv6 $(hostname) | head -n1 | cut -d' ' -f1) || true)"
-  echo "INFO: Detected IPv6 address: ${curr_ipv6}" >&2
-  if [ -f /kind/old-ipv6 ]; then
-      old_ipv6=$(cat /kind/old-ipv6)
-      echo "INFO: Detected old IPv6 address: ${old_ipv6}" >&2
-      # sanity check that we have a current address
-      if [[ -z $curr_ipv6 ]]; then
-        echo "ERROR: Have an old IPv6 address but no current IPv6 address (!)" >&2
-      fi
-      # kubernetes manifests are only present on control-plane nodes
-      sed -i "s#${old_ipv6}#${curr_ipv6}#" /etc/kubernetes/manifests/*.yaml || true
-      # this is no longer required with autodiscovery
-      sed -i "s#${old_ipv6}#${curr_ipv6}#" /var/lib/kubelet/kubeadm-flags.env || true
-  fi
-  if [[ -n $curr_ipv6 ]]; then
-    echo -n "${curr_ipv6}" >/kind/old-ipv6
-  fi
-}
-
-# run pre-init fixups
-fix_kmsg
-fix_mount
-fix_cgroup
-fix_machine_id
-fix_product_name
-fix_product_uuid
-configure_proxy
-select_iptables
-enable_network_magic
-
-# we want the command (expected to be systemd) to be PID1, so exec to it
-exec "$@"

From 06ac270741da5840245b27891e1a31fdb95cd9b8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= <anders.f.bjorklund@gmail.com>
Date: Sun, 27 Sep 2020 15:26:35 +0200
Subject: [PATCH 34/65] The ISO build doesn't need Go installed on host

Buildroot will bootstrap the Go compiler anyway
---
 .github/workflows/iso.yml | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/.github/workflows/iso.yml b/.github/workflows/iso.yml
index 785ab71a14..7294872fc2 100644
--- a/.github/workflows/iso.yml
+++ b/.github/workflows/iso.yml
@@ -64,11 +64,6 @@ jobs:
           echo workspace $GITHUB_WORKSPACE
           echo "end of debug stuff"
           echo $(which jq)
-      # iso needs golang 1.11.3 
-      - uses: actions/setup-go@v2
-        with:
-          go-version: '1.11.13'
-          stable: true
       - name: Build ISO
         run: |
           whoami
@@ -146,4 +141,4 @@ jobs:
           if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
           if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
           if [ "$numPass" -lt 32 ];then echo "*** Failed to pass at least 32 ! ***";exit 2;fi
-          if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
\ No newline at end of file
+          if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi

From 5ae6ce03bb65becb1a6058c23f98569270c1e5f3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= <anders.f.bjorklund@gmail.com>
Date: Sun, 27 Sep 2020 17:35:49 +0200
Subject: [PATCH 35/65] Use newer version of the go-bindata tool

Use the same "vendor" as kubernetes does
---
 Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 2a87f19f67..95420d333b 100644
--- a/Makefile
+++ b/Makefile
@@ -331,7 +331,7 @@ pkg/minikube/assets/assets.go: $(shell find "deploy/addons" -type f)
 ifeq ($(MINIKUBE_BUILD_IN_DOCKER),y)
 	$(call DOCKER,$(BUILD_IMAGE),/usr/bin/make $@)
 endif
-	@which go-bindata >/dev/null 2>&1 || GO111MODULE=off GOBIN="$(GOPATH)$(DIRSEP)bin" go get github.com/jteeuwen/go-bindata/...
+	@which go-bindata >/dev/null 2>&1 || GO111MODULE=off GOBIN="$(GOPATH)$(DIRSEP)bin" go get github.com/go-bindata/go-bindata/...
 	$(if $(quiet),@echo "  GEN      $@")
 	$(Q)PATH="$(PATH)$(PATHSEP)$(GOPATH)$(DIRSEP)bin" go-bindata -nomemcopy -o $@ -pkg assets deploy/addons/...
 	$(Q)-gofmt -s -w $@
@@ -344,7 +344,7 @@ pkg/minikube/translate/translations.go: $(shell find "translations/" -type f)
 ifeq ($(MINIKUBE_BUILD_IN_DOCKER),y)
 	$(call DOCKER,$(BUILD_IMAGE),/usr/bin/make $@)
 endif
-	@which go-bindata >/dev/null 2>&1 || GO111MODULE=off GOBIN="$(GOPATH)$(DIRSEP)bin" go get github.com/jteeuwen/go-bindata/...
+	@which go-bindata >/dev/null 2>&1 || GO111MODULE=off GOBIN="$(GOPATH)$(DIRSEP)bin" go get github.com/go-bindata/go-bindata/...
 	$(if $(quiet),@echo "  GEN      $@")
 	$(Q)PATH="$(PATH)$(PATHSEP)$(GOPATH)$(DIRSEP)bin" go-bindata -nomemcopy -o $@ -pkg translate translations/...
 	$(Q)-gofmt -s -w $@

From e6b510543ef7667502fbe62017596e4e96068640 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= <anders.f.bjorklund@gmail.com>
Date: Sun, 27 Sep 2020 16:29:39 +0200
Subject: [PATCH 36/65] Don't trim whitespace while extracting strings

Remove some leading and trailing whitespace from non-string
constructs like description and automatic extraction filters.

But don't change all the translations or the translations here,
since that would make this code change impossible to review.
---
 cmd/minikube/cmd/completion.go              |  3 +-
 cmd/minikube/cmd/config/configure.go        |  4 +-
 cmd/minikube/cmd/start.go                   |  2 +-
 pkg/minikube/extract/extract.go             |  5 +-
 pkg/minikube/machine/advice.go              | 26 ++++-----
 site/content/en/docs/commands/addons.md     |  4 +-
 site/content/en/docs/commands/completion.md |  3 +-
 translations/strings.txt                    | 59 +++++++++++----------
 8 files changed, 53 insertions(+), 53 deletions(-)

diff --git a/cmd/minikube/cmd/completion.go b/cmd/minikube/cmd/completion.go
index c2e083e6a5..0035800fec 100644
--- a/cmd/minikube/cmd/completion.go
+++ b/cmd/minikube/cmd/completion.go
@@ -28,8 +28,7 @@ import (
 	"k8s.io/minikube/pkg/minikube/reason"
 )
 
-const longDescription = `
-	Outputs minikube shell completion for the given shell (bash, zsh or fish)
+const longDescription = `Outputs minikube shell completion for the given shell (bash, zsh or fish)
 
 	This depends on the bash-completion binary.  Example installation instructions:
 	OS X:
diff --git a/cmd/minikube/cmd/config/configure.go b/cmd/minikube/cmd/config/configure.go
index 25679e2d38..5b4d5301b6 100644
--- a/cmd/minikube/cmd/config/configure.go
+++ b/cmd/minikube/cmd/config/configure.go
@@ -32,8 +32,8 @@ import (
 
 var addonsConfigureCmd = &cobra.Command{
 	Use:   "configure ADDON_NAME",
-	Short: "Configures the addon w/ADDON_NAME within minikube (example: minikube addons configure registry-creds). For a list of available addons use: minikube addons list ",
-	Long:  "Configures the addon w/ADDON_NAME within minikube (example: minikube addons configure registry-creds). For a list of available addons use: minikube addons list ",
+	Short: "Configures the addon w/ADDON_NAME within minikube (example: minikube addons configure registry-creds). For a list of available addons use: minikube addons list",
+	Long:  "Configures the addon w/ADDON_NAME within minikube (example: minikube addons configure registry-creds). For a list of available addons use: minikube addons list",
 	Run: func(cmd *cobra.Command, args []string) {
 		if len(args) != 1 {
 			exit.Message(reason.Usage, "usage: minikube addons configure ADDON_NAME")
diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 1803210ccc..8975cd787b 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -794,7 +794,7 @@ func validateUser(drvName string) {
 
 	out.ErrT(style.Stopped, `The "{{.driver_name}}" driver should not be used with root privileges.`, out.V{"driver_name": drvName})
 	out.ErrT(style.Tip, "If you are running minikube within a VM, consider using --driver=none:")
-	out.ErrT(style.Documentation, "  https://minikube.sigs.k8s.io/docs/reference/drivers/none/")
+	out.ErrT(style.Documentation, "  {{.url}}", out.V{"url": "https://minikube.sigs.k8s.io/docs/reference/drivers/none/"})
 
 	cname := ClusterFlagValue()
 	_, err = config.Load(cname)
diff --git a/pkg/minikube/extract/extract.go b/pkg/minikube/extract/extract.go
index 5d00ecbaa7..4ae7b08dd7 100644
--- a/pkg/minikube/extract/extract.go
+++ b/pkg/minikube/extract/extract.go
@@ -42,6 +42,7 @@ import (
 var exclude = []string{
 	"{{.error}}",
 	"{{.url}}",
+	"  {{.url}}",
 	"{{.msg}}: {{.err}}",
 	"{{.key}}={{.value}}",
 	"opt {{.docker_option}}",
@@ -50,6 +51,7 @@ var exclude = []string{
 	"\\n",
 	"==\u003e {{.name}} \u003c==",
 	"- {{.profile}}",
+	"    - {{.profile}}",
 }
 
 // ErrMapFile is a constant to refer to the err_map file, which contains the Advice strings.
@@ -333,9 +335,6 @@ func checkString(s string) string {
 	// Parse out quote marks
 	stringToTranslate := s[1 : len(s)-1]
 
-	// Trim whitespace
-	stringToTranslate = strings.TrimSpace(stringToTranslate)
-
 	// Don't translate integers
 	if _, err := strconv.Atoi(stringToTranslate); err == nil {
 		return ""
diff --git a/pkg/minikube/machine/advice.go b/pkg/minikube/machine/advice.go
index 268ab56788..495d55a1ef 100644
--- a/pkg/minikube/machine/advice.go
+++ b/pkg/minikube/machine/advice.go
@@ -40,27 +40,27 @@ func MaybeDisplayAdvice(err error, driver string) {
 	if errors.Is(err, oci.ErrExitedUnexpectedly) || errors.Is(err, oci.ErrDaemonInfo) {
 		out.T(style.Tip, "If you are still interested to make {{.driver_name}} driver work. The following suggestions might help you get passed this issue:", out.V{"driver_name": driver})
 		if driver == oci.Docker || driver == oci.Podman {
-			out.T(style.Empty, `
-	- Prune unused {{.driver_name}} images, volumes, networks and abandoned containers.
+			out.String("\n\t")
+			out.T(style.Empty, `- Prune unused {{.driver_name}} images, volumes, networks and abandoned containers.
 
-		{{.driver_name}} system prune --volumes`, out.V{"driver_name": driver})
+				{{.driver_name}} system prune --volumes`, out.V{"driver_name": driver})
 		}
-		out.T(style.Empty, `
-	- Restart your {{.driver_name}} service`, out.V{"driver_name": driver})
+		out.String("\n\t")
+		out.T(style.Empty, `- Restart your {{.driver_name}} service`, out.V{"driver_name": driver})
 		if runtime.GOOS != "linux" {
-			out.T(style.Empty, `
-	- Ensure your {{.driver_name}} daemon has access to enough CPU/memory resources. `, out.V{"driver_name": driver})
+			out.String("\n\t")
+			out.T(style.Empty, `- Ensure your {{.driver_name}} daemon has access to enough CPU/memory resources.`, out.V{"driver_name": driver})
 			if runtime.GOOS == "darwin" && driver == oci.Docker {
-				out.T(style.Empty, `
-	- Docs https://docs.docker.com/docker-for-mac/#resources`, out.V{"driver_name": driver})
+				out.String("\n\t")
+				out.T(style.Empty, `- Docs https://docs.docker.com/docker-for-mac/#resources`, out.V{"driver_name": driver})
 			}
 			if runtime.GOOS == "windows" && driver == oci.Docker {
-				out.T(style.Empty, `
-	- Docs https://docs.docker.com/docker-for-windows/#resources`, out.V{"driver_name": driver})
+				out.String("\n\t")
+				out.T(style.Empty, `- Docs https://docs.docker.com/docker-for-windows/#resources`, out.V{"driver_name": driver})
 			}
 		}
-		out.T(style.Empty, `
-	- Delete and recreate minikube cluster
+		out.String("\n\t")
+		out.T(style.Empty, `- Delete and recreate minikube cluster
 		minikube delete
 		minikube start --driver={{.driver_name}}`, out.V{"driver_name": driver})
 		// TODO #8348: maybe advice user if to set the --force-systemd https://github.com/kubernetes/minikube/issues/8348
diff --git a/site/content/en/docs/commands/addons.md b/site/content/en/docs/commands/addons.md
index b35589cfe4..dd8af6140f 100644
--- a/site/content/en/docs/commands/addons.md
+++ b/site/content/en/docs/commands/addons.md
@@ -39,11 +39,11 @@ minikube addons SUBCOMMAND [flags]
 
 ## minikube addons configure
 
-Configures the addon w/ADDON_NAME within minikube (example: minikube addons configure registry-creds). For a list of available addons use: minikube addons list 
+Configures the addon w/ADDON_NAME within minikube (example: minikube addons configure registry-creds). For a list of available addons use: minikube addons list
 
 ### Synopsis
 
-Configures the addon w/ADDON_NAME within minikube (example: minikube addons configure registry-creds). For a list of available addons use: minikube addons list 
+Configures the addon w/ADDON_NAME within minikube (example: minikube addons configure registry-creds). For a list of available addons use: minikube addons list
 
 ```
 minikube addons configure ADDON_NAME [flags]
diff --git a/site/content/en/docs/commands/completion.md b/site/content/en/docs/commands/completion.md
index 09f118c9f0..dbe36b99d3 100644
--- a/site/content/en/docs/commands/completion.md
+++ b/site/content/en/docs/commands/completion.md
@@ -11,8 +11,7 @@ Generate command completion for a shell
 
 ### Synopsis
 
-
-	Outputs minikube shell completion for the given shell (bash, zsh or fish)
+Outputs minikube shell completion for the given shell (bash, zsh or fish)
 
 	This depends on the bash-completion binary.  Example installation instructions:
 	OS X:
diff --git a/translations/strings.txt b/translations/strings.txt
index c80809c9d9..6abccd32da 100644
--- a/translations/strings.txt
+++ b/translations/strings.txt
@@ -11,7 +11,7 @@
 	"- Docs https://docs.docker.com/docker-for-mac/#resources": "",
 	"- Docs https://docs.docker.com/docker-for-windows/#resources": "",
 	"- Ensure your {{.driver_name}} daemon has access to enough CPU/memory resources.": "",
-	"- Prune unused {{.driver_name}} images, volumes and abandoned containers.": "",
+	"- Prune unused {{.driver_name}} images, volumes, networks and abandoned containers.\n\n\t\t\t\t{{.driver_name}} system prune --volumes": "",
 	"- Restart your {{.driver_name}} service": "",
 	"A VPN or firewall is interfering with HTTP access to the minikube VM. Alternatively, try a different VM driver: https://minikube.sigs.k8s.io/docs/start/": "",
 	"A firewall is blocking Docker the minikube VM from reaching the image repository. You may need to select --image-repository, or use a proxy.": "",
@@ -45,7 +45,7 @@
 	"Basic Commands:": "",
 	"Because you are using a Docker driver on {{.operating_system}}, the terminal needs to be open to run it.": "",
 	"Bind Address: {{.Address}}": "",
-	"Both driver={{.driver}} and vm-driver={{.vmd}} have been set.\n\n    Since vm-driver is deprecated, minikube will default to driver={{.driver}}.\n\n    If vm-driver is set in the global config, please run \"minikube config unset vm-driver\" to resolve this warning.": "",
+	"Both driver={{.driver}} and vm-driver={{.vmd}} have been set.\n\n    Since vm-driver is deprecated, minikube will default to driver={{.driver}}.\n\n    If vm-driver is set in the global config, please run \"minikube config unset vm-driver\" to resolve this warning.\n\t\t\t": "",
 	"CNI plug-in to use. Valid options: auto, bridge, calico, cilium, flannel, kindnet, or path to a CNI manifest (default: auto)": "",
 	"Cannot find directory {{.path}} for mount": "",
 	"Cannot use both --output and --format options": "",
@@ -68,7 +68,7 @@
 	"Confirm that you have a working internet connection and that your VM has not run out of resources by using: 'minikube logs'": "",
 	"Confirm that you have supplied the correct value to --hyperv-virtual-switch using the 'Get-VMSwitch' command": "",
 	"Connect to LoadBalancer services": "",
-	"Consider creating a cluster with larger memory size using `minikube start --memory SIZE_MB`": "",
+	"Consider creating a cluster with larger memory size using `minikube start --memory SIZE_MB` ": "",
 	"Consider increasing Docker Desktop's memory size.": "",
 	"Could not determine a Google Cloud project, which might be ok.": "",
 	"Could not find any GCP credentials. Either run `gcloud auth login` or set the GOOGLE_APPLICATION_CREDENTIALS environment variable to the path of your credentials file.": "",
@@ -78,6 +78,7 @@
 	"Creating mount {{.name}} ...": "",
 	"Creating {{.driver_name}} {{.machine_type}} (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB) ...": "",
 	"Creating {{.driver_name}} {{.machine_type}} (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...": "",
+	"Current context is \"{{.context}}\"": "",
 	"DEPRECATED, use `driver` instead.": "",
 	"DEPRECATED: Replaced by --cni=bridge": "",
 	"Default group id used for the mount": "",
@@ -91,7 +92,7 @@
 	"Deleting node {{.name}} from cluster {{.cluster}}": "",
 	"Disable checking for the availability of hardware virtualization before the vm is started (virtualbox driver only)": "",
 	"Disable dynamic memory in your VM manager, or pass in a larger --memory value": "",
-	"Disables the addon w/ADDON_NAME within minikube (example: minikube addons disable dashboard). For a list of available addons use: minikube addons list": "",
+	"Disables the addon w/ADDON_NAME within minikube (example: minikube addons disable dashboard). For a list of available addons use: minikube addons list ": "",
 	"Disables the filesystem mounts provided by the hypervisors": "",
 	"Disk size allocated to the minikube VM (format: \u003cnumber\u003e[\u003cunit\u003e], where unit = b, k, m or g).": "",
 	"Display dashboard URL instead of opening a browser": "",
@@ -102,10 +103,9 @@
 	"Docker Desktop has less than 2 CPUs configured, but Kubernetes requires at least 2 to be available": "",
 	"Docker Desktop is configured for Windows containers, but Linux containers are required for minikube": "",
 	"Docker Desktop only has {{.size}}MiB available, less than the required {{.req}}MiB for Kubernetes": "",
+	"Docker Desktop only has {{.size}}MiB available, you may encounter application deployment failures.": "",
 	"Docker has less than 2 CPUs available, but Kubernetes requires at least 2 to be available": "",
 	"Docker inside the VM is unavailable. Try running 'minikube delete' to reset the VM.": "",
-	"Docker is nearly out of disk space, which may cause deployments to fail! ({{.p}}% of capacity)": "",
-	"Docker is out of disk space! (/var is at {{.p}}% of capacity)": "",
 	"Docs have been saved at - {{.path}}": "",
 	"Documentation: {{.url}}": "",
 	"Done! kubectl is now configured to use \"{{.name}}\" by default": "",
@@ -127,7 +127,7 @@
 	"Enable or disable a minikube addon": "",
 	"Enable proxy for NAT DNS requests (virtualbox driver only)": "",
 	"Enabled addons: {{.addons}}": "",
-	"Enables the addon w/ADDON_NAME within minikube (example: minikube addons enable dashboard). For a list of available addons use: minikube addons list": "",
+	"Enables the addon w/ADDON_NAME within minikube (example: minikube addons enable dashboard). For a list of available addons use: minikube addons list ": "",
 	"Enabling '{{.name}}' returned an error: {{.error}}": "",
 	"Enabling dashboard ...": "",
 	"Ensure that CRI-O is installed and healthy: Run 'sudo systemctl start crio' and 'journalctl -u crio'. Alternatively, use --container-runtime=docker": "",
@@ -161,6 +161,7 @@
 	"Error starting cluster": "",
 	"Error starting mount": "",
 	"Error while setting kubectl current context :  {{.error}}": "",
+	"Error while setting kubectl current context:  {{.error}}": "",
 	"Error writing mount pid": "",
 	"Examples": "",
 	"Executing \"{{.command}}\" took an unusually long time: {{.duration}}": "",
@@ -180,7 +181,6 @@
 	"Failed to delete images": "",
 	"Failed to delete images from config": "",
 	"Failed to enable container runtime": "",
-	"Failed to get API Server URL": "",
 	"Failed to get bootstrapper": "",
 	"Failed to get command runner": "",
 	"Failed to get image map": "",
@@ -208,7 +208,7 @@
 	"Force minikube to perform possibly dangerous operations": "",
 	"Format to print stdout in. Options include: [text,json]": "",
 	"Found network options:": "",
-	"Found {{.number}} invalid profile(s) !": "",
+	"Found {{.number}} invalid profile(s) ! ": "",
 	"Generate command completion for a shell": "",
 	"Generate unable to parse disk size '{{.diskSize}}': {{.error}}": "",
 	"Generate unable to parse memory '{{.memory}}': {{.error}}": "",
@@ -232,7 +232,7 @@
 	"If set, install addons. Defaults to true.": "",
 	"If set, pause all namespaces": "",
 	"If set, unpause all namespaces": "",
-	"If the above advice does not help, please let us know:": "",
+	"If the above advice does not help, please let us know: ": "",
 	"If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --driver=none.": "",
 	"If true, only download and cache files for later use - don't install or start anything.": "",
 	"If true, the added node will be marked for work. Defaults to true.": "",
@@ -277,7 +277,7 @@
 	"Mounts the specified directory into minikube.": "",
 	"Multi-node clusters are currently experimental and might exhibit unintended behavior.": "",
 	"Multiple errors deleting profiles": "",
-	"Multiple minikube profiles were found -": "",
+	"Multiple minikube profiles were found - ": "",
 	"NIC Type used for host only network. One of Am79C970A, Am79C973, 82540EM, 82543GC, 82545EM, or virtio (virtualbox driver only)": "",
 	"NIC Type used for nat network. One of Am79C970A, Am79C973, 82540EM, 82543GC, 82545EM, or virtio (virtualbox driver only)": "",
 	"NOTE: This process must stay alive for the mount to be accessible ...": "",
@@ -299,21 +299,21 @@
 	"One of 'yaml' or 'json'.": "",
 	"Only alphanumeric and dashes '-' are permitted. Minimum 1 character, starting with alphanumeric.": "",
 	"Open the addons URL with https instead of http": "",
-	"Open the service URL with https instead of http": "",
+	"Open the service URL with https instead of http (defaults to \\\"false\\\")": "",
 	"Opening Kubernetes service  {{.namespace_name}}/{{.service_name}} in default browser...": "",
 	"Opening service {{.namespace_name}}/{{.service_name}} in default browser...": "",
 	"Opening {{.url}} in your default browser...": "",
-	"Opens the addon w/ADDON_NAME within minikube (example: minikube addons open dashboard). For a list of available addons use: minikube addons list": "",
+	"Opens the addon w/ADDON_NAME within minikube (example: minikube addons open dashboard). For a list of available addons use: minikube addons list ": "",
 	"Operations on nodes": "",
 	"Options:      {{.options}}": "",
 	"Output format. Accepted values: [json]": "",
-	"Outputs minikube shell completion for the given shell (bash, zsh or fish)\n\n\tThis depends on the bash-completion binary.  Example installation instructions:\n\tOS X:\n\t\t$ brew install bash-completion\n\t\t$ source $(brew --prefix)/etc/bash_completion\n\t\t$ minikube completion bash \u003e ~/.minikube-completion  # for bash users\n\t\t$ minikube completion zsh \u003e ~/.minikube-completion  # for zsh users\n\t\t$ source ~/.minikube-completion\n\t\t$ minikube completion fish \u003e ~/.config/fish/completions/minikube.fish # for fish users\n\tUbuntu:\n\t\t$ apt-get install bash-completion\n\t\t$ source /etc/bash-completion\n\t\t$ source \u003c(minikube completion bash) # for bash users\n\t\t$ source \u003c(minikube completion zsh) # for zsh users\n\t\t$ minikube completion fish \u003e ~/.config/fish/completions/minikube.fish # for fish users\n\n\tAdditionally, you may want to output the completion to a file and source in your .bashrc\n\n\tNote for zsh users: [1] zsh completions are only supported in versions of zsh \u003e= 5.2\n\tNote for fish users: [2] please refer to this docs for more details https://fishshell.com/docs/current/#tab-completion": "",
+	"Outputs minikube shell completion for the given shell (bash, zsh or fish)\n\n\tThis depends on the bash-completion binary.  Example installation instructions:\n\tOS X:\n\t\t$ brew install bash-completion\n\t\t$ source $(brew --prefix)/etc/bash_completion\n\t\t$ minikube completion bash \u003e ~/.minikube-completion  # for bash users\n\t\t$ minikube completion zsh \u003e ~/.minikube-completion  # for zsh users\n\t\t$ source ~/.minikube-completion\n\t\t$ minikube completion fish \u003e ~/.config/fish/completions/minikube.fish # for fish users\n\tUbuntu:\n\t\t$ apt-get install bash-completion\n\t\t$ source /etc/bash-completion\n\t\t$ source \u003c(minikube completion bash) # for bash users\n\t\t$ source \u003c(minikube completion zsh) # for zsh users\n\t\t$ minikube completion fish \u003e ~/.config/fish/completions/minikube.fish # for fish users\n\n\tAdditionally, you may want to output the completion to a file and source in your .bashrc\n\n\tNote for zsh users: [1] zsh completions are only supported in versions of zsh \u003e= 5.2\n\tNote for fish users: [2] please refer to this docs for more details https://fishshell.com/docs/current/#tab-completion\n": "",
 	"Pause": "",
 	"Paused {{.count}} containers": "",
 	"Paused {{.count}} containers in: {{.namespaces}}": "",
-	"Pausing node {{.name}} ...": "",
+	"Pausing node {{.name}} ... ": "",
 	"Permissions:  {{.octalMode}} ({{.writtenMode}})": "",
-	"Please create a cluster with bigger disk size: `minikube start --disk SIZE_MB`": "",
+	"Please create a cluster with bigger disk size: `minikube start --disk SIZE_MB` ": "",
 	"Please either authenticate to the registry or use --base-image flag to use a different registry.": "",
 	"Please enter a value:": "",
 	"Please free up disk or prune images.": "",
@@ -321,11 +321,11 @@
 	"Please install the minikube hyperkit VM driver, or select an alternative --driver": "",
 	"Please install the minikube kvm2 VM driver, or select an alternative --driver": "",
 	"Please make sure the service you are looking for is deployed or is in the correct namespace.": "",
-	"Please re-eval your docker-env, To ensure your environment variables have updated ports:\n\n\t'minikube -p {{.profile_name}} docker-env'": "",
-	"Please re-eval your podman-env, To ensure your environment variables have updated ports:\n\n\t'minikube -p {{.profile_name}} podman-env'": "",
+	"Please re-eval your docker-env, To ensure your environment variables have updated ports:\n\n\t'minikube -p {{.profile_name}} docker-env'\n\n\t": "",
+	"Please re-eval your podman-env, To ensure your environment variables have updated ports:\n\n\t'minikube -p {{.profile_name}} podman-env'\n\n\t": "",
 	"Please see {{.documentation_url}} for more details": "",
 	"Please specify the directory to be mounted: \n\tminikube mount \u003csource directory\u003e:\u003ctarget directory\u003e   (example: \"/host-home:/vm-home\")": "",
-	"Please visit the following link for documentation around this: \n\thttps://help.github.com/en/packages/using-github-packages-with-your-projects-ecosystem/configuring-docker-for-use-with-github-packages#authenticating-to-github-packages": "",
+	"Please visit the following link for documentation around this: \n\thttps://help.github.com/en/packages/using-github-packages-with-your-projects-ecosystem/configuring-docker-for-use-with-github-packages#authenticating-to-github-packages\n": "",
 	"Populates the specified folder with documentation in markdown about minikube": "",
 	"PowerShell is running in constrained mode, which is incompatible with Hyper-V scripting.": "",
 	"Powering off \"{{.profile_name}}\" via SSH ...": "",
@@ -400,7 +400,7 @@
 	"Show only log entries which point to known problems": "",
 	"Show only the most recent journal entries, and continuously print new entries as they are appended to the journal.": "",
 	"Skipped switching kubectl context for {{.profile_name}} because --keep-context was set.": "",
-	"Some dashboard features require the metrics-server addon. To enable all features please run:\n\n\tminikube{{.profileArg}} addons enable metrics-server": "",
+	"Some dashboard features require the metrics-server addon. To enable all features please run:\n\n\tminikube{{.profileArg}} addons enable metrics-server\t\n\n": "",
 	"Sorry, Kubernetes {{.k8sVersion}} requires conntrack to be installed in root's path": "",
 	"Sorry, completion support is not yet implemented for {{.name}}": "",
 	"Sorry, please set the --output flag to one of the following valid options: [text,json]": "",
@@ -555,12 +555,12 @@
 	"Unable to safely downgrade existing Kubernetes v{{.old}} cluster to v{{.new}}": "",
 	"Unable to stop VM": "",
 	"Unable to update {{.driver}} driver: {{.error}}": "",
-	"Unfortunately, could not download the base image {{.image_name}}": "",
+	"Unfortunately, could not download the base image {{.image_name}} ": "",
 	"Uninstalling Kubernetes {{.kubernetes_version}} using {{.bootstrapper_name}} ...": "",
 	"Unmounting {{.path}} ...": "",
 	"Unpaused {{.count}} containers": "",
 	"Unpaused {{.count}} containers in: {{.namespaces}}": "",
-	"Unpausing node {{.name}} ...": "",
+	"Unpausing node {{.name}} ... ": "",
 	"Unset the KUBECONFIG environment variable, or verify that it does not point to an empty or otherwise invalid path": "",
 	"Unset variables instead of setting them": "",
 	"Update kubeconfig in case of an IP or port change": "",
@@ -583,7 +583,7 @@
 	"Use native Golang SSH client (default true). Set to 'false' to use the command line 'ssh' command when accessing the docker machine. Useful for the machine drivers when they will not start with 'Waiting for SSH'.": "",
 	"User ID:      {{.userID}}": "",
 	"Userspace file server is shutdown": "",
-	"Userspace file server:": "",
+	"Userspace file server: ": "",
 	"Using image repository {{.name}}": "",
 	"Using podman 2 is not supported yet. your version is \"{{.currentVersion}}\". minikube might not work. use at your own risk.": "",
 	"Using the '{{.runtime}}' runtime with the 'none' driver is an untested configuration!": "",
@@ -608,7 +608,7 @@
 	"Where to root the NFS Shares, defaults to /nfsshares (hyperkit driver only)": "",
 	"Whether to use external switch over Default Switch if virtual switch not explicitly specified. (hyperv driver only)": "",
 	"You appear to be using a proxy, but your NO_PROXY environment does not include the minikube IP ({{.ip_address}}).": "",
-	"You can delete them using the following command(s):": "",
+	"You can delete them using the following command(s): ": "",
 	"You can force an unsupported Kubernetes version via the --force flag": "",
 	"You cannot change the CPUs for an existing minikube cluster. Please first delete the cluster.": "",
 	"You cannot change the Disk size for an exiting minikube cluster. Please first delete the cluster.": "",
@@ -622,13 +622,13 @@
 	"Your host is failing to route packets to the minikube VM. If you have VPN software, try turning it off or configuring it so that it does not re-route traffic to the VM IP. If not, check your VM environment routing options.": "",
 	"Your minikube config refers to an unsupported driver. Erase ~/.minikube, and try again.": "",
 	"Your minikube vm is not running, try minikube start.": "",
-	"Your user lacks permissions to the minikube profile directory. Run: 'sudo chown -R $USER $HOME/.minikube; chmod -R u+wrx $HOME/.minikube' to fix": "",
+	"[WARNING] For full functionality, the 'csi-hostpath-driver' addon requires the 'volumesnapshots' addon to be enabled.\n\nYou can enable 'volumesnapshots' addon by running: 'minikube addons enable volumesnapshots'\n": "",
 	"addon '{{.name}}' is currently not enabled.\nTo enable this addon run:\nminikube addons enable {{.name}}": "",
 	"addon '{{.name}}' is not a valid addon packaged with minikube.\nTo see the list of available addons run:\nminikube addons list": "",
 	"addons modifies minikube addons files using subcommands like \"minikube addons enable dashboard\"": "",
 	"bash completion failed": "",
 	"call with cleanup=true to remove old tunnels": "",
-	"config modifies minikube config files using subcommands like \"minikube config set driver kvm\"\nConfigurable fields:\\n\\n": "",
+	"config modifies minikube config files using subcommands like \"minikube config set driver kvm\"\nConfigurable fields: \\n\\n": "",
 	"config view failed": "",
 	"dashboard service is not running: {{.error}}": "",
 	"deleting node": "",
@@ -660,14 +660,14 @@
 	"kubectl not found. If you need it, try: 'minikube kubectl -- get pods -A'": "",
 	"kubectl proxy": "",
 	"libmachine failed": "",
-	"list displays all valid default settings for PROPERTY_NAME\nAcceptable fields:\\n\\n": "",
+	"list displays all valid default settings for PROPERTY_NAME\nAcceptable fields: \\n\\n": "",
 	"logdir set failed": "",
 	"max time to wait per Kubernetes or host to be healthy.": "",
 	"minikube addons list --output OUTPUT. json, list": "",
 	"minikube is exiting due to an error. If the above message is not useful, open an issue:": "",
 	"minikube is missing files relating to your guest environment. This can be fixed by running 'minikube delete'": "",
 	"minikube is unable to access the Google Container Registry. You may need to configure it to use a HTTP proxy.": "",
-	"minikube is unable to connect to the VM: {{.error}}\n\n\tThis is likely due to one of two reasons:\n\n\t- VPN or firewall interference\n\t- {{.hypervisor}} network configuration issue\n\n\tSuggested workarounds:\n\n\t- Disable your local VPN or firewall software\n\t- Configure your local VPN or firewall to allow access to {{.ip}}\n\t- Restart or reinstall {{.hypervisor}}\n\t- Use an alternative --vm-driver\n\t- Use --force to override this connectivity check": "",
+	"minikube is unable to connect to the VM: {{.error}}\n\n\tThis is likely due to one of two reasons:\n\n\t- VPN or firewall interference\n\t- {{.hypervisor}} network configuration issue\n\n\tSuggested workarounds:\n\n\t- Disable your local VPN or firewall software\n\t- Configure your local VPN or firewall to allow access to {{.ip}}\n\t- Restart or reinstall {{.hypervisor}}\n\t- Use an alternative --vm-driver\n\t- Use --force to override this connectivity check\n\t": "",
 	"minikube profile was successfully set to {{.profile_name}}": "",
 	"minikube provisions and manages local Kubernetes clusters optimized for development workflows.": "",
 	"minikube quickly sets up a local Kubernetes cluster": "",
@@ -715,6 +715,7 @@
 	"version yaml failure": "",
 	"zsh completion failed": "",
 	"{{ .name }}: {{ .rejection }}": "",
+	"{{.Driver}} is currently using the {{.StorageDriver}} storage driver, consider switching to overlay2 for better performance": "",
 	"{{.count}} nodes stopped.": "",
 	"{{.driver_name}} \"{{.cluster}}\" {{.machine_type}} is missing, will recreate.": "",
 	"{{.driver_name}} couldn't proceed because {{.driver_name}} service is not healthy.": "",
@@ -725,6 +726,8 @@
 	"{{.name}} has no available configuration options": "",
 	"{{.name}} is already running": "",
 	"{{.name}} was successfully configured": "",
+	"{{.n}} is nearly out of disk space, which may cause deployments to fail! ({{.p}}% of capacity)": "",
+	"{{.n}} is out of disk space! (/var is at {{.p}}% of capacity)": "",
 	"{{.ocibin}} is taking an unsually long time to respond, consider restarting {{.ocibin}}": "",
 	"{{.path}} is version {{.client_version}}, which may have incompatibilites with Kubernetes {{.cluster_version}}.": "",
 	"{{.prefix}}minikube {{.version}} on {{.platform}}": "",

From 256a1c2b920b3c2836b647fbb54b2783cef9636e Mon Sep 17 00:00:00 2001
From: "Jituri, Pranav" <blueelvisrocks@gmail.com>
Date: Mon, 28 Sep 2020 18:19:36 +0530
Subject: [PATCH 37/65] Added target for Windows HTML & switched encoding

---
 .github/workflows/master.yml | 8 ++++----
 .github/workflows/pr.yml     | 8 ++++----
 Makefile                     | 9 +++++++++
 3 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index 5521378f3b..0c8714c538 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -362,7 +362,7 @@ jobs:
           $env:KUBECONFIG="${pwd}\testhome\kubeconfig"
           $env:MINIKUBE_HOME="${pwd}\testhome"
           $ErrorActionPreference = "SilentlyContinue"
-          .\e2e-windows-amd64.exe --minikube-start-args="--driver=docker" --test.timeout=15m --timeout-multiplier=1 --test.v --test.run=TestFunctional --binary=./minikube-windows-amd64.exe | Tee-Object -FilePath ".\report\testout.txt"
+          .\e2e-windows-amd64.exe --minikube-start-args="--driver=docker" --test.timeout=15m --timeout-multiplier=1 --test.v --test.run=TestFunctional --binary=./minikube-windows-amd64.exe | Out-File -Encoding utf8 -FilePath ".\report\testout.txt"
           $END_TIME=(GET-DATE)
           echo $END_TIME
           $DURATION=(NEW-TIMESPAN -Start $START_TIME -End $END_TIME)
@@ -379,7 +379,7 @@ jobs:
         shell: powershell
         run: |
           cd minikube_binaries
-          Get-Content .\report\testout.txt -Encoding ASCII  | go tool test2json -t | Out-File -FilePath .\report\testout.json -Encoding ASCII
+          Get-Content .\report\testout.txt -Encoding UTF8  | go tool test2json -t | Out-File -FilePath .\report\testout.json -Encoding utf8
           $STAT=(gopogh -in .\report\testout.json -out .\report\testout.html -name "${Env:JOB_NAME} ${Env:GITHUB_REF}" -repo "${Env:GITHUB_REPOSITORY}"  -details "${Env:GITHUB_SHA}")
           echo status: ${STAT}
           $FailNum=$(echo $STAT | jq '.NumberOfFail')
@@ -499,7 +499,7 @@ jobs:
           $env:KUBECONFIG="${pwd}\testhome\kubeconfig"
           $env:MINIKUBE_HOME="${pwd}\testhome"
           $ErrorActionPreference = "SilentlyContinue"
-          .\e2e-windows-amd64.exe --minikube-start-args="--driver=hyperv" --test.timeout=20m --timeout-multiplier=1.5 --test.v --test.run=TestFunctional --binary=./minikube-windows-amd64.exe | Tee-Object -FilePath ".\report\testout.txt"
+          .\e2e-windows-amd64.exe --minikube-start-args="--driver=hyperv" --test.timeout=20m --timeout-multiplier=1.5 --test.v --test.run=TestFunctional --binary=./minikube-windows-amd64.exe | Out-File -Encoding utf8 -FilePath ".\report\testout.txt"
           $END_TIME=(GET-DATE)
           echo $END_TIME
           $DURATION=(NEW-TIMESPAN -Start $START_TIME -End $END_TIME)
@@ -516,7 +516,7 @@ jobs:
         shell: powershell
         run: |
           cd minikube_binaries
-          Get-Content .\report\testout.txt -Encoding ASCII  | go tool test2json -t | Out-File -FilePath .\report\testout.json -Encoding ASCII
+          Get-Content .\report\testout.txt -Encoding UTF8  | go tool test2json -t | Out-File -FilePath .\report\testout.json -Encoding utf8
           $STAT=(gopogh -in .\report\testout.json -out .\report\testout.html -name "${Env:JOB_NAME} ${Env:GITHUB_REF}" -repo "${Env:GITHUB_REPOSITORY}"  -details "${Env:GITHUB_SHA}")
           echo status: ${STAT}
           $FailNum=$(echo $STAT | jq '.NumberOfFail')
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 21cd5e5a04..9a663222aa 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -360,7 +360,7 @@ jobs:
           $env:KUBECONFIG="${pwd}\testhome\kubeconfig"
           $env:MINIKUBE_HOME="${pwd}\testhome"
           $ErrorActionPreference = "SilentlyContinue"
-          .\e2e-windows-amd64.exe --minikube-start-args="--driver=docker" --test.timeout=15m --timeout-multiplier=1 --test.v --test.run=TestFunctional --binary=./minikube-windows-amd64.exe | Tee-Object -FilePath ".\report\testout.txt"
+          .\e2e-windows-amd64.exe --minikube-start-args="--driver=docker" --test.timeout=15m --timeout-multiplier=1 --test.v --test.run=TestFunctional --binary=./minikube-windows-amd64.exe | Out-File -Encoding utf8 -FilePath ".\report\testout.txt"
           $END_TIME=(GET-DATE)
           echo $END_TIME
           $DURATION=(NEW-TIMESPAN -Start $START_TIME -End $END_TIME)
@@ -377,7 +377,7 @@ jobs:
         shell: powershell
         run: |
           cd minikube_binaries
-          Get-Content .\report\testout.txt -Encoding ASCII  | go tool test2json -t | Out-File -FilePath .\report\testout.json -Encoding ASCII
+          Get-Content .\report\testout.txt -Encoding UTF8  | go tool test2json -t | Out-File -FilePath .\report\testout.json -Encoding utf8
           $STAT=(gopogh -in .\report\testout.json -out .\report\testout.html -name "${Env:JOB_NAME} ${Env:GITHUB_REF}" -repo "${Env:GITHUB_REPOSITORY}"  -details "${Env:GITHUB_SHA}")
           echo status: ${STAT}
           $FailNum=$(echo $STAT | jq '.NumberOfFail')
@@ -497,7 +497,7 @@ jobs:
           $env:KUBECONFIG="${pwd}\testhome\kubeconfig"
           $env:MINIKUBE_HOME="${pwd}\testhome"
           $ErrorActionPreference = "SilentlyContinue"
-          .\e2e-windows-amd64.exe --minikube-start-args="--driver=hyperv" --test.timeout=20m --timeout-multiplier=1.5 --test.v --test.run=TestFunctional --binary=./minikube-windows-amd64.exe | Tee-Object -FilePath ".\report\testout.txt"
+          .\e2e-windows-amd64.exe --minikube-start-args="--driver=hyperv" --test.timeout=20m --timeout-multiplier=1.5 --test.v --test.run=TestFunctional --binary=./minikube-windows-amd64.exe | Out-File -Encoding utf8 -FilePath ".\report\testout.txt"
           $END_TIME=(GET-DATE)
           echo $END_TIME
           $DURATION=(NEW-TIMESPAN -Start $START_TIME -End $END_TIME)
@@ -514,7 +514,7 @@ jobs:
         shell: powershell
         run: |
           cd minikube_binaries
-          Get-Content .\report\testout.txt -Encoding ASCII  | go tool test2json -t | Out-File -FilePath .\report\testout.json -Encoding ASCII
+          Get-Content .\report\testout.txt -Encoding UTF8  | go tool test2json -t | Out-File -FilePath .\report\testout.json -Encoding utf8
           $STAT=(gopogh -in .\report\testout.json -out .\report\testout.html -name "${Env:JOB_NAME} ${Env:GITHUB_REF}" -repo "${Env:GITHUB_REPOSITORY}"  -details "${Env:GITHUB_SHA}")
           echo status: ${STAT}
           $FailNum=$(echo $STAT | jq '.NumberOfFail')
diff --git a/Makefile b/Makefile
index 1592fe5f0a..c48f308e30 100644
--- a/Makefile
+++ b/Makefile
@@ -309,6 +309,15 @@ html_report: ## Generate HTML  report out of the last ran integration test logs.
 	@echo "-----------------------------------------------------------------------------------"
 	@open $(CURDIR)/out/testout_$(COMMIT_SHORT).html || true
 
+.PHONY: html_report_windows
+html_report_windows: ## Generate the HTML Report out of the last ran integration test logs on Windows.
+	@go tool test2json -t < "./out/testout_$(COMMIT_SHORT).txt" > "./out/testout_$(COMMIT_SHORT).json"
+	@gopogh -in "./out/testout_$(COMMIT_SHORT).json" -out ./out/testout_$(COMMIT_SHORT).html -name "$(shell git rev-parse --abbrev-ref HEAD)" -pr "" -repo github.com/kubernetes/minikube/ -details "${COMMIT_SHORT}"
+	@echo "-------------------------- Open HTML Report in Browser: ---------------------------"
+	@echo $(CURDIR)/out/testout_$(COMMIT_SHORT).html
+	@echo "-----------------------------------------------------------------------------------"
+	@start $(CURDIR)/out/testout_$(COMMIT_SHORT).html || true
+
 .PHONY: test
 test: pkg/minikube/assets/assets.go pkg/minikube/translate/translations.go ## Trigger minikube test
 	MINIKUBE_LDFLAGS="${MINIKUBE_LDFLAGS}" ./test.sh

From d34a24020fc5547ae7a181bde4750c191897bc5b Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Mon, 28 Sep 2020 09:56:19 -0700
Subject: [PATCH 38/65] hyperv: Make interface search name match
 case-insensitive

---
 pkg/minikube/cluster/ip.go | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/pkg/minikube/cluster/ip.go b/pkg/minikube/cluster/ip.go
index f853848399..2dd4548020 100644
--- a/pkg/minikube/cluster/ip.go
+++ b/pkg/minikube/cluster/ip.go
@@ -49,6 +49,7 @@ func HostIP(host *host.Host) (net.IP, error) {
 		for i := 0; i < v.NumField(); i++ {
 			if v.Type().Field(i).Name == "VSwitch" {
 				hypervVirtualSwitch = v.Field(i).Interface().(string)
+
 				break
 			}
 		}
@@ -59,6 +60,7 @@ func HostIP(host *host.Host) (net.IP, error) {
 		if err != nil {
 			return []byte{}, errors.Wrap(err, fmt.Sprintf("ip for interface (%s)", hypervVirtualSwitch))
 		}
+
 		return ip, nil
 	case driver.VirtualBox:
 		vBoxManageCmd := driver.VBoxManagePath()
@@ -74,6 +76,7 @@ func HostIP(host *host.Host) (net.IP, error) {
 		}
 		re = regexp.MustCompile(`(?sm)Name:\s*` + iface + `\s*$.+?IPAddress:\s*(\S+)`)
 		ip := re.FindStringSubmatch(string(ipList))[1]
+
 		return net.ParseIP(ip), nil
 	case driver.Parallels:
 		bin := "prlsrvctl"
@@ -93,6 +96,7 @@ func HostIP(host *host.Host) (net.IP, error) {
 			return []byte{}, errors.Wrap(err, "Error getting the IP address of Parallels Shared network interface")
 		}
 		ip := ipMatch[1]
+
 		return net.ParseIP(ip), nil
 	case driver.HyperKit:
 		return net.ParseIP("192.168.64.1"), nil
@@ -136,6 +140,7 @@ func DriverIP(api libmachine.API, machineName string) (net.IP, error) {
 
 // Based on code from http://stackoverflow.com/questions/23529663/how-to-get-all-addresses-and-masks-from-local-interfaces-in-go
 func getIPForInterface(name string) (net.IP, error) {
+	glog.Infof("getIPForInterface: searching for %q", name)
 	ints, err := net.Interfaces()
 	if err != nil {
 		return nil, err
@@ -143,19 +148,25 @@ func getIPForInterface(name string) (net.IP, error) {
 
 	var i net.Interface
 	for _, in := range ints {
-		if strings.HasPrefix(in.Name, name) {
+		if strings.HasPrefix(strings.ToLower(in.Name), strings.ToLower(name)) {
+			glog.Infof("found prefix matching interface for %q: %q", name, in.Name)
 			i = in
+
 			break
 		}
+		glog.Infof("%q does not match prefix %q", in.Name, name)
 	}
 
 	// Didn't find prefix, let's try any substring
 	if i.Name == "" {
 		for _, in := range ints {
-			if strings.Contains(in.Name, name) {
+			if strings.Contains(strings.ToLower(in.Name), strings.ToLower(name)) {
+				glog.Infof("found substring matching interface for %q: %q", name, in.Name)
 				i = in
+
 				break
 			}
+			glog.Infof("%q does not match substring %q", in.Name, name)
 		}
 	}
 
@@ -164,14 +175,15 @@ func getIPForInterface(name string) (net.IP, error) {
 		return nil, errors.Errorf("Could not find interface %s inside %+v", name, ints)
 	}
 
-	glog.Infof("Found hyperv interface: %+v\n", i)
+	glog.Infof("Found interface: %+v\n", i)
 	addrs, _ := i.Addrs()
 	for _, a := range addrs {
+		glog.Infof("interface addr: %+v", a)
 		if ipnet, ok := a.(*net.IPNet); ok {
 			if ip := ipnet.IP.To4(); ip != nil {
 				return ip, nil
 			}
 		}
 	}
-	return nil, errors.Errorf("Error finding IPV4 address for %s", name)
+	return nil, errors.Errorf("Unable to find a IPv4 address for interface %q", name)
 }

From 96e35e0d70b3ec7d2f842abf0e292e2ff30213be Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Mon, 28 Sep 2020 11:17:59 -0700
Subject: [PATCH 39/65] Add 'docker status' check, improve deadline-exceeded
 error message

---
 pkg/minikube/registry/drvs/docker/docker.go | 73 +++++++++++++++------
 1 file changed, 54 insertions(+), 19 deletions(-)

diff --git a/pkg/minikube/registry/drvs/docker/docker.go b/pkg/minikube/registry/drvs/docker/docker.go
index ed2716a2df..d2a4f88e8a 100644
--- a/pkg/minikube/registry/drvs/docker/docker.go
+++ b/pkg/minikube/registry/drvs/docker/docker.go
@@ -27,6 +27,7 @@ import (
 
 	"github.com/docker/machine/libmachine/drivers"
 	"github.com/golang/glog"
+	"github.com/pkg/errors"
 	"k8s.io/minikube/pkg/drivers/kic"
 	"k8s.io/minikube/pkg/drivers/kic/oci"
 	"k8s.io/minikube/pkg/minikube/config"
@@ -86,34 +87,60 @@ func status() registry.State {
 	ctx, cancel := context.WithTimeout(context.Background(), 6*time.Second)
 	defer cancel()
 
-	// Quickly returns an error code if server is not running
 	cmd := exec.CommandContext(ctx, oci.Docker, "version", "--format", "{{.Server.Os}}-{{.Server.Version}}")
 	o, err := cmd.Output()
-	output := string(o)
-	if strings.Contains(output, "windows-") {
-		return registry.State{Error: oci.ErrWindowsContainers, Installed: true, Healthy: false, Fix: "Change container type to \"linux\" in Docker Desktop settings", Doc: docURL + "#verify-docker-container-type-is-linux"}
+	if err != nil {
+		if ctx.Err() == context.DeadlineExceeded {
+			err = errors.Wrapf(err, "deadline exceeded running %q", strings.Join(cmd.Args, " "))
+		}
 
-	}
-	if err == nil {
-		glog.Infof("docker version: %s", output)
-		return checkNeedsImprovement()
-	}
+		glog.Warningf("docker version returned error: %v", err)
 
-	glog.Warningf("docker returned error: %v", err)
+		if exitErr, ok := err.(*exec.ExitError); ok {
+			stderr := strings.TrimSpace(string(exitErr.Stderr))
+			newErr := fmt.Errorf(`%q %v: %s`, strings.Join(cmd.Args, " "), exitErr, stderr)
+
+			return suggestFix(stderr, newErr)
+		}
 
-	// Basic timeout
-	if ctx.Err() == context.DeadlineExceeded {
-		glog.Warningf("%q timed out. ", strings.Join(cmd.Args, " "))
 		return registry.State{Error: err, Installed: true, Healthy: false, Fix: "Restart the Docker service", Doc: docURL}
 	}
 
-	if exitErr, ok := err.(*exec.ExitError); ok {
-		stderr := strings.TrimSpace(string(exitErr.Stderr))
-		newErr := fmt.Errorf(`%q %v: %s`, strings.Join(cmd.Args, " "), exitErr, stderr)
-		return suggestFix(stderr, newErr)
+	glog.Infof("docker version: %s", o)
+	if strings.Contains(string(o), "windows-") {
+		return registry.State{Error: oci.ErrWindowsContainers, Installed: true, Healthy: false, Fix: "Change container type to \"linux\" in Docker Desktop settings", Doc: docURL + "#verify-docker-container-type-is-linux"}
 	}
 
-	return registry.State{Error: err, Installed: true, Healthy: false, Doc: docURL}
+	ctx, cancel = context.WithTimeout(context.Background(), 6*time.Second)
+	defer cancel()
+
+	// Why run "info"? On some releases (Docker Desktop v19.03.12), "version" returns exit code 0 even if Docker is down.
+	cmd = exec.CommandContext(ctx, oci.Docker, "info")
+	o, err = cmd.Output()
+	if err != nil {
+		if ctx.Err() == context.DeadlineExceeded {
+			err = errors.Wrapf(err, "deadline exceeded running %q", strings.Join(cmd.Args, " "))
+		}
+
+		glog.Warningf("docker info returned error: %v", err)
+
+		if exitErr, ok := err.(*exec.ExitError); ok {
+			stderr := strings.TrimSpace(string(exitErr.Stderr))
+			newErr := fmt.Errorf(`%q %v: %s`, strings.Join(cmd.Args, " "), exitErr, stderr)
+
+			if strings.Contains(stderr, "errors pretty printing info") || dockerNotRunning(string(o)) {
+				return registry.State{Error: err, Installed: true, Running: false, Healthy: false, Fix: "Start the Docker service", Doc: docURL}
+			}
+
+			return suggestFix(stderr, newErr)
+		}
+
+		return registry.State{Error: err, Installed: true, Healthy: false, Fix: "Restart the Docker service", Doc: docURL}
+	}
+
+	glog.Infof("docker info: %s", o)
+
+	return checkNeedsImprovement()
 }
 
 // checkNeedsImprovement if overlay mod is installed on a system
@@ -121,6 +148,7 @@ func checkNeedsImprovement() registry.State {
 	if runtime.GOOS == "linux" {
 		return checkOverlayMod()
 	}
+
 	return registry.State{Installed: true, Healthy: true}
 }
 
@@ -128,15 +156,18 @@ func checkNeedsImprovement() registry.State {
 func checkOverlayMod() registry.State {
 	if _, err := os.Stat("/sys/module/overlay"); err == nil {
 		glog.Info("overlay module found")
+
 		return registry.State{Installed: true, Healthy: true}
 	}
 
 	if _, err := os.Stat("/sys/module/overlay2"); err == nil {
 		glog.Info("overlay2 module found")
+
 		return registry.State{Installed: true, Healthy: true}
 	}
 
 	glog.Warningf("overlay modules were not found")
+
 	return registry.State{NeedsImprovement: true, Installed: true, Healthy: true, Fix: "enable the overlay Linux kernel module using 'modprobe overlay'"}
 }
 
@@ -150,10 +181,14 @@ func suggestFix(stderr string, err error) registry.State {
 		return registry.State{Error: err, Installed: true, Running: false, Healthy: false, Fix: "Start the Docker service. If Docker is already running, you may need to reset Docker to factory settings with: Settings > Reset.", Doc: "https://github.com/docker/for-win/issues/1825#issuecomment-450501157"}
 	}
 
-	if strings.Contains(stderr, "Cannot connect") || strings.Contains(stderr, "refused") || strings.Contains(stderr, "Is the docker daemon running") || strings.Contains(stderr, "docker daemon is not running") {
+	if dockerNotRunning(stderr) {
 		return registry.State{Error: err, Installed: true, Running: false, Healthy: false, Fix: "Start the Docker service", Doc: docURL}
 	}
 
 	// We don't have good advice, but at least we can provide a good error message
 	return registry.State{Error: err, Installed: true, Running: true, Healthy: false, Doc: docURL}
 }
+
+func dockerNotRunning(s string) bool {
+	return strings.Contains(s, "Cannot connect") || strings.Contains(s, "refused") || strings.Contains(s, "Is the docker daemon running") || strings.Contains(s, "docker daemon is not running")
+}

From 1ffd800c299bf75da0a019173651866d158a6b18 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Mon, 28 Sep 2020 11:27:05 -0700
Subject: [PATCH 40/65] Add comment

---
 pkg/minikube/registry/drvs/docker/docker.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pkg/minikube/registry/drvs/docker/docker.go b/pkg/minikube/registry/drvs/docker/docker.go
index d2a4f88e8a..acbff3010d 100644
--- a/pkg/minikube/registry/drvs/docker/docker.go
+++ b/pkg/minikube/registry/drvs/docker/docker.go
@@ -128,6 +128,7 @@ func status() registry.State {
 			stderr := strings.TrimSpace(string(exitErr.Stderr))
 			newErr := fmt.Errorf(`%q %v: %s`, strings.Join(cmd.Args, " "), exitErr, stderr)
 
+			// Oddly enough, the underlying connection error is in stdout rather than stderr
 			if strings.Contains(stderr, "errors pretty printing info") || dockerNotRunning(string(o)) {
 				return registry.State{Error: err, Installed: true, Running: false, Healthy: false, Fix: "Start the Docker service", Doc: docURL}
 			}

From 647c857fe1f2c525d28af6aa2f133c20b0cdb169 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= <anders.f.bjorklund@gmail.com>
Date: Mon, 28 Sep 2020 21:40:14 +0200
Subject: [PATCH 41/65] Fix error in unittest, as pointed out by warning

"conversion from int to string yields a string of one rune,
not a string of digits (did you mean fmt.Sprint(x)?)"
---
 pkg/drivers/kic/kic.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go
index 6c01be18ba..e296b327b8 100644
--- a/pkg/drivers/kic/kic.go
+++ b/pkg/drivers/kic/kic.go
@@ -403,7 +403,7 @@ func killAPIServerProc(runner command.Runner) error {
 		pid, err := strconv.Atoi(rr.Stdout.String())
 		if err == nil { // this means we have a valid pid
 			glog.Warningf("Found a kube-apiserver running with pid %d, will try to kill the proc", pid)
-			if _, err = runner.RunCmd(exec.Command("pkill", "-9", string(pid))); err != nil {
+			if _, err = runner.RunCmd(exec.Command("pkill", "-9", fmt.Sprint(pid))); err != nil {
 				return errors.Wrap(err, "kill")
 			}
 		}

From cf7df774a17a273036ece6074f085712751eb279 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= <anders.f.bjorklund@gmail.com>
Date: Mon, 28 Sep 2020 21:42:46 +0200
Subject: [PATCH 42/65] Run gofmt on code, to avoid modified workspace

---
 pkg/minikube/out/register/register.go | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/pkg/minikube/out/register/register.go b/pkg/minikube/out/register/register.go
index d4ff06af5e..22cbb5dcfe 100644
--- a/pkg/minikube/out/register/register.go
+++ b/pkg/minikube/out/register/register.go
@@ -62,7 +62,7 @@ func init() {
 	Reg = Register{
 		// Expected step orders, organized by the initial step seen
 		steps: map[RegStep][]RegStep{
-			InitialSetup: []RegStep{
+			InitialSetup: {
 				InitialSetup,
 				SelectingDriver,
 				DownloadingArtifacts,
@@ -78,10 +78,10 @@ func init() {
 				Done,
 			},
 
-			Stopping:  []RegStep{Stopping, Done},
-			Pausing:   []RegStep{Pausing, Done},
-			Unpausing: []RegStep{Unpausing, Done},
-			Deleting:  []RegStep{Deleting, Stopping, Deleting, Done},
+			Stopping:  {Stopping, Done},
+			Pausing:   {Pausing, Done},
+			Unpausing: {Unpausing, Done},
+			Deleting:  {Deleting, Stopping, Deleting, Done},
 		},
 	}
 }

From c2642c3a5c3185e3d83b6cc6763349ed7053673a Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Mon, 28 Sep 2020 16:35:33 -0400
Subject: [PATCH 43/65] Compare fully qualified image names when pulling kic

Fixes a small bug where we were comparing one fully qualified image (tag+digest) to an image that had only tag specified. This resulted in an unnecessary warning, which only showed up when the base image was loaded from a tarball. This happens on Cloud Shell.
---
 pkg/minikube/node/cache.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkg/minikube/node/cache.go b/pkg/minikube/node/cache.go
index bcb3b7807a..859c6456b0 100644
--- a/pkg/minikube/node/cache.go
+++ b/pkg/minikube/node/cache.go
@@ -127,7 +127,7 @@ func beginDownloadKicBaseImage(g *errgroup.Group, cc *config.ClusterConfig, down
 		// If we end up using a fallback image, notify the user
 		defer func() {
 			if finalImg != "" && finalImg != baseImg {
-				out.WarningT(fmt.Sprintf("minikube was unable to download %s, but successfully downloaded %s as a fallback image", image.Tag(cc.KicBaseImage), image.Tag(finalImg)))
+				out.WarningT(fmt.Sprintf("minikube was unable to download %s, but successfully downloaded %s as a fallback image", image.Tag(baseImg), image.Tag(finalImg)))
 				cc.KicBaseImage = finalImg
 			}
 		}()
@@ -136,7 +136,7 @@ func beginDownloadKicBaseImage(g *errgroup.Group, cc *config.ClusterConfig, down
 				glog.Infof("successfully loaded %s from cached tarball", img)
 				// strip the digest from the img before saving it in the config
 				// because loading an image from tarball to daemon doesn't load the digest
-				finalImg = image.Tag(img)
+				finalImg = img
 				return nil
 			}
 			glog.Infof("Downloading %s to local daemon", img)

From 5dce77b9cb81e8846e78e727497a28bd05d5743b Mon Sep 17 00:00:00 2001
From: "Jituri, Pranav" <blueelvisrocks@gmail.com>
Date: Tue, 29 Sep 2020 02:12:51 +0530
Subject: [PATCH 44/65] Make 1 target for html report and remove changes to
 github actions

---
 .github/workflows/master.yml | 30 +++++++++++++++---------------
 .github/workflows/pr.yml     | 30 +++++++++++++++---------------
 Makefile                     | 15 ++++++---------
 3 files changed, 36 insertions(+), 39 deletions(-)

diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index 0c8714c538..7aea4691f4 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -362,7 +362,7 @@ jobs:
           $env:KUBECONFIG="${pwd}\testhome\kubeconfig"
           $env:MINIKUBE_HOME="${pwd}\testhome"
           $ErrorActionPreference = "SilentlyContinue"
-          .\e2e-windows-amd64.exe --minikube-start-args="--driver=docker" --test.timeout=15m --timeout-multiplier=1 --test.v --test.run=TestFunctional --binary=./minikube-windows-amd64.exe | Out-File -Encoding utf8 -FilePath ".\report\testout.txt"
+          .\e2e-windows-amd64.exe --minikube-start-args="--driver=docker" --test.timeout=15m --timeout-multiplier=1 --test.v --test.run=TestFunctional --binary=./minikube-windows-amd64.exe | Tee-Object -FilePath ".\report\testout.txt"
           $END_TIME=(GET-DATE)
           echo $END_TIME
           $DURATION=(NEW-TIMESPAN -Start $START_TIME -End $END_TIME)
@@ -379,7 +379,7 @@ jobs:
         shell: powershell
         run: |
           cd minikube_binaries
-          Get-Content .\report\testout.txt -Encoding UTF8  | go tool test2json -t | Out-File -FilePath .\report\testout.json -Encoding utf8
+          Get-Content .\report\testout.txt -Encoding ASCII  | go tool test2json -t | Out-File -FilePath .\report\testout.json -Encoding ASCII
           $STAT=(gopogh -in .\report\testout.json -out .\report\testout.html -name "${Env:JOB_NAME} ${Env:GITHUB_REF}" -repo "${Env:GITHUB_REPOSITORY}"  -details "${Env:GITHUB_SHA}")
           echo status: ${STAT}
           $FailNum=$(echo $STAT | jq '.NumberOfFail')
@@ -412,7 +412,7 @@ jobs:
           echo "---------------- ${numFail} Failures :( ----------------------------"
           echo $failedTests
           echo "-------------------------------------------------------"
-          If ($numFail -gt 0){ exit 2 } 
+          If ($numFail -gt 0){ exit 2 }
           If ($numPass -eq 0){ exit 2 }
           If ($numPass -lt 33){ exit 2 }
           If ($numFail -eq 0){ exit 0 }
@@ -429,7 +429,7 @@ jobs:
         shell: powershell
         run: |
           echo $env:computerName
-          ls 
+          ls
           $ErrorActionPreference = "SilentlyContinue"
           cd minikube_binaries
           ls
@@ -464,14 +464,14 @@ jobs:
           $docker_running = $?
           }
           Write-Output "Docker is running"
-          docker system prune -f 
+          docker system prune -f
       - name: Info
         continue-on-error: true
         shell: powershell
         run: |
           $ErrorActionPreference = "SilentlyContinue"
           cd minikube_binaries
-          ls 
+          ls
           echo $env:computername
           Get-WmiObject -class Win32_ComputerSystem
       - uses: actions/setup-go@v2
@@ -483,7 +483,7 @@ jobs:
         shell: powershell
         run: |
           $ErrorActionPreference = "SilentlyContinue"
-          (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe")          
+          (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe")
           choco install -y kubernetes-cli
           choco install -y jq
           choco install -y caffeine
@@ -499,7 +499,7 @@ jobs:
           $env:KUBECONFIG="${pwd}\testhome\kubeconfig"
           $env:MINIKUBE_HOME="${pwd}\testhome"
           $ErrorActionPreference = "SilentlyContinue"
-          .\e2e-windows-amd64.exe --minikube-start-args="--driver=hyperv" --test.timeout=20m --timeout-multiplier=1.5 --test.v --test.run=TestFunctional --binary=./minikube-windows-amd64.exe | Out-File -Encoding utf8 -FilePath ".\report\testout.txt"
+          .\e2e-windows-amd64.exe --minikube-start-args="--driver=hyperv" --test.timeout=20m --timeout-multiplier=1.5 --test.v --test.run=TestFunctional --binary=./minikube-windows-amd64.exe | Tee-Object -FilePath ".\report\testout.txt"
           $END_TIME=(GET-DATE)
           echo $END_TIME
           $DURATION=(NEW-TIMESPAN -Start $START_TIME -End $END_TIME)
@@ -516,7 +516,7 @@ jobs:
         shell: powershell
         run: |
           cd minikube_binaries
-          Get-Content .\report\testout.txt -Encoding UTF8  | go tool test2json -t | Out-File -FilePath .\report\testout.json -Encoding utf8
+          Get-Content .\report\testout.txt -Encoding ASCII  | go tool test2json -t | Out-File -FilePath .\report\testout.json -Encoding ASCII
           $STAT=(gopogh -in .\report\testout.json -out .\report\testout.html -name "${Env:JOB_NAME} ${Env:GITHUB_REF}" -repo "${Env:GITHUB_REPOSITORY}"  -details "${Env:GITHUB_SHA}")
           echo status: ${STAT}
           $FailNum=$(echo $STAT | jq '.NumberOfFail')
@@ -549,7 +549,7 @@ jobs:
           echo "---------------- ${numFail} Failures :( ----------------------------"
           echo $failedTests
           echo "-------------------------------------------------------"
-          If ($numFail -gt 0){ exit 2 } 
+          If ($numFail -gt 0){ exit 2 }
           If ($numPass -eq 0){ exit 2 }
           If ($numPass -lt 33){ exit 2 }
           If ($numFail -eq 0){ exit 0 }
@@ -775,11 +775,11 @@ jobs:
         run: |
           hostname
           VBoxManage --version
-          sysctl hw.physicalcpu hw.logicalcpu 
+          sysctl hw.physicalcpu hw.logicalcpu
       - name: Disable firewall
         run: |
           sudo /usr/libexec/ApplicationFirewall/socketfilterfw --setglobalstate off
-          sudo /usr/libexec/ApplicationFirewall/socketfilterfw -k      
+          sudo /usr/libexec/ApplicationFirewall/socketfilterfw -k
       - name: Download Binaries
         uses: actions/download-artifact@v1
         with:
@@ -966,7 +966,7 @@ jobs:
         run: |
           hostname
           VBoxManage --version
-          sysctl hw.physicalcpu hw.logicalcpu 
+          sysctl hw.physicalcpu hw.logicalcpu
       - name: Disable firewall
         run: |
           sudo /usr/libexec/ApplicationFirewall/socketfilterfw --setglobalstate off
@@ -1150,7 +1150,7 @@ jobs:
         run: |
           hostname
           VBoxManage --version
-          sysctl hw.physicalcpu hw.logicalcpu 
+          sysctl hw.physicalcpu hw.logicalcpu
       - name: Disable firewall
         run: |
           sudo /usr/libexec/ApplicationFirewall/socketfilterfw --setglobalstate off
@@ -1247,4 +1247,4 @@ jobs:
       - uses: actions/upload-artifact@v1
         with:
           name: all_reports
-          path: all_reports
+          path: all_reports
\ No newline at end of file
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 9a663222aa..cbb271de0d 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -360,7 +360,7 @@ jobs:
           $env:KUBECONFIG="${pwd}\testhome\kubeconfig"
           $env:MINIKUBE_HOME="${pwd}\testhome"
           $ErrorActionPreference = "SilentlyContinue"
-          .\e2e-windows-amd64.exe --minikube-start-args="--driver=docker" --test.timeout=15m --timeout-multiplier=1 --test.v --test.run=TestFunctional --binary=./minikube-windows-amd64.exe | Out-File -Encoding utf8 -FilePath ".\report\testout.txt"
+          .\e2e-windows-amd64.exe --minikube-start-args="--driver=docker" --test.timeout=15m --timeout-multiplier=1 --test.v --test.run=TestFunctional --binary=./minikube-windows-amd64.exe | Tee-Object -FilePath ".\report\testout.txt"
           $END_TIME=(GET-DATE)
           echo $END_TIME
           $DURATION=(NEW-TIMESPAN -Start $START_TIME -End $END_TIME)
@@ -377,7 +377,7 @@ jobs:
         shell: powershell
         run: |
           cd minikube_binaries
-          Get-Content .\report\testout.txt -Encoding UTF8  | go tool test2json -t | Out-File -FilePath .\report\testout.json -Encoding utf8
+          Get-Content .\report\testout.txt -Encoding ASCII  | go tool test2json -t | Out-File -FilePath .\report\testout.json -Encoding ASCII
           $STAT=(gopogh -in .\report\testout.json -out .\report\testout.html -name "${Env:JOB_NAME} ${Env:GITHUB_REF}" -repo "${Env:GITHUB_REPOSITORY}"  -details "${Env:GITHUB_SHA}")
           echo status: ${STAT}
           $FailNum=$(echo $STAT | jq '.NumberOfFail')
@@ -410,7 +410,7 @@ jobs:
           echo "---------------- ${numFail} Failures :( ----------------------------"
           echo $failedTests
           echo "-------------------------------------------------------"
-          If ($numFail -gt 0){ exit 2 } 
+          If ($numFail -gt 0){ exit 2 }
           If ($numPass -eq 0){ exit 2 }
           If ($numPass -lt 33){ exit 2 }
           If ($numFail -eq 0){ exit 0 }
@@ -427,7 +427,7 @@ jobs:
         shell: powershell
         run: |
           echo $env:computerName
-          ls 
+          ls
           $ErrorActionPreference = "SilentlyContinue"
           cd minikube_binaries
           ls
@@ -462,14 +462,14 @@ jobs:
           $docker_running = $?
           }
           Write-Output "Docker is running"
-          docker system prune -f 
+          docker system prune -f
       - name: Info
         continue-on-error: true
         shell: powershell
         run: |
           $ErrorActionPreference = "SilentlyContinue"
           cd minikube_binaries
-          ls 
+          ls
           echo $env:computername
           Get-WmiObject -class Win32_ComputerSystem
       - uses: actions/setup-go@v2
@@ -481,7 +481,7 @@ jobs:
         shell: powershell
         run: |
           $ErrorActionPreference = "SilentlyContinue"
-          (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe")          
+          (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe")
           choco install -y kubernetes-cli
           choco install -y jq
           choco install -y caffeine
@@ -497,7 +497,7 @@ jobs:
           $env:KUBECONFIG="${pwd}\testhome\kubeconfig"
           $env:MINIKUBE_HOME="${pwd}\testhome"
           $ErrorActionPreference = "SilentlyContinue"
-          .\e2e-windows-amd64.exe --minikube-start-args="--driver=hyperv" --test.timeout=20m --timeout-multiplier=1.5 --test.v --test.run=TestFunctional --binary=./minikube-windows-amd64.exe | Out-File -Encoding utf8 -FilePath ".\report\testout.txt"
+          .\e2e-windows-amd64.exe --minikube-start-args="--driver=hyperv" --test.timeout=20m --timeout-multiplier=1.5 --test.v --test.run=TestFunctional --binary=./minikube-windows-amd64.exe | Tee-Object -FilePath ".\report\testout.txt"
           $END_TIME=(GET-DATE)
           echo $END_TIME
           $DURATION=(NEW-TIMESPAN -Start $START_TIME -End $END_TIME)
@@ -514,7 +514,7 @@ jobs:
         shell: powershell
         run: |
           cd minikube_binaries
-          Get-Content .\report\testout.txt -Encoding UTF8  | go tool test2json -t | Out-File -FilePath .\report\testout.json -Encoding utf8
+          Get-Content .\report\testout.txt -Encoding ASCII  | go tool test2json -t | Out-File -FilePath .\report\testout.json -Encoding ASCII
           $STAT=(gopogh -in .\report\testout.json -out .\report\testout.html -name "${Env:JOB_NAME} ${Env:GITHUB_REF}" -repo "${Env:GITHUB_REPOSITORY}"  -details "${Env:GITHUB_SHA}")
           echo status: ${STAT}
           $FailNum=$(echo $STAT | jq '.NumberOfFail')
@@ -547,7 +547,7 @@ jobs:
           echo "---------------- ${numFail} Failures :( ----------------------------"
           echo $failedTests
           echo "-------------------------------------------------------"
-          If ($numFail -gt 0){ exit 2 } 
+          If ($numFail -gt 0){ exit 2 }
           If ($numPass -eq 0){ exit 2 }
           If ($numPass -lt 33){ exit 2 }
           If ($numFail -eq 0){ exit 0 }
@@ -773,11 +773,11 @@ jobs:
         run: |
           hostname
           VBoxManage --version
-          sysctl hw.physicalcpu hw.logicalcpu 
+          sysctl hw.physicalcpu hw.logicalcpu
       - name: Disable firewall
         run: |
           sudo /usr/libexec/ApplicationFirewall/socketfilterfw --setglobalstate off
-          sudo /usr/libexec/ApplicationFirewall/socketfilterfw -k      
+          sudo /usr/libexec/ApplicationFirewall/socketfilterfw -k
       - name: Download Binaries
         uses: actions/download-artifact@v1
         with:
@@ -964,7 +964,7 @@ jobs:
         run: |
           hostname
           VBoxManage --version
-          sysctl hw.physicalcpu hw.logicalcpu 
+          sysctl hw.physicalcpu hw.logicalcpu
       - name: Disable firewall
         run: |
           sudo /usr/libexec/ApplicationFirewall/socketfilterfw --setglobalstate off
@@ -1148,7 +1148,7 @@ jobs:
         run: |
           hostname
           VBoxManage --version
-          sysctl hw.physicalcpu hw.logicalcpu 
+          sysctl hw.physicalcpu hw.logicalcpu
       - name: Disable firewall
         run: |
           sudo /usr/libexec/ApplicationFirewall/socketfilterfw --setglobalstate off
@@ -1245,4 +1245,4 @@ jobs:
       - uses: actions/upload-artifact@v1
         with:
           name: all_reports
-          path: all_reports
+          path: all_reports
\ No newline at end of file
diff --git a/Makefile b/Makefile
index c48f308e30..6a028bb312 100644
--- a/Makefile
+++ b/Makefile
@@ -305,18 +305,15 @@ html_report: ## Generate HTML  report out of the last ran integration test logs.
 	@go tool test2json -t < "./out/testout_$(COMMIT_SHORT).txt" > "./out/testout_$(COMMIT_SHORT).json"
 	@gopogh -in "./out/testout_$(COMMIT_SHORT).json" -out ./out/testout_$(COMMIT_SHORT).html -name "$(shell git rev-parse --abbrev-ref HEAD)" -pr "" -repo github.com/kubernetes/minikube/  -details "${COMMIT_SHORT}"
 	@echo "-------------------------- Open HTML Report in Browser: ---------------------------"
+ifeq ($(GOOS),windows)
+	@echo start $(CURDIR)/out/testout_$(COMMIT_SHORT).html
+	@echo "-----------------------------------------------------------------------------------"
+	@start $(CURDIR)/out/testout_$(COMMIT_SHORT).html || true
+else
 	@echo open $(CURDIR)/out/testout_$(COMMIT_SHORT).html
 	@echo "-----------------------------------------------------------------------------------"
 	@open $(CURDIR)/out/testout_$(COMMIT_SHORT).html || true
-
-.PHONY: html_report_windows
-html_report_windows: ## Generate the HTML Report out of the last ran integration test logs on Windows.
-	@go tool test2json -t < "./out/testout_$(COMMIT_SHORT).txt" > "./out/testout_$(COMMIT_SHORT).json"
-	@gopogh -in "./out/testout_$(COMMIT_SHORT).json" -out ./out/testout_$(COMMIT_SHORT).html -name "$(shell git rev-parse --abbrev-ref HEAD)" -pr "" -repo github.com/kubernetes/minikube/ -details "${COMMIT_SHORT}"
-	@echo "-------------------------- Open HTML Report in Browser: ---------------------------"
-	@echo $(CURDIR)/out/testout_$(COMMIT_SHORT).html
-	@echo "-----------------------------------------------------------------------------------"
-	@start $(CURDIR)/out/testout_$(COMMIT_SHORT).html || true
+endif
 
 .PHONY: test
 test: pkg/minikube/assets/assets.go pkg/minikube/translate/translations.go ## Trigger minikube test

From 5965b687337cda97ab8919aff5249b9c0a0f4e2d Mon Sep 17 00:00:00 2001
From: Till Hoffmann <till.hoffmann@enforge.de>
Date: Tue, 29 Sep 2020 10:31:34 +0200
Subject: [PATCH 45/65] Improved file-filter for github-release-assets

---
 hack/jenkins/release_github_page.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hack/jenkins/release_github_page.sh b/hack/jenkins/release_github_page.sh
index 676fea3942..49cbf197a3 100755
--- a/hack/jenkins/release_github_page.sh
+++ b/hack/jenkins/release_github_page.sh
@@ -81,7 +81,7 @@ for path in $(gsutil ls "gs://${ISO_BUCKET}/minikube-v${VERSION}*" || true); do
 done
  
 # Upload all end-user assets other than preload files, as they are release independent
-for file in out/minikube[_-]* out/docker-machine-*; do
+for file in $( find out \( -name "minikube[_-]*" -or -name "docker-machine-*"  \) -and ! -name "*latest*"); do
     n=0
     until [ $n -ge 5 ]
     do

From 8eaa3ee4f3bd102f379f94b9303f1929edba9f28 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Tue, 29 Sep 2020 09:19:14 -0700
Subject: [PATCH 46/65] Use oci.CachedDaemonInfo for status check

---
 pkg/drivers/kic/oci/cli_runner.go           |  6 ++--
 pkg/drivers/kic/oci/info.go                 | 23 +++++++++------
 pkg/minikube/registry/drvs/docker/docker.go | 32 ++++-----------------
 3 files changed, 25 insertions(+), 36 deletions(-)

diff --git a/pkg/drivers/kic/oci/cli_runner.go b/pkg/drivers/kic/oci/cli_runner.go
index 9a5cabd567..9799c7391b 100644
--- a/pkg/drivers/kic/oci/cli_runner.go
+++ b/pkg/drivers/kic/oci/cli_runner.go
@@ -144,9 +144,11 @@ func runCmd(cmd *exec.Cmd, warnSlow ...bool) (*RunResult, error) {
 		}
 	}
 
-	if exitError, ok := err.(*exec.ExitError); ok {
-		rr.ExitCode = exitError.ExitCode()
+	if ex, ok := err.(*exec.ExitError); ok {
+		glog.Warningf("%s returned with exit code %s", rr.Command(), ex.ExitCode())
+		rr.ExitCode = ex.ExitCode()
 	}
+
 	// Decrease log spam
 	if elapsed > (1 * time.Second) {
 		glog.Infof("Completed: %s: (%s)", rr.Command(), elapsed)
diff --git a/pkg/drivers/kic/oci/info.go b/pkg/drivers/kic/oci/info.go
index e13d641a1e..e58a122967 100644
--- a/pkg/drivers/kic/oci/info.go
+++ b/pkg/drivers/kic/oci/info.go
@@ -22,20 +22,24 @@ import (
 	"strings"
 	"time"
 
+	"github.com/golang/glog"
 	"github.com/pkg/errors"
 )
 
 // SysInfo Info represents common system Information between docker and podman that minikube cares
 type SysInfo struct {
-	CPUs          int    // CPUs is Number of CPUs
-	TotalMemory   int64  // TotalMemory Total available ram
-	OSType        string // container's OsType (windows or linux)
-	Swarm         bool   // Weather or not the docker swarm is active
-	StorageDriver string // the storage driver for the daemon  (for example overlay2)
+	CPUs          int      // CPUs is Number of CPUs
+	TotalMemory   int64    // TotalMemory Total available ram
+	OSType        string   // container's OsType (windows or linux)
+	Swarm         bool     // Weather or not the docker swarm is active
+	StorageDriver string   // the storage driver for the daemon  (for example overlay2)
+	Errors        []string // any server issues
 }
 
-var cachedSysInfo *SysInfo
-var cachedSysInfoErr *error
+var (
+	cachedSysInfo    *SysInfo
+	cachedSysInfoErr *error
+)
 
 // CachedDaemonInfo will run and return a docker/podman info only once per minikube run time. to avoid performance
 func CachedDaemonInfo(ociBin string) (SysInfo, error) {
@@ -58,7 +62,7 @@ func DaemonInfo(ociBin string) (SysInfo, error) {
 		return *cachedSysInfo, err
 	}
 	d, err := dockerSystemInfo()
-	cachedSysInfo = &SysInfo{CPUs: d.NCPU, TotalMemory: d.MemTotal, OSType: d.OSType, Swarm: d.Swarm.LocalNodeState == "active", StorageDriver: d.Driver}
+	cachedSysInfo = &SysInfo{CPUs: d.NCPU, TotalMemory: d.MemTotal, OSType: d.OSType, Swarm: d.Swarm.LocalNodeState == "active", StorageDriver: d.Driver, Errors: d.ServerErrors}
 	return *cachedSysInfo, err
 }
 
@@ -163,6 +167,7 @@ type dockerSysInfo struct {
 	SecurityOptions []string    `json:"SecurityOptions"`
 	ProductLicense  string      `json:"ProductLicense"`
 	Warnings        interface{} `json:"Warnings"`
+	ServerErrors    []string
 	ClientInfo      struct {
 		Debug    bool          `json:"Debug"`
 		Plugins  []interface{} `json:"Plugins"`
@@ -245,6 +250,7 @@ func dockerSystemInfo() (dockerSysInfo, error) {
 		return ds, errors.Wrapf(err, "unmarshal docker system info")
 	}
 
+	glog.Infof("docker info: %+v", ds)
 	return ds, nil
 }
 
@@ -264,5 +270,6 @@ func podmanSystemInfo() (podmanSysInfo, error) {
 	if err := json.Unmarshal([]byte(strings.TrimSpace(rawJSON)), &ps); err != nil {
 		return ps, errors.Wrapf(err, "unmarshal podman system info")
 	}
+	glog.Infof("podman info: %+v", ps)
 	return ps, nil
 }
diff --git a/pkg/minikube/registry/drvs/docker/docker.go b/pkg/minikube/registry/drvs/docker/docker.go
index acbff3010d..38a325a6aa 100644
--- a/pkg/minikube/registry/drvs/docker/docker.go
+++ b/pkg/minikube/registry/drvs/docker/docker.go
@@ -111,35 +111,15 @@ func status() registry.State {
 		return registry.State{Error: oci.ErrWindowsContainers, Installed: true, Healthy: false, Fix: "Change container type to \"linux\" in Docker Desktop settings", Doc: docURL + "#verify-docker-container-type-is-linux"}
 	}
 
-	ctx, cancel = context.WithTimeout(context.Background(), 6*time.Second)
-	defer cancel()
-
-	// Why run "info"? On some releases (Docker Desktop v19.03.12), "version" returns exit code 0 even if Docker is down.
-	cmd = exec.CommandContext(ctx, oci.Docker, "info")
-	o, err = cmd.Output()
+	si, err := oci.CachedDaemonInfo("docker")
 	if err != nil {
-		if ctx.Err() == context.DeadlineExceeded {
-			err = errors.Wrapf(err, "deadline exceeded running %q", strings.Join(cmd.Args, " "))
-		}
-
-		glog.Warningf("docker info returned error: %v", err)
-
-		if exitErr, ok := err.(*exec.ExitError); ok {
-			stderr := strings.TrimSpace(string(exitErr.Stderr))
-			newErr := fmt.Errorf(`%q %v: %s`, strings.Join(cmd.Args, " "), exitErr, stderr)
-
-			// Oddly enough, the underlying connection error is in stdout rather than stderr
-			if strings.Contains(stderr, "errors pretty printing info") || dockerNotRunning(string(o)) {
-				return registry.State{Error: err, Installed: true, Running: false, Healthy: false, Fix: "Start the Docker service", Doc: docURL}
-			}
-
-			return suggestFix(stderr, newErr)
-		}
-
-		return registry.State{Error: err, Installed: true, Healthy: false, Fix: "Restart the Docker service", Doc: docURL}
+		// No known fix because we haven't yet seen a failure here
+		return registry.State{Error: errors.Wrap(err, "docker info"), Installed: true, Healthy: false, Doc: docURL}
 	}
 
-	glog.Infof("docker info: %s", o)
+	for _, serr := range si.Errors {
+		return suggestFix(serr, fmt.Errorf("docker info error: %s", serr))
+	}
 
 	return checkNeedsImprovement()
 }

From 91a32541e827528bc84b0f933a234e551debda54 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Tue, 29 Sep 2020 09:49:30 -0700
Subject: [PATCH 47/65] Use %d for exit code

---
 pkg/drivers/kic/oci/cli_runner.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/drivers/kic/oci/cli_runner.go b/pkg/drivers/kic/oci/cli_runner.go
index 9799c7391b..7169e12bdb 100644
--- a/pkg/drivers/kic/oci/cli_runner.go
+++ b/pkg/drivers/kic/oci/cli_runner.go
@@ -145,7 +145,7 @@ func runCmd(cmd *exec.Cmd, warnSlow ...bool) (*RunResult, error) {
 	}
 
 	if ex, ok := err.(*exec.ExitError); ok {
-		glog.Warningf("%s returned with exit code %s", rr.Command(), ex.ExitCode())
+		glog.Warningf("%s returned with exit code %d", rr.Command(), ex.ExitCode())
 		rr.ExitCode = ex.ExitCode()
 	}
 

From 5ab4bc64fc2265a3f66915cb81088b61f0c90358 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= <anders.f.bjorklund@gmail.com>
Date: Tue, 29 Sep 2020 20:13:05 +0200
Subject: [PATCH 48/65] Remove the base target from kicbase docker build

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 2a87f19f67..cf86209e3b 100644
--- a/Makefile
+++ b/Makefile
@@ -579,7 +579,7 @@ storage-provisioner-image: out/storage-provisioner-$(GOARCH) ## Build storage-pr
 .PHONY: kic-base-image
 kic-base-image: ## builds the base image used for kic.
 	docker rmi -f $(KIC_BASE_IMAGE_GCR)-snapshot || true
-	docker build -f ./deploy/kicbase/Dockerfile -t local/kicbase:$(KIC_VERSION)-snapshot  --build-arg COMMIT_SHA=${VERSION}-$(COMMIT) --cache-from $(KIC_BASE_IMAGE_GCR) --target base ./deploy/kicbase
+	docker build -f ./deploy/kicbase/Dockerfile -t local/kicbase:$(KIC_VERSION)-snapshot  --build-arg COMMIT_SHA=${VERSION}-$(COMMIT) --cache-from $(KIC_BASE_IMAGE_GCR) ./deploy/kicbase
 	docker tag local/kicbase:$(KIC_VERSION)-snapshot $(KIC_BASE_IMAGE_GCR)-snapshot
 	docker tag local/kicbase:$(KIC_VERSION)-snapshot $(KIC_BASE_IMAGE_GCR)
 	docker tag local/kicbase:$(KIC_VERSION)-snapshot $(KIC_BASE_IMAGE_HUB)

From b3401405f09c3e879ec9bc61a8b5d4723b6881d0 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Tue, 29 Sep 2020 13:03:11 -0700
Subject: [PATCH 49/65] update kicbase image to ubuntu-based

---
 pkg/drivers/kic/types.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go
index 5d138a0ea8..7e51ce945c 100644
--- a/pkg/drivers/kic/types.go
+++ b/pkg/drivers/kic/types.go
@@ -24,9 +24,9 @@ import (
 
 const (
 	// Version is the current version of kic
-	Version = "v0.0.12-snapshot3"
+	Version = "v0.0.13-snapshot1"
 	// SHA of the kic base image
-	baseImageSHA = "1d687ba53e19dbe5fafe4cc18aa07f269ecc4b7b622f2251b5bf569ddb474e9b"
+	baseImageSHA = "4d43acbd0050148d4bc399931f1b15253b5e73815b63a67b8ab4a5c9e523403f"
 )
 
 var (

From 23eb1690557403e1a578a7b17d24d794ad1cbc4e Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Tue, 29 Sep 2020 13:16:37 -0700
Subject: [PATCH 50/65] generate docs

---
 site/content/en/docs/commands/start.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/site/content/en/docs/commands/start.md b/site/content/en/docs/commands/start.md
index 72163a3365..87538fb59c 100644
--- a/site/content/en/docs/commands/start.md
+++ b/site/content/en/docs/commands/start.md
@@ -26,7 +26,7 @@ minikube start [flags]
       --apiserver-names stringArray       A set of apiserver names which are used in the generated certificate for kubernetes.  This can be used if you want to make the apiserver available from outside the machine
       --apiserver-port int                The apiserver listening port (default 8443)
       --auto-update-drivers               If set, automatically updates drivers to the latest version. Defaults to true. (default true)
-      --base-image string                 The base image to use for docker/podman drivers. Intended for local development. (default "gcr.io/k8s-minikube/kicbase:v0.0.12-snapshot3@sha256:1d687ba53e19dbe5fafe4cc18aa07f269ecc4b7b622f2251b5bf569ddb474e9b")
+      --base-image string                 The base image to use for docker/podman drivers. Intended for local development. (default "gcr.io/k8s-minikube/kicbase:v0.0.13-snapshot1@sha256:4d43acbd0050148d4bc399931f1b15253b5e73815b63a67b8ab4a5c9e523403f")
       --cache-images                      If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --driver=none. (default true)
       --cni string                        CNI plug-in to use. Valid options: auto, bridge, calico, cilium, flannel, kindnet, or path to a CNI manifest (default: auto)
       --container-runtime string          The container runtime to be used (docker, cri-o, containerd). (default "docker")

From ff051f9a3357947386f35b85e1a6c85d2620f517 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Tue, 29 Sep 2020 13:29:14 -0700
Subject: [PATCH 51/65] Make sure gcp-auth addon can be enabled on startup
 (#9318)

* fix documentation for gcp-auth addon

* make sure kube-system pods are up before enabling gcp-auth

* fix lint

* add failurePolicy for webhook

* only install addons if asked

* better comment

* slightly less hacky code

* defer addons properly

* simplify code for performance
---
 deploy/addons/gcp-auth/gcp-auth-webhook.yaml  |  7 ++++
 pkg/addons/addons.go                          | 40 +++++++++++++++++--
 pkg/addons/config.go                          |  2 +-
 pkg/addons/gcpauth/enable.go                  | 15 +------
 .../en/docs/handbook/addons/gcp-auth.md       |  2 +-
 5 files changed, 46 insertions(+), 20 deletions(-)

diff --git a/deploy/addons/gcp-auth/gcp-auth-webhook.yaml b/deploy/addons/gcp-auth/gcp-auth-webhook.yaml
index cb06808cc7..81f4de2cc1 100644
--- a/deploy/addons/gcp-auth/gcp-auth-webhook.yaml
+++ b/deploy/addons/gcp-auth/gcp-auth-webhook.yaml
@@ -131,10 +131,17 @@ metadata:
     app: gcp-auth
 webhooks:
 - name: gcp-auth-mutate.k8s.io
+  failurePolicy: Fail
   objectSelector:
     matchExpressions:
       - key: gcp-auth-skip-secret
         operator: DoesNotExist
+  namespaceSelector:
+    matchExpressions:
+      - key: name
+        operator: NotIn
+        values:
+        - kube-system
   sideEffects: None
   admissionReviewVersions: ["v1","v1beta1"]
   clientConfig:
diff --git a/pkg/addons/addons.go b/pkg/addons/addons.go
index 8235e0cfd9..a7ca64a21e 100644
--- a/pkg/addons/addons.go
+++ b/pkg/addons/addons.go
@@ -324,7 +324,18 @@ func verifyAddonStatus(cc *config.ClusterConfig, name string, val string) error
 }
 
 func verifyGCPAuthAddon(cc *config.ClusterConfig, name string, val string) error {
-	return verifyAddonStatusInternal(cc, name, val, "gcp-auth")
+	enable, err := strconv.ParseBool(val)
+	if err != nil {
+		return errors.Wrapf(err, "parsing bool: %s", name)
+	}
+	err = verifyAddonStatusInternal(cc, name, val, "gcp-auth")
+
+	if enable && err == nil {
+		out.T(style.Notice, "Your GCP credentials will now be mounted into every pod created in the {{.name}} cluster.", out.V{"name": cc.Name})
+		out.T(style.Notice, "If you don't want your credentials mounted into a specific pod, add a label with the `gcp-auth-skip-secret` key to your pod configuration.")
+	}
+
+	return err
 }
 
 func verifyAddonStatusInternal(cc *config.ClusterConfig, name string, val string, ns string) error {
@@ -394,16 +405,26 @@ func Start(wg *sync.WaitGroup, cc *config.ClusterConfig, toEnable map[string]boo
 
 	var awg sync.WaitGroup
 
-	defer func() { // making it show after verifications( not perfect till #7613 is closed)
+	enabledAddons := []string{}
+	deferredAddons := []string{}
+
+	defer func() { // making it show after verifications (see #7613)
 		register.Reg.SetStep(register.EnablingAddons)
-		out.T(style.AddonEnable, "Enabled addons: {{.addons}}", out.V{"addons": strings.Join(toEnableList, ", ")})
+		out.T(style.AddonEnable, "Enabled addons: {{.addons}}", out.V{"addons": strings.Join(enabledAddons, ", ")})
 	}()
 	for _, a := range toEnableList {
+		if a == "gcp-auth" {
+			deferredAddons = append(deferredAddons, a)
+			continue
+		}
+
 		awg.Add(1)
 		go func(name string) {
 			err := RunCallbacks(cc, name, "true")
 			if err != nil {
 				out.WarningT("Enabling '{{.name}}' returned an error: {{.error}}", out.V{"name": name, "error": err})
+			} else {
+				enabledAddons = append(enabledAddons, name)
 			}
 			awg.Done()
 		}(a)
@@ -411,7 +432,18 @@ func Start(wg *sync.WaitGroup, cc *config.ClusterConfig, toEnable map[string]boo
 
 	// Wait until all of the addons are enabled before updating the config (not thread safe)
 	awg.Wait()
-	for _, a := range toEnableList {
+
+	// Now run the deferred addons
+	for _, a := range deferredAddons {
+		err := RunCallbacks(cc, a, "true")
+		if err != nil {
+			out.WarningT("Enabling '{{.name}}' returned an error: {{.error}}", out.V{"name": a, "error": err})
+		} else {
+			enabledAddons = append(enabledAddons, a)
+		}
+	}
+
+	for _, a := range enabledAddons {
 		if err := Set(cc, a, "true"); err != nil {
 			glog.Errorf("store failed: %v", err)
 		}
diff --git a/pkg/addons/config.go b/pkg/addons/config.go
index 088698d91a..980915f407 100644
--- a/pkg/addons/config.go
+++ b/pkg/addons/config.go
@@ -169,7 +169,7 @@ var Addons = []*Addon{
 	{
 		name:      "gcp-auth",
 		set:       SetBool,
-		callbacks: []setFn{gcpauth.EnableOrDisable, enableOrDisableAddon, verifyGCPAuthAddon, gcpauth.DisplayAddonMessage},
+		callbacks: []setFn{gcpauth.EnableOrDisable, enableOrDisableAddon, verifyGCPAuthAddon},
 	},
 	{
 		name:      "volumesnapshots",
diff --git a/pkg/addons/gcpauth/enable.go b/pkg/addons/gcpauth/enable.go
index 46c9ca0ec3..a394d481ef 100644
--- a/pkg/addons/gcpauth/enable.go
+++ b/pkg/addons/gcpauth/enable.go
@@ -60,7 +60,7 @@ func enableAddon(cfg *config.ClusterConfig) error {
 	ctx := context.Background()
 	creds, err := google.FindDefaultCredentials(ctx)
 	if err != nil {
-		exit.Message(reason.InternalCredsNotFound, "Could not find any GCP credentials. Either run `gcloud auth login` or set the GOOGLE_APPLICATION_CREDENTIALS environment variable to the path of your credentials file.")
+		exit.Message(reason.InternalCredsNotFound, "Could not find any GCP credentials. Either run `gcloud auth application-default login` or set the GOOGLE_APPLICATION_CREDENTIALS environment variable to the path of your credentials file.")
 	}
 
 	f := assets.NewMemoryAssetTarget(creds.JSON, credentialsPath, "0444")
@@ -116,16 +116,3 @@ func disableAddon(cfg *config.ClusterConfig) error {
 
 	return nil
 }
-
-// DisplayAddonMessage display an gcp auth addon specific message to the user
-func DisplayAddonMessage(cfg *config.ClusterConfig, name string, val string) error {
-	enable, err := strconv.ParseBool(val)
-	if err != nil {
-		return errors.Wrapf(err, "parsing bool: %s", name)
-	}
-	if enable {
-		out.T(style.Notice, "Your GCP credentials will now be mounted into every pod created in the {{.name}} cluster.", out.V{"name": cfg.Name})
-		out.T(style.Notice, "If you don't want your credentials mounted into a specific pod, add a label with the `gcp-auth-skip-secret` key to your pod configuration.")
-	}
-	return nil
-}
diff --git a/site/content/en/docs/handbook/addons/gcp-auth.md b/site/content/en/docs/handbook/addons/gcp-auth.md
index 14194909e9..19aabd3920 100644
--- a/site/content/en/docs/handbook/addons/gcp-auth.md
+++ b/site/content/en/docs/handbook/addons/gcp-auth.md
@@ -5,7 +5,7 @@ weight: 1
 date: 2020-07-15
 ---
 
-If you have a containerized GCP app with a Kubernetes yaml, you can automatically add your credentials to all your deployed pods dynamically with this minikube addon. You just need to have a credentials file, which can be generated with `gcloud auth login`. If you already have a json credentials file you want specify, use the GOOGLE_APPLICATION_CREDENTIALS environment variable.
+If you have a containerized GCP app with a Kubernetes yaml, you can automatically add your credentials to all your deployed pods dynamically with this minikube addon. You just need to have a credentials file, which can be generated with `gcloud auth application-default login`. If you already have a json credentials file you want specify, use the GOOGLE_APPLICATION_CREDENTIALS environment variable.
 
 - Start a cluster:
 ```

From 9cd42163b1679e9cf38f2816a40c92cd5b6ddaef Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Mon, 21 Sep 2020 15:56:08 -0700
Subject: [PATCH 52/65] add network create funcs

---
 pkg/drivers/kic/oci/errors.go         |  15 ++++
 pkg/drivers/kic/oci/network_create.go | 125 ++++++++++++++++++++++++++
 2 files changed, 140 insertions(+)
 create mode 100644 pkg/drivers/kic/oci/network_create.go

diff --git a/pkg/drivers/kic/oci/errors.go b/pkg/drivers/kic/oci/errors.go
index 8c30201f97..0889358bc0 100644
--- a/pkg/drivers/kic/oci/errors.go
+++ b/pkg/drivers/kic/oci/errors.go
@@ -39,12 +39,27 @@ var ErrWindowsContainers = &FailFastError{errors.New("docker container type is w
 // ErrCPUCountLimit is thrown when docker daemon doesn't have enough CPUs for the requested container
 var ErrCPUCountLimit = &FailFastError{errors.New("not enough CPUs is available for container")}
 
+// ErrIPinUse is thrown when the container been given an IP used by another container
+var ErrIPinUse = &FailFastError{errors.New("can't create with that IP, address already in use")}
+
 // ErrExitedUnexpectedly is thrown when container is created/started without error but later it exists and it's status is not running anymore.
 var ErrExitedUnexpectedly = errors.New("container exited unexpectedly")
 
 // ErrDaemonInfo is thrown when docker/podman info is failing or not responding
 var ErrDaemonInfo = errors.New("daemon info not responding")
 
+// ErrNetworkSubnetTaken is thrown when a subnet is taken by another network
+var ErrNetworkSubnetTaken = errors.New("subnet is taken")
+
+// ErrNetworkNotFound is when given network was not found
+var ErrNetworkNotFound = errors.New("kic network not found")
+
+// ErrNetworkGatewayTaken is when given network gatway is taken
+var ErrNetworkGatewayTaken = errors.New("network gateway is taken")
+
+// ErrNetworkInUse is when trying to delete a network which is attached to another container
+var ErrNetworkInUse = errors.New("can't delete network attached to a running container")
+
 // LogContainerDebug will print relevant docker/podman infos after a container fails
 func LogContainerDebug(ociBin string, name string) string {
 	rr, err := containerInspect(ociBin, name)
diff --git a/pkg/drivers/kic/oci/network_create.go b/pkg/drivers/kic/oci/network_create.go
new file mode 100644
index 0000000000..3df9660814
--- /dev/null
+++ b/pkg/drivers/kic/oci/network_create.go
@@ -0,0 +1,125 @@
+/*
+Copyright 2019 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package oci
+
+import (
+	"fmt"
+	"net"
+	"os/exec"
+	"strings"
+
+	"github.com/golang/glog"
+	"github.com/pkg/errors"
+)
+
+// DefaultSubnet subnet to be used on first cluster
+const defaultSubnet = "192.168.39.0/24"
+
+// CreateNetwork creates a network returns gateway and error, minikube creates one network per cluster
+func CreateNetwork(ociBin string, name string) (net.IP, error) {
+	if ociBin != Docker {
+		return nil, fmt.Errorf("%s network not implemented yet", ociBin)
+	}
+	return createDockerNetwork(name)
+}
+
+func createDockerNetwork(name string) (net.IP, error) {
+	// check if the network already exists
+	subnet, gateway, err := dockerNetworkInspect(name)
+	if err == nil {
+		glog.Infof("Found existing network with subnet %s and gateway %s.", subnet, gateway)
+		return gateway, nil
+	}
+	// simple way to create networks, subnet is taken, try one bigger
+	attempt := 0
+	_, subnet, err = net.ParseCIDR(defaultSubnet)
+	if err != nil {
+		return nil, errors.Wrapf(err, "parse default subnet %s", defaultSubnet)
+	}
+
+	gateway, err = tryCreateDockerNetwork(subnet, name)
+	if err != nil {
+		if err != ErrNetworkSubnetTaken {
+			return nil, errors.Wrapf(err, "error creating network")
+		}
+		// try up to 13 times
+		// we can try up to 255
+		for attempt < 13 {
+			attempt++
+			glog.Infof("Couldn't create network %q at %q subnet will try again with a new subnet ...", name, subnet)
+			// increase 3nd digit by 10 each time
+			// 13 times adding 10 defaultSubnet "192.168.39.0/24"
+			// at most it will add up to 169 which is still less than max allowed 255
+			// this is large enough to try more and not too small to not try enough
+			// can be tuned in the next iterations
+			subnet.IP.To4()[2] += 10
+			gateway, err := tryCreateDockerNetwork(subnet, name)
+			if err == nil {
+				return gateway, nil
+			}
+			if err == ErrNetworkSubnetTaken {
+				continue
+			}
+		}
+
+	}
+	return gateway, nil
+}
+
+func tryCreateDockerNetwork(subnet *net.IPNet, name string) (net.IP, error) {
+	gateway := subnet.IP.To4()
+	gateway[3]++ // first ip for gateway
+	glog.Infof("attempt to create network %q with subnet: %s and gateway %s...", subnet, name, gateway)
+	// options documentation https://docs.docker.com/engine/reference/commandline/network_create/#bridge-driver-options
+	rr, err := runCmd(exec.Command(Docker, "network", "create", "--driver=bridge", fmt.Sprintf("--subnet=%s", subnet), fmt.Sprintf("--gateway=%s", gateway), "-o", "--ip-masq", "-o", "--icc", fmt.Sprintf("--label=%s=%s", CreatedByLabelKey, "true"), name))
+	if err != nil {
+		if strings.Contains(rr.Output(), "Pool overlaps with other one on this address space") {
+			return nil, ErrNetworkSubnetTaken
+		}
+		if strings.Contains(rr.Output(), "failed to allocate gateway") && strings.Contains(rr.Output(), "Address already in use") {
+			return nil, ErrNetworkGatewayTaken
+		}
+		return nil, errors.Wrapf(err, "error creating network")
+	}
+	return gateway, nil
+}
+
+// returns subnet and gate if exists
+func dockerNetworkInspect(name string) (*net.IPNet, net.IP, error) {
+	rr, err := runCmd(exec.Command(Docker, "network", "inspect", name, "--format", "{{(index .IPAM.Config 0).Subnet}},{{(index .IPAM.Config 0).Gateway}}"))
+	if err != nil {
+		if strings.Contains(rr.Output(), "No such network:") {
+			return nil, nil, ErrNetworkNotFound
+		}
+		return nil, nil, err
+	}
+	// results looks like 172.17.0.0/16,172.17.0.1
+	ips := strings.Split(strings.TrimSpace(rr.Stdout.String()), ",")
+	if len(ips) == 0 {
+		return nil, nil, fmt.Errorf("invalid network info")
+	}
+
+	_, subnet, err := net.ParseCIDR(ips[0])
+	if err != nil {
+		return nil, nil, errors.Wrapf(err, "parse subnet for %s", name)
+	}
+	var gateway net.IP
+	if len(ips) > 0 {
+		gateway = net.ParseIP(ips[1])
+	}
+	return subnet, gateway, nil
+}

From f0f10d6135d7b98951369af6a2e4b37fb8b57926 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Mon, 21 Sep 2020 17:26:55 -0700
Subject: [PATCH 53/65] first version

---
 cmd/minikube/cmd/delete.go                  |  5 ++
 cmd/minikube/cmd/mount.go                   |  2 +-
 hack/preload-images/generate.go             |  1 +
 pkg/drivers/kic/kic.go                      | 32 ++++++++++
 pkg/drivers/kic/kic_test.go                 | 59 +++++++++++++++++
 pkg/drivers/kic/oci/network.go              | 64 +++----------------
 pkg/drivers/kic/oci/network_create.go       | 71 +++++++++++++++++++++
 pkg/drivers/kic/oci/oci.go                  |  9 +++
 pkg/drivers/kic/oci/types.go                |  3 +
 pkg/drivers/kic/types.go                    |  1 +
 pkg/minikube/cluster/ip.go                  |  6 +-
 pkg/minikube/node/start.go                  |  2 +-
 pkg/minikube/registry/drvs/docker/docker.go |  1 +
 pkg/minikube/registry/drvs/podman/podman.go |  1 +
 14 files changed, 198 insertions(+), 59 deletions(-)
 create mode 100644 pkg/drivers/kic/kic_test.go

diff --git a/cmd/minikube/cmd/delete.go b/cmd/minikube/cmd/delete.go
index 6b50e46316..b8ac93a568 100644
--- a/cmd/minikube/cmd/delete.go
+++ b/cmd/minikube/cmd/delete.go
@@ -260,6 +260,11 @@ func deletePossibleKicLeftOver(cname string, driverName string) {
 		glog.Warningf("error deleting volumes (might be okay).\nTo see the list of volumes run: 'docker volume ls'\n:%v", errs)
 	}
 
+	errs = oci.DeleteAllNetworksByKIC()
+	if errs != nil {
+		glog.Warningf("error deleting left over networks (might be okay).\nTo see the list of netowrks: 'docker network ls'\n:%v", errs)
+	}
+
 	if bin == oci.Podman {
 		// podman prune does not support --filter
 		return
diff --git a/cmd/minikube/cmd/mount.go b/cmd/minikube/cmd/mount.go
index 8d268d1cc2..f59b3ca4a7 100644
--- a/cmd/minikube/cmd/mount.go
+++ b/cmd/minikube/cmd/mount.go
@@ -110,7 +110,7 @@ var mountCmd = &cobra.Command{
 		var ip net.IP
 		var err error
 		if mountIP == "" {
-			ip, err = cluster.HostIP(co.CP.Host)
+			ip, err = cluster.HostIP(co.CP.Host, co.Config.Name)
 			if err != nil {
 				exit.Error(reason.IfHostIP, "Error getting the host IP address to use from within the VM", err)
 			}
diff --git a/hack/preload-images/generate.go b/hack/preload-images/generate.go
index ec3de2ece5..e1d6061d8b 100644
--- a/hack/preload-images/generate.go
+++ b/hack/preload-images/generate.go
@@ -39,6 +39,7 @@ import (
 
 func generateTarball(kubernetesVersion, containerRuntime, tarballFilename string) error {
 	driver := kic.NewDriver(kic.Config{
+		ClusterName:       profile,
 		KubernetesVersion: kubernetesVersion,
 		ContainerRuntime:  containerRuntime,
 		OCIBinary:         oci.Docker,
diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go
index e296b327b8..d63dfcd943 100644
--- a/pkg/drivers/kic/kic.go
+++ b/pkg/drivers/kic/kic.go
@@ -37,6 +37,7 @@ import (
 	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/cruntime"
 	"k8s.io/minikube/pkg/minikube/download"
+	"k8s.io/minikube/pkg/minikube/out"
 	"k8s.io/minikube/pkg/minikube/sysinit"
 	"k8s.io/minikube/pkg/util/retry"
 )
@@ -65,6 +66,18 @@ func NewDriver(c Config) *Driver {
 	return d
 }
 
+// machineOrder returns the order of the container based on it is name
+func machineOrder(machineName string) int {
+	// minikube-m02
+	sp := strings.Split(machineName, "-")
+	m := strings.Trim(sp[len(sp)-1], "m") // m02
+	i, err := strconv.Atoi(m)
+	if err != nil {
+		return 1
+	}
+	return i
+}
+
 // Create a host using the driver's config
 func (d *Driver) Create() error {
 	params := oci.CreateParams{
@@ -81,6 +94,20 @@ func (d *Driver) Create() error {
 		APIServerPort: d.NodeConfig.APIServerPort,
 	}
 
+	// one network bridge per cluster.
+	defaultNetwork := d.NodeConfig.ClusterName
+	if gateway, err := oci.CreateNetwork(d.OCIBinary, defaultNetwork); err != nil {
+		glog.Warningf("failed to create network: %v", err)
+		out.WarningT("Unable to create dedicated network, This might result in cluster IP change after restart.")
+	} else {
+		params.Network = defaultNetwork
+		ip := gateway.To4()
+		// calculate the container IP based on its machine order
+		ip[3] += byte(machineOrder(d.NodeConfig.MachineName))
+		glog.Infof("calculated static IP %q for the %q container ", ip.String(), d.NodeConfig.MachineName)
+		params.IP = ip.String()
+	}
+
 	// control plane specific options
 	params.PortMappings = append(params.PortMappings, oci.PortMapping{
 		ListenAddress: oci.DefaultBindIPV4,
@@ -289,6 +316,11 @@ func (d *Driver) Remove() error {
 	if id, err := oci.ContainerID(d.OCIBinary, d.MachineName); err == nil && id != "" {
 		return fmt.Errorf("expected no container ID be found for %q after delete. but got %q", d.MachineName, id)
 	}
+
+	if err := oci.RemoveNetwork(d.NodeConfig.ClusterName); err != nil {
+		//TODO: Ingore error if this is a multinode cluster and first container is trying to delete network while other containers are attached to it
+		glog.Warningf("failed to remove network (which might be okay) %s: %v", d.NodeConfig.ClusterName, err)
+	}
 	return nil
 }
 
diff --git a/pkg/drivers/kic/kic_test.go b/pkg/drivers/kic/kic_test.go
new file mode 100644
index 0000000000..380879183e
--- /dev/null
+++ b/pkg/drivers/kic/kic_test.go
@@ -0,0 +1,59 @@
+/*
+Copyright 2019 The Kubernetes Authors All rights reserved.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kic
+
+import (
+	"testing"
+)
+
+func TestMachineOrder(t *testing.T) {
+	testCases := []struct {
+		Name        string
+		MachineName string
+		Want        int
+	}{
+		{
+			Name:        "default",
+			MachineName: "minikube",
+			Want:        1},
+		{
+			Name:        "second-node",
+			MachineName: "minikube-m02",
+			Want:        2},
+		{
+			Name:        "dash-profile",
+			MachineName: "my-dashy-minikube",
+			Want:        1},
+
+		{
+			Name:        "dash-profile-second-node",
+			MachineName: "my-dashy-minikube-m02",
+			Want:        2},
+		{
+			Name:        "michivious-user",
+			MachineName: "michivious-user-m02-m03",
+			Want:        3},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.Name, func(t *testing.T) {
+			got := machineOrder(tc.MachineName)
+			if got != tc.Want {
+				t.Errorf("want order %q but got %q", tc.Want, got)
+
+			}
+		})
+
+	}
+}
diff --git a/pkg/drivers/kic/oci/network.go b/pkg/drivers/kic/oci/network.go
index ca9e53bdb6..e292cbad13 100644
--- a/pkg/drivers/kic/oci/network.go
+++ b/pkg/drivers/kic/oci/network.go
@@ -31,17 +31,21 @@ import (
 
 // RoutableHostIPFromInside returns the ip/dns of the host that container lives on
 // is routable from inside the container
-func RoutableHostIPFromInside(ociBin string, containerName string) (net.IP, error) {
+func RoutableHostIPFromInside(ociBin string, clusterName string, containerName string) (net.IP, error) {
 	if ociBin == Docker {
 		if runtime.GOOS == "linux" {
-			return dockerGatewayIP(containerName)
+			_, gateway, err := dockerNetworkInspect(clusterName)
+			if err != nil {
+				return gateway, errors.Wrap(err, "network inspect")
+			}
+			return gateway, nil
 		}
 		// for windows and mac, the gateway ip is not routable so we use dns trick.
 		return digDNS(ociBin, containerName, "host.docker.internal")
 	}
 
 	if runtime.GOOS == "linux" {
-		return containerGatewayIP(ociBin, containerName)
+		return podmanGatewayIP(containerName)
 	}
 
 	return nil, fmt.Errorf("RoutableHostIPFromInside is currently only implemented for linux")
@@ -59,57 +63,9 @@ func digDNS(ociBin, containerName, dns string) (net.IP, error) {
 	return ip, nil
 }
 
-// profileInContainers checks whether the profile is within the containers list
-func profileInContainers(profile string, containers []string) bool {
-	for _, container := range containers {
-		if container == profile {
-			return true
-		}
-	}
-	return false
-}
-
-// dockerGatewayIP gets the default gateway ip for the docker bridge on the user's host machine
-// gets the ip from user's host docker
-func dockerGatewayIP(profile string) (net.IP, error) {
-	var bridgeID string
-	rr, err := runCmd(exec.Command(Docker, "network", "ls", "--filter", "name=bridge", "--format", "{{.ID}}"))
-	if err != nil {
-		return nil, errors.Wrapf(err, "get network bridge")
-	}
-	networksOutput := strings.TrimSpace(rr.Stdout.String())
-	networksSlice := strings.Fields(networksOutput)
-	// Look for the minikube container within each docker network
-	for _, net := range networksSlice {
-		// get all containers in the network
-		rs, err := runCmd(exec.Command(Docker, "network", "inspect", net, "-f", "{{range $k, $v := .Containers}}{{$v.Name}} {{end}}"))
-		if err != nil {
-			return nil, errors.Wrapf(err, "get containers in network")
-		}
-		containersSlice := strings.Fields(rs.Stdout.String())
-		if profileInContainers(profile, containersSlice) {
-			bridgeID = net
-			break
-		}
-	}
-
-	if bridgeID == "" {
-		return nil, errors.Errorf("unable to determine bridge network id from %q", networksOutput)
-	}
-	rr, err = runCmd(exec.Command(Docker, "network", "inspect",
-		"--format", "{{(index .IPAM.Config 0).Gateway}}", bridgeID))
-	if err != nil {
-		return nil, errors.Wrapf(err, "inspect IP bridge network %q.", bridgeID)
-	}
-
-	ip := net.ParseIP(strings.TrimSpace(rr.Stdout.String()))
-	glog.Infof("got host ip for mount in container by inspect docker network: %s", ip.String())
-	return ip, nil
-}
-
-// containerGatewayIP gets the default gateway ip for the container
-func containerGatewayIP(ociBin, containerName string) (net.IP, error) {
-	rr, err := runCmd(exec.Command(ociBin, "container", "inspect", "--format", "{{.NetworkSettings.Gateway}}", containerName))
+// podmanGatewayIP gets the default gateway ip for the container
+func podmanGatewayIP(containerName string) (net.IP, error) {
+	rr, err := runCmd(exec.Command(Podman, "container", "inspect", "--format", "{{.NetworkSettings.Gateway}}", containerName))
 	if err != nil {
 		return nil, errors.Wrapf(err, "inspect gateway")
 	}
diff --git a/pkg/drivers/kic/oci/network_create.go b/pkg/drivers/kic/oci/network_create.go
index 3df9660814..58f15066bd 100644
--- a/pkg/drivers/kic/oci/network_create.go
+++ b/pkg/drivers/kic/oci/network_create.go
@@ -17,6 +17,8 @@ limitations under the License.
 package oci
 
 import (
+	"bufio"
+	"bytes"
 	"fmt"
 	"net"
 	"os/exec"
@@ -123,3 +125,72 @@ func dockerNetworkInspect(name string) (*net.IPNet, net.IP, error) {
 	}
 	return subnet, gateway, nil
 }
+
+// RemoveNetwork removes a network
+func RemoveNetwork(name string) error {
+	if !networkExists(name) {
+		return nil
+	}
+	rr, err := runCmd(exec.Command(Docker, "network", "remove", name))
+	if err != nil {
+		if strings.Contains(rr.Output(), "No such network:") {
+			return ErrNetworkNotFound
+		}
+		// Error response from daemon: error while removing network: network mynet123 id f9e1c50b89feb0b8f4b687f3501a81b618252c9907bc20666e386d0928322387 has active endpoints
+		if strings.Contains(rr.Output(), "has active endpoints") {
+			return ErrNetworkInUse
+		}
+	}
+
+	return err
+}
+
+func networkExists(name string) bool {
+	if _, _, err := dockerNetworkInspect(name); err != nil {
+		if err == ErrNetworkNotFound {
+			return false
+		}
+		glog.Warningf("error inspecting network %s: %v", name, err)
+		return false
+	}
+	return true
+}
+
+// returns all network names created by a label
+func allNetworkByLabel(ociBin string, label string) ([]string, error) {
+	if ociBin != Docker {
+		return nil, fmt.Errorf("%s not supported", ociBin)
+	}
+
+	// docker network ls --filter='label=created_by.minikube.sigs.k8s.io=true' --format '{{.Name}}
+	rr, err := runCmd(exec.Command(Docker, "network", "ls", fmt.Sprintf("--filter=label=%s", label), "--format", "{{.Name}}"))
+	if err != nil {
+		return nil, err
+	}
+	var lines []string
+	scanner := bufio.NewScanner(bytes.NewReader(rr.Stdout.Bytes()))
+	for scanner.Scan() {
+		lines = append(lines, strings.TrimSpace(scanner.Text()))
+	}
+
+	return lines, nil
+}
+
+// DeleteAllNetworksByKIC deletes all networks created by kic
+func DeleteAllNetworksByKIC() []error {
+	var errs []error
+	ns, err := allNetworkByLabel(Docker, CreatedByLabelKey+"=true")
+	if err != nil {
+		return []error{errors.Wrap(err, "list all volume")}
+	}
+	for _, n := range ns {
+		err := RemoveNetwork(n)
+		if err != nil {
+			errs = append(errs, err)
+		}
+	}
+	if len(errs) > 0 {
+		return errs
+	}
+	return nil
+}
diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go
index 878189e387..ac7c3ee87b 100644
--- a/pkg/drivers/kic/oci/oci.go
+++ b/pkg/drivers/kic/oci/oci.go
@@ -169,6 +169,11 @@ func CreateContainerNode(p CreateParams) error {
 		virtualization = "podman" // VIRTUALIZATION_PODMAN
 	}
 	if p.OCIBinary == Docker {
+		// to provide a static IP for docker
+		if p.Network != "" && p.IP != "" {
+			runArgs = append(runArgs, "--network", p.Network)
+			runArgs = append(runArgs, "--ip", p.IP)
+		}
 		runArgs = append(runArgs, "--volume", fmt.Sprintf("%s:/var", p.Name))
 		// ignore apparmore github actions docker: https://github.com/kubernetes/minikube/issues/7624
 		runArgs = append(runArgs, "--security-opt", "apparmor=unconfined")
@@ -285,6 +290,10 @@ func createContainer(ociBin string, image string, opts ...createOpt) error {
 		if strings.Contains(rr.Output(), "Range of CPUs is from") && strings.Contains(rr.Output(), "CPUs available") { // CPUs available
 			return ErrCPUCountLimit
 		}
+		// example: docker: Error response from daemon: Address already in use.
+		if strings.Contains(rr.Output(), "Address already in use") {
+			return ErrIPinUse
+		}
 		return err
 	}
 
diff --git a/pkg/drivers/kic/oci/types.go b/pkg/drivers/kic/oci/types.go
index b2dad8de4a..09a42d4da1 100644
--- a/pkg/drivers/kic/oci/types.go
+++ b/pkg/drivers/kic/oci/types.go
@@ -43,6 +43,7 @@ const (
 
 // CreateParams are parameters needed to create a container
 type CreateParams struct {
+	ClusterName   string            // cluster(profile name) that this container belongs to
 	Name          string            // used for container name and hostname
 	Image         string            // container image to use to create the node.
 	ClusterLabel  string            // label the clusters we create using minikube so we can clean up
@@ -56,6 +57,8 @@ type CreateParams struct {
 	Envs          map[string]string // environment variables to pass to the container
 	ExtraArgs     []string          // a list of any extra option to pass to oci binary during creation time, for example --expose 8080...
 	OCIBinary     string            // docker or podman
+	Network       string            // network to use for the containe
+	IP            string            // IP to assign for th container in the work
 }
 
 // createOpt is an option for Create
diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go
index 5d138a0ea8..67ab4da09e 100644
--- a/pkg/drivers/kic/types.go
+++ b/pkg/drivers/kic/types.go
@@ -48,6 +48,7 @@ var (
 
 // Config is configuration for the kic driver used by registry
 type Config struct {
+	ClusterName       string            // The cluster the container belongs to
 	MachineName       string            // maps to the container name being created
 	CPU               int               // Number of CPU cores assigned to the container
 	Memory            int               // max memory in MB
diff --git a/pkg/minikube/cluster/ip.go b/pkg/minikube/cluster/ip.go
index 2dd4548020..caf34e2bb2 100644
--- a/pkg/minikube/cluster/ip.go
+++ b/pkg/minikube/cluster/ip.go
@@ -34,12 +34,12 @@ import (
 )
 
 // HostIP gets the ip address to be used for mapping host -> VM and VM -> host
-func HostIP(host *host.Host) (net.IP, error) {
+func HostIP(host *host.Host, clusterName string) (net.IP, error) {
 	switch host.DriverName {
 	case driver.Docker:
-		return oci.RoutableHostIPFromInside(oci.Docker, host.Name)
+		return oci.RoutableHostIPFromInside(oci.Docker, clusterName, host.Name)
 	case driver.Podman:
-		return oci.RoutableHostIPFromInside(oci.Podman, host.Name)
+		return oci.RoutableHostIPFromInside(oci.Podman, clusterName, host.Name)
 	case driver.KVM2:
 		return net.ParseIP("192.168.39.1"), nil
 	case driver.HyperV:
diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go
index 6982f10135..c00aa6cadc 100644
--- a/pkg/minikube/node/start.go
+++ b/pkg/minikube/node/start.go
@@ -94,7 +94,7 @@ func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) {
 	showVersionInfo(starter.Node.KubernetesVersion, cr)
 
 	// Add "host.minikube.internal" DNS alias (intentionally non-fatal)
-	hostIP, err := cluster.HostIP(starter.Host)
+	hostIP, err := cluster.HostIP(starter.Host, starter.Cfg.Name)
 	if err != nil {
 		glog.Errorf("Unable to get host IP: %v", err)
 	} else if err := machine.AddHostAlias(starter.Runner, constants.HostAlias, hostIP); err != nil {
diff --git a/pkg/minikube/registry/drvs/docker/docker.go b/pkg/minikube/registry/drvs/docker/docker.go
index 38a325a6aa..98d961e1d2 100644
--- a/pkg/minikube/registry/drvs/docker/docker.go
+++ b/pkg/minikube/registry/drvs/docker/docker.go
@@ -61,6 +61,7 @@ func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) {
 	}
 
 	return kic.NewDriver(kic.Config{
+		ClusterName:       cc.Name,
 		MachineName:       driver.MachineName(cc, n),
 		StorePath:         localpath.MiniPath(),
 		ImageDigest:       cc.KicBaseImage,
diff --git a/pkg/minikube/registry/drvs/podman/podman.go b/pkg/minikube/registry/drvs/podman/podman.go
index 166fd9e6d5..fdf912f5e2 100644
--- a/pkg/minikube/registry/drvs/podman/podman.go
+++ b/pkg/minikube/registry/drvs/podman/podman.go
@@ -73,6 +73,7 @@ func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) {
 	}
 
 	return kic.NewDriver(kic.Config{
+		ClusterName:       cc.Name,
 		MachineName:       driver.MachineName(cc, n),
 		StorePath:         localpath.MiniPath(),
 		ImageDigest:       strings.Split(cc.KicBaseImage, "@")[0], // for podman does not support docker images references with both a tag and digest.

From fa0afb220d0c1162e9b36d0ec9350414bfa8913e Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Mon, 21 Sep 2020 17:52:06 -0700
Subject: [PATCH 54/65] lint

---
 pkg/drivers/kic/kic_test.go           | 3 +++
 pkg/drivers/kic/oci/network_create.go | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/pkg/drivers/kic/kic_test.go b/pkg/drivers/kic/kic_test.go
index 380879183e..d607bbc7f2 100644
--- a/pkg/drivers/kic/kic_test.go
+++ b/pkg/drivers/kic/kic_test.go
@@ -1,9 +1,12 @@
 /*
 Copyright 2019 The Kubernetes Authors All rights reserved.
+
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
+
     http://www.apache.org/licenses/LICENSE-2.0
+
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
diff --git a/pkg/drivers/kic/oci/network_create.go b/pkg/drivers/kic/oci/network_create.go
index 58f15066bd..259efc7f35 100644
--- a/pkg/drivers/kic/oci/network_create.go
+++ b/pkg/drivers/kic/oci/network_create.go
@@ -1,5 +1,5 @@
 /*
-Copyright 2019 The Kubernetes Authors All rights reserved.
+Copyright 2020 The Kubernetes Authors All rights reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

From 2766235a5c05406d333df73cb3ca929f1b252856 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Wed, 23 Sep 2020 17:08:56 -0700
Subject: [PATCH 55/65] try alternate if no network

---
 pkg/drivers/kic/oci/network.go | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/pkg/drivers/kic/oci/network.go b/pkg/drivers/kic/oci/network.go
index e292cbad13..2dcf22cc56 100644
--- a/pkg/drivers/kic/oci/network.go
+++ b/pkg/drivers/kic/oci/network.go
@@ -36,6 +36,10 @@ func RoutableHostIPFromInside(ociBin string, clusterName string, containerName s
 		if runtime.GOOS == "linux" {
 			_, gateway, err := dockerNetworkInspect(clusterName)
 			if err != nil {
+				if errors.Is(err, ErrNetworkNotFound) {
+					glog.Infof("The container %s is not attached to a network, this could be because the cluster was created by an older than v.1.14 minikube, will try to get the IP using container gatway", containerName)
+					return containerGatewayIP(Docker, containerName)
+				}
 				return gateway, errors.Wrap(err, "network inspect")
 			}
 			return gateway, nil
@@ -43,9 +47,9 @@ func RoutableHostIPFromInside(ociBin string, clusterName string, containerName s
 		// for windows and mac, the gateway ip is not routable so we use dns trick.
 		return digDNS(ociBin, containerName, "host.docker.internal")
 	}
-
+	//	podman
 	if runtime.GOOS == "linux" {
-		return podmanGatewayIP(containerName)
+		return containerGatewayIP(Podman, containerName)
 	}
 
 	return nil, fmt.Errorf("RoutableHostIPFromInside is currently only implemented for linux")
@@ -63,9 +67,9 @@ func digDNS(ociBin, containerName, dns string) (net.IP, error) {
 	return ip, nil
 }
 
-// podmanGatewayIP gets the default gateway ip for the container
-func podmanGatewayIP(containerName string) (net.IP, error) {
-	rr, err := runCmd(exec.Command(Podman, "container", "inspect", "--format", "{{.NetworkSettings.Gateway}}", containerName))
+// containerGatewayIP gets the default gateway ip for the container
+func containerGatewayIP(ociBin string, containerName string) (net.IP, error) {
+	rr, err := runCmd(exec.Command(ociBin, "container", "inspect", "--format", "{{.NetworkSettings.Gateway}}", containerName))
 	if err != nil {
 		return nil, errors.Wrapf(err, "inspect gateway")
 	}

From f163cf9e15b38e165f3663c1c3562f1e910864a1 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Wed, 23 Sep 2020 20:03:11 -0700
Subject: [PATCH 56/65] update with new logic

---
 pkg/drivers/kic/kic.go                |  6 ++---
 pkg/drivers/kic/kic_test.go           |  5 ++++
 pkg/drivers/kic/oci/network_create.go | 37 ++++++++++++++-------------
 3 files changed, 26 insertions(+), 22 deletions(-)

diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go
index d63dfcd943..9183fd6098 100644
--- a/pkg/drivers/kic/kic.go
+++ b/pkg/drivers/kic/kic.go
@@ -94,13 +94,11 @@ func (d *Driver) Create() error {
 		APIServerPort: d.NodeConfig.APIServerPort,
 	}
 
-	// one network bridge per cluster.
-	defaultNetwork := d.NodeConfig.ClusterName
-	if gateway, err := oci.CreateNetwork(d.OCIBinary, defaultNetwork); err != nil {
+	if gateway, err := oci.CreateNetwork(d.OCIBinary, d.NodeConfig.ClusterName); err != nil {
 		glog.Warningf("failed to create network: %v", err)
 		out.WarningT("Unable to create dedicated network, This might result in cluster IP change after restart.")
 	} else {
-		params.Network = defaultNetwork
+		params.Network = d.NodeConfig.ClusterName
 		ip := gateway.To4()
 		// calculate the container IP based on its machine order
 		ip[3] += byte(machineOrder(d.NodeConfig.MachineName))
diff --git a/pkg/drivers/kic/kic_test.go b/pkg/drivers/kic/kic_test.go
index d607bbc7f2..a5bb6c9783 100644
--- a/pkg/drivers/kic/kic_test.go
+++ b/pkg/drivers/kic/kic_test.go
@@ -34,6 +34,11 @@ func TestMachineOrder(t *testing.T) {
 			Name:        "second-node",
 			MachineName: "minikube-m02",
 			Want:        2},
+		{
+			Name:        "funny",
+			MachineName: "hahaha",
+			Want:        1},
+
 		{
 			Name:        "dash-profile",
 			MachineName: "my-dashy-minikube",
diff --git a/pkg/drivers/kic/oci/network_create.go b/pkg/drivers/kic/oci/network_create.go
index 259efc7f35..04ba730a61 100644
--- a/pkg/drivers/kic/oci/network_create.go
+++ b/pkg/drivers/kic/oci/network_create.go
@@ -29,7 +29,10 @@ import (
 )
 
 // DefaultSubnet subnet to be used on first cluster
-const defaultSubnet = "192.168.39.0/24"
+const defaultSubnetAddr = "192.168.39.0"
+
+// big enough for a cluster of 256 nodes
+const defaultSubnetRange = 24
 
 // CreateNetwork creates a network returns gateway and error, minikube creates one network per cluster
 func CreateNetwork(ociBin string, name string) (net.IP, error) {
@@ -39,21 +42,17 @@ func CreateNetwork(ociBin string, name string) (net.IP, error) {
 	return createDockerNetwork(name)
 }
 
-func createDockerNetwork(name string) (net.IP, error) {
+func createDockerNetwork(clusterName string) (net.IP, error) {
 	// check if the network already exists
-	subnet, gateway, err := dockerNetworkInspect(name)
+	subnet, gateway, err := dockerNetworkInspect(clusterName)
 	if err == nil {
 		glog.Infof("Found existing network with subnet %s and gateway %s.", subnet, gateway)
 		return gateway, nil
 	}
 	// simple way to create networks, subnet is taken, try one bigger
 	attempt := 0
-	_, subnet, err = net.ParseCIDR(defaultSubnet)
-	if err != nil {
-		return nil, errors.Wrapf(err, "parse default subnet %s", defaultSubnet)
-	}
-
-	gateway, err = tryCreateDockerNetwork(subnet, name)
+	subnetAddr := defaultSubnetAddr
+	gateway, err = tryCreateDockerNetwork(subnetAddr, defaultSubnetRange, clusterName)
 	if err != nil {
 		if err != ErrNetworkSubnetTaken {
 			return nil, errors.Wrapf(err, "error creating network")
@@ -62,18 +61,20 @@ func createDockerNetwork(name string) (net.IP, error) {
 		// we can try up to 255
 		for attempt < 13 {
 			attempt++
-			glog.Infof("Couldn't create network %q at %q subnet will try again with a new subnet ...", name, subnet)
+			glog.Infof("Couldn't create network %q at %q subnet will try again with a new subnet ...", clusterName, subnetAddr)
 			// increase 3nd digit by 10 each time
 			// 13 times adding 10 defaultSubnet "192.168.39.0/24"
 			// at most it will add up to 169 which is still less than max allowed 255
 			// this is large enough to try more and not too small to not try enough
 			// can be tuned in the next iterations
-			subnet.IP.To4()[2] += 10
-			gateway, err := tryCreateDockerNetwork(subnet, name)
+			ip := net.ParseIP(subnetAddr).To4()
+			ip[2] += byte(9 + attempt)
+
+			gateway, err = tryCreateDockerNetwork(ip.String(), defaultSubnetRange, clusterName)
 			if err == nil {
 				return gateway, nil
 			}
-			if err == ErrNetworkSubnetTaken {
+			if errors.Is(err, ErrNetworkSubnetTaken) || errors.Is(err, ErrNetworkGatewayTaken) {
 				continue
 			}
 		}
@@ -82,12 +83,12 @@ func createDockerNetwork(name string) (net.IP, error) {
 	return gateway, nil
 }
 
-func tryCreateDockerNetwork(subnet *net.IPNet, name string) (net.IP, error) {
-	gateway := subnet.IP.To4()
-	gateway[3]++ // first ip for gateway
-	glog.Infof("attempt to create network %q with subnet: %s and gateway %s...", subnet, name, gateway)
+func tryCreateDockerNetwork(subnetAddr string, subnetRange int, name string) (net.IP, error) {
+	gateway := net.ParseIP(subnetAddr)
+	gateway.To4()[3]++ // first ip for gateway
+	glog.Infof("attempt to create network %q with subnet: %s and gateway %s...", subnetAddr, name, gateway)
 	// options documentation https://docs.docker.com/engine/reference/commandline/network_create/#bridge-driver-options
-	rr, err := runCmd(exec.Command(Docker, "network", "create", "--driver=bridge", fmt.Sprintf("--subnet=%s", subnet), fmt.Sprintf("--gateway=%s", gateway), "-o", "--ip-masq", "-o", "--icc", fmt.Sprintf("--label=%s=%s", CreatedByLabelKey, "true"), name))
+	rr, err := runCmd(exec.Command(Docker, "network", "create", "--driver=bridge", fmt.Sprintf("--subnet=%s", fmt.Sprintf("%s/%d", subnetAddr, subnetRange)), fmt.Sprintf("--gateway=%s", gateway), "-o", "--ip-masq", "-o", "--icc", fmt.Sprintf("--label=%s=%s", CreatedByLabelKey, "true"), name))
 	if err != nil {
 		if strings.Contains(rr.Output(), "Pool overlaps with other one on this address space") {
 			return nil, ErrNetworkSubnetTaken

From b7780ec815695a533062c3b656dc2d6cce07e58d Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Mon, 28 Sep 2020 12:55:02 -0700
Subject: [PATCH 57/65] address review comments

---
 cmd/minikube/cmd/delete.go                    |  2 +-
 pkg/drivers/kic/kic.go                        | 14 +++++------
 pkg/drivers/kic/kic_test.go                   |  4 ++--
 pkg/drivers/kic/oci/network_create.go         | 24 ++++++++++---------
 pkg/drivers/kic/oci/types.go                  |  4 ++--
 .../content/en/docs/handbook/vpn_and_proxy.md |  1 +
 6 files changed, 25 insertions(+), 24 deletions(-)

diff --git a/cmd/minikube/cmd/delete.go b/cmd/minikube/cmd/delete.go
index b8ac93a568..93ca41fd34 100644
--- a/cmd/minikube/cmd/delete.go
+++ b/cmd/minikube/cmd/delete.go
@@ -262,7 +262,7 @@ func deletePossibleKicLeftOver(cname string, driverName string) {
 
 	errs = oci.DeleteAllNetworksByKIC()
 	if errs != nil {
-		glog.Warningf("error deleting left over networks (might be okay).\nTo see the list of netowrks: 'docker network ls'\n:%v", errs)
+		glog.Warningf("error deleting leftover networks (might be okay).\nTo see the list of networks: 'docker network ls'\n:%v", errs)
 	}
 
 	if bin == oci.Podman {
diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go
index 9183fd6098..08eef23fd4 100644
--- a/pkg/drivers/kic/kic.go
+++ b/pkg/drivers/kic/kic.go
@@ -66,8 +66,8 @@ func NewDriver(c Config) *Driver {
 	return d
 }
 
-// machineOrder returns the order of the container based on it is name
-func machineOrder(machineName string) int {
+// machineIndex returns the order of the container based on it is name
+func machineIndex(machineName string) int {
 	// minikube-m02
 	sp := strings.Split(machineName, "-")
 	m := strings.Trim(sp[len(sp)-1], "m") // m02
@@ -95,14 +95,13 @@ func (d *Driver) Create() error {
 	}
 
 	if gateway, err := oci.CreateNetwork(d.OCIBinary, d.NodeConfig.ClusterName); err != nil {
-		glog.Warningf("failed to create network: %v", err)
-		out.WarningT("Unable to create dedicated network, This might result in cluster IP change after restart.")
+		out.WarningT("Unable to create dedicated network, This might result in cluster IP change after restart. {{.error}}", out.V{"error": err})
 	} else {
 		params.Network = d.NodeConfig.ClusterName
 		ip := gateway.To4()
-		// calculate the container IP based on its machine order
-		ip[3] += byte(machineOrder(d.NodeConfig.MachineName))
-		glog.Infof("calculated static IP %q for the %q container ", ip.String(), d.NodeConfig.MachineName)
+		// calculate the container IP based on guessing the machine index
+		ip[3] += byte(machineIndex(d.NodeConfig.MachineName))
+		glog.Infof("calculated static IP %q for the %q container", ip.String(), d.NodeConfig.MachineName)
 		params.IP = ip.String()
 	}
 
@@ -316,7 +315,6 @@ func (d *Driver) Remove() error {
 	}
 
 	if err := oci.RemoveNetwork(d.NodeConfig.ClusterName); err != nil {
-		//TODO: Ingore error if this is a multinode cluster and first container is trying to delete network while other containers are attached to it
 		glog.Warningf("failed to remove network (which might be okay) %s: %v", d.NodeConfig.ClusterName, err)
 	}
 	return nil
diff --git a/pkg/drivers/kic/kic_test.go b/pkg/drivers/kic/kic_test.go
index a5bb6c9783..47af3763fe 100644
--- a/pkg/drivers/kic/kic_test.go
+++ b/pkg/drivers/kic/kic_test.go
@@ -20,7 +20,7 @@ import (
 	"testing"
 )
 
-func TestMachineOrder(t *testing.T) {
+func TestMachineIndex(t *testing.T) {
 	testCases := []struct {
 		Name        string
 		MachineName string
@@ -56,7 +56,7 @@ func TestMachineOrder(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.Name, func(t *testing.T) {
-			got := machineOrder(tc.MachineName)
+			got := machineIndex(tc.MachineName)
 			if got != tc.Want {
 				t.Errorf("want order %q but got %q", tc.Want, got)
 
diff --git a/pkg/drivers/kic/oci/network_create.go b/pkg/drivers/kic/oci/network_create.go
index 04ba730a61..bc31c9b55a 100644
--- a/pkg/drivers/kic/oci/network_create.go
+++ b/pkg/drivers/kic/oci/network_create.go
@@ -28,8 +28,9 @@ import (
 	"github.com/pkg/errors"
 )
 
-// DefaultSubnet subnet to be used on first cluster
-const defaultSubnetAddr = "192.168.39.0"
+// firstSubnetAddr subnet to be used on first kic cluster
+// it is one octet more than the one used by KVM to avoid possible conflict
+const firstSubnetAddr = "192.168.49.0"
 
 // big enough for a cluster of 256 nodes
 const defaultSubnetRange = 24
@@ -51,19 +52,19 @@ func createDockerNetwork(clusterName string) (net.IP, error) {
 	}
 	// simple way to create networks, subnet is taken, try one bigger
 	attempt := 0
-	subnetAddr := defaultSubnetAddr
+	subnetAddr := firstSubnetAddr
 	gateway, err = tryCreateDockerNetwork(subnetAddr, defaultSubnetRange, clusterName)
 	if err != nil {
 		if err != ErrNetworkSubnetTaken {
 			return nil, errors.Wrapf(err, "error creating network")
 		}
-		// try up to 13 times
-		// we can try up to 255
-		for attempt < 13 {
+		// try up to 20 times, third octet would go like 49,59 ,69,..., 239
+		// max we could go is 255 we stop at 239
+		for attempt < 20 {
 			attempt++
 			glog.Infof("Couldn't create network %q at %q subnet will try again with a new subnet ...", clusterName, subnetAddr)
-			// increase 3nd digit by 10 each time
-			// 13 times adding 10 defaultSubnet "192.168.39.0/24"
+			// Find an open subnet by incrementing the 3rd octet by 10 for each try
+			// 13 times adding 10 firstSubnetAddr "192.168.49.0/24"
 			// at most it will add up to 169 which is still less than max allowed 255
 			// this is large enough to try more and not too small to not try enough
 			// can be tuned in the next iterations
@@ -77,8 +78,8 @@ func createDockerNetwork(clusterName string) (net.IP, error) {
 			if errors.Is(err, ErrNetworkSubnetTaken) || errors.Is(err, ErrNetworkGatewayTaken) {
 				continue
 			}
+			glog.Errorf("unexpected error while trying to create network, will try again anyways: %v", err)
 		}
-
 	}
 	return gateway, nil
 }
@@ -90,7 +91,8 @@ func tryCreateDockerNetwork(subnetAddr string, subnetRange int, name string) (ne
 	// options documentation https://docs.docker.com/engine/reference/commandline/network_create/#bridge-driver-options
 	rr, err := runCmd(exec.Command(Docker, "network", "create", "--driver=bridge", fmt.Sprintf("--subnet=%s", fmt.Sprintf("%s/%d", subnetAddr, subnetRange)), fmt.Sprintf("--gateway=%s", gateway), "-o", "--ip-masq", "-o", "--icc", fmt.Sprintf("--label=%s=%s", CreatedByLabelKey, "true"), name))
 	if err != nil {
-		if strings.Contains(rr.Output(), "Pool overlaps with other one on this address space") {
+		// Pool overlaps with other one on this address space
+		if strings.Contains(rr.Output(), "Pool overlaps") {
 			return nil, ErrNetworkSubnetTaken
 		}
 		if strings.Contains(rr.Output(), "failed to allocate gateway") && strings.Contains(rr.Output(), "Address already in use") {
@@ -113,7 +115,7 @@ func dockerNetworkInspect(name string) (*net.IPNet, net.IP, error) {
 	// results looks like 172.17.0.0/16,172.17.0.1
 	ips := strings.Split(strings.TrimSpace(rr.Stdout.String()), ",")
 	if len(ips) == 0 {
-		return nil, nil, fmt.Errorf("invalid network info")
+		return nil, nil, fmt.Errorf("empty IP list parsed from: %q", rr.Output())
 	}
 
 	_, subnet, err := net.ParseCIDR(ips[0])
diff --git a/pkg/drivers/kic/oci/types.go b/pkg/drivers/kic/oci/types.go
index 09a42d4da1..0e6de567f5 100644
--- a/pkg/drivers/kic/oci/types.go
+++ b/pkg/drivers/kic/oci/types.go
@@ -57,8 +57,8 @@ type CreateParams struct {
 	Envs          map[string]string // environment variables to pass to the container
 	ExtraArgs     []string          // a list of any extra option to pass to oci binary during creation time, for example --expose 8080...
 	OCIBinary     string            // docker or podman
-	Network       string            // network to use for the containe
-	IP            string            // IP to assign for th container in the work
+	Network       string            // docker/podman network that the container will attach to
+	IP            string            // static IP to assign for th container in the cluster network
 }
 
 // createOpt is an option for Create
diff --git a/site/content/en/docs/handbook/vpn_and_proxy.md b/site/content/en/docs/handbook/vpn_and_proxy.md
index 3d430b10b4..41174d9999 100644
--- a/site/content/en/docs/handbook/vpn_and_proxy.md
+++ b/site/content/en/docs/handbook/vpn_and_proxy.md
@@ -22,6 +22,7 @@ The NO_PROXY variable here is important: Without setting it, minikube may not be
 
 * **192.168.99.0/24**: Used by the minikube VM. Configurable for some hypervisors via `--host-only-cidr`
 * **192.168.39.0/24**: Used by the minikube kvm2 driver.
+* **192.168.49.0/24**: Used by the minikube docker driver.
 * **10.96.0.0/12**: Used by service cluster IP's. Configurable via  `--service-cluster-ip-range`
 
 One important note: If NO_PROXY is required by non-Kubernetes applications, such as Firefox or Chrome, you may want to specifically add the minikube IP to the comma-separated list, as they may not understand IP ranges ([#3827](https://github.com/kubernetes/minikube/issues/3827)).

From 058adb775ca4ed4354937425d7db70bae4f7f185 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Mon, 28 Sep 2020 12:56:51 -0700
Subject: [PATCH 58/65] address review comments

---
 pkg/drivers/kic/oci/network_create.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkg/drivers/kic/oci/network_create.go b/pkg/drivers/kic/oci/network_create.go
index bc31c9b55a..0525b616bb 100644
--- a/pkg/drivers/kic/oci/network_create.go
+++ b/pkg/drivers/kic/oci/network_create.go
@@ -107,7 +107,7 @@ func tryCreateDockerNetwork(subnetAddr string, subnetRange int, name string) (ne
 func dockerNetworkInspect(name string) (*net.IPNet, net.IP, error) {
 	rr, err := runCmd(exec.Command(Docker, "network", "inspect", name, "--format", "{{(index .IPAM.Config 0).Subnet}},{{(index .IPAM.Config 0).Gateway}}"))
 	if err != nil {
-		if strings.Contains(rr.Output(), "No such network:") {
+		if strings.Contains(rr.Output(), "No such network") {
 			return nil, nil, ErrNetworkNotFound
 		}
 		return nil, nil, err
@@ -136,7 +136,7 @@ func RemoveNetwork(name string) error {
 	}
 	rr, err := runCmd(exec.Command(Docker, "network", "remove", name))
 	if err != nil {
-		if strings.Contains(rr.Output(), "No such network:") {
+		if strings.Contains(rr.Output(), "No such network") {
 			return ErrNetworkNotFound
 		}
 		// Error response from daemon: error while removing network: network mynet123 id f9e1c50b89feb0b8f4b687f3501a81b618252c9907bc20666e386d0928322387 has active endpoints

From 31e175c6f84e9cfe8bd68b6e8e72ac8e7c4e325c Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Tue, 29 Sep 2020 14:27:37 -0700
Subject: [PATCH 59/65] move index machine to driver package

---
 pkg/drivers/kic/kic.go             | 15 ++-------
 pkg/drivers/kic/kic_test.go        | 50 ------------------------------
 pkg/minikube/driver/driver.go      | 13 ++++++++
 pkg/minikube/driver/driver_test.go | 46 +++++++++++++++++++++++++++
 4 files changed, 61 insertions(+), 63 deletions(-)

diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go
index 08eef23fd4..0235f2d9e7 100644
--- a/pkg/drivers/kic/kic.go
+++ b/pkg/drivers/kic/kic.go
@@ -37,6 +37,7 @@ import (
 	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/cruntime"
 	"k8s.io/minikube/pkg/minikube/download"
+	"k8s.io/minikube/pkg/minikube/driver"
 	"k8s.io/minikube/pkg/minikube/out"
 	"k8s.io/minikube/pkg/minikube/sysinit"
 	"k8s.io/minikube/pkg/util/retry"
@@ -66,18 +67,6 @@ func NewDriver(c Config) *Driver {
 	return d
 }
 
-// machineIndex returns the order of the container based on it is name
-func machineIndex(machineName string) int {
-	// minikube-m02
-	sp := strings.Split(machineName, "-")
-	m := strings.Trim(sp[len(sp)-1], "m") // m02
-	i, err := strconv.Atoi(m)
-	if err != nil {
-		return 1
-	}
-	return i
-}
-
 // Create a host using the driver's config
 func (d *Driver) Create() error {
 	params := oci.CreateParams{
@@ -100,7 +89,7 @@ func (d *Driver) Create() error {
 		params.Network = d.NodeConfig.ClusterName
 		ip := gateway.To4()
 		// calculate the container IP based on guessing the machine index
-		ip[3] += byte(machineIndex(d.NodeConfig.MachineName))
+		ip[3] += byte(driver.IndexFromMachineName(d.NodeConfig.MachineName))
 		glog.Infof("calculated static IP %q for the %q container", ip.String(), d.NodeConfig.MachineName)
 		params.IP = ip.String()
 	}
diff --git a/pkg/drivers/kic/kic_test.go b/pkg/drivers/kic/kic_test.go
index 47af3763fe..0f77824b29 100644
--- a/pkg/drivers/kic/kic_test.go
+++ b/pkg/drivers/kic/kic_test.go
@@ -15,53 +15,3 @@ limitations under the License.
 */
 
 package kic
-
-import (
-	"testing"
-)
-
-func TestMachineIndex(t *testing.T) {
-	testCases := []struct {
-		Name        string
-		MachineName string
-		Want        int
-	}{
-		{
-			Name:        "default",
-			MachineName: "minikube",
-			Want:        1},
-		{
-			Name:        "second-node",
-			MachineName: "minikube-m02",
-			Want:        2},
-		{
-			Name:        "funny",
-			MachineName: "hahaha",
-			Want:        1},
-
-		{
-			Name:        "dash-profile",
-			MachineName: "my-dashy-minikube",
-			Want:        1},
-
-		{
-			Name:        "dash-profile-second-node",
-			MachineName: "my-dashy-minikube-m02",
-			Want:        2},
-		{
-			Name:        "michivious-user",
-			MachineName: "michivious-user-m02-m03",
-			Want:        3},
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.Name, func(t *testing.T) {
-			got := machineIndex(tc.MachineName)
-			if got != tc.Want {
-				t.Errorf("want order %q but got %q", tc.Want, got)
-
-			}
-		})
-
-	}
-}
diff --git a/pkg/minikube/driver/driver.go b/pkg/minikube/driver/driver.go
index 791b3d0bd3..2002ea3afd 100644
--- a/pkg/minikube/driver/driver.go
+++ b/pkg/minikube/driver/driver.go
@@ -21,6 +21,7 @@ import (
 	"os"
 	"runtime"
 	"sort"
+	"strconv"
 	"strings"
 
 	"github.com/golang/glog"
@@ -297,3 +298,15 @@ func MachineName(cc config.ClusterConfig, n config.Node) string {
 	}
 	return fmt.Sprintf("%s-%s", cc.Name, n.Name)
 }
+
+// IndexFromMachineName returns the order of the container based on it is name
+func IndexFromMachineName(machineName string) int {
+	// minikube-m02
+	sp := strings.Split(machineName, "-")
+	m := strings.Trim(sp[len(sp)-1], "m") // m02
+	i, err := strconv.Atoi(m)
+	if err != nil {
+		return 1
+	}
+	return i
+}
diff --git a/pkg/minikube/driver/driver_test.go b/pkg/minikube/driver/driver_test.go
index b6afd6a62c..713efbdd26 100644
--- a/pkg/minikube/driver/driver_test.go
+++ b/pkg/minikube/driver/driver_test.go
@@ -201,3 +201,49 @@ func TestSuggest(t *testing.T) {
 		})
 	}
 }
+
+func TestIndexFromMachineName(t *testing.T) {
+	testCases := []struct {
+		Name        string
+		MachineName string
+		Want        int
+	}{
+		{
+			Name:        "default",
+			MachineName: "minikube",
+			Want:        1},
+		{
+			Name:        "second-node",
+			MachineName: "minikube-m02",
+			Want:        2},
+		{
+			Name:        "funny",
+			MachineName: "hahaha",
+			Want:        1},
+
+		{
+			Name:        "dash-profile",
+			MachineName: "my-dashy-minikube",
+			Want:        1},
+
+		{
+			Name:        "dash-profile-second-node",
+			MachineName: "my-dashy-minikube-m02",
+			Want:        2},
+		{
+			Name:        "michivious-user",
+			MachineName: "michivious-user-m02-m03",
+			Want:        3},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.Name, func(t *testing.T) {
+			got := IndexFromMachineName(tc.MachineName)
+			if got != tc.Want {
+				t.Errorf("want order %q but got %q", tc.Want, got)
+
+			}
+		})
+
+	}
+}

From 82f39d1387ec5d81abe9bbfdb6bf27ba6e470b7e Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Tue, 29 Sep 2020 14:29:34 -0700
Subject: [PATCH 60/65] clarify comment

---
 pkg/drivers/kic/oci/types.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/drivers/kic/oci/types.go b/pkg/drivers/kic/oci/types.go
index 0e6de567f5..b21da438ec 100644
--- a/pkg/drivers/kic/oci/types.go
+++ b/pkg/drivers/kic/oci/types.go
@@ -57,7 +57,7 @@ type CreateParams struct {
 	Envs          map[string]string // environment variables to pass to the container
 	ExtraArgs     []string          // a list of any extra option to pass to oci binary during creation time, for example --expose 8080...
 	OCIBinary     string            // docker or podman
-	Network       string            // docker/podman network that the container will attach to
+	Network       string            // network name that the container will attach to
 	IP            string            // static IP to assign for th container in the cluster network
 }
 

From c48e8e5ec960689fa89c88fd1ca3f1a4b188a112 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Tue, 29 Sep 2020 15:00:44 -0700
Subject: [PATCH 61/65] add more unit tests

---
 pkg/minikube/driver/driver_test.go | 112 +++++++++++++++++++++++++++++
 1 file changed, 112 insertions(+)

diff --git a/pkg/minikube/driver/driver_test.go b/pkg/minikube/driver/driver_test.go
index 713efbdd26..ba47bf7c59 100644
--- a/pkg/minikube/driver/driver_test.go
+++ b/pkg/minikube/driver/driver_test.go
@@ -23,6 +23,7 @@ import (
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
+	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/registry"
 )
 
@@ -202,6 +203,60 @@ func TestSuggest(t *testing.T) {
 	}
 }
 
+func TestMachineName(t *testing.T) {
+	testsCases := []struct {
+		ClusterConfig config.ClusterConfig
+		Want          string
+	}{
+		{
+			ClusterConfig: config.ClusterConfig{Name: "minikube",
+				Nodes: []config.Node{
+					config.Node{
+						Name:              "",
+						IP:                "172.17.0.3",
+						Port:              8443,
+						KubernetesVersion: "v1.19.2",
+						ControlPlane:      true,
+						Worker:            true,
+					},
+				},
+			},
+			Want: "minikube",
+		},
+
+		{
+			ClusterConfig: config.ClusterConfig{Name: "p2",
+				Nodes: []config.Node{
+					config.Node{
+						Name:              "",
+						IP:                "172.17.0.3",
+						Port:              8443,
+						KubernetesVersion: "v1.19.2",
+						ControlPlane:      true,
+						Worker:            true,
+					},
+					config.Node{
+						Name:              "m2",
+						IP:                "172.17.0.4",
+						Port:              0,
+						KubernetesVersion: "v1.19.2",
+						ControlPlane:      false,
+						Worker:            true,
+					},
+				},
+			},
+			Want: "p2-m2",
+		},
+	}
+
+	for _, tc := range testsCases {
+		got := MachineName(tc.ClusterConfig, tc.ClusterConfig.Nodes[len(tc.ClusterConfig.Nodes)-1])
+		if got != tc.Want {
+			t.Errorf("Expected MachineName to be %q but got %q", tc.Want, got)
+		}
+	}
+}
+
 func TestIndexFromMachineName(t *testing.T) {
 	testCases := []struct {
 		Name        string
@@ -247,3 +302,60 @@ func TestIndexFromMachineName(t *testing.T) {
 
 	}
 }
+
+// test against cluster config
+func TestIndexFromMachineName2(t *testing.T) {
+
+	testsCases := []struct {
+		ClusterConfig config.ClusterConfig
+		Want          int
+	}{
+		{
+			ClusterConfig: config.ClusterConfig{Name: "minikube",
+				Nodes: []config.Node{
+					config.Node{
+						Name:              "",
+						IP:                "172.17.0.3",
+						Port:              8443,
+						KubernetesVersion: "v1.19.2",
+						ControlPlane:      true,
+						Worker:            true,
+					},
+				},
+			},
+			Want: 1,
+		},
+
+		{
+			ClusterConfig: config.ClusterConfig{Name: "p2",
+				Nodes: []config.Node{
+					config.Node{
+						Name:              "",
+						IP:                "172.17.0.3",
+						Port:              8443,
+						KubernetesVersion: "v1.19.2",
+						ControlPlane:      true,
+						Worker:            true,
+					},
+					config.Node{
+						Name:              "m2",
+						IP:                "172.17.0.4",
+						Port:              0,
+						KubernetesVersion: "v1.19.2",
+						ControlPlane:      false,
+						Worker:            true,
+					},
+				},
+			},
+			Want: 2,
+		},
+	}
+
+	for _, tc := range testsCases {
+		got := IndexFromMachineName(MachineName(tc.ClusterConfig, tc.ClusterConfig.Nodes[len(tc.ClusterConfig.Nodes)-1]))
+		if got != tc.Want {
+			t.Errorf("expected IndexFromMachineName to be %d but got %d", tc.Want, got)
+		}
+
+	}
+}

From b84dedc652a67ef4a461ab8f228827846c7f5b3c Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Tue, 29 Sep 2020 15:07:06 -0700
Subject: [PATCH 62/65] address review comments

---
 cmd/minikube/cmd/delete.go            |  2 +-
 pkg/drivers/kic/oci/errors.go         |  2 +-
 pkg/drivers/kic/oci/network.go        |  5 +++--
 pkg/drivers/kic/oci/network_create.go | 26 +++++++++++++-------------
 4 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/cmd/minikube/cmd/delete.go b/cmd/minikube/cmd/delete.go
index 93ca41fd34..e819fec0f3 100644
--- a/cmd/minikube/cmd/delete.go
+++ b/cmd/minikube/cmd/delete.go
@@ -260,7 +260,7 @@ func deletePossibleKicLeftOver(cname string, driverName string) {
 		glog.Warningf("error deleting volumes (might be okay).\nTo see the list of volumes run: 'docker volume ls'\n:%v", errs)
 	}
 
-	errs = oci.DeleteAllNetworksByKIC()
+	errs = oci.DeleteKICNetworks()
 	if errs != nil {
 		glog.Warningf("error deleting leftover networks (might be okay).\nTo see the list of networks: 'docker network ls'\n:%v", errs)
 	}
diff --git a/pkg/drivers/kic/oci/errors.go b/pkg/drivers/kic/oci/errors.go
index 0889358bc0..f8e65b7b1b 100644
--- a/pkg/drivers/kic/oci/errors.go
+++ b/pkg/drivers/kic/oci/errors.go
@@ -58,7 +58,7 @@ var ErrNetworkNotFound = errors.New("kic network not found")
 var ErrNetworkGatewayTaken = errors.New("network gateway is taken")
 
 // ErrNetworkInUse is when trying to delete a network which is attached to another container
-var ErrNetworkInUse = errors.New("can't delete network attached to a running container")
+var ErrNetworkInUse = errors.New("unable to delete a network that is attached to a running container")
 
 // LogContainerDebug will print relevant docker/podman infos after a container fails
 func LogContainerDebug(ociBin string, name string) string {
diff --git a/pkg/drivers/kic/oci/network.go b/pkg/drivers/kic/oci/network.go
index 2dcf22cc56..137063996a 100644
--- a/pkg/drivers/kic/oci/network.go
+++ b/pkg/drivers/kic/oci/network.go
@@ -37,7 +37,8 @@ func RoutableHostIPFromInside(ociBin string, clusterName string, containerName s
 			_, gateway, err := dockerNetworkInspect(clusterName)
 			if err != nil {
 				if errors.Is(err, ErrNetworkNotFound) {
-					glog.Infof("The container %s is not attached to a network, this could be because the cluster was created by an older than v.1.14 minikube, will try to get the IP using container gatway", containerName)
+					glog.Infof("The container %s is not attached to a network, this could be because the cluster was created by minikube <v1.14, will try to get the IP using container gatway", containerName)
+
 					return containerGatewayIP(Docker, containerName)
 				}
 				return gateway, errors.Wrap(err, "network inspect")
@@ -47,7 +48,7 @@ func RoutableHostIPFromInside(ociBin string, clusterName string, containerName s
 		// for windows and mac, the gateway ip is not routable so we use dns trick.
 		return digDNS(ociBin, containerName, "host.docker.internal")
 	}
-	//	podman
+	// podman
 	if runtime.GOOS == "linux" {
 		return containerGatewayIP(Podman, containerName)
 	}
diff --git a/pkg/drivers/kic/oci/network_create.go b/pkg/drivers/kic/oci/network_create.go
index 0525b616bb..bac03140b7 100644
--- a/pkg/drivers/kic/oci/network_create.go
+++ b/pkg/drivers/kic/oci/network_create.go
@@ -32,8 +32,8 @@ import (
 // it is one octet more than the one used by KVM to avoid possible conflict
 const firstSubnetAddr = "192.168.49.0"
 
-// big enough for a cluster of 256 nodes
-const defaultSubnetRange = 24
+// big enough for a cluster of 254 nodes
+const defaultSubnetMask = 24
 
 // CreateNetwork creates a network returns gateway and error, minikube creates one network per cluster
 func CreateNetwork(ociBin string, name string) (net.IP, error) {
@@ -53,13 +53,13 @@ func createDockerNetwork(clusterName string) (net.IP, error) {
 	// simple way to create networks, subnet is taken, try one bigger
 	attempt := 0
 	subnetAddr := firstSubnetAddr
-	gateway, err = tryCreateDockerNetwork(subnetAddr, defaultSubnetRange, clusterName)
+	gateway, err = tryCreateDockerNetwork(subnetAddr, defaultSubnetMask, clusterName)
 	if err != nil {
 		if err != ErrNetworkSubnetTaken {
 			return nil, errors.Wrapf(err, "error creating network")
 		}
+		// Rather than iterate through all of the valid subnets, give up at 20 to avoid a lengthy user delay for something that is unlikely to work.
 		// try up to 20 times, third octet would go like 49,59 ,69,..., 239
-		// max we could go is 255 we stop at 239
 		for attempt < 20 {
 			attempt++
 			glog.Infof("Couldn't create network %q at %q subnet will try again with a new subnet ...", clusterName, subnetAddr)
@@ -71,7 +71,7 @@ func createDockerNetwork(clusterName string) (net.IP, error) {
 			ip := net.ParseIP(subnetAddr).To4()
 			ip[2] += byte(9 + attempt)
 
-			gateway, err = tryCreateDockerNetwork(ip.String(), defaultSubnetRange, clusterName)
+			gateway, err = tryCreateDockerNetwork(ip.String(), defaultSubnetMask, clusterName)
 			if err == nil {
 				return gateway, nil
 			}
@@ -84,12 +84,12 @@ func createDockerNetwork(clusterName string) (net.IP, error) {
 	return gateway, nil
 }
 
-func tryCreateDockerNetwork(subnetAddr string, subnetRange int, name string) (net.IP, error) {
+func tryCreateDockerNetwork(subnetAddr string, subnetMask int, name string) (net.IP, error) {
 	gateway := net.ParseIP(subnetAddr)
 	gateway.To4()[3]++ // first ip for gateway
 	glog.Infof("attempt to create network %q with subnet: %s and gateway %s...", subnetAddr, name, gateway)
 	// options documentation https://docs.docker.com/engine/reference/commandline/network_create/#bridge-driver-options
-	rr, err := runCmd(exec.Command(Docker, "network", "create", "--driver=bridge", fmt.Sprintf("--subnet=%s", fmt.Sprintf("%s/%d", subnetAddr, subnetRange)), fmt.Sprintf("--gateway=%s", gateway), "-o", "--ip-masq", "-o", "--icc", fmt.Sprintf("--label=%s=%s", CreatedByLabelKey, "true"), name))
+	rr, err := runCmd(exec.Command(Docker, "network", "create", "--driver=bridge", fmt.Sprintf("--subnet=%s", fmt.Sprintf("%s/%d", subnetAddr, subnetMask)), fmt.Sprintf("--gateway=%s", gateway), "-o", "--ip-masq", "-o", "--icc", fmt.Sprintf("--label=%s=%s", CreatedByLabelKey, "true"), name))
 	if err != nil {
 		// Pool overlaps with other one on this address space
 		if strings.Contains(rr.Output(), "Pool overlaps") {
@@ -159,13 +159,13 @@ func networkExists(name string) bool {
 	return true
 }
 
-// returns all network names created by a label
-func allNetworkByLabel(ociBin string, label string) ([]string, error) {
+// networkNamesByLabel returns all network names created by a label
+func networkNamesByLabel(ociBin string, label string) ([]string, error) {
 	if ociBin != Docker {
 		return nil, fmt.Errorf("%s not supported", ociBin)
 	}
 
-	// docker network ls --filter='label=created_by.minikube.sigs.k8s.io=true' --format '{{.Name}}
+	// docker network ls --filter='label=created_by.minikube.sigs.k8s.io=true' --format '{{.Name}}'
 	rr, err := runCmd(exec.Command(Docker, "network", "ls", fmt.Sprintf("--filter=label=%s", label), "--format", "{{.Name}}"))
 	if err != nil {
 		return nil, err
@@ -179,10 +179,10 @@ func allNetworkByLabel(ociBin string, label string) ([]string, error) {
 	return lines, nil
 }
 
-// DeleteAllNetworksByKIC deletes all networks created by kic
-func DeleteAllNetworksByKIC() []error {
+// DeleteKICNetworks deletes all networks created by kic
+func DeleteKICNetworks() []error {
 	var errs []error
-	ns, err := allNetworkByLabel(Docker, CreatedByLabelKey+"=true")
+	ns, err := networkNamesByLabel(Docker, CreatedByLabelKey+"=true")
 	if err != nil {
 		return []error{errors.Wrap(err, "list all volume")}
 	}

From 790dd2483619c25653d9decb30ed5021b0bd9c53 Mon Sep 17 00:00:00 2001
From: zhanwang <zhanw15@gmail.com>
Date: Wed, 30 Sep 2020 09:56:33 +0800
Subject: [PATCH 63/65] pkg machine: correct 'deleting' spelling errors

---
 pkg/minikube/machine/delete.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/minikube/machine/delete.go b/pkg/minikube/machine/delete.go
index f4bfa83f47..e4ad04b058 100644
--- a/pkg/minikube/machine/delete.go
+++ b/pkg/minikube/machine/delete.go
@@ -46,7 +46,7 @@ func deleteOrphanedKIC(ociBin string, name string) {
 		glog.Infof("couldn't inspect container %q before deleting: %v", name, err)
 		return
 	}
-	// allow no more than 5 seconds for delting the container
+	// allow no more than 5 seconds for deleting the container
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()
 

From f807bcf8c90ea0a11dc731d29859e67f27f1335d Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Wed, 30 Sep 2020 14:58:20 -0700
Subject: [PATCH 64/65] remove kic test

---
 pkg/drivers/kic/kic_test.go           | 17 ---------
 pkg/drivers/kic/oci/network_create.go | 55 ++++++++++++---------------
 2 files changed, 25 insertions(+), 47 deletions(-)
 delete mode 100644 pkg/drivers/kic/kic_test.go

diff --git a/pkg/drivers/kic/kic_test.go b/pkg/drivers/kic/kic_test.go
deleted file mode 100644
index 0f77824b29..0000000000
--- a/pkg/drivers/kic/kic_test.go
+++ /dev/null
@@ -1,17 +0,0 @@
-/*
-Copyright 2019 The Kubernetes Authors All rights reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package kic
diff --git a/pkg/drivers/kic/oci/network_create.go b/pkg/drivers/kic/oci/network_create.go
index bac03140b7..af8bb7b337 100644
--- a/pkg/drivers/kic/oci/network_create.go
+++ b/pkg/drivers/kic/oci/network_create.go
@@ -50,36 +50,31 @@ func createDockerNetwork(clusterName string) (net.IP, error) {
 		glog.Infof("Found existing network with subnet %s and gateway %s.", subnet, gateway)
 		return gateway, nil
 	}
-	// simple way to create networks, subnet is taken, try one bigger
-	attempt := 0
-	subnetAddr := firstSubnetAddr
-	gateway, err = tryCreateDockerNetwork(subnetAddr, defaultSubnetMask, clusterName)
-	if err != nil {
-		if err != ErrNetworkSubnetTaken {
-			return nil, errors.Wrapf(err, "error creating network")
-		}
-		// Rather than iterate through all of the valid subnets, give up at 20 to avoid a lengthy user delay for something that is unlikely to work.
-		// try up to 20 times, third octet would go like 49,59 ,69,..., 239
-		for attempt < 20 {
-			attempt++
-			glog.Infof("Couldn't create network %q at %q subnet will try again with a new subnet ...", clusterName, subnetAddr)
-			// Find an open subnet by incrementing the 3rd octet by 10 for each try
-			// 13 times adding 10 firstSubnetAddr "192.168.49.0/24"
-			// at most it will add up to 169 which is still less than max allowed 255
-			// this is large enough to try more and not too small to not try enough
-			// can be tuned in the next iterations
-			ip := net.ParseIP(subnetAddr).To4()
-			ip[2] += byte(9 + attempt)
 
-			gateway, err = tryCreateDockerNetwork(ip.String(), defaultSubnetMask, clusterName)
-			if err == nil {
-				return gateway, nil
-			}
-			if errors.Is(err, ErrNetworkSubnetTaken) || errors.Is(err, ErrNetworkGatewayTaken) {
-				continue
-			}
-			glog.Errorf("unexpected error while trying to create network, will try again anyways: %v", err)
+	attempts := 0
+	subnetAddr := firstSubnetAddr
+	// Rather than iterate through all of the valid subnets, give up at 20 to avoid a lengthy user delay for something that is unlikely to work.
+	// will be like like 192.168.49.0/24 ,...,192.168.239.0/24
+	for attempts < 20 {
+		gateway, err = tryCreateDockerNetwork(subnetAddr, defaultSubnetMask, clusterName)
+		if err == nil {
+			return gateway, nil
 		}
+
+		// don't retry if error is not adddress is taken
+		if !(errors.Is(err, ErrNetworkSubnetTaken) || errors.Is(err, ErrNetworkGatewayTaken)) {
+			glog.Errorf("error while trying to create network %v", err)
+			return nil, errors.Wrap(err, "un-retryable")
+		}
+		attempts++
+		// Find an open subnet by incrementing the 3rd octet by 10 for each try
+		// 13 times adding 10 firstSubnetAddr "192.168.49.0/24"
+		// at most it will add up to 169 which is still less than max allowed 255
+		// this is large enough to try more and not too small to not try enough
+		// can be tuned in the next iterations
+		newSubnet := net.ParseIP(subnetAddr).To4()
+		newSubnet[2] += byte(9 + attempts)
+		subnetAddr = newSubnet.String()
 	}
 	return gateway, nil
 }
@@ -87,7 +82,7 @@ func createDockerNetwork(clusterName string) (net.IP, error) {
 func tryCreateDockerNetwork(subnetAddr string, subnetMask int, name string) (net.IP, error) {
 	gateway := net.ParseIP(subnetAddr)
 	gateway.To4()[3]++ // first ip for gateway
-	glog.Infof("attempt to create network %q with subnet: %s and gateway %s...", subnetAddr, name, gateway)
+	glog.Infof("attempt to create network %s/%d with subnet: %s and gateway %s...", subnetAddr, subnetMask, name, gateway)
 	// options documentation https://docs.docker.com/engine/reference/commandline/network_create/#bridge-driver-options
 	rr, err := runCmd(exec.Command(Docker, "network", "create", "--driver=bridge", fmt.Sprintf("--subnet=%s", fmt.Sprintf("%s/%d", subnetAddr, subnetMask)), fmt.Sprintf("--gateway=%s", gateway), "-o", "--ip-masq", "-o", "--icc", fmt.Sprintf("--label=%s=%s", CreatedByLabelKey, "true"), name))
 	if err != nil {
@@ -98,7 +93,7 @@ func tryCreateDockerNetwork(subnetAddr string, subnetMask int, name string) (net
 		if strings.Contains(rr.Output(), "failed to allocate gateway") && strings.Contains(rr.Output(), "Address already in use") {
 			return nil, ErrNetworkGatewayTaken
 		}
-		return nil, errors.Wrapf(err, "error creating network")
+		return nil, errors.Wrapf(err, "create network %s", fmt.Sprintf("%s %s/%d", name, subnetAddr, subnetMask))
 	}
 	return gateway, nil
 }

From 73d77e2463a2710a0518fbd5ac13ebadb348072a Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Wed, 30 Sep 2020 16:42:42 -0700
Subject: [PATCH 65/65] address review comments

---
 pkg/drivers/kic/kic.go                         |  2 +-
 pkg/drivers/kic/oci/network_create.go          | 15 ++++++---------
 pkg/minikube/driver/driver_test.go             |  4 ++--
 site/content/en/docs/handbook/vpn_and_proxy.md |  2 +-
 4 files changed, 10 insertions(+), 13 deletions(-)

diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go
index 0235f2d9e7..120d64a9b6 100644
--- a/pkg/drivers/kic/kic.go
+++ b/pkg/drivers/kic/kic.go
@@ -84,7 +84,7 @@ func (d *Driver) Create() error {
 	}
 
 	if gateway, err := oci.CreateNetwork(d.OCIBinary, d.NodeConfig.ClusterName); err != nil {
-		out.WarningT("Unable to create dedicated network, This might result in cluster IP change after restart. {{.error}}", out.V{"error": err})
+		out.WarningT("Unable to create dedicated network, this might result in cluster IP change after restart: {{.error}}", out.V{"error": err})
 	} else {
 		params.Network = d.NodeConfig.ClusterName
 		ip := gateway.To4()
diff --git a/pkg/drivers/kic/oci/network_create.go b/pkg/drivers/kic/oci/network_create.go
index af8bb7b337..dbbbe15115 100644
--- a/pkg/drivers/kic/oci/network_create.go
+++ b/pkg/drivers/kic/oci/network_create.go
@@ -54,7 +54,7 @@ func createDockerNetwork(clusterName string) (net.IP, error) {
 	attempts := 0
 	subnetAddr := firstSubnetAddr
 	// Rather than iterate through all of the valid subnets, give up at 20 to avoid a lengthy user delay for something that is unlikely to work.
-	// will be like like 192.168.49.0/24 ,...,192.168.239.0/24
+	// will be like 192.168.49.0/24 ,...,192.168.239.0/24
 	for attempts < 20 {
 		gateway, err = tryCreateDockerNetwork(subnetAddr, defaultSubnetMask, clusterName)
 		if err == nil {
@@ -76,7 +76,7 @@ func createDockerNetwork(clusterName string) (net.IP, error) {
 		newSubnet[2] += byte(9 + attempts)
 		subnetAddr = newSubnet.String()
 	}
-	return gateway, nil
+	return gateway, fmt.Errorf("failed to create network after 20 attempts")
 }
 
 func tryCreateDockerNetwork(subnetAddr string, subnetMask int, name string) (net.IP, error) {
@@ -144,14 +144,11 @@ func RemoveNetwork(name string) error {
 }
 
 func networkExists(name string) bool {
-	if _, _, err := dockerNetworkInspect(name); err != nil {
-		if err == ErrNetworkNotFound {
-			return false
-		}
-		glog.Warningf("error inspecting network %s: %v", name, err)
-		return false
+	_, _, err := dockerNetworkInspect(name)
+	if err != nil && !errors.Is(err, ErrNetworkNotFound) { // log unexpected error
+		glog.Warningf("Error inspecting docker network %s: %v", name, err)
 	}
-	return true
+	return err == nil
 }
 
 // networkNamesByLabel returns all network names created by a label
diff --git a/pkg/minikube/driver/driver_test.go b/pkg/minikube/driver/driver_test.go
index ba47bf7c59..953ff53691 100644
--- a/pkg/minikube/driver/driver_test.go
+++ b/pkg/minikube/driver/driver_test.go
@@ -303,8 +303,8 @@ func TestIndexFromMachineName(t *testing.T) {
 	}
 }
 
-// test against cluster config
-func TestIndexFromMachineName2(t *testing.T) {
+// test indexFroMachine against cluster config
+func TestIndexFromMachineNameClusterConfig(t *testing.T) {
 
 	testsCases := []struct {
 		ClusterConfig config.ClusterConfig
diff --git a/site/content/en/docs/handbook/vpn_and_proxy.md b/site/content/en/docs/handbook/vpn_and_proxy.md
index 41174d9999..bcd92ee5c8 100644
--- a/site/content/en/docs/handbook/vpn_and_proxy.md
+++ b/site/content/en/docs/handbook/vpn_and_proxy.md
@@ -22,7 +22,7 @@ The NO_PROXY variable here is important: Without setting it, minikube may not be
 
 * **192.168.99.0/24**: Used by the minikube VM. Configurable for some hypervisors via `--host-only-cidr`
 * **192.168.39.0/24**: Used by the minikube kvm2 driver.
-* **192.168.49.0/24**: Used by the minikube docker driver.
+* **192.168.49.0/24**: Used by the minikube docker driver's first cluster.
 * **10.96.0.0/12**: Used by service cluster IP's. Configurable via  `--service-cluster-ip-range`
 
 One important note: If NO_PROXY is required by non-Kubernetes applications, such as Firefox or Chrome, you may want to specifically add the minikube IP to the comma-separated list, as they may not understand IP ranges ([#3827](https://github.com/kubernetes/minikube/issues/3827)).