From cda2d13d211f0b143902c2981139bbdec78ec161 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Wed, 28 Oct 2020 12:19:26 -0700
Subject: [PATCH 001/114] Turn on ingress addon on linux baremetal

---
 pkg/addons/addons.go            | 9 ++-------
 test/integration/addons_test.go | 4 ++--
 2 files changed, 4 insertions(+), 9 deletions(-)

diff --git a/pkg/addons/addons.go b/pkg/addons/addons.go
index b86184ae42..a3fb175d1f 100644
--- a/pkg/addons/addons.go
+++ b/pkg/addons/addons.go
@@ -144,19 +144,14 @@ func enableOrDisableAddon(cc *config.ClusterConfig, name string, val string) err
 	}
 
 	// to match both ingress and ingress-dns addons
-	if strings.HasPrefix(name, "ingress") && enable {
-		if driver.IsKIC(cc.Driver) && runtime.GOOS != "linux" {
-			exit.Message(reason.Usage, `Due to networking limitations of driver {{.driver_name}} on {{.os_name}}, {{.addon_name}} addon is not supported.
+	if strings.HasPrefix(name, "ingress") && enable && driver.IsKIC(cc.Driver) && runtime.GOOS != "linux" {
+		exit.Message(reason.Usage, `Due to networking limitations of driver {{.driver_name}} on {{.os_name}}, {{.addon_name}} addon is not supported.
 Alternatively to use this addon you can use a vm-based driver:
 
 	'minikube start --vm=true'
 
 To track the update on this work in progress feature please check:
 https://github.com/kubernetes/minikube/issues/7332`, out.V{"driver_name": cc.Driver, "os_name": runtime.GOOS, "addon_name": name})
-		} else if driver.BareMetal(cc.Driver) {
-			exit.Message(reason.Usage, `Due to networking limitations of driver {{.driver_name}}, {{.addon_name}} addon is not supported. Try using a different driver.`,
-				out.V{"driver_name": cc.Driver, "addon_name": name})
-		}
 	}
 
 	if strings.HasPrefix(name, "istio") && enable {
diff --git a/test/integration/addons_test.go b/test/integration/addons_test.go
index d9c686dd2d..95c1fe89eb 100644
--- a/test/integration/addons_test.go
+++ b/test/integration/addons_test.go
@@ -111,8 +111,8 @@ func TestAddons(t *testing.T) {
 func validateIngressAddon(ctx context.Context, t *testing.T, profile string) {
 	defer PostMortemLogs(t, profile)
 
-	if NoneDriver() || (runtime.GOOS == "darwin" && KicDriver()) {
-		t.Skipf("skipping: ssh unsupported by none")
+	if runtime.GOOS == "darwin" && KicDriver() {
+		t.Skipf("skipping: ingress not supported on macOS docker driver")
 	}
 
 	client, err := kapi.Client(profile)

From 3a506605446e62c72fb44106e47d27b14b7d8264 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Wed, 28 Oct 2020 17:35:42 -0700
Subject: [PATCH 002/114] fix ingress test

---
 pkg/minikube/machine/start.go   | 2 +-
 test/integration/addons_test.go | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkg/minikube/machine/start.go b/pkg/minikube/machine/start.go
index 16034e6e0b..b5224a06dc 100644
--- a/pkg/minikube/machine/start.go
+++ b/pkg/minikube/machine/start.go
@@ -299,7 +299,7 @@ func postStartSetup(h *host.Host, mc config.ClusterConfig) error {
 // acquireMachinesLock protects against code that is not parallel-safe (libmachine, cert setup)
 func acquireMachinesLock(name string, drv string) (mutex.Releaser, error) {
 	lockPath := filepath.Join(localpath.MiniPath(), "machines", drv)
-	// "With KIC, it's safe to provision multiple hosts simultaneously"
+	// With KIC, it's safe to provision multiple hosts simultaneously
 	if driver.IsKIC(drv) {
 		lockPath = filepath.Join(localpath.MiniPath(), "machines", drv, name)
 	}
diff --git a/test/integration/addons_test.go b/test/integration/addons_test.go
index 95c1fe89eb..e17eccb244 100644
--- a/test/integration/addons_test.go
+++ b/test/integration/addons_test.go
@@ -59,7 +59,7 @@ func TestAddons(t *testing.T) {
 	}
 
 	args := append([]string{"start", "-p", profile, "--wait=false", "--memory=2600", "--alsologtostderr", "--addons=registry", "--addons=metrics-server", "--addons=helm-tiller", "--addons=olm", "--addons=volumesnapshots", "--addons=csi-hostpath-driver", "--addons=gcp-auth"}, StartArgs()...)
-	if !NoneDriver() && !(runtime.GOOS == "darwin" && KicDriver()) { // none doesn't support ingress
+	if !(runtime.GOOS == "darwin" && KicDriver()) { // docker driver on macos doesn't support ingress
 		args = append(args, "--addons=ingress")
 	}
 	rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))

From a904da4e26d22d08b6855e0afdc4702ed7bf7796 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Wed, 28 Oct 2020 18:38:30 -0700
Subject: [PATCH 003/114] don't use minikube ssh for none driver

---
 test/integration/addons_test.go | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/test/integration/addons_test.go b/test/integration/addons_test.go
index e17eccb244..93b20093d2 100644
--- a/test/integration/addons_test.go
+++ b/test/integration/addons_test.go
@@ -157,9 +157,17 @@ func validateIngressAddon(ctx context.Context, t *testing.T, profile string) {
 	want := "Welcome to nginx!"
 	addr := "http://127.0.0.1/"
 	checkIngress := func() error {
-		rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", fmt.Sprintf("curl -s %s -H 'Host: nginx.example.com'", addr)))
-		if err != nil {
-			return err
+		var rr RunResult
+		if NoneDriver() { // just run curl directly on the none driver
+			rr, err := Run(t, exec.CommandContext(ctx, fmt.Sprintf("curl -s %s -H 'Host: nginx.example.com'", addr))
+			if err != nil {
+				return err
+			}
+		} else {
+			rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", fmt.Sprintf("curl -s %s -H 'Host: nginx.example.com'", addr)))
+			if err != nil {
+				return err
+			}
 		}
 
 		stderr := rr.Stderr.String()

From a9a1c4a71e917455609528ae340fa6f4f7c44454 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Wed, 28 Oct 2020 18:49:08 -0700
Subject: [PATCH 004/114] syntax

---
 test/integration/addons_test.go | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/test/integration/addons_test.go b/test/integration/addons_test.go
index 93b20093d2..e0d92d0520 100644
--- a/test/integration/addons_test.go
+++ b/test/integration/addons_test.go
@@ -157,14 +157,15 @@ func validateIngressAddon(ctx context.Context, t *testing.T, profile string) {
 	want := "Welcome to nginx!"
 	addr := "http://127.0.0.1/"
 	checkIngress := func() error {
-		var rr RunResult
+		var rr *RunResult
+		var err error
 		if NoneDriver() { // just run curl directly on the none driver
-			rr, err := Run(t, exec.CommandContext(ctx, fmt.Sprintf("curl -s %s -H 'Host: nginx.example.com'", addr))
+			rr, err = Run(t, exec.CommandContext(ctx, fmt.Sprintf("curl -s %s -H 'Host: nginx.example.com'", addr)))
 			if err != nil {
 				return err
 			}
 		} else {
-			rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", fmt.Sprintf("curl -s %s -H 'Host: nginx.example.com'", addr)))
+			rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", fmt.Sprintf("curl -s %s -H 'Host: nginx.example.com'", addr)))
 			if err != nil {
 				return err
 			}

From d52d6e440d6468a300e2fce4aeaecf58ddd689ec Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Fri, 30 Oct 2020 10:34:19 -0700
Subject: [PATCH 005/114] run command properly

---
 pkg/minikube/bootstrapper/bsutil/kverify/default_sa.go | 2 +-
 test/integration/addons_test.go                        | 4 ++--
 test/integration/testdata/nginx-ing.yaml               | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/default_sa.go b/pkg/minikube/bootstrapper/bsutil/kverify/default_sa.go
index 7efc5215d3..1f4cb0a84d 100644
--- a/pkg/minikube/bootstrapper/bsutil/kverify/default_sa.go
+++ b/pkg/minikube/bootstrapper/bsutil/kverify/default_sa.go
@@ -36,7 +36,7 @@ func WaitForDefaultSA(cs *kubernetes.Clientset, timeout time.Duration) error {
 		// equivalent to manual check of 'kubectl --context profile get serviceaccount default'
 		sas, err := cs.CoreV1().ServiceAccounts("default").List(meta.ListOptions{})
 		if err != nil {
-			klog.Infof("temproary error waiting for default SA: %v", err)
+			klog.Infof("temporary error waiting for default SA: %v", err)
 			return false, nil
 		}
 		for _, sa := range sas.Items {
diff --git a/test/integration/addons_test.go b/test/integration/addons_test.go
index e0d92d0520..553d01dd3e 100644
--- a/test/integration/addons_test.go
+++ b/test/integration/addons_test.go
@@ -133,7 +133,7 @@ func validateIngressAddon(ctx context.Context, t *testing.T, profile string) {
 			return err
 		}
 		if rr.Stderr.String() != "" {
-			t.Logf("%v: unexpected stderr: %s (may be temproary)", rr.Command(), rr.Stderr)
+			t.Logf("%v: unexpected stderr: %s (may be temporary)", rr.Command(), rr.Stderr)
 		}
 		return nil
 	}
@@ -160,7 +160,7 @@ func validateIngressAddon(ctx context.Context, t *testing.T, profile string) {
 		var rr *RunResult
 		var err error
 		if NoneDriver() { // just run curl directly on the none driver
-			rr, err = Run(t, exec.CommandContext(ctx, fmt.Sprintf("curl -s %s -H 'Host: nginx.example.com'", addr)))
+			rr, err = Run(t, exec.CommandContext(ctx, "curl", "-s", addr, "-H", "'Host: nginx.example.com'"))
 			if err != nil {
 				return err
 			}
diff --git a/test/integration/testdata/nginx-ing.yaml b/test/integration/testdata/nginx-ing.yaml
index 30cef8139d..16f442e205 100644
--- a/test/integration/testdata/nginx-ing.yaml
+++ b/test/integration/testdata/nginx-ing.yaml
@@ -1,4 +1,4 @@
-apiVersion: extensions/v1beta1
+apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
   name: nginx-ingress

From 492ae13a4af8b2d198a992cd2a146d6f0aaca515 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Fri, 30 Oct 2020 11:02:50 -0700
Subject: [PATCH 006/114] undo yaml change

---
 test/integration/testdata/nginx-ing.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/integration/testdata/nginx-ing.yaml b/test/integration/testdata/nginx-ing.yaml
index 16f442e205..30cef8139d 100644
--- a/test/integration/testdata/nginx-ing.yaml
+++ b/test/integration/testdata/nginx-ing.yaml
@@ -1,4 +1,4 @@
-apiVersion: networking.k8s.io/v1
+apiVersion: extensions/v1beta1
 kind: Ingress
 metadata:
   name: nginx-ingress

From 98f46777623f26088edd083fbb389bc12cb332d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= <anders.f.bjorklund@gmail.com>
Date: Wed, 30 Dec 2020 15:04:17 +0100
Subject: [PATCH 007/114] MEP: Standard Linux Distribution ("distro")

---
 .../standard-distro.md                        | 41 +++++++++++++++++++
 1 file changed, 41 insertions(+)
 create mode 100644 enhancements/proposed/20201230-standard-distro/standard-distro.md

diff --git a/enhancements/proposed/20201230-standard-distro/standard-distro.md b/enhancements/proposed/20201230-standard-distro/standard-distro.md
new file mode 100644
index 0000000000..e1aa5e1d14
--- /dev/null
+++ b/enhancements/proposed/20201230-standard-distro/standard-distro.md
@@ -0,0 +1,41 @@
+# Standard Linux Distribution
+
+* First proposed: 2020-12-17
+* Authors: Anders F Björklund (@afbjorklund)
+
+## Reviewer Priorities
+
+Please review this proposal with the following priorities:
+
+*   Does this fit with minikube's [principles](https://minikube.sigs.k8s.io/docs/concepts/principles/)?
+*   Are there other approaches to consider?
+*   Could the implementation be made simpler?
+*   Are there usability, reliability, or technical debt concerns?
+
+Please leave the above text in your proposal as instructions to the reader.
+
+## Summary
+
+Change the distribution (OS) for the minikube ISO, from Buildroot to Ubuntu.
+
+## Goals
+
+* Use one of the supported Kubernetes OS, like Ubuntu 20.04
+* Use the same operating system for KIC base and ISO image
+
+## Non-Goals
+
+* Making major changes to the new standard operating system
+* Support production deployments, still intended for learning
+
+## Design Details
+
+Use external system image and external packages, same as for KIC image.
+
+Keep both images available (one being default), during transition period.
+
+## Alternatives Considered
+
+Continue to support custom distro, instead of using a standard distro.
+
+Make current Buildroot OS into standard supported Kubernetes distribution.

From 7e8aaa8a91ecde181b1acfcc9aae276ed43fb890 Mon Sep 17 00:00:00 2001
From: Hari Udhayakumar <h.udhayakumar@salesforce.com>
Date: Fri, 22 Jan 2021 16:47:11 -0800
Subject: [PATCH 008/114] Add google group link to triaging doc

---
 site/content/en/docs/contrib/triage.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/site/content/en/docs/contrib/triage.md b/site/content/en/docs/contrib/triage.md
index bc35a25bcd..e8627516de 100644
--- a/site/content/en/docs/contrib/triage.md
+++ b/site/content/en/docs/contrib/triage.md
@@ -8,7 +8,9 @@ description: >
 ---
 
 Community triage takes place **every Wednesday** from **11AM-12PM PST**.
-Hangouts link: https://meet.google.com/ikf-fvrs-eer
+
+- Hangouts link: https://meet.google.com/ikf-fvrs-eer
+- Google Group: https://groups.google.com/forum/#!forum/minikube-dev
 
 All community members are welcome and encouraged to join and help us triage minikube!
 

From 1fe4da9643c74c9db3f4327ccbe80bd7d24b2cee Mon Sep 17 00:00:00 2001
From: Ilya Zuyev <izuyev@google.com>
Date: Mon, 1 Feb 2021 16:11:16 -0800
Subject: [PATCH 009/114] Fix builderx creation

---
 Makefile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index f39055678c..e985fcd3d7 100644
--- a/Makefile
+++ b/Makefile
@@ -635,12 +635,12 @@ X_BUILD_ENV ?= DOCKER_CLI_EXPERIMENTAL=enabled
 docker-multi-arch-builder:
 	env $(X_BUILD_ENV) docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
 	env $(X_BUILD_ENV) docker buildx rm --builder $(X_DOCKER_BUILDER) || true
-	env $(X_BUILD_ENV) docker buildx create --name kicbase-builder --buildkitd-flags '--debug' --use || true
+	env $(X_BUILD_ENV) docker buildx create --name $(X_DOCKER_BUILDER) --buildkitd-flags '--debug' || true
 
 KICBASE_ARCH = linux/arm64,linux/amd64
 KICBASE_IMAGE_REGISTRIES ?= $(REGISTRY)/kicbase:$(KIC_VERSION) $(REGISTRY_GH)/kicbase:$(KIC_VERSION) kicbase/stable:$(KIC_VERSION)
 
-.PHONY: push-kic-base-image 
+.PHONY: ppush-kic-base-image 
 push-kic-base-image: docker-multi-arch-builder ## Push multi-arch local/kicbase:latest to all remote registries
 ifdef AUTOPUSH
 	docker login gcr.io/k8s-minikube
@@ -652,7 +652,7 @@ endif
 ifndef AUTOPUSH
 	$(call user_confirm, 'Are you sure you want to push $(KICBASE_IMAGE_REGISTRIES) ?')
 endif
-	env $(X_BUILD_ENV) docker buildx build --platform $(KICBASE_ARCH) $(addprefix -t ,$(KICBASE_IMAGE_REGISTRIES)) --push  --build-arg COMMIT_SHA=${VERSION}-$(COMMIT) ./deploy/kicbase
+	env $(X_BUILD_ENV) docker buildx build --builder $(X_DOCKER_BUILDER) --platform $(KICBASE_ARCH) $(addprefix -t ,$(KICBASE_IMAGE_REGISTRIES)) --push  --build-arg COMMIT_SHA=${VERSION}-$(COMMIT) ./deploy/kicbase
 
 .PHONY: upload-preloaded-images-tar
 upload-preloaded-images-tar: out/minikube # Upload the preloaded images for oldest supported, newest supported, and default kubernetes versions to GCS.

From 9eee59bb22cc3975cf0a2f6b99fe405d65480793 Mon Sep 17 00:00:00 2001
From: Steven Powell <spowellgrr@gmail.com>
Date: Tue, 2 Feb 2021 16:43:57 -0700
Subject: [PATCH 010/114] Added audit logs to minikube logs output

---
 pkg/minikube/audit/audit.go   | 31 ++++++++++++++++++++++++++
 pkg/minikube/audit/entry.go   | 42 ++++++++++++++++++++++++++++++++++-
 pkg/minikube/audit/logFile.go |  4 ++--
 pkg/minikube/logs/logs.go     | 16 +++++++++++++
 4 files changed, 90 insertions(+), 3 deletions(-)

diff --git a/pkg/minikube/audit/audit.go b/pkg/minikube/audit/audit.go
index 83b0ee98c2..4c3f22b41f 100644
--- a/pkg/minikube/audit/audit.go
+++ b/pkg/minikube/audit/audit.go
@@ -17,6 +17,8 @@ limitations under the License.
 package audit
 
 import (
+	"bufio"
+	"fmt"
 	"os"
 	"os/user"
 	"strings"
@@ -76,3 +78,32 @@ func shouldLog() bool {
 	}
 	return true
 }
+
+// Retrieve the last n lines from the log.
+func Retrieve(n int) (string, error) {
+	if n <= 0 {
+		return "", fmt.Errorf("number of lines to retrieve must be 1 or greater")
+	}
+	if currentLogFile == nil {
+		if err := setLogFile(); err != nil {
+			return "", fmt.Errorf("failed to set the log file: %v", err)
+		}
+	}
+	var l []string
+	s := bufio.NewScanner(currentLogFile)
+	for s.Scan() {
+		// pop off the earliest line if already at desired log length
+		if len(l) == n {
+			l = l[1:]
+		}
+		l = append(l, s.Text())
+	}
+	if err := s.Err(); err != nil {
+		return "", fmt.Errorf("failed to read from audit file: %v", err)
+	}
+	t, err := linesToTable(l)
+	if err != nil {
+		return "", fmt.Errorf("failed to convert lines to table: %v", err)
+	}
+	return t, nil
+}
diff --git a/pkg/minikube/audit/entry.go b/pkg/minikube/audit/entry.go
index 75b32af9a0..08c2773c14 100644
--- a/pkg/minikube/audit/entry.go
+++ b/pkg/minikube/audit/entry.go
@@ -17,8 +17,12 @@ limitations under the License.
 package audit
 
 import (
+	"bytes"
+	"encoding/json"
+	"fmt"
 	"time"
 
+	"github.com/olekukonko/tablewriter"
 	"github.com/spf13/viper"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
@@ -26,7 +30,7 @@ import (
 
 // entry represents the execution of a command.
 type entry struct {
-	data map[string]string
+	Data map[string]string `json:"data"`
 }
 
 // Type returns the cloud events compatible type of this struct.
@@ -47,3 +51,39 @@ func newEntry(command string, args string, user string, startTime time.Time, end
 		},
 	}
 }
+
+// toFields converts an entry to an array of fields.
+func (e *entry) toFields() []string {
+	d := e.Data
+	return []string{d["command"], d["args"], d["profile"], d["user"], d["startTime"], d["endTime"]}
+}
+
+// linesToFields converts audit lines into arrays of fields.
+func linesToFields(lines []string) ([][]string, error) {
+	c := [][]string{}
+	for _, l := range lines {
+		e := &entry{}
+		if err := json.Unmarshal([]byte(l), e); err != nil {
+			return nil, fmt.Errorf("failed to unmarshal %q: %v", l, err)
+		}
+		c = append(c, e.toFields())
+	}
+	return c, nil
+}
+
+// linesToTable converts audit lines into a formatted table.
+func linesToTable(lines []string) (string, error) {
+	f, err := linesToFields(lines)
+	if err != nil {
+		return "", fmt.Errorf("failed to convert lines to fields: %v", err)
+	}
+	b := new(bytes.Buffer)
+	t := tablewriter.NewWriter(b)
+	t.SetHeader([]string{"Command", "Args", "Profile", "User", "Start Time", "End Time"})
+	t.SetAutoFormatHeaders(false)
+	t.SetBorders(tablewriter.Border{Left: true, Top: true, Right: true, Bottom: true})
+	t.SetCenterSeparator("|")
+	t.AppendBulk(f)
+	t.Render()
+	return b.String(), nil
+}
diff --git a/pkg/minikube/audit/logFile.go b/pkg/minikube/audit/logFile.go
index 1d24284412..79febd6927 100644
--- a/pkg/minikube/audit/logFile.go
+++ b/pkg/minikube/audit/logFile.go
@@ -30,7 +30,7 @@ var currentLogFile *os.File
 // setLogFile sets the logPath and creates the log file if it doesn't exist.
 func setLogFile() error {
 	lp := localpath.AuditLog()
-	f, err := os.OpenFile(lp, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	f, err := os.OpenFile(lp, os.O_APPEND|os.O_CREATE|os.O_RDWR, 0644)
 	if err != nil {
 		return fmt.Errorf("unable to open %s: %v", lp, err)
 	}
@@ -45,7 +45,7 @@ func appendToLog(entry *entry) error {
 			return err
 		}
 	}
-	e := register.CloudEvent(entry, entry.data)
+	e := register.CloudEvent(entry, entry.Data)
 	bs, err := e.MarshalJSON()
 	if err != nil {
 		return fmt.Errorf("error marshalling event: %v", err)
diff --git a/pkg/minikube/logs/logs.go b/pkg/minikube/logs/logs.go
index 43b4c08e67..a9063308f2 100644
--- a/pkg/minikube/logs/logs.go
+++ b/pkg/minikube/logs/logs.go
@@ -29,6 +29,7 @@ import (
 
 	"github.com/pkg/errors"
 	"k8s.io/klog/v2"
+	"k8s.io/minikube/pkg/minikube/audit"
 	"k8s.io/minikube/pkg/minikube/bootstrapper"
 	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/minikube/config"
@@ -188,12 +189,27 @@ func Output(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg config.Cluster
 		}
 	}
 
+	outputAudit(lines, failed)
+
 	if len(failed) > 0 {
 		return fmt.Errorf("unable to fetch logs for: %s", strings.Join(failed, ", "))
 	}
 	return nil
 }
 
+// outputAudit displays the audit logs.
+func outputAudit(lines int, failed []string) {
+	out.Step(style.Empty, "")
+	out.Step(style.Empty, "==> Audit <==")
+	a, err := audit.Retrieve(lines)
+	if err != nil {
+		klog.Errorf("failed to get audit logs with error: %v", err)
+		failed = append(failed, "audit")
+		return
+	}
+	out.Step(style.Empty, a)
+}
+
 // logCommands returns a list of commands that would be run to receive the anticipated logs
 func logCommands(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg config.ClusterConfig, length int, follow bool) map[string]string {
 	cmds := bs.LogCommands(cfg, bootstrapper.LogOptions{Lines: length, Follow: follow})

From 69ffef11a88aafe578dbde38eeb780f614af26ce Mon Sep 17 00:00:00 2001
From: Steven Powell <spowellgrr@gmail.com>
Date: Thu, 4 Feb 2021 11:37:27 -0700
Subject: [PATCH 011/114] Added unit tests and refactored report making

---
 pkg/minikube/audit/audit.go       | 31 -----------
 pkg/minikube/audit/entry.go       |  4 +-
 pkg/minikube/audit/entry_test.go  | 88 +++++++++++++++++++++++++++++++
 pkg/minikube/audit/report.go      | 65 +++++++++++++++++++++++
 pkg/minikube/audit/report_test.go | 62 ++++++++++++++++++++++
 pkg/minikube/logs/logs.go         | 20 ++++---
 6 files changed, 230 insertions(+), 40 deletions(-)
 create mode 100644 pkg/minikube/audit/entry_test.go
 create mode 100644 pkg/minikube/audit/report.go
 create mode 100644 pkg/minikube/audit/report_test.go

diff --git a/pkg/minikube/audit/audit.go b/pkg/minikube/audit/audit.go
index 4c3f22b41f..83b0ee98c2 100644
--- a/pkg/minikube/audit/audit.go
+++ b/pkg/minikube/audit/audit.go
@@ -17,8 +17,6 @@ limitations under the License.
 package audit
 
 import (
-	"bufio"
-	"fmt"
 	"os"
 	"os/user"
 	"strings"
@@ -78,32 +76,3 @@ func shouldLog() bool {
 	}
 	return true
 }
-
-// Retrieve the last n lines from the log.
-func Retrieve(n int) (string, error) {
-	if n <= 0 {
-		return "", fmt.Errorf("number of lines to retrieve must be 1 or greater")
-	}
-	if currentLogFile == nil {
-		if err := setLogFile(); err != nil {
-			return "", fmt.Errorf("failed to set the log file: %v", err)
-		}
-	}
-	var l []string
-	s := bufio.NewScanner(currentLogFile)
-	for s.Scan() {
-		// pop off the earliest line if already at desired log length
-		if len(l) == n {
-			l = l[1:]
-		}
-		l = append(l, s.Text())
-	}
-	if err := s.Err(); err != nil {
-		return "", fmt.Errorf("failed to read from audit file: %v", err)
-	}
-	t, err := linesToTable(l)
-	if err != nil {
-		return "", fmt.Errorf("failed to convert lines to table: %v", err)
-	}
-	return t, nil
-}
diff --git a/pkg/minikube/audit/entry.go b/pkg/minikube/audit/entry.go
index 08c2773c14..1d14ded9a4 100644
--- a/pkg/minikube/audit/entry.go
+++ b/pkg/minikube/audit/entry.go
@@ -72,14 +72,14 @@ func linesToFields(lines []string) ([][]string, error) {
 }
 
 // linesToTable converts audit lines into a formatted table.
-func linesToTable(lines []string) (string, error) {
+func linesToTable(lines []string, headers []string) (string, error) {
 	f, err := linesToFields(lines)
 	if err != nil {
 		return "", fmt.Errorf("failed to convert lines to fields: %v", err)
 	}
 	b := new(bytes.Buffer)
 	t := tablewriter.NewWriter(b)
-	t.SetHeader([]string{"Command", "Args", "Profile", "User", "Start Time", "End Time"})
+	t.SetHeader(headers)
 	t.SetAutoFormatHeaders(false)
 	t.SetBorders(tablewriter.Border{Left: true, Top: true, Right: true, Bottom: true})
 	t.SetCenterSeparator("|")
diff --git a/pkg/minikube/audit/entry_test.go b/pkg/minikube/audit/entry_test.go
new file mode 100644
index 0000000000..1d7867014a
--- /dev/null
+++ b/pkg/minikube/audit/entry_test.go
@@ -0,0 +1,88 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package audit
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/spf13/viper"
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/constants"
+)
+
+func TestEntry(t *testing.T) {
+	c := "start"
+	a := "--alsologtostderr"
+	p := "profile1"
+	u := "user1"
+	st := time.Now()
+	stFormatted := st.Format(constants.TimeFormat)
+	et := time.Now()
+	etFormatted := et.Format(constants.TimeFormat)
+
+	// save current profile in case something depends on it
+	oldProfile := viper.GetString(config.ProfileName)
+	viper.Set(config.ProfileName, p)
+	e := newEntry(c, a, u, st, et)
+	viper.Set(config.ProfileName, oldProfile)
+
+	t.Run("TestNewEntry", func(t *testing.T) {
+		d := e.Data
+
+		tests := []struct {
+			key  string
+			want string
+		}{
+			{"command", c},
+			{"args", a},
+			{"profile", p},
+			{"user", u},
+			{"startTime", stFormatted},
+			{"endTime", etFormatted},
+		}
+
+		for _, tt := range tests {
+			got := d[tt.key]
+
+			if got != tt.want {
+				t.Errorf("Data[%q] = %s; want %s", tt.key, got, tt.want)
+			}
+		}
+	})
+
+	t.Run("TestType", func(t *testing.T) {
+		got := e.Type()
+		want := "io.k8s.sigs.minikube.audit"
+
+		if got != want {
+			t.Errorf("Type() = %s; want %s", got, want)
+		}
+	})
+
+	t.Run("TestToField", func(t *testing.T) {
+		got := e.toFields()
+		gotString := strings.Join(got, ",")
+		want := []string{c, a, p, u, stFormatted, etFormatted}
+		wantString := strings.Join(want, ",")
+
+		if gotString != wantString {
+			t.Errorf("toFields() = %s; want %s", gotString, wantString)
+		}
+	})
+}
diff --git a/pkg/minikube/audit/report.go b/pkg/minikube/audit/report.go
new file mode 100644
index 0000000000..1ef770f12d
--- /dev/null
+++ b/pkg/minikube/audit/report.go
@@ -0,0 +1,65 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package audit
+
+import (
+	"bufio"
+	"fmt"
+)
+
+type Data struct {
+	headers []string
+	logs    []string
+}
+
+// Report is created from the log file.
+func Report(lines int) (*Data, error) {
+	if lines <= 0 {
+		return nil, fmt.Errorf("number of lines must be 1 or greater")
+	}
+	if currentLogFile == nil {
+		if err := setLogFile(); err != nil {
+			return nil, fmt.Errorf("failed to set the log file: %v", err)
+		}
+	}
+	var l []string
+	s := bufio.NewScanner(currentLogFile)
+	for s.Scan() {
+		// pop off the earliest line if already at desired log length
+		if len(l) == lines {
+			l = l[1:]
+		}
+		l = append(l, s.Text())
+	}
+	if err := s.Err(); err != nil {
+		return nil, fmt.Errorf("failed to read from audit file: %v", err)
+	}
+	r := &Data{
+		[]string{"Command", "Args", "Profile", "User", "Start Time", "End Time"},
+		l,
+	}
+	return r, nil
+}
+
+// Table creates a formatted table using last n lines from the report.
+func (r *Data) Table() (string, error) {
+	t, err := linesToTable(r.logs, r.headers)
+	if err != nil {
+		return "", fmt.Errorf("failed to convert lines to table: %v", err)
+	}
+	return t, nil
+}
diff --git a/pkg/minikube/audit/report_test.go b/pkg/minikube/audit/report_test.go
new file mode 100644
index 0000000000..4dd727ecdd
--- /dev/null
+++ b/pkg/minikube/audit/report_test.go
@@ -0,0 +1,62 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package audit
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+func TestAuditReport(t *testing.T) {
+	f, err := ioutil.TempFile("", "audit.json")
+	if err != nil {
+		t.Fatalf("failed creating temporary file: %v", err)
+	}
+	defer os.Remove(f.Name())
+
+	s := `{"data":{"args":"-p mini1","command":"start","endTime":"Wed, 03 Feb 2021 15:33:05 MST","profile":"mini1","startTime":"Wed, 03 Feb 2021 15:30:33 MST","user":"user1"},"datacontenttype":"application/json","id":"9b7593cb-fbec-49e5-a3ce-bdc2d0bfb208","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.si  gs.minikube.audit"}
+{"data":{"args":"-p mini1","command":"start","endTime":"Wed, 03 Feb 2021 15:33:05 MST","profile":"mini1","startTime":"Wed, 03 Feb 2021 15:30:33 MST","user":"user1"},"datacontenttype":"application/json","id":"9b7593cb-fbec-49e5-a3ce-bdc2d0bfb208","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.si  gs.minikube.audit"}
+{"data":{"args":"--user user2","command":"logs","endTime":"Tue, 02 Feb 2021 16:46:20 MST","profile":"minikube","startTime":"Tue, 02 Feb 2021 16:46:00 MST","user":"user2"},"datacontenttype":"application/json","id":"fec03227-2484-48b6-880a-88fd010b5efd","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.audit"}`
+
+	if _, err := f.WriteString(s); err != nil {
+		t.Fatalf("failed writing to file: %v", err)
+	}
+	if _, err := f.Seek(0, 0); err != nil {
+		t.Fatalf("failed seeking to start of file: %v", err)
+	}
+
+	oldLogFile := *currentLogFile
+	defer func() { currentLogFile = &oldLogFile }()
+	currentLogFile = f
+
+	wantedLines := 2
+	r, err := Report(wantedLines)
+	if err != nil {
+		t.Fatalf("failed to create report: %v", err)
+	}
+
+	if len(r.logs) != wantedLines {
+		t.Fatalf("report has %d lines of logs, want %d", len(r.logs), wantedLines)
+	}
+
+	t.Run("TestTable", func(t *testing.T) {
+		if _, err := r.Table(); err != nil {
+			t.Errorf("failed to create table from report: %v", err)
+		}
+	})
+}
diff --git a/pkg/minikube/logs/logs.go b/pkg/minikube/logs/logs.go
index a9063308f2..4fcac53b7e 100644
--- a/pkg/minikube/logs/logs.go
+++ b/pkg/minikube/logs/logs.go
@@ -189,7 +189,10 @@ func Output(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg config.Cluster
 		}
 	}
 
-	outputAudit(lines, failed)
+	if err := outputAudit(lines); err != nil {
+		klog.Error(err)
+		failed = append(failed, "audit")
+	}
 
 	if len(failed) > 0 {
 		return fmt.Errorf("unable to fetch logs for: %s", strings.Join(failed, ", "))
@@ -198,16 +201,19 @@ func Output(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg config.Cluster
 }
 
 // outputAudit displays the audit logs.
-func outputAudit(lines int, failed []string) {
+func outputAudit(lines int) error {
 	out.Step(style.Empty, "")
 	out.Step(style.Empty, "==> Audit <==")
-	a, err := audit.Retrieve(lines)
+	r, err := audit.Report(lines)
 	if err != nil {
-		klog.Errorf("failed to get audit logs with error: %v", err)
-		failed = append(failed, "audit")
-		return
+		return fmt.Errorf("failed to create audit report with error: %v", err)
 	}
-	out.Step(style.Empty, a)
+	t, err := r.Table()
+	if err != nil {
+		return fmt.Errorf("failed to create audit table with error: %v", err)
+	}
+	out.Step(style.Empty, t)
+	return nil
 }
 
 // logCommands returns a list of commands that would be run to receive the anticipated logs

From 210accb328f7c3105a9d27cd104511a62b1f3dd3 Mon Sep 17 00:00:00 2001
From: Steven Powell <spowellgrr@gmail.com>
Date: Thu, 4 Feb 2021 12:56:10 -0700
Subject: [PATCH 012/114] renamed lines to logs

---
 pkg/minikube/audit/entry.go  | 14 +++++++-------
 pkg/minikube/audit/report.go |  6 +++---
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/pkg/minikube/audit/entry.go b/pkg/minikube/audit/entry.go
index 1d14ded9a4..7664a65d04 100644
--- a/pkg/minikube/audit/entry.go
+++ b/pkg/minikube/audit/entry.go
@@ -58,10 +58,10 @@ func (e *entry) toFields() []string {
 	return []string{d["command"], d["args"], d["profile"], d["user"], d["startTime"], d["endTime"]}
 }
 
-// linesToFields converts audit lines into arrays of fields.
-func linesToFields(lines []string) ([][]string, error) {
+// logsToFields converts audit logs into arrays of fields.
+func logsToFields(logs []string) ([][]string, error) {
 	c := [][]string{}
-	for _, l := range lines {
+	for _, l := range logs {
 		e := &entry{}
 		if err := json.Unmarshal([]byte(l), e); err != nil {
 			return nil, fmt.Errorf("failed to unmarshal %q: %v", l, err)
@@ -71,11 +71,11 @@ func linesToFields(lines []string) ([][]string, error) {
 	return c, nil
 }
 
-// linesToTable converts audit lines into a formatted table.
-func linesToTable(lines []string, headers []string) (string, error) {
-	f, err := linesToFields(lines)
+// logsToTable converts audit lines into a formatted table.
+func logsToTable(logs []string, headers []string) (string, error) {
+	f, err := logsToFields(logs)
 	if err != nil {
-		return "", fmt.Errorf("failed to convert lines to fields: %v", err)
+		return "", fmt.Errorf("failed to convert logs to fields: %v", err)
 	}
 	b := new(bytes.Buffer)
 	t := tablewriter.NewWriter(b)
diff --git a/pkg/minikube/audit/report.go b/pkg/minikube/audit/report.go
index 1ef770f12d..5b48d6da1e 100644
--- a/pkg/minikube/audit/report.go
+++ b/pkg/minikube/audit/report.go
@@ -55,11 +55,11 @@ func Report(lines int) (*Data, error) {
 	return r, nil
 }
 
-// Table creates a formatted table using last n lines from the report.
+// Table creates a formatted table using last n logs from the report.
 func (r *Data) Table() (string, error) {
-	t, err := linesToTable(r.logs, r.headers)
+	t, err := logsToTable(r.logs, r.headers)
 	if err != nil {
-		return "", fmt.Errorf("failed to convert lines to table: %v", err)
+		return "", fmt.Errorf("failed to convert logs to table: %v", err)
 	}
 	return t, nil
 }

From cc2bf6aa15b4c8bd1691675b3a4b96d1517ef507 Mon Sep 17 00:00:00 2001
From: BLasan <benuraab@gmail.com>
Date: Sun, 7 Feb 2021 01:08:08 +0530
Subject: [PATCH 013/114] Add condition to check --cpus count with available
 cpu count

---
 cmd/minikube/cmd/start.go | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 08b9b0a212..a5db46f97e 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -995,7 +995,7 @@ func validateRequestedMemorySize(req int, drvName string) {
 	}
 }
 
-// validateCPUCount validates the cpu count matches the minimum recommended
+// validateCPUCount validates the cpu count matches the minimum recommended & not exceeding the available cpu count
 func validateCPUCount(drvName string) {
 	var cpuCount int
 	if driver.BareMetal(drvName) {
@@ -1029,6 +1029,10 @@ func validateCPUCount(drvName string) {
 
 	}
 
+	if si.CPUs < cpuCount {
+		exitIfNotForced(reason.RsrcInsufficientCores, "Requested cpu count {{.requested_cpus}} is greater than the available cpus of {{.avail_cpus}}", out.V{"requested_cpus": cpuCount, "avail_cpus": si.CPUs})
+	}
+
 	// looks good
 	if si.CPUs >= 2 {
 		return

From 6564d1d72a32bc5704e96d7994a5f13e4f012e06 Mon Sep 17 00:00:00 2001
From: John Losito <lositojohnj@gmail.com>
Date: Sun, 7 Feb 2021 17:59:40 -0500
Subject: [PATCH 014/114] Allow dependabot to check go dependencies

---
 .github/dependabot.yml | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 .github/dependabot.yml

diff --git a/.github/dependabot.yml b/.github/dependabot.yml
new file mode 100644
index 0000000000..61634d5a45
--- /dev/null
+++ b/.github/dependabot.yml
@@ -0,0 +1,7 @@
+---
+version: 2
+updates:
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: "weekly"

From 948178711b97915d424c65b89f4ec5e9328567fd Mon Sep 17 00:00:00 2001
From: BLasan <benuraab@gmail.com>
Date: Mon, 8 Feb 2021 23:22:48 +0530
Subject: [PATCH 015/114] Add advice message to increase cpus

---
 cmd/minikube/cmd/start.go | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index a5db46f97e..56965e164c 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -1030,6 +1030,18 @@ func validateCPUCount(drvName string) {
 	}
 
 	if si.CPUs < cpuCount {
+
+		if driver.IsDockerDesktop(drvName) {
+			out.Step(style.Empty, `- Ensure your {{.driver_name}} daemon has access to enough CPU/memory resources.`, out.V{"driver_name": drvName})
+			if runtime.GOOS == "darwin" {
+				out.Step(style.Empty, `- Docs https://docs.docker.com/docker-for-mac/#resources`, out.V{"driver_name": drvName})
+			}
+			if runtime.GOOS == "windows" {
+				out.String("\n\t")
+				out.Step(style.Empty, `- Docs https://docs.docker.com/docker-for-windows/#resources`, out.V{"driver_name": drvName})
+			}
+		}
+
 		exitIfNotForced(reason.RsrcInsufficientCores, "Requested cpu count {{.requested_cpus}} is greater than the available cpus of {{.avail_cpus}}", out.V{"requested_cpus": cpuCount, "avail_cpus": si.CPUs})
 	}
 

From 789c5244314cc67a8c4e2a0e6f15823d23d13cad Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Mon, 8 Feb 2021 12:52:28 -0800
Subject: [PATCH 016/114] change to dockerfile to test autobuild

---
 deploy/kicbase/Dockerfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/deploy/kicbase/Dockerfile b/deploy/kicbase/Dockerfile
index 98a7e01018..77ca503aaf 100644
--- a/deploy/kicbase/Dockerfile
+++ b/deploy/kicbase/Dockerfile
@@ -23,6 +23,7 @@ FROM ubuntu:focal-20201106
 
 ARG BUILDKIT_VERSION="v0.8.1"
 
+
 # copy in static files (configs, scripts)
 COPY 10-network-security.conf /etc/sysctl.d/10-network-security.conf
 COPY 11-tcp-mtu-probing.conf /etc/sysctl.d/11-tcp-mtu-probing.conf

From e34354f6ea10e828bd0a16968e31843cc0f938f4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= <anders.f.bjorklund@gmail.com>
Date: Tue, 9 Feb 2021 17:49:00 +0100
Subject: [PATCH 017/114] Update kicbase base image and sync with kind

---
 deploy/kicbase/Dockerfile |  8 +++--
 deploy/kicbase/entrypoint | 72 ++++++++++++++++++++++++++++++++++++---
 2 files changed, 73 insertions(+), 7 deletions(-)

diff --git a/deploy/kicbase/Dockerfile b/deploy/kicbase/Dockerfile
index 98a7e01018..51226fb6c4 100644
--- a/deploy/kicbase/Dockerfile
+++ b/deploy/kicbase/Dockerfile
@@ -19,7 +19,7 @@
 
 # start from ubuntu 20.04, this image is reasonably small as a starting point
 # for a kubernetes node image, it doesn't contain much we don't need
-FROM ubuntu:focal-20201106
+FROM ubuntu:focal-20210119
 
 ARG BUILDKIT_VERSION="v0.8.1"
 
@@ -46,7 +46,7 @@ COPY entrypoint /usr/local/bin/entrypoint
 # - disabling kmsg in journald (these log entries would be confusing)
 #
 # Next we ensure the /etc/kubernetes/manifests directory exists. Normally
-# a kubeadm debain / rpm package would ensure that this exists but we install
+# a kubeadm debian / rpm package would ensure that this exists but we install
 # freshly built binaries directly when we build the node image.
 #
 # Finally we adjust tempfiles cleanup to be 1 minute after "boot" instead of 15m
@@ -74,6 +74,8 @@ RUN echo "Ensuring scripts are executable ..." \
     && mkdir -p /etc/kubernetes/manifests \
  && echo "Adjusting systemd-tmpfiles timer" \
     && sed -i /usr/lib/systemd/system/systemd-tmpfiles-clean.timer -e 's#OnBootSec=.*#OnBootSec=1min#' \
+ && echo "Disabling udev" \
+    && systemctl disable udev.service \
  && echo "Modifying /etc/nsswitch.conf to prefer hosts" \
     && sed -i /etc/nsswitch.conf -re 's#^(hosts:\s*).*#\1dns files#'
 
@@ -87,7 +89,7 @@ STOPSIGNAL SIGRTMIN+3
 ENTRYPOINT [ "/usr/local/bin/entrypoint", "/sbin/init" ]
 
 ARG COMMIT_SHA
-# using base image created by kind https://github.com/kubernetes-sigs/kind/blob/2c0eee40/images/base/Dockerfile
+# using base image created by kind https://github.com/kubernetes-sigs/kind/blob/1da0c5e6/images/base/Dockerfile
 # which is an ubuntu 20.04 with an entry-point that helps running systemd
 # could be changed to any debian that can run systemd
 USER root
diff --git a/deploy/kicbase/entrypoint b/deploy/kicbase/entrypoint
index 0ff55d7802..2ddfc3dcbf 100755
--- a/deploy/kicbase/entrypoint
+++ b/deploy/kicbase/entrypoint
@@ -19,6 +19,13 @@ set -o nounset
 set -o pipefail
 set -x
 
+configure_containerd() {
+  # we need to switch to the 'native' snapshotter on zfs
+  if [[ "$(stat -f -c %T /kind)" == 'zfs' ]]; then
+    sed -i 's/snapshotter = "overlayfs"/snapshotter = "native"/' /etc/containerd/config.toml
+  fi
+}
+
 configure_proxy() {
   # ensure all processes receive the proxy settings by default
   # https://www.freedesktop.org/software/systemd/man/systemd-system.conf.html
@@ -78,8 +85,54 @@ fix_mount() {
   mount --make-rshared /
 }
 
-fix_cgroup_mounts() {
+# helper used by fix_cgroup
+mount_kubelet_cgroup_root() {
+  local cgroup_root=$1
+  local subsystem=$2
+  if [ -z "${cgroup_root}" ]; then
+    return 0
+  fi
+  mkdir -p "${subsystem}/${cgroup_root}"
+  if [ "${subsystem}" == "/sys/fs/cgroup/cpuset" ]; then
+    # This is needed. Otherwise, assigning process to the cgroup
+    # (or any nested cgroup) would result in ENOSPC.
+    cat "${subsystem}/cpuset.cpus" > "${subsystem}/${cgroup_root}/cpuset.cpus"
+    cat "${subsystem}/cpuset.mems" > "${subsystem}/${cgroup_root}/cpuset.mems"
+  fi
+  # We need to perform a self bind mount here because otherwise,
+  # systemd might delete the cgroup unintentionally before the
+  # kubelet starts.
+  mount --bind "${subsystem}/${cgroup_root}" "${subsystem}/${cgroup_root}"
+}
+
+fix_cgroup() {
+  if [[ -f "/sys/fs/cgroup/cgroup.controllers" ]]; then
+    echo 'INFO: detected cgroup v2'
+    # Both Docker and Podman enable CgroupNS on cgroup v2 hosts by default.
+    #
+    # So mostly we do not need to mess around with the cgroup path stuff,
+    # however, we still need to create the "/kubelet" cgroup at least.
+    # (Otherwise kubelet fails with `cgroup-root ["kubelet"] doesn't exist` error, see #1969)
+    #
+    # The "/kubelet" cgroup is created in ExecStartPre of the kubeadm service.
+    #
+    # [FAQ: Why not create "/kubelet" cgroup here?]
+    # We can't create the cgroup with controllers here, because /sys/fs/cgroup/cgroup.subtree_control is empty.
+    # And yet we can't write controllers to /sys/fs/cgroup/cgroup.subtree_control by ourselves either, because
+    # /sys/fs/cgroup/cgroup.procs is not empty at this moment.
+    #
+    # After switching from this entrypoint script to systemd, systemd evacuates the processes in the root
+    # group to "/init.scope" group, so we can write the root subtree_control and create "/kubelet" cgroup.
+    return
+  fi
+  echo 'INFO: detected cgroup v1'
   echo 'INFO: fix cgroup mounts for all subsystems'
+  # see: https://d2iq.com/blog/running-kind-inside-a-kubernetes-cluster-for-continuous-integration
+  # capture initial state before modifying
+  local current_cgroup
+  current_cgroup=$(grep systemd /proc/self/cgroup | cut -d: -f3)
+  local cgroup_subsystems
+  cgroup_subsystems=$(findmnt -lun -o source,target -t cgroup | grep "${current_cgroup}" | awk '{print $2}')
   # For each cgroup subsystem, Docker does a bind mount from the current
   # cgroup to the root of the cgroup subsystem. For instance:
   #   /sys/fs/cgroup/memory/docker/<cid> -> /sys/fs/cgroup/memory
@@ -96,6 +149,7 @@ fix_cgroup_mounts() {
   # This regexp finds all /sys/fs/cgroup mounts that are cgroupfs and mounted somewhere other than / - extracting fields 4+
   # See https://man7.org/linux/man-pages/man5/proc.5.html for field names
 
+  # xref: https://github.com/kubernetes/minikube/pull/9508
   # Example inputs:
   #
   # Docker:               /docker/562a56986a84b3cd38d6a32ac43fdfcc8ad4d2473acf2839cbf549273f35c206 /sys/fs/cgroup/devices rw,nosuid,nodev,noexec,relatime shared:143 master:23 - cgroup devices rw,devices
@@ -120,11 +174,20 @@ fix_cgroup_mounts() {
       fi
     done
   fi
+  # kubelet will try to manage cgroups / pods that are not owned by it when
+  # "nesting" clusters, unless we instruct it to use a different cgroup root.
+  # We do this, and when doing so we must fixup this alternative root
+  # currently this is hardcoded to be /kubelet
+  mount --make-rprivate /sys/fs/cgroup
+  echo "${cgroup_subsystems}" |
+  while IFS= read -r subsystem; do
+    mount_kubelet_cgroup_root "/kubelet" "${subsystem}"
+  done
 }
 
-retryable_fix_cgroup_mounts() {
+retryable_fix_cgroup() {
     for i in $(seq 0 10); do
-      fix_cgroup_mounts && return || echo "fix_cgroup failed with exit code $? (retry $i)"
+      fix_cgroup && return || echo "fix_cgroup failed with exit code $? (retry $i)"
       echo "fix_cgroup diagnostics information below:"
       mount
       sleep 1
@@ -273,10 +336,11 @@ enable_network_magic(){
 
 # run pre-init fixups
 # NOTE: it's important that we do configure* first in this order to avoid races
+configure_containerd
 configure_proxy
 fix_kmsg
 fix_mount
-retryable_fix_cgroup_mounts
+retryable_fix_cgroup
 fix_machine_id
 fix_product_name
 fix_product_uuid

From b71728b27f430122f4309637dfe1d3803f9f2307 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Wed, 10 Feb 2021 12:16:06 -0800
Subject: [PATCH 018/114] fix syntax

---
 pkg/addons/addons.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pkg/addons/addons.go b/pkg/addons/addons.go
index be16d66928..4d137b5840 100644
--- a/pkg/addons/addons.go
+++ b/pkg/addons/addons.go
@@ -157,6 +157,7 @@ Alternatively to use this addon you can use a vm-based driver:
 To track the update on this work in progress feature please check:
 https://github.com/kubernetes/minikube/issues/7332`, out.V{"driver_name": cc.Driver, "os_name": runtime.GOOS, "addon_name": name})
 			}
+		}
 	}
 
 	if strings.HasPrefix(name, "istio") && enable {

From d7beb72025ad6e6f1437f52505a6ce6c0c1401a9 Mon Sep 17 00:00:00 2001
From: Ilya Zuyev <izuyev@google.com>
Date: Wed, 10 Feb 2021 13:47:46 -0800
Subject: [PATCH 019/114] disable hyperkit driver on arm64

---
 cmd/drivers/hyperkit/main.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cmd/drivers/hyperkit/main.go b/cmd/drivers/hyperkit/main.go
index 32788b0c2b..9869932053 100644
--- a/cmd/drivers/hyperkit/main.go
+++ b/cmd/drivers/hyperkit/main.go
@@ -1,4 +1,5 @@
 // +build darwin
+// -build arm64
 
 /*
 Copyright 2016 The Kubernetes Authors All rights reserved.

From e0b9dce3bf0695a4f9227f0b52002c1039ac1792 Mon Sep 17 00:00:00 2001
From: Steven Powell <spowellgrr@gmail.com>
Date: Wed, 10 Feb 2021 17:09:07 -0700
Subject: [PATCH 020/114] Added version to audit logs & logs output and changes
 entry structure

---
 pkg/minikube/audit/audit.go   |  3 +-
 pkg/minikube/audit/entry.go   | 90 ++++++++++++++++++++++++-----------
 pkg/minikube/audit/logFile.go |  4 +-
 pkg/minikube/audit/report.go  | 22 +++++----
 4 files changed, 78 insertions(+), 41 deletions(-)

diff --git a/pkg/minikube/audit/audit.go b/pkg/minikube/audit/audit.go
index 83b0ee98c2..7b7e9d2d95 100644
--- a/pkg/minikube/audit/audit.go
+++ b/pkg/minikube/audit/audit.go
@@ -25,6 +25,7 @@ import (
 	"github.com/spf13/viper"
 	"k8s.io/klog"
 	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/version"
 )
 
 // userName pulls the user flag, if empty gets the os username.
@@ -54,7 +55,7 @@ func Log(startTime time.Time) {
 	if !shouldLog() {
 		return
 	}
-	e := newEntry(os.Args[1], args(), userName(), startTime, time.Now())
+	e := newEntry(os.Args[1], args(), userName(), version.GetVersion(), startTime, time.Now())
 	if err := appendToLog(e); err != nil {
 		klog.Error(err)
 	}
diff --git a/pkg/minikube/audit/entry.go b/pkg/minikube/audit/entry.go
index 7664a65d04..33b44dfb3f 100644
--- a/pkg/minikube/audit/entry.go
+++ b/pkg/minikube/audit/entry.go
@@ -24,66 +24,98 @@ import (
 
 	"github.com/olekukonko/tablewriter"
 	"github.com/spf13/viper"
+	"k8s.io/klog"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
 )
 
-// entry represents the execution of a command.
-type entry struct {
-	Data map[string]string `json:"data"`
+// singleEntry is the log entry of a single command.
+type singleEntry struct {
+	args      string
+	command   string
+	endTime   string
+	profile   string
+	startTime string
+	user      string
+	version   string
+	Data      map[string]string `json:"data"`
 }
 
 // Type returns the cloud events compatible type of this struct.
-func (e *entry) Type() string {
+func (e *singleEntry) Type() string {
 	return "io.k8s.sigs.minikube.audit"
 }
 
+// assignFields converts the map values to their proper fields
+func (e *singleEntry) assignFields() {
+	e.args = e.Data["args"]
+	e.command = e.Data["command"]
+	e.endTime = e.Data["endTime"]
+	e.profile = e.Data["profile"]
+	e.startTime = e.Data["startTime"]
+	e.user = e.Data["user"]
+	e.version = e.Data["version"]
+}
+
+// toMap combines fields into a string map
+func (e *singleEntry) toMap() map[string]string {
+	return map[string]string{
+		"args":      e.args,
+		"command":   e.command,
+		"endTime":   e.endTime,
+		"profile":   e.profile,
+		"startTime": e.startTime,
+		"user":      e.user,
+		"version":   e.version,
+	}
+}
+
 // newEntry returns a new audit type.
-func newEntry(command string, args string, user string, startTime time.Time, endTime time.Time) *entry {
-	return &entry{
-		map[string]string{
-			"args":      args,
-			"command":   command,
-			"endTime":   endTime.Format(constants.TimeFormat),
-			"profile":   viper.GetString(config.ProfileName),
-			"startTime": startTime.Format(constants.TimeFormat),
-			"user":      user,
-		},
+func newEntry(command string, args string, user string, version string, startTime time.Time, endTime time.Time) *singleEntry {
+	return &singleEntry{
+		args:      args,
+		command:   command,
+		endTime:   endTime.Format(constants.TimeFormat),
+		profile:   viper.GetString(config.ProfileName),
+		startTime: startTime.Format(constants.TimeFormat),
+		user:      user,
+		version:   version,
 	}
 }
 
 // toFields converts an entry to an array of fields.
-func (e *entry) toFields() []string {
-	d := e.Data
-	return []string{d["command"], d["args"], d["profile"], d["user"], d["startTime"], d["endTime"]}
+func (e *singleEntry) toFields() []string {
+	return []string{e.command, e.args, e.profile, e.user, e.version, e.startTime, e.endTime}
 }
 
-// logsToFields converts audit logs into arrays of fields.
-func logsToFields(logs []string) ([][]string, error) {
-	c := [][]string{}
+// logsToEntries converts audit logs into arrays of entries.
+func logsToEntries(logs []string) ([]singleEntry, error) {
+	c := []singleEntry{}
 	for _, l := range logs {
-		e := &entry{}
-		if err := json.Unmarshal([]byte(l), e); err != nil {
+		e := singleEntry{}
+		if err := json.Unmarshal([]byte(l), &e); err != nil {
 			return nil, fmt.Errorf("failed to unmarshal %q: %v", l, err)
 		}
-		c = append(c, e.toFields())
+		e.assignFields()
+		c = append(c, e)
 	}
 	return c, nil
 }
 
-// logsToTable converts audit lines into a formatted table.
-func logsToTable(logs []string, headers []string) (string, error) {
-	f, err := logsToFields(logs)
-	if err != nil {
-		return "", fmt.Errorf("failed to convert logs to fields: %v", err)
+// entriesToTable converts audit lines into a formatted table.
+func entriesToTable(entries []singleEntry, headers []string) (string, error) {
+	c := [][]string{}
+	for _, e := range entries {
+		c = append(c, e.toFields())
 	}
+	klog.Info(c)
 	b := new(bytes.Buffer)
 	t := tablewriter.NewWriter(b)
 	t.SetHeader(headers)
 	t.SetAutoFormatHeaders(false)
 	t.SetBorders(tablewriter.Border{Left: true, Top: true, Right: true, Bottom: true})
 	t.SetCenterSeparator("|")
-	t.AppendBulk(f)
+	t.AppendBulk(c)
 	t.Render()
 	return b.String(), nil
 }
diff --git a/pkg/minikube/audit/logFile.go b/pkg/minikube/audit/logFile.go
index 79febd6927..af6e2741a9 100644
--- a/pkg/minikube/audit/logFile.go
+++ b/pkg/minikube/audit/logFile.go
@@ -39,13 +39,13 @@ func setLogFile() error {
 }
 
 // appendToLog appends the audit entry to the log file.
-func appendToLog(entry *entry) error {
+func appendToLog(entry *singleEntry) error {
 	if currentLogFile == nil {
 		if err := setLogFile(); err != nil {
 			return err
 		}
 	}
-	e := register.CloudEvent(entry, entry.Data)
+	e := register.CloudEvent(entry, entry.toMap())
 	bs, err := e.MarshalJSON()
 	if err != nil {
 		return fmt.Errorf("error marshalling event: %v", err)
diff --git a/pkg/minikube/audit/report.go b/pkg/minikube/audit/report.go
index 5b48d6da1e..3b316e86dd 100644
--- a/pkg/minikube/audit/report.go
+++ b/pkg/minikube/audit/report.go
@@ -23,7 +23,7 @@ import (
 
 type Data struct {
 	headers []string
-	logs    []string
+	entries []singleEntry
 }
 
 // Report is created from the log file.
@@ -36,28 +36,32 @@ func Report(lines int) (*Data, error) {
 			return nil, fmt.Errorf("failed to set the log file: %v", err)
 		}
 	}
-	var l []string
+	var logs []string
 	s := bufio.NewScanner(currentLogFile)
 	for s.Scan() {
 		// pop off the earliest line if already at desired log length
-		if len(l) == lines {
-			l = l[1:]
+		if len(logs) == lines {
+			logs = logs[1:]
 		}
-		l = append(l, s.Text())
+		logs = append(logs, s.Text())
 	}
 	if err := s.Err(); err != nil {
 		return nil, fmt.Errorf("failed to read from audit file: %v", err)
 	}
+	e, err := logsToEntries(logs)
+	if err != nil {
+		return nil, fmt.Errorf("failed to convert logs to entries: %v", err)
+	}
 	r := &Data{
-		[]string{"Command", "Args", "Profile", "User", "Start Time", "End Time"},
-		l,
+		[]string{"Command", "Args", "Profile", "User", "Version", "Start Time", "End Time"},
+		e,
 	}
 	return r, nil
 }
 
-// Table creates a formatted table using last n logs from the report.
+// Table creates a formatted table using entries from the report.
 func (r *Data) Table() (string, error) {
-	t, err := logsToTable(r.logs, r.headers)
+	t, err := entriesToTable(r.entries, r.headers)
 	if err != nil {
 		return "", fmt.Errorf("failed to convert logs to table: %v", err)
 	}

From 7536f4436c82dbade982713dcf16d9d0a8ebd143 Mon Sep 17 00:00:00 2001
From: Ilya Zuyev <izuyev@google.com>
Date: Wed, 10 Feb 2021 16:15:28 -0800
Subject: [PATCH 021/114] Fix build constraints

---
 cmd/drivers/hyperkit/main.go | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/cmd/drivers/hyperkit/main.go b/cmd/drivers/hyperkit/main.go
index 9869932053..ca8f90ce6a 100644
--- a/cmd/drivers/hyperkit/main.go
+++ b/cmd/drivers/hyperkit/main.go
@@ -1,5 +1,4 @@
-// +build darwin
-// -build arm64
+// +build darwin,!arm64
 
 /*
 Copyright 2016 The Kubernetes Authors All rights reserved.

From 60bafd00fae4d87c6b3d41ad05cee2e37d87675a Mon Sep 17 00:00:00 2001
From: Ilya Zuyev <izuyev@google.com>
Date: Wed, 10 Feb 2021 16:56:35 -0800
Subject: [PATCH 022/114] fix supported drivers list for darwin/arm64

---
 pkg/minikube/driver/driver_darwin.go | 32 +++++++++++++++++++---------
 1 file changed, 22 insertions(+), 10 deletions(-)

diff --git a/pkg/minikube/driver/driver_darwin.go b/pkg/minikube/driver/driver_darwin.go
index 1eef33635d..168b3084d9 100644
--- a/pkg/minikube/driver/driver_darwin.go
+++ b/pkg/minikube/driver/driver_darwin.go
@@ -16,18 +16,30 @@ limitations under the License.
 
 package driver
 
-import "os/exec"
+import (
+	"os/exec"
+	"runtime"
+)
 
 // supportedDrivers is a list of supported drivers on Darwin.
-var supportedDrivers = []string{
-	VirtualBox,
-	Parallels,
-	VMwareFusion,
-	HyperKit,
-	VMware,
-	Docker,
-	SSH,
-}
+var supportedDrivers []string = func() []string {
+	if runtime.GOARCH == "arm64" {
+		// on darwin/arm64 only docker and ssh are supported yet
+		return []string{
+			Docker,
+			SSH,
+		}
+	}
+	return []string{
+		VirtualBox,
+		Parallels,
+		VMwareFusion,
+		HyperKit,
+		VMware,
+		Docker,
+		SSH,
+	}
+}()
 
 func VBoxManagePath() string {
 	cmd := "VBoxManage"

From af5b44df126bbfd772f9b1b5650b07392e163a74 Mon Sep 17 00:00:00 2001
From: Ilya Zuyev <izuyev@google.com>
Date: Wed, 10 Feb 2021 17:03:53 -0800
Subject: [PATCH 023/114] Fix "The driver {{.driver}} is not supported on
 {{.os}}/{{.arch}}" message

---
 cmd/minikube/cmd/start.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 08b9b0a212..af2d485dd7 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -573,7 +573,7 @@ func selectDriver(existing *config.ClusterConfig) (registry.DriverState, []regis
 		}
 		ds := driver.Status(d)
 		if ds.Name == "" {
-			exit.Message(reason.DrvUnsupportedOS, "The driver '{{.driver}}' is not supported on {{.os}}", out.V{"driver": d, "os": runtime.GOOS})
+			exit.Message(reason.DrvUnsupportedOS, "The driver '{{.driver}}' is not supported on {{.os}}/{{.arch}}", out.V{"driver": d, "os": runtime.GOOS, "arch": runtime.GOARCH})
 		}
 		out.Step(style.Sparkle, `Using the {{.driver}} driver based on user configuration`, out.V{"driver": ds.String()})
 		return ds, nil, true
@@ -583,7 +583,7 @@ func selectDriver(existing *config.ClusterConfig) (registry.DriverState, []regis
 	if d := viper.GetString("vm-driver"); d != "" {
 		ds := driver.Status(viper.GetString("vm-driver"))
 		if ds.Name == "" {
-			exit.Message(reason.DrvUnsupportedOS, "The driver '{{.driver}}' is not supported on {{.os}}", out.V{"driver": d, "os": runtime.GOOS})
+			exit.Message(reason.DrvUnsupportedOS, "The driver '{{.driver}}' is not supported on {{.os}}/{{.arch}}", out.V{"driver": d, "os": runtime.GOOS, "arch": runtime.GOARCH})
 		}
 		out.Step(style.Sparkle, `Using the {{.driver}} driver based on user configuration`, out.V{"driver": ds.String()})
 		return ds, nil, true
@@ -722,7 +722,7 @@ func validateDriver(ds registry.DriverState, existing *config.ClusterConfig) {
 	name := ds.Name
 	klog.Infof("validating driver %q against %+v", name, existing)
 	if !driver.Supported(name) {
-		exit.Message(reason.DrvUnsupportedOS, "The driver '{{.driver}}' is not supported on {{.os}}", out.V{"driver": name, "os": runtime.GOOS})
+		exit.Message(reason.DrvUnsupportedOS, "The driver '{{.driver}}' is not supported on {{.os}}/{{.arch}}", out.V{"driver": d, "os": runtime.GOOS, "arch": runtime.GOARCH}
 	}
 
 	// if we are only downloading artifacts for a driver, we can stop validation here

From 3caf9c0f057ab880faa8d715fcfbf3b30625fc12 Mon Sep 17 00:00:00 2001
From: Ilya Zuyev <izuyev@google.com>
Date: Wed, 10 Feb 2021 17:05:16 -0800
Subject: [PATCH 024/114] Fix "The driver {{.driver}} is not supported on
 {{.os}}/{{.arch}}" message

---
 cmd/minikube/cmd/start.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index af2d485dd7..4271d70d17 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -722,7 +722,7 @@ func validateDriver(ds registry.DriverState, existing *config.ClusterConfig) {
 	name := ds.Name
 	klog.Infof("validating driver %q against %+v", name, existing)
 	if !driver.Supported(name) {
-		exit.Message(reason.DrvUnsupportedOS, "The driver '{{.driver}}' is not supported on {{.os}}/{{.arch}}", out.V{"driver": d, "os": runtime.GOOS, "arch": runtime.GOARCH}
+		exit.Message(reason.DrvUnsupportedOS, "The driver '{{.driver}}' is not supported on {{.os}}/{{.arch}}", out.V{"driver": d, "os": runtime.GOOS, "arch": runtime.GOARCH})
 	}
 
 	// if we are only downloading artifacts for a driver, we can stop validation here

From a99d545a54f6ca25249c58f2e1815aa52123684c Mon Sep 17 00:00:00 2001
From: Ilya Zuyev <izuyev@google.com>
Date: Wed, 10 Feb 2021 17:06:20 -0800
Subject: [PATCH 025/114] Fix "The driver {{.driver}} is not supported on
 {{.os}}/{{.arch}}" message

---
 cmd/minikube/cmd/start.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 4271d70d17..61a6758820 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -722,7 +722,7 @@ func validateDriver(ds registry.DriverState, existing *config.ClusterConfig) {
 	name := ds.Name
 	klog.Infof("validating driver %q against %+v", name, existing)
 	if !driver.Supported(name) {
-		exit.Message(reason.DrvUnsupportedOS, "The driver '{{.driver}}' is not supported on {{.os}}/{{.arch}}", out.V{"driver": d, "os": runtime.GOOS, "arch": runtime.GOARCH})
+		exit.Message(reason.DrvUnsupportedOS, "The driver '{{.driver}}' is not supported on {{.os}}/{{.arch}}", out.V{"driver": name, "os": runtime.GOOS, "arch": runtime.GOARCH})
 	}
 
 	// if we are only downloading artifacts for a driver, we can stop validation here

From 306e7fedc4cb6a28eb73bba3844018e00c4dc943 Mon Sep 17 00:00:00 2001
From: Ilya Zuyev <izuyev@google.com>
Date: Thu, 11 Feb 2021 12:22:27 -0800
Subject: [PATCH 026/114] fix translations for unsupported driver changes

---
 translations/de.json     | 2 +-
 translations/es.json     | 2 +-
 translations/fr.json     | 2 +-
 translations/ja.json     | 2 +-
 translations/ko.json     | 2 +-
 translations/pl.json     | 2 +-
 translations/strings.txt | 2 +-
 translations/zh-CN.json  | 2 +-
 8 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/translations/de.json b/translations/de.json
index 2aaee48299..86d0c28552 100644
--- a/translations/de.json
+++ b/translations/de.json
@@ -510,7 +510,7 @@
 	"The cri socket path to be used.": "",
 	"The docker-env command is incompatible with multi-node clusters. Use the 'registry' add-on: https://minikube.sigs.k8s.io/docs/handbook/registry/": "",
 	"The docker-env command is only compatible with the \"docker\" runtime, but this cluster was configured to use the \"{{.runtime}}\" runtime.": "",
-	"The driver '{{.driver}}' is not supported on {{.os}}": "Der Treiber '{{.driver}}' wird auf {{.os}} nicht unterstützt",
+	"The driver '{{.driver}}' is not supported on {{.os}}/{{.arch}}": "Der Treiber '{{.driver}}' wird auf {{.os}}/{{.arch}} nicht unterstützt",
 	"The existing \"{{.name}}\" cluster was created using the \"{{.old}}\" driver, which is incompatible with requested \"{{.new}}\" driver.": "",
 	"The existing node configuration appears to be corrupt. Run 'minikube delete'": "",
 	"The heapster addon is depreciated. please try to disable metrics-server instead": "",
diff --git a/translations/es.json b/translations/es.json
index 00a20bee7d..eacfaf1c1f 100644
--- a/translations/es.json
+++ b/translations/es.json
@@ -511,7 +511,7 @@
 	"The cri socket path to be used.": "",
 	"The docker-env command is incompatible with multi-node clusters. Use the 'registry' add-on: https://minikube.sigs.k8s.io/docs/handbook/registry/": "",
 	"The docker-env command is only compatible with the \"docker\" runtime, but this cluster was configured to use the \"{{.runtime}}\" runtime.": "",
-	"The driver '{{.driver}}' is not supported on {{.os}}": "El controlador \"{{.driver}}\" no se puede utilizar en {{.os}}",
+	"The driver '{{.driver}}' is not supported on {{.os}}/{{.arch}}": "El controlador \"{{.driver}}\" no se puede utilizar en {{.os}}/{{.arch}}",
 	"The existing \"{{.name}}\" cluster was created using the \"{{.old}}\" driver, which is incompatible with requested \"{{.new}}\" driver.": "",
 	"The existing node configuration appears to be corrupt. Run 'minikube delete'": "",
 	"The heapster addon is depreciated. please try to disable metrics-server instead": "",
diff --git a/translations/fr.json b/translations/fr.json
index 27629f8b05..d5734e4ef1 100644
--- a/translations/fr.json
+++ b/translations/fr.json
@@ -513,7 +513,7 @@
 	"The cri socket path to be used.": "",
 	"The docker-env command is incompatible with multi-node clusters. Use the 'registry' add-on: https://minikube.sigs.k8s.io/docs/handbook/registry/": "",
 	"The docker-env command is only compatible with the \"docker\" runtime, but this cluster was configured to use the \"{{.runtime}}\" runtime.": "",
-	"The driver '{{.driver}}' is not supported on {{.os}}": "Le pilote \"{{.driver}}\" n'est pas compatible avec {{.os}}.",
+	"The driver '{{.driver}}' is not supported on {{.os}}/{{.arch}}": "Le pilote \"{{.driver}}\" n'est pas compatible avec {{.os}}/{{.arch}}.",
 	"The existing \"{{.name}}\" cluster was created using the \"{{.old}}\" driver, which is incompatible with requested \"{{.new}}\" driver.": "",
 	"The existing node configuration appears to be corrupt. Run 'minikube delete'": "",
 	"The heapster addon is depreciated. please try to disable metrics-server instead": "",
diff --git a/translations/ja.json b/translations/ja.json
index 1a13fdd8ff..993c78d726 100644
--- a/translations/ja.json
+++ b/translations/ja.json
@@ -507,7 +507,7 @@
 	"The cri socket path to be used.": "",
 	"The docker-env command is incompatible with multi-node clusters. Use the 'registry' add-on: https://minikube.sigs.k8s.io/docs/handbook/registry/": "",
 	"The docker-env command is only compatible with the \"docker\" runtime, but this cluster was configured to use the \"{{.runtime}}\" runtime.": "",
-	"The driver '{{.driver}}' is not supported on {{.os}}": "ドライバ「{{.driver}}」は、{{.os}} ではサポートされていません",
+	"The driver '{{.driver}}' is not supported on {{.os}}/{{.arch}}": "ドライバ「{{.driver}}」は、{{.os}}/{{.arch}} ではサポートされていません",
 	"The existing \"{{.name}}\" cluster was created using the \"{{.old}}\" driver, which is incompatible with requested \"{{.new}}\" driver.": "",
 	"The existing node configuration appears to be corrupt. Run 'minikube delete'": "",
 	"The heapster addon is depreciated. please try to disable metrics-server instead": "",
diff --git a/translations/ko.json b/translations/ko.json
index 8f3fe07c87..1fb76383b3 100644
--- a/translations/ko.json
+++ b/translations/ko.json
@@ -480,7 +480,7 @@
 	"The cri socket path to be used.": "",
 	"The docker-env command is incompatible with multi-node clusters. Use the 'registry' add-on: https://minikube.sigs.k8s.io/docs/handbook/registry/": "",
 	"The docker-env command is only compatible with the \"docker\" runtime, but this cluster was configured to use the \"{{.runtime}}\" runtime.": "",
-	"The driver '{{.driver}}' is not supported on {{.os}}": "",
+	"The driver '{{.driver}}' is not supported on {{.os}}/{{.arch}}": "",
 	"The existing \"{{.name}}\" cluster was created using the \"{{.old}}\" driver, which is incompatible with requested \"{{.new}}\" driver.": "",
 	"The heapster addon is depreciated. please try to disable metrics-server instead": "",
 	"The hyperv virtual switch name. Defaults to first found. (hyperv driver only)": "",
diff --git a/translations/pl.json b/translations/pl.json
index d52ba3813c..121292080d 100644
--- a/translations/pl.json
+++ b/translations/pl.json
@@ -524,7 +524,7 @@
 	"The docker service is currently not active": "Serwis docker jest nieaktywny",
 	"The docker-env command is incompatible with multi-node clusters. Use the 'registry' add-on: https://minikube.sigs.k8s.io/docs/handbook/registry/": "",
 	"The docker-env command is only compatible with the \"docker\" runtime, but this cluster was configured to use the \"{{.runtime}}\" runtime.": "",
-	"The driver '{{.driver}}' is not supported on {{.os}}": "Sterownik '{{.driver}} jest niewspierany przez system {{.os}}",
+	"The driver '{{.driver}}' is not supported on {{.os}}/{{.arch}}": "Sterownik '{{.driver}} jest niewspierany przez system {{.os}}/{{.arch}}",
 	"The existing \"{{.name}}\" cluster was created using the \"{{.old}}\" driver, which is incompatible with requested \"{{.new}}\" driver.": "",
 	"The existing node configuration appears to be corrupt. Run 'minikube delete'": "",
 	"The heapster addon is depreciated. please try to disable metrics-server instead": "",
diff --git a/translations/strings.txt b/translations/strings.txt
index 1187b0a510..c131b03eca 100644
--- a/translations/strings.txt
+++ b/translations/strings.txt
@@ -426,7 +426,7 @@
 	"The cri socket path to be used.": "",
 	"The docker-env command is incompatible with multi-node clusters. Use the 'registry' add-on: https://minikube.sigs.k8s.io/docs/handbook/registry/": "",
 	"The docker-env command is only compatible with the \"docker\" runtime, but this cluster was configured to use the \"{{.runtime}}\" runtime.": "",
-	"The driver '{{.driver}}' is not supported on {{.os}}": "",
+	"The driver '{{.driver}}' is not supported on {{.os}}/{{.arch}}": "",
 	"The existing \"{{.name}}\" cluster was created using the \"{{.old}}\" driver, which is incompatible with requested \"{{.new}}\" driver.": "",
 	"The heapster addon is depreciated. please try to disable metrics-server instead": "",
 	"The hyperv virtual switch name. Defaults to first found. (hyperv driver only)": "",
diff --git a/translations/zh-CN.json b/translations/zh-CN.json
index 59c8e8787a..353234dc00 100644
--- a/translations/zh-CN.json
+++ b/translations/zh-CN.json
@@ -613,7 +613,7 @@
 	"The cri socket path to be used.": "",
 	"The docker-env command is incompatible with multi-node clusters. Use the 'registry' add-on: https://minikube.sigs.k8s.io/docs/handbook/registry/": "",
 	"The docker-env command is only compatible with the \"docker\" runtime, but this cluster was configured to use the \"{{.runtime}}\" runtime.": "",
-	"The driver '{{.driver}}' is not supported on {{.os}}": "{{.os}} 不支持驱动程序“{{.driver}}”",
+	"The driver '{{.driver}}' is not supported on {{.os}}/{{.arch}}": "{{.os}} 不支持驱动程序“{{.driver}}/{{.arch}}”",
 	"The existing \"{{.name}}\" cluster was created using the \"{{.old}}\" driver, which is incompatible with requested \"{{.new}}\" driver.": "",
 	"The existing node configuration appears to be corrupt. Run 'minikube delete'": "",
 	"The heapster addon is depreciated. please try to disable metrics-server instead": "",

From 3c7d2e0351d22f2cb6f6cb69c65b388f6c0b6c27 Mon Sep 17 00:00:00 2001
From: Predrag Rogic <prezha@users.noreply.github.com>
Date: Tue, 9 Feb 2021 19:51:26 +0000
Subject: [PATCH 027/114] fix WaitForPod by waiting for component Ready instead
 of pod Running status

---
 .../bootstrapper/bsutil/kverify/kverify.go    |   9 ++
 .../bootstrapper/bsutil/kverify/pod_ready.go  | 139 ++++++++++++++++++
 .../bsutil/kverify/system_pods.go             |  37 ++---
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go  |   8 +-
 test/integration/functional_test.go           |  18 ++-
 5 files changed, 175 insertions(+), 36 deletions(-)
 create mode 100644 pkg/minikube/bootstrapper/bsutil/kverify/pod_ready.go

diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
index e12f2d7074..98db0a125a 100644
--- a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
+++ b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
@@ -60,6 +60,15 @@ var (
 		"kube-proxy",
 		"kube-scheduler",
 	}
+	// SystemPodsList is a list of essential pods for running kurnetes to wait for them to be Ready
+	SystemPodsList = []string{
+		"kube-dns", // coredns
+		"etcd",
+		"kube-apiserver",
+		"kube-controller-manager",
+		"kube-proxy",
+		"kube-scheduler",
+	}
 )
 
 // ShouldWait will return true if the config says need to wait
diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/pod_ready.go b/pkg/minikube/bootstrapper/bsutil/kverify/pod_ready.go
new file mode 100644
index 0000000000..2cade76409
--- /dev/null
+++ b/pkg/minikube/bootstrapper/bsutil/kverify/pod_ready.go
@@ -0,0 +1,139 @@
+/*
+Copyright 2021 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package kverify verifies a running Kubernetes cluster is healthy
+package kverify
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+	core "k8s.io/api/core/v1"
+	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/klog/v2"
+	kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+)
+
+// WaitForPodReadyByLabel waits for pod with label ([key:]val) in a namespace to be in Ready condition.
+// If namespace is not provided, it defaults to "kube-system".
+// If label key is not provided, it will try with "component" and "k8s-app".
+func WaitForPodReadyByLabel(cs *kubernetes.Clientset, label, namespace string, timeout time.Duration) error {
+	klog.Infof("waiting %v for pod with %q label in %q namespace to be Ready ...", timeout, label, namespace)
+	start := time.Now()
+	defer func() {
+		klog.Infof("duration metric: took %v to run WaitForPodReadyByLabel for pod with %q label in %q namespace ...", time.Since(start), label, namespace)
+	}()
+
+	if namespace == "" {
+		namespace = "kube-system"
+	}
+
+	lkey := ""
+	lval := ""
+	l := strings.Split(label, ":")
+	switch len(l) {
+	case 1: // treat as no label key provided, just val
+		lval = strings.TrimSpace(l[0])
+	case 2:
+		lkey = strings.TrimSpace(l[0])
+		lval = strings.TrimSpace(l[1])
+	default:
+		return fmt.Errorf("pod label %q is malformed", label)
+	}
+
+	checkReady := func() (bool, error) {
+		if time.Since(start) > timeout {
+			return false, fmt.Errorf("wait for pod with %q label in %q namespace to be Ready timed out", label, namespace)
+		}
+
+		pods, err := cs.CoreV1().Pods(namespace).List(meta.ListOptions{})
+		if err != nil {
+			klog.Infof("error listing pods in %q namespace, will retry: %v", namespace, err)
+			return false, nil
+		}
+		for _, pod := range pods.Items {
+			for k, v := range pod.ObjectMeta.Labels {
+				if ((lkey == "" && (k == "component" || k == "k8s-app")) || lkey == k) && v == lval {
+					return checkPodStatus(&pod)
+				}
+			}
+		}
+		klog.Infof("pod with %q label in %q namespace was not found, will retry", label, namespace)
+		return false, nil
+	}
+
+	if err := wait.PollImmediate(kconst.APICallRetryInterval, kconst.DefaultControlPlaneTimeout, checkReady); err != nil {
+		return errors.Wrapf(err, "wait pod Ready")
+	}
+
+	return nil
+}
+
+// WaitForPodReadyByName waits for pod with name in a namespace to be in Ready condition.
+// If namespace is not provided, it defaults to "kube-system".
+func WaitForPodReadyByName(cs *kubernetes.Clientset, name, namespace string, timeout time.Duration) error {
+	klog.Infof("waiting %v for pod %q in %q namespace to be Ready ...", timeout, name, namespace)
+	start := time.Now()
+	defer func() {
+		klog.Infof("duration metric: took %v to run WaitForPodReadyByName for pod %q in %q namespace ...", time.Since(start), name, namespace)
+	}()
+
+	if namespace == "" {
+		namespace = "kube-system"
+	}
+
+	checkReady := func() (bool, error) {
+		if time.Since(start) > timeout {
+			return false, fmt.Errorf("wait for pod %q in %q namespace to be Ready timed out", name, namespace)
+		}
+
+		pod, err := cs.CoreV1().Pods(namespace).Get(name, meta.GetOptions{})
+		if err != nil {
+			klog.Infof("error getting pod %q in %q namespace, will retry: %v", name, namespace, err)
+			return false, nil
+		}
+		return checkPodStatus(pod)
+	}
+
+	if err := wait.PollImmediate(kconst.APICallRetryInterval, kconst.DefaultControlPlaneTimeout, checkReady); err != nil {
+		return errors.Wrapf(err, "wait pod Ready")
+	}
+
+	return nil
+}
+
+// checkPodStatus returns if pod is Ready and any error occurred.
+func checkPodStatus(pod *core.Pod) (bool, error) {
+	if pod.Status.Phase != core.PodRunning {
+		klog.Infof("pod %q in %q namespace is not Running, will retry: %+v", pod.Name, pod.Namespace, pod.Status)
+		return false, nil
+	}
+	for _, c := range pod.Status.Conditions {
+		if c.Type == core.PodReady {
+			if c.Status != core.ConditionTrue {
+				klog.Infof("pod %q in %q namespace is not Ready, will retry: %+v", pod.Name, pod.Namespace, c)
+				return false, nil
+			}
+			klog.Infof("pod %q in %q namespace is Ready ...", pod.Name, pod.Namespace)
+			return true, nil
+		}
+	}
+	return false, fmt.Errorf("pod %q in %q namespace does not have %q status: %+v", pod.Name, pod.Namespace, core.PodReady, pod.Status)
+}
diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/system_pods.go b/pkg/minikube/bootstrapper/bsutil/kverify/system_pods.go
index 740586b803..56cbabc02c 100644
--- a/pkg/minikube/bootstrapper/bsutil/kverify/system_pods.go
+++ b/pkg/minikube/bootstrapper/bsutil/kverify/system_pods.go
@@ -36,40 +36,25 @@ import (
 	"k8s.io/minikube/pkg/util/retry"
 )
 
-// WaitForSystemPods verifies essential pods for running kurnetes is running
+// WaitForSystemPods verifies essential pods for running kurnetes are Ready
 func WaitForSystemPods(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg config.ClusterConfig, cr command.Runner, client *kubernetes.Clientset, start time.Time, timeout time.Duration) error {
-	klog.Info("waiting for kube-system pods to appear ...")
+	klog.Info("waiting for kube-system pods to be Ready ...")
 	pStart := time.Now()
+	defer func() {
+		klog.Infof("duration metric: took %s for waiting for kube-system pods to be Ready ...", time.Since(pStart))
+	}()
 
-	podList := func() error {
-		if time.Since(start) > minLogCheckTime {
-			announceProblems(r, bs, cfg, cr)
-			time.Sleep(kconst.APICallRetryInterval * 5)
-		}
+	if time.Since(start) > minLogCheckTime {
+		announceProblems(r, bs, cfg, cr)
+		time.Sleep(kconst.APICallRetryInterval * 5)
+	}
 
-		// Wait for any system pod, as waiting for apiserver may block until etcd
-		pods, err := client.CoreV1().Pods("kube-system").List(meta.ListOptions{})
-		if err != nil {
-			klog.Warningf("pod list returned error: %v", err)
+	for _, label := range SystemPodsList {
+		if err := WaitForPodReadyByLabel(client, label, "kube-system", timeout); err != nil {
 			return err
 		}
-
-		klog.Infof("%d kube-system pods found", len(pods.Items))
-		for _, pod := range pods.Items {
-			klog.Infof(podStatusMsg(pod))
-		}
-
-		if len(pods.Items) < 2 {
-			return fmt.Errorf("only %d pod(s) have shown up", len(pods.Items))
-		}
-
-		return nil
 	}
 
-	if err := retry.Local(podList, timeout); err != nil {
-		return fmt.Errorf("apiserver never returned a pod list")
-	}
-	klog.Infof("duration metric: took %s to wait for pod list to return data ...", time.Since(pStart))
 	return nil
 }
 
diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index ad0348aa40..062d09832b 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -477,12 +477,8 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time
 
 	if n.ControlPlane {
 		if cfg.VerifyComponents[kverify.APIServerWaitKey] {
-			if err := kverify.WaitForAPIServerProcess(cr, k, cfg, k.c, start, timeout); err != nil {
-				return errors.Wrap(err, "wait for apiserver proc")
-			}
-
-			if err := kverify.WaitForHealthyAPIServer(cr, k, cfg, k.c, client, start, hostname, port, timeout); err != nil {
-				return errors.Wrap(err, "wait for healthy API server")
+			if err := kverify.WaitForPodReadyByLabel(client, "component: kube-apiserver", "kube-system", timeout); err != nil {
+				return errors.Wrapf(err, "waiting for API server pod to be Ready")
 			}
 		}
 
diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go
index c63c6f94a8..8932911184 100644
--- a/test/integration/functional_test.go
+++ b/test/integration/functional_test.go
@@ -474,12 +474,22 @@ func validateComponentHealth(ctx context.Context, t *testing.T, profile string)
 
 	for _, i := range cs.Items {
 		for _, l := range i.Labels {
-			t.Logf("%s phase: %s", l, i.Status.Phase)
-			_, ok := found[l]
-			if ok {
+			if _, ok := found[l]; ok { // skip irrelevant (eg, repeating/redundant '"tier": "control-plane"') labels
 				found[l] = true
-				if i.Status.Phase != "Running" {
+				t.Logf("%s phase: %s", l, i.Status.Phase)
+				if i.Status.Phase != api.PodRunning {
 					t.Errorf("%s is not Running: %+v", l, i.Status)
+					continue
+				}
+				for _, c := range i.Status.Conditions {
+					if c.Type == api.PodReady {
+						if c.Status != api.ConditionTrue {
+							t.Errorf("%s is not Ready: %+v", l, i.Status)
+						} else {
+							t.Logf("%s status: %s", l, c.Type)
+						}
+						break
+					}
 				}
 			}
 		}

From 93e98dee6852d7aad7dfef8eca425430519ca9ec Mon Sep 17 00:00:00 2001
From: Predrag Rogic <prezha@users.noreply.github.com>
Date: Fri, 12 Feb 2021 00:40:42 +0000
Subject: [PATCH 028/114] wait CorePodsList components to be Ready only if
 --wait=all

---
 .../bootstrapper/bsutil/kverify/kverify.go    | 10 +++--
 .../bootstrapper/bsutil/kverify/pod_ready.go  | 21 +++++++++++
 .../bsutil/kverify/system_pods.go             | 37 +++++++++++++------
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go  | 14 ++++++-
 4 files changed, 65 insertions(+), 17 deletions(-)

diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
index 98db0a125a..f6abca315f 100644
--- a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
+++ b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
@@ -37,6 +37,8 @@ const (
 	NodeReadyKey = "node_ready"
 	// KubeletKey is the name used in the flags for waiting for the kubelet status to be ready
 	KubeletKey = "kubelet"
+	// OperationalKey is the name used for waiting for pods in CorePodsList to be Ready
+	OperationalKey = "operational"
 )
 
 //  vars related to the --wait flag
@@ -44,9 +46,9 @@ var (
 	// DefaultComponents is map of the the default components to wait for
 	DefaultComponents = map[string]bool{APIServerWaitKey: true, SystemPodsWaitKey: true}
 	// NoWaitComponents is map of componets to wait for if specified 'none' or 'false'
-	NoComponents = map[string]bool{APIServerWaitKey: false, SystemPodsWaitKey: false, DefaultSAWaitKey: false, AppsRunningKey: false, NodeReadyKey: false, KubeletKey: false}
+	NoComponents = map[string]bool{APIServerWaitKey: false, SystemPodsWaitKey: false, DefaultSAWaitKey: false, AppsRunningKey: false, NodeReadyKey: false, KubeletKey: false, OperationalKey: false}
 	// AllComponents is map for waiting for all components.
-	AllComponents = map[string]bool{APIServerWaitKey: true, SystemPodsWaitKey: true, DefaultSAWaitKey: true, AppsRunningKey: true, NodeReadyKey: true, KubeletKey: true}
+	AllComponents = map[string]bool{APIServerWaitKey: true, SystemPodsWaitKey: true, DefaultSAWaitKey: true, AppsRunningKey: true, NodeReadyKey: true, KubeletKey: true, OperationalKey: true}
 	// DefaultWaitList is list of all default components to wait for. only names to be used for start flags.
 	DefaultWaitList = []string{APIServerWaitKey, SystemPodsWaitKey}
 	// AllComponentsList list of all valid components keys to wait for. only names to be used used for start flags.
@@ -60,8 +62,8 @@ var (
 		"kube-proxy",
 		"kube-scheduler",
 	}
-	// SystemPodsList is a list of essential pods for running kurnetes to wait for them to be Ready
-	SystemPodsList = []string{
+	// CorePodsList is a list of essential pods for running kurnetes to wait for them to be operational ("Ready")
+	CorePodsList = []string{
 		"kube-dns", // coredns
 		"etcd",
 		"kube-apiserver",
diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/pod_ready.go b/pkg/minikube/bootstrapper/bsutil/kverify/pod_ready.go
index 2cade76409..d9c4c70539 100644
--- a/pkg/minikube/bootstrapper/bsutil/kverify/pod_ready.go
+++ b/pkg/minikube/bootstrapper/bsutil/kverify/pod_ready.go
@@ -31,6 +31,27 @@ import (
 	kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 )
 
+// WaitOperational calls WaitForPodReadyByLabel for each pod in labels list and returns any errors occurred.
+func WaitOperational(cs *kubernetes.Clientset, labels []string, timeout time.Duration) error {
+	klog.Info("waiting for kube-system core pods %s to be Ready ...", labels)
+	pStart := time.Now()
+	defer func() {
+		klog.Infof("duration metric: took %s for waiting for kube-system core pods to be Ready ...", time.Since(pStart))
+	}()
+
+	var errs []string
+	for _, label := range labels {
+		if err := WaitForPodReadyByLabel(cs, label, "kube-system", timeout); err != nil {
+			errs = append(errs, fmt.Sprintf("%q: %q", label, err.Error()))
+		}
+	}
+	if errs != nil {
+		return fmt.Errorf(strings.Join(errs, ", "))
+	}
+
+	return nil
+}
+
 // WaitForPodReadyByLabel waits for pod with label ([key:]val) in a namespace to be in Ready condition.
 // If namespace is not provided, it defaults to "kube-system".
 // If label key is not provided, it will try with "component" and "k8s-app".
diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/system_pods.go b/pkg/minikube/bootstrapper/bsutil/kverify/system_pods.go
index 56cbabc02c..740586b803 100644
--- a/pkg/minikube/bootstrapper/bsutil/kverify/system_pods.go
+++ b/pkg/minikube/bootstrapper/bsutil/kverify/system_pods.go
@@ -36,25 +36,40 @@ import (
 	"k8s.io/minikube/pkg/util/retry"
 )
 
-// WaitForSystemPods verifies essential pods for running kurnetes are Ready
+// WaitForSystemPods verifies essential pods for running kurnetes is running
 func WaitForSystemPods(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg config.ClusterConfig, cr command.Runner, client *kubernetes.Clientset, start time.Time, timeout time.Duration) error {
-	klog.Info("waiting for kube-system pods to be Ready ...")
+	klog.Info("waiting for kube-system pods to appear ...")
 	pStart := time.Now()
-	defer func() {
-		klog.Infof("duration metric: took %s for waiting for kube-system pods to be Ready ...", time.Since(pStart))
-	}()
 
-	if time.Since(start) > minLogCheckTime {
-		announceProblems(r, bs, cfg, cr)
-		time.Sleep(kconst.APICallRetryInterval * 5)
-	}
+	podList := func() error {
+		if time.Since(start) > minLogCheckTime {
+			announceProblems(r, bs, cfg, cr)
+			time.Sleep(kconst.APICallRetryInterval * 5)
+		}
 
-	for _, label := range SystemPodsList {
-		if err := WaitForPodReadyByLabel(client, label, "kube-system", timeout); err != nil {
+		// Wait for any system pod, as waiting for apiserver may block until etcd
+		pods, err := client.CoreV1().Pods("kube-system").List(meta.ListOptions{})
+		if err != nil {
+			klog.Warningf("pod list returned error: %v", err)
 			return err
 		}
+
+		klog.Infof("%d kube-system pods found", len(pods.Items))
+		for _, pod := range pods.Items {
+			klog.Infof(podStatusMsg(pod))
+		}
+
+		if len(pods.Items) < 2 {
+			return fmt.Errorf("only %d pod(s) have shown up", len(pods.Items))
+		}
+
+		return nil
 	}
 
+	if err := retry.Local(podList, timeout); err != nil {
+		return fmt.Errorf("apiserver never returned a pod list")
+	}
+	klog.Infof("duration metric: took %s to wait for pod list to return data ...", time.Since(pStart))
 	return nil
 }
 
diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index 062d09832b..6efdd1dbd7 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -477,8 +477,12 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time
 
 	if n.ControlPlane {
 		if cfg.VerifyComponents[kverify.APIServerWaitKey] {
-			if err := kverify.WaitForPodReadyByLabel(client, "component: kube-apiserver", "kube-system", timeout); err != nil {
-				return errors.Wrapf(err, "waiting for API server pod to be Ready")
+			if err := kverify.WaitForAPIServerProcess(cr, k, cfg, k.c, start, timeout); err != nil {
+				return errors.Wrap(err, "wait for apiserver proc")
+			}
+
+			if err := kverify.WaitForHealthyAPIServer(cr, k, cfg, k.c, client, start, hostname, port, timeout); err != nil {
+				return errors.Wrap(err, "wait for healthy API server")
 			}
 		}
 
@@ -499,6 +503,12 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time
 				return errors.Wrap(err, "waiting for apps_running")
 			}
 		}
+
+		if cfg.VerifyComponents[kverify.OperationalKey] {
+			if err := kverify.WaitOperational(client, kverify.CorePodsList, timeout); err != nil {
+				return errors.Wrap(err, "waiting for operational status")
+			}
+		}
 	}
 	if cfg.VerifyComponents[kverify.KubeletKey] {
 		if err := kverify.WaitForService(k.c, "kubelet", timeout); err != nil {

From ec97f1db4b059406c3b2e66525cc7252183958f2 Mon Sep 17 00:00:00 2001
From: Predrag Rogic <prezha@users.noreply.github.com>
Date: Fri, 12 Feb 2021 02:13:06 +0000
Subject: [PATCH 029/114] fix integration.validateExtraConfig test adding
 --wait=all

---
 test/integration/functional_test.go | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go
index 8932911184..0f90a5cbc7 100644
--- a/test/integration/functional_test.go
+++ b/test/integration/functional_test.go
@@ -256,7 +256,6 @@ func validateDockerEnv(ctx context.Context, t *testing.T, profile string) {
 	if !strings.Contains(rr.Output(), expectedImgInside) {
 		t.Fatalf("expected 'docker images' to have %q inside minikube. but the output is: *%s*", expectedImgInside, rr.Output())
 	}
-
 }
 
 func validateStartWithProxy(ctx context.Context, t *testing.T, profile string) {
@@ -269,7 +268,7 @@ func validateStartWithProxy(ctx context.Context, t *testing.T, profile string) {
 
 	// Use more memory so that we may reliably fit MySQL and nginx
 	// changing api server so later in soft start we verify it didn't change
-	startArgs := append([]string{"start", "-p", profile, "--memory=4000", fmt.Sprintf("--apiserver-port=%d", apiPortTest), "--wait=true"}, StartArgs()...)
+	startArgs := append([]string{"start", "-p", profile, "--memory=4000", fmt.Sprintf("--apiserver-port=%d", apiPortTest), "--wait=all"}, StartArgs()...)
 	c := exec.CommandContext(ctx, Target(), startArgs...)
 	env := os.Environ()
 	env = append(env, fmt.Sprintf("HTTP_PROXY=%s", srv.Addr))
@@ -401,7 +400,6 @@ func validateMinikubeKubectlDirectCall(ctx context.Context, t *testing.T, profil
 	if err != nil {
 		t.Fatalf("failed to run kubectl directly. args %q: %v", rr.Command(), err)
 	}
-
 }
 
 func validateExtraConfig(ctx context.Context, t *testing.T, profile string) {
@@ -409,7 +407,7 @@ func validateExtraConfig(ctx context.Context, t *testing.T, profile string) {
 
 	start := time.Now()
 	// The tests before this already created a profile, starting minikube with different --extra-config cmdline option.
-	startArgs := []string{"start", "-p", profile, "--extra-config=apiserver.enable-admission-plugins=NamespaceAutoProvision"}
+	startArgs := []string{"start", "-p", profile, "--extra-config=apiserver.enable-admission-plugins=NamespaceAutoProvision", "--wait=all"}
 	c := exec.CommandContext(ctx, Target(), startArgs...)
 	rr, err := Run(t, c)
 	if err != nil {
@@ -427,7 +425,6 @@ func validateExtraConfig(ctx context.Context, t *testing.T, profile string) {
 	if !strings.Contains(afterCfg.Config.KubernetesConfig.ExtraOptions.String(), expectedExtraOptions) {
 		t.Errorf("expected ExtraOptions to contain %s but got %s", expectedExtraOptions, afterCfg.Config.KubernetesConfig.ExtraOptions.String())
 	}
-
 }
 
 // imageID returns a docker image id for image `image` and current architecture

From 88f60b94f457ece436f139e630d673d9fe1ce0fd Mon Sep 17 00:00:00 2001
From: Kent Iso <kentaiso@google.com>
Date: Sat, 13 Feb 2021 00:14:37 +0900
Subject: [PATCH 030/114] Fix TestCheckDockerVersion Assertion

---
 pkg/minikube/registry/drvs/docker/docker_test.go | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/pkg/minikube/registry/drvs/docker/docker_test.go b/pkg/minikube/registry/drvs/docker/docker_test.go
index e8b228e993..f1e5d05ce7 100644
--- a/pkg/minikube/registry/drvs/docker/docker_test.go
+++ b/pkg/minikube/registry/drvs/docker/docker_test.go
@@ -86,8 +86,10 @@ func TestCheckDockerVersion(t *testing.T) {
 	for _, c := range tc {
 		t.Run("checkDockerVersion test", func(t *testing.T) {
 			s := checkDockerVersion(c.version)
-			if c.expect != s.Reason {
-				t.Errorf("Error %v expected. but got %q. (version string : %s)", c.expect, s.Reason, c.version)
+			if s.Error != nil {
+				if c.expect != s.Reason {
+					t.Errorf("Error %v expected. but got %q. (version string : %s)", c.expect, s.Reason, c.version)
+				}
 			}
 		})
 	}

From dbafdd984e384b197edfe4505ef493bb40d6a6d9 Mon Sep 17 00:00:00 2001
From: Steven Powell <spowellgrr@gmail.com>
Date: Fri, 12 Feb 2021 12:56:15 -0700
Subject: [PATCH 031/114] Made profile optional param and added more unit tests

---
 pkg/minikube/audit/entry.go        | 12 ++--
 pkg/minikube/audit/entry_test.go   | 91 +++++++++++++++++++++++-------
 pkg/minikube/audit/logFile_test.go |  2 +-
 pkg/minikube/audit/report.go       |  8 +--
 pkg/minikube/audit/report_test.go  | 10 +---
 pkg/minikube/logs/logs.go          |  6 +-
 6 files changed, 85 insertions(+), 44 deletions(-)

diff --git a/pkg/minikube/audit/entry.go b/pkg/minikube/audit/entry.go
index 33b44dfb3f..58503be55c 100644
--- a/pkg/minikube/audit/entry.go
+++ b/pkg/minikube/audit/entry.go
@@ -71,12 +71,16 @@ func (e *singleEntry) toMap() map[string]string {
 }
 
 // newEntry returns a new audit type.
-func newEntry(command string, args string, user string, version string, startTime time.Time, endTime time.Time) *singleEntry {
+func newEntry(command string, args string, user string, version string, startTime time.Time, endTime time.Time, profile ...string) *singleEntry {
+	p := viper.GetString(config.ProfileName)
+	if len(profile) > 0 {
+		p = profile[0]
+	}
 	return &singleEntry{
 		args:      args,
 		command:   command,
 		endTime:   endTime.Format(constants.TimeFormat),
-		profile:   viper.GetString(config.ProfileName),
+		profile:   p,
 		startTime: startTime.Format(constants.TimeFormat),
 		user:      user,
 		version:   version,
@@ -103,7 +107,7 @@ func logsToEntries(logs []string) ([]singleEntry, error) {
 }
 
 // entriesToTable converts audit lines into a formatted table.
-func entriesToTable(entries []singleEntry, headers []string) (string, error) {
+func entriesToTable(entries []singleEntry, headers []string) string {
 	c := [][]string{}
 	for _, e := range entries {
 		c = append(c, e.toFields())
@@ -117,5 +121,5 @@ func entriesToTable(entries []singleEntry, headers []string) (string, error) {
 	t.SetCenterSeparator("|")
 	t.AppendBulk(c)
 	t.Render()
-	return b.String(), nil
+	return b.String()
 }
diff --git a/pkg/minikube/audit/entry_test.go b/pkg/minikube/audit/entry_test.go
index 1d7867014a..c8d8b6c562 100644
--- a/pkg/minikube/audit/entry_test.go
+++ b/pkg/minikube/audit/entry_test.go
@@ -17,12 +17,12 @@ limitations under the License.
 package audit
 
 import (
+	"encoding/json"
+	"fmt"
 	"strings"
 	"testing"
 	"time"
 
-	"github.com/spf13/viper"
-	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
 )
 
@@ -31,37 +31,32 @@ func TestEntry(t *testing.T) {
 	a := "--alsologtostderr"
 	p := "profile1"
 	u := "user1"
+	v := "v0.17.1"
 	st := time.Now()
 	stFormatted := st.Format(constants.TimeFormat)
 	et := time.Now()
 	etFormatted := et.Format(constants.TimeFormat)
 
-	// save current profile in case something depends on it
-	oldProfile := viper.GetString(config.ProfileName)
-	viper.Set(config.ProfileName, p)
-	e := newEntry(c, a, u, st, et)
-	viper.Set(config.ProfileName, oldProfile)
+	e := newEntry(c, a, u, v, st, et, p)
 
 	t.Run("TestNewEntry", func(t *testing.T) {
-		d := e.Data
-
 		tests := []struct {
 			key  string
+			got  string
 			want string
 		}{
-			{"command", c},
-			{"args", a},
-			{"profile", p},
-			{"user", u},
-			{"startTime", stFormatted},
-			{"endTime", etFormatted},
+			{"command", e.command, c},
+			{"args", e.args, a},
+			{"profile", e.profile, p},
+			{"user", e.user, u},
+			{"version", e.version, v},
+			{"startTime", e.startTime, stFormatted},
+			{"endTime", e.endTime, etFormatted},
 		}
 
 		for _, tt := range tests {
-			got := d[tt.key]
-
-			if got != tt.want {
-				t.Errorf("Data[%q] = %s; want %s", tt.key, got, tt.want)
+			if tt.got != tt.want {
+				t.Errorf("singleEntry.%s = %s; want %s", tt.key, tt.got, tt.want)
 			}
 		}
 	})
@@ -75,14 +70,70 @@ func TestEntry(t *testing.T) {
 		}
 	})
 
+	t.Run("TestToMap", func(t *testing.T) {
+		m := e.toMap()
+
+		tests := []struct {
+			key  string
+			want string
+		}{
+			{"command", c},
+			{"args", a},
+			{"profile", p},
+			{"user", u},
+			{"version", v},
+			{"startTime", stFormatted},
+			{"endTime", etFormatted},
+		}
+
+		for _, tt := range tests {
+			got := m[tt.key]
+			if got != tt.want {
+				t.Errorf("map[%s] = %s; want %s", tt.key, got, tt.want)
+			}
+		}
+	})
+
 	t.Run("TestToField", func(t *testing.T) {
 		got := e.toFields()
 		gotString := strings.Join(got, ",")
-		want := []string{c, a, p, u, stFormatted, etFormatted}
+		want := []string{c, a, p, u, v, stFormatted, etFormatted}
 		wantString := strings.Join(want, ",")
 
 		if gotString != wantString {
 			t.Errorf("toFields() = %s; want %s", gotString, wantString)
 		}
 	})
+
+	t.Run("TestAssignFields", func(t *testing.T) {
+		l := fmt.Sprintf(`{"data":{"args":"%s","command":"%s","endTime":"%s","profile":"%s","startTime":"%s","user":"%s","version":"v0.17.1"},"datacontenttype":"application/json","id":"bc6ec9d4-0d08-4b57-ac3b-db8d67774768","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.audit"}`, a, c, etFormatted, p, stFormatted, u)
+
+		e := &singleEntry{}
+		if err := json.Unmarshal([]byte(l), e); err != nil {
+			t.Fatalf("failed to unmarshal log:: %v", err)
+		}
+
+		e.assignFields()
+
+		tests := []struct {
+			key  string
+			got  string
+			want string
+		}{
+			{"command", e.command, c},
+			{"args", e.args, a},
+			{"profile", e.profile, p},
+			{"user", e.user, u},
+			{"version", e.version, v},
+			{"startTime", e.startTime, stFormatted},
+			{"endTime", e.endTime, etFormatted},
+		}
+
+		for _, tt := range tests {
+			if tt.got != tt.want {
+				t.Errorf("singleEntry.%s = %s; want %s", tt.key, tt.got, tt.want)
+
+			}
+		}
+	})
 }
diff --git a/pkg/minikube/audit/logFile_test.go b/pkg/minikube/audit/logFile_test.go
index ccea362767..90c8faba4f 100644
--- a/pkg/minikube/audit/logFile_test.go
+++ b/pkg/minikube/audit/logFile_test.go
@@ -42,7 +42,7 @@ func TestLogFile(t *testing.T) {
 		defer func() { currentLogFile = &oldLogFile }()
 		currentLogFile = f
 
-		e := newEntry("start", "-v", "user1", time.Now(), time.Now())
+		e := newEntry("start", "-v", "user1", "v0.17.1", time.Now(), time.Now())
 		if err := appendToLog(e); err != nil {
 			t.Fatalf("Error appendingToLog: %v", err)
 		}
diff --git a/pkg/minikube/audit/report.go b/pkg/minikube/audit/report.go
index 3b316e86dd..c4e8a211e9 100644
--- a/pkg/minikube/audit/report.go
+++ b/pkg/minikube/audit/report.go
@@ -60,10 +60,6 @@ func Report(lines int) (*Data, error) {
 }
 
 // Table creates a formatted table using entries from the report.
-func (r *Data) Table() (string, error) {
-	t, err := entriesToTable(r.entries, r.headers)
-	if err != nil {
-		return "", fmt.Errorf("failed to convert logs to table: %v", err)
-	}
-	return t, nil
+func (r *Data) Table() string {
+	return entriesToTable(r.entries, r.headers)
 }
diff --git a/pkg/minikube/audit/report_test.go b/pkg/minikube/audit/report_test.go
index 4dd727ecdd..a6547b0d2b 100644
--- a/pkg/minikube/audit/report_test.go
+++ b/pkg/minikube/audit/report_test.go
@@ -50,13 +50,7 @@ func TestAuditReport(t *testing.T) {
 		t.Fatalf("failed to create report: %v", err)
 	}
 
-	if len(r.logs) != wantedLines {
-		t.Fatalf("report has %d lines of logs, want %d", len(r.logs), wantedLines)
+	if len(r.entries) != wantedLines {
+		t.Fatalf("report has %d lines of logs, want %d", len(r.entries), wantedLines)
 	}
-
-	t.Run("TestTable", func(t *testing.T) {
-		if _, err := r.Table(); err != nil {
-			t.Errorf("failed to create table from report: %v", err)
-		}
-	})
 }
diff --git a/pkg/minikube/logs/logs.go b/pkg/minikube/logs/logs.go
index 4fcac53b7e..568f26ee7a 100644
--- a/pkg/minikube/logs/logs.go
+++ b/pkg/minikube/logs/logs.go
@@ -208,11 +208,7 @@ func outputAudit(lines int) error {
 	if err != nil {
 		return fmt.Errorf("failed to create audit report with error: %v", err)
 	}
-	t, err := r.Table()
-	if err != nil {
-		return fmt.Errorf("failed to create audit table with error: %v", err)
-	}
-	out.Step(style.Empty, t)
+	out.Step(style.Empty, r.Table())
 	return nil
 }
 

From 4001c0a26744ccb855d14a889592f2353e4ec0bb Mon Sep 17 00:00:00 2001
From: Ilya Zuyev <izuyev@google.com>
Date: Fri, 12 Feb 2021 15:07:11 -0800
Subject: [PATCH 032/114] Fix TestErrorJSONOutput test

---
 test/integration/json_output_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/integration/json_output_test.go b/test/integration/json_output_test.go
index e91147f231..f120f64279 100644
--- a/test/integration/json_output_test.go
+++ b/test/integration/json_output_test.go
@@ -155,7 +155,7 @@ func TestErrorJSONOutput(t *testing.T) {
 		t.Fatalf("last cloud event is not of type error: %v", last)
 	}
 	last.validateData(t, "exitcode", fmt.Sprintf("%v", reason.ExDriverUnsupported))
-	last.validateData(t, "message", fmt.Sprintf("The driver 'fail' is not supported on %s", runtime.GOOS))
+	last.validateData(t, "message", fmt.Sprintf("The driver 'fail' is not supported on %s/%s", runtime.GOOS, runtime.GOARCH))
 }
 
 type cloudEvent struct {

From 1888b5a1b225c5d4a14804b1a4f53dba5046f4a1 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Fri, 12 Feb 2021 15:11:26 -0800
Subject: [PATCH 033/114] do not send info to stderr

---
 pkg/minikube/assets/addons.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go
index 4b4db916ab..6a971fbfbc 100644
--- a/pkg/minikube/assets/addons.go
+++ b/pkg/minikube/assets/addons.go
@@ -691,17 +691,17 @@ func GenerateTemplateData(addon *Addon, cfg config.KubernetesConfig) interface{}
 
 		// Send messages to stderr due to some tests rely on stdout
 		if override, ok := opts.CustomRegistries[name]; ok {
-			out.ErrT(style.Option, "Using image {{.registry}}{{.image}}", out.V{
+			out.Step(style.Option, "Using image {{.registry}}{{.image}}", out.V{
 				"registry": override,
 				"image":    image,
 			})
 		} else if opts.ImageRepository != "" {
-			out.ErrT(style.Option, "Using image {{.registry}}{{.image}} (global image repository)", out.V{
+			out.Step(style.Option, "Using image {{.registry}}{{.image}} (global image repository)", out.V{
 				"registry": opts.ImageRepository,
 				"image":    image,
 			})
 		} else {
-			out.ErrT(style.Option, "Using image {{.registry}}{{.image}}", out.V{
+			out.Step(style.Option, "Using image {{.registry}}{{.image}}", out.V{
 				"registry": opts.Registries[name],
 				"image":    image,
 			})

From 50138e6bbe17331d30874fd0235f9396c76eacea Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Fri, 12 Feb 2021 15:16:49 -0800
Subject: [PATCH 034/114] remove stale comment

---
 pkg/minikube/assets/addons.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go
index 6a971fbfbc..41050358ab 100644
--- a/pkg/minikube/assets/addons.go
+++ b/pkg/minikube/assets/addons.go
@@ -689,7 +689,6 @@ func GenerateTemplateData(addon *Addon, cfg config.KubernetesConfig) interface{}
 			opts.Registries[name] = "" // Avoid nil access when rendering
 		}
 
-		// Send messages to stderr due to some tests rely on stdout
 		if override, ok := opts.CustomRegistries[name]; ok {
 			out.Step(style.Option, "Using image {{.registry}}{{.image}}", out.V{
 				"registry": override,

From 400456b3003c762a50ab2de7c40d89953340d2a1 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Fri, 12 Feb 2021 17:09:46 -0800
Subject: [PATCH 035/114] no need for bash

---
 .github/workflows/kic_image.yml | 121 --------------------------------
 deploy/kicbase/Dockerfile       |   3 +-
 2 files changed, 1 insertion(+), 123 deletions(-)
 delete mode 100644 .github/workflows/kic_image.yml

diff --git a/.github/workflows/kic_image.yml b/.github/workflows/kic_image.yml
deleted file mode 100644
index 85028af43f..0000000000
--- a/.github/workflows/kic_image.yml
+++ /dev/null
@@ -1,121 +0,0 @@
-name: KIC_IMAGE
-on:
-  pull_request:
-    paths:
-      - "deploy/kicbase/**"
-env:
-  GOPROXY: https://proxy.golang.org
-jobs:
-  build_test_kic_image:
-    runs-on: [self-hosted, debian9, baremetal, equinix]
-    steps:
-      - name: Clean up 
-        shell: bash
-        run: |
-          pwd
-          ls -lah
-          rm -rf out
-          ls -lah
-          df -h
-          sudo rm -f /etc/cron.hourly/cleanup_and_reboot || true
-      - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
-        with:
-          go-version: '1.15.5'
-          stable: true
-      - name: Download Dependencies
-        run: go mod download
-      - name: Build Binaries
-        run: |
-          sudo apt-get update	
-          sudo apt-get install -y make build-essential
-          make linux
-          make e2e-linux-amd64
-          cp -r test/integration/testdata ./out
-          whoami
-          echo github ref $GITHUB_REF
-          echo workflow $GITHUB_WORKFLOW
-          echo home $HOME
-          echo event name $GITHUB_EVENT_NAME
-          echo workspace $GITHUB_WORKSPACE
-          echo "end of debug stuff"
-          echo $(which jq)
-      - name: Build Image
-        run: |          
-          docker images
-          make kic-base-image
-          docker images
-      - name: Info
-        shell: bash
-        run: |
-          hostname
-          uname -r
-          lsb_release -a
-      - name: Install kubectl
-        shell: bash
-        run: |
-          curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/amd64/kubectl
-          sudo install kubectl /usr/local/bin/kubectl
-          kubectl version --client=true
-      - name: Install gopogh
-        shell: bash
-        run: |
-          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-linux-amd64
-          sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
-          sudo apt-get install -y jq
-          rm -f gopogh-linux-amd64 || true
-      - name: Run Integration Test
-        continue-on-error: false
-        # bash {0} to allow test to continue to next step. in case of
-        shell: bash {0}
-        run: |
-          KIC_VERSION=$(egrep "Version =" pkg/drivers/kic/types.go | cut -d \" -f2)
-          KIC_IMG_HEAD="local/kicbase:${KIC_VERSION}-snapshot"
-          cd out
-          mkdir -p report
-          mkdir -p testhome
-          chmod a+x e2e-*
-          chmod a+x minikube-*
-          START_TIME=$(date -u +%s)
-          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args="--vm-driver=docker   --base-image=${KIC_IMG_HEAD}"  -test.v -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
-          END_TIME=$(date -u +%s)
-          TIME_ELAPSED=$(($END_TIME-$START_TIME))
-          min=$((${TIME_ELAPSED}/60))
-          sec=$((${TIME_ELAPSED}%60))
-          TIME_ELAPSED="${min} min $sec seconds "
-          echo "TIME_ELAPSED=${TIME_ELAPSED}" >> $GITHUB_ENV
-      - name: Generate HTML Report
-        shell: bash
-        run: |
-          cd out
-          export PATH=${PATH}:`go env GOPATH`/bin
-          go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
-          STAT=$(/usr/local/bin/gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
-          echo status: ${STAT}
-          FailNum=$(echo $STAT | jq '.NumberOfFail' || true)
-          TestsNum=$(echo $STAT | jq '.NumberOfTests' || true)
-          GOPOGH_RESULT="${JOB_NAME} : completed with ${FailNum} / ${TestsNum} failures in ${TIME_ELAPSED}"
-          echo "GOPOGH_RESULT=${GOPOGH_RESULT}" >> $GITHUB_ENV
-          echo 'STAT<<EOF' >> $GITHUB_ENV
-          echo "${STAT}" >> $GITHUB_ENV
-          echo 'EOF' >> $GITHUB_ENV
-      - uses: actions/upload-artifact@v1
-        with:
-          name: kic_image_functional_test_docker_ubuntu
-          path: out/report
-      - name: The End Result build_test_kic_image_docker_ubuntu
-        shell: bash
-        run: |
-          echo ${GOPOGH_RESULT}
-          numFail=$(echo $STAT | jq '.NumberOfFail')
-          numPass=$(echo $STAT | jq '.NumberOfPass')
-          echo "*******************${numPass} Passes :) *******************"
-          echo $STAT | jq '.PassedTests' || true
-          echo "*******************************************************"
-          echo "---------------- ${numFail} Failures :( ----------------------------"
-          echo $STAT | jq '.FailedTests' || true
-          echo "-------------------------------------------------------"
-          numPass=$(echo $STAT | jq '.NumberOfPass')
-          if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
-          if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
-          if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
diff --git a/deploy/kicbase/Dockerfile b/deploy/kicbase/Dockerfile
index 44e6261e4e..1d1d6403ea 100644
--- a/deploy/kicbase/Dockerfile
+++ b/deploy/kicbase/Dockerfile
@@ -23,7 +23,6 @@ FROM ubuntu:focal-20201106
 
 ARG BUILDKIT_VERSION="v0.8.1"
 
-
 # copy in static files (configs, scripts)
 COPY 10-network-security.conf /etc/sysctl.d/10-network-security.conf
 COPY 11-tcp-mtu-probing.conf /etc/sysctl.d/11-tcp-mtu-probing.conf
@@ -56,7 +55,7 @@ COPY entrypoint /usr/local/bin/entrypoint
 RUN echo "Ensuring scripts are executable ..." \
     && chmod +x /usr/local/bin/clean-install /usr/local/bin/entrypoint \
  && echo "Installing Packages ..." \
-    && DEBIAN_FRONTEND=noninteractive /bin/bash clean-install \
+    && DEBIAN_FRONTEND=noninteractive clean-install \
       systemd \
       conntrack iptables iproute2 ethtool socat util-linux mount ebtables udev kmod \
       libseccomp2 pigz \

From 3406be89f88f31f6a9e4e49a1b4cbf7fe4a41d2c Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Fri, 12 Feb 2021 17:18:40 -0800
Subject: [PATCH 036/114] fix final line of dockerfile

---
 deploy/kicbase/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deploy/kicbase/Dockerfile b/deploy/kicbase/Dockerfile
index 1d1d6403ea..788f47a971 100644
--- a/deploy/kicbase/Dockerfile
+++ b/deploy/kicbase/Dockerfile
@@ -187,5 +187,5 @@ RUN mkdir -p /kind
 RUN rm -rf \
   /usr/share/doc/* \
   /usr/share/man/* \
-  /usr/share/local/* \
+  /usr/share/local/* 
 RUN echo "kic! Build: ${COMMIT_SHA} Time :$(date)" > "/kic.txt"

From eb079e66bb508dfd2739e5adf882a49ed05ee669 Mon Sep 17 00:00:00 2001
From: ashwanth1109 <ashwanth1109@gmail.com>
Date: Sat, 13 Feb 2021 10:03:12 +0000
Subject: [PATCH 037/114] Accessing apps documentation follows convention for
 dynamic values

---
 site/content/en/docs/handbook/accessing.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/site/content/en/docs/handbook/accessing.md b/site/content/en/docs/handbook/accessing.md
index cb50e6f8a7..09a77f9a2c 100644
--- a/site/content/en/docs/handbook/accessing.md
+++ b/site/content/en/docs/handbook/accessing.md
@@ -25,7 +25,7 @@ A NodePort service is the most basic way to get external traffic directly to you
 We also have a shortcut for fetching the minikube IP and a service's `NodePort`:
 
 ```shell
-minikube service --url $SERVICE
+minikube service --url <service-name>
 ```
 
 ## Getting the NodePort using kubectl
@@ -35,7 +35,7 @@ The minikube VM is exposed to the host system via a host-only IP address, that c
 To determine the NodePort for your service, you can use a `kubectl` command like this (note that `nodePort` begins with lowercase `n` in JSON output):
 
 ```shell
-kubectl get service $SERVICE --output='jsonpath="{.spec.ports[0].nodePort}"'
+kubectl get service <service-name> --output='jsonpath="{.spec.ports[0].nodePort}"'
 ```
 
 ### Increasing the NodePort range

From 2c87f6e1ad11f91fdc51b3c642d9871287bb698b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= <anders.f.bjorklund@gmail.com>
Date: Sat, 13 Feb 2021 13:35:08 +0100
Subject: [PATCH 038/114] Make sure to show debian warning also for cgroup 2

Move check to function, to cut down on complexity
---
 pkg/drivers/kic/oci/cgroups_linux.go | 34 ++++++++++++++++++++++++++++
 pkg/drivers/kic/oci/cgroups_other.go | 30 ++++++++++++++++++++++++
 pkg/drivers/kic/oci/oci.go           | 27 +++++++++++++++-------
 3 files changed, 83 insertions(+), 8 deletions(-)
 create mode 100644 pkg/drivers/kic/oci/cgroups_linux.go
 create mode 100644 pkg/drivers/kic/oci/cgroups_other.go

diff --git a/pkg/drivers/kic/oci/cgroups_linux.go b/pkg/drivers/kic/oci/cgroups_linux.go
new file mode 100644
index 0000000000..c56251fa18
--- /dev/null
+++ b/pkg/drivers/kic/oci/cgroups_linux.go
@@ -0,0 +1,34 @@
+// +build linux
+
+/*
+Copyright 2021 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package oci
+
+import (
+	"syscall"
+
+	"golang.org/x/sys/unix"
+)
+
+// IsCgroup2UnifiedMode returns whether we are running in cgroup 2 cgroup2 mode.
+func IsCgroup2UnifiedMode() (bool, error) {
+	var st syscall.Statfs_t
+	if err := syscall.Statfs("/sys/fs/cgroup", &st); err != nil {
+		return false, err
+	}
+	return st.Type == unix.CGROUP2_SUPER_MAGIC, nil
+}
diff --git a/pkg/drivers/kic/oci/cgroups_other.go b/pkg/drivers/kic/oci/cgroups_other.go
new file mode 100644
index 0000000000..7d91d8d2c3
--- /dev/null
+++ b/pkg/drivers/kic/oci/cgroups_other.go
@@ -0,0 +1,30 @@
+// +build !linux
+
+/*
+Copyright 2021 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package oci
+
+import (
+	"runtime"
+
+	"github.com/pkg/errors"
+)
+
+// IsCgroup2UnifiedMode returns whether we are running in cgroup 2 cgroup2 mode.
+func IsCgroup2UnifiedMode() (bool, error) {
+	return false, errors.Errorf("Not supported on %s", runtime.GOOS)
+}
diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go
index 27f7a49a73..03a99cdc2c 100644
--- a/pkg/drivers/kic/oci/oci.go
+++ b/pkg/drivers/kic/oci/oci.go
@@ -107,6 +107,24 @@ func PrepareContainerNode(p CreateParams) error {
 	return nil
 }
 
+func hasMemorySwapCgroup() bool {
+	memcgSwap := true
+	if runtime.GOOS == "linux" {
+		var memoryswap string
+		if cgroup2, err := IsCgroup2UnifiedMode(); err == nil && cgroup2 {
+			memoryswap = "/sys/fs/cgroup/memory/memory.swap.max"
+		} else {
+			memoryswap = "/sys/fs/cgroup/memory/memsw.limit_in_bytes"
+		}
+		if _, err := os.Stat(memoryswap); os.IsNotExist(err) {
+			// requires CONFIG_MEMCG_SWAP_ENABLED or cgroup_enable=memory in grub
+			klog.Warning("Your kernel does not support swap limit capabilities or the cgroup is not mounted.")
+			memcgSwap = false
+		}
+	}
+	return memcgSwap
+}
+
 // CreateContainerNode creates a new container node
 func CreateContainerNode(p CreateParams) error {
 	// on windows os, if docker desktop is using Windows Containers. Exit early with error
@@ -152,14 +170,7 @@ func CreateContainerNode(p CreateParams) error {
 		runArgs = append(runArgs, "--ip", p.IP)
 	}
 
-	memcgSwap := true
-	if runtime.GOOS == "linux" {
-		if _, err := os.Stat("/sys/fs/cgroup/memory/memsw.limit_in_bytes"); os.IsNotExist(err) {
-			// requires CONFIG_MEMCG_SWAP_ENABLED or cgroup_enable=memory in grub
-			klog.Warning("Your kernel does not support swap limit capabilities or the cgroup is not mounted.")
-			memcgSwap = false
-		}
-	}
+	memcgSwap := hasMemorySwapCgroup()
 
 	// https://www.freedesktop.org/wiki/Software/systemd/ContainerInterface/
 	var virtualization string

From 4a441a48bffb14bb125789b1a8d7491935c04b2d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= <anders.f.bjorklund@gmail.com>
Date: Sat, 13 Feb 2021 20:36:08 +0100
Subject: [PATCH 039/114] The checksum of the podman source tarball changed

Due to it being auto-generated by "git archive",
and not being an actual and archived distfile...

The actual code contents are the same, though.
---
 deploy/iso/minikube-iso/package/podman/podman.hash | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deploy/iso/minikube-iso/package/podman/podman.hash b/deploy/iso/minikube-iso/package/podman/podman.hash
index 7f0122541b..4f5158b977 100644
--- a/deploy/iso/minikube-iso/package/podman/podman.hash
+++ b/deploy/iso/minikube-iso/package/podman/podman.hash
@@ -1,4 +1,4 @@
 sha256 a16846fe076aaf2c9ea2e854c3baba9fb838d916be7fb4b5be332e6c92d907d4 v1.9.3.tar.gz
 sha256 5ebaa6e0dbd7fd1863f70d2bc71dc8a94e195c3339c17e3cac4560c9ec5747f8 v2.1.1.tar.gz
 sha256 ec5473e51fa28f29af323473fc484f742dc7df23d06d8ba9f217f13382893a71 v2.2.0.tar.gz
-sha256 bd86b181251e2308cb52f18410fb52d89df7f130cecf0298bbf9a848fe7daf60 v2.2.1.tar.gz
+sha256 3212bad60d945c1169b27da03959f36d92d1d8964645c701a5a82a89118e96d1 v2.2.1.tar.gz

From c8266c03182a10ec61c9478f508f78497690d736 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= <anders.f.bjorklund@gmail.com>
Date: Sun, 14 Feb 2021 13:19:34 +0100
Subject: [PATCH 040/114] Upgrade crio to 1.20.0

---
 .../package/crio-bin/crio-bin.hash            |  2 +
 .../minikube-iso/package/crio-bin/crio-bin.mk |  4 +-
 .../minikube-iso/package/crio-bin/crio.conf   | 41 +++++++++++++------
 .../package/crio-bin/crio.conf.default        | 41 +++++++++++++------
 4 files changed, 62 insertions(+), 26 deletions(-)

diff --git a/deploy/iso/minikube-iso/package/crio-bin/crio-bin.hash b/deploy/iso/minikube-iso/package/crio-bin/crio-bin.hash
index 93ddf49dab..4fb4c559b8 100644
--- a/deploy/iso/minikube-iso/package/crio-bin/crio-bin.hash
+++ b/deploy/iso/minikube-iso/package/crio-bin/crio-bin.hash
@@ -19,3 +19,5 @@ sha256 25dc558fbabc987bd58c7eab5230121b258a7b0eb34a49dc6595f1c6f3969116 v1.18.2.
 sha256 d5c6442e3990938badc966cdd1eb9ebe2fc11345452c233aa0d87ca38fbeed81 v1.18.3.tar.gz
 sha256 74a4e916acddc6cf47ab5752bdebb6732ce2c028505ef57b7edc21d2da9039b6 v1.18.4.tar.gz
 sha256 fc8a8e61375e3ce30563eeb0fd6534c4f48fc20300a72e6ff51cc99cb2703516 v1.19.0.tar.gz
+sha256 6165c5b8212ea03be2a465403177318bfe25a54c3e8d66d720344643913a0223 v1.19.1.tar.gz
+sha256 76fd7543bc92d4364a11060f43a5131893a76c6e6e9d6de3a6bb6292c110b631 v1.20.0.tar.gz
diff --git a/deploy/iso/minikube-iso/package/crio-bin/crio-bin.mk b/deploy/iso/minikube-iso/package/crio-bin/crio-bin.mk
index 0be5317687..41ec5044d2 100644
--- a/deploy/iso/minikube-iso/package/crio-bin/crio-bin.mk
+++ b/deploy/iso/minikube-iso/package/crio-bin/crio-bin.mk
@@ -4,8 +4,8 @@
 #
 ################################################################################
 
-CRIO_BIN_VERSION = v1.19.0
-CRIO_BIN_COMMIT = 99c925bebdd9e392f2d575e25f2e6a1082e6c232
+CRIO_BIN_VERSION = v1.20.0
+CRIO_BIN_COMMIT = d388528dbed26b93c5bc1c89623607a1e597aa57
 CRIO_BIN_SITE = https://github.com/cri-o/cri-o/archive
 CRIO_BIN_SOURCE = $(CRIO_BIN_VERSION).tar.gz
 CRIO_BIN_DEPENDENCIES = host-go libgpgme
diff --git a/deploy/iso/minikube-iso/package/crio-bin/crio.conf b/deploy/iso/minikube-iso/package/crio-bin/crio.conf
index a7e010c9ea..fafaed67bc 100644
--- a/deploy/iso/minikube-iso/package/crio-bin/crio.conf
+++ b/deploy/iso/minikube-iso/package/crio-bin/crio.conf
@@ -29,6 +29,7 @@ storage_driver = "overlay"
 # List to pass options to the storage driver. Please refer to
 # containers-storage.conf(5) to see all available storage options.
 #storage_option = [
+#	"overlay.mountopt=nodev,metacopy=on",
 #]
 
 # The default log directory where all logs will go unless directly specified by
@@ -92,11 +93,6 @@ grpc_max_recv_msg_size = 16777216
 #default_ulimits = [
 #]
 
-# default_runtime is the _name_ of the OCI runtime to be used as the default.
-# The name is matched against the runtimes map below. If this value is changed,
-# the corresponding existing entry from the runtimes map below will be ignored.
-default_runtime = "runc"
-
 # If true, the runtime will not use pivot_root, but instead use MS_MOVE.
 no_pivot = false
 
@@ -131,6 +127,12 @@ selinux = false
 # will be used. This option supports live configuration reload.
 seccomp_profile = ""
 
+# Changes the meaning of an empty seccomp profile. By default
+# (and according to CRI spec), an empty profile means unconfined.
+# This option tells CRI-O to treat an empty profile as the default profile,
+# which might increase security.
+seccomp_use_default_when_empty = false
+
 # Used to change the name of the default AppArmor profile of CRI-O. The default
 # profile name is "crio-default". This profile only takes effect if the user
 # does not specify a profile via the Kubernetes Pod's metadata annotation. If
@@ -141,6 +143,9 @@ apparmor_profile = "crio-default"
 # Cgroup management implementation used for the runtime.
 cgroup_manager = "systemd"
 
+# Specify whether the image pull must be performed in a separate cgroup.
+separate_pull_cgroup = ""
+
 # List of default capabilities for containers. If it is empty or commented out,
 # only the capabilities defined in the containers json file by the user/kube
 # will be added.
@@ -174,11 +179,6 @@ hooks_dir = [
 	"/usr/share/containers/oci/hooks.d",
 ]
 
-# List of default mounts for each container. **Deprecated:** this option will
-# be removed in future versions in favor of default_mounts_file.
-default_mounts = [
-]
-
 # Path to the file specifying the defaults mounts for each container. The
 # format of the config is /SRC:/DST, one mount per line. Notice that CRI-O reads
 # its default mounts from the following two files:
@@ -243,7 +243,8 @@ gid_mappings = ""
 ctr_stop_timeout = 30
 
 # manage_ns_lifecycle determines whether we pin and remove namespaces
-# and manage their lifecycle
+# and manage their lifecycle.
+# This option is being deprecated, and will be unconditionally true in the future.
 manage_ns_lifecycle = true
 
 # drop_infra_ctr determines whether CRI-O drops the infra container
@@ -259,6 +260,11 @@ namespaces_dir = "/var/run"
 # pinns_path is the path to find the pinns binary, which is needed to manage namespace lifecycle
 pinns_path = "/usr/bin/pinns"
 
+# default_runtime is the _name_ of the OCI runtime to be used as the default.
+# The name is matched against the runtimes map below. If this value is changed,
+# the corresponding existing entry from the runtimes map below will be ignored.
+default_runtime = "runc"
+
 # The "crio.runtime.runtimes" table defines a list of OCI compatible runtimes.
 # The runtime to use is picked based on the runtime_handler provided by the CRI.
 # If no runtime_handler is provided, the runtime will be picked based on the level
@@ -268,7 +274,8 @@ pinns_path = "/usr/bin/pinns"
 #  runtime_path = "/path/to/the/executable"
 #  runtime_type = "oci"
 #  runtime_root = "/path/to/the/root"
-#
+#  privileged_without_host_devices = false
+#  allowed_annotations = []
 # Where:
 # - runtime-handler: name used to identify the runtime
 # - runtime_path (optional, string): absolute path to the runtime executable in
@@ -279,6 +286,14 @@ pinns_path = "/usr/bin/pinns"
 #   omitted, an "oci" runtime is assumed.
 # - runtime_root (optional, string): root directory for storage of containers
 #   state.
+# - privileged_without_host_devices (optional, bool): an option for restricting
+#   host devices from being passed to privileged containers.
+# - allowed_annotations (optional, array of strings): an option for specifying
+#   a list of experimental annotations that this runtime handler is allowed to process.
+#   The currently recognized values are:
+#   "io.kubernetes.cri-o.userns-mode" for configuring a user namespace for the pod.
+#   "io.kubernetes.cri-o.Devices" for configuring devices for the pod.
+#   "io.kubernetes.cri-o.ShmSize" for configuring the size of /dev/shm.
 
 
 [crio.runtime.runtimes.runc]
@@ -287,6 +302,8 @@ runtime_type = "oci"
 runtime_root = "/run/runc"
 
 
+
+
 # crun is a fast and lightweight fully featured OCI runtime and C library for
 # running containers
 #[crio.runtime.runtimes.crun]
diff --git a/deploy/iso/minikube-iso/package/crio-bin/crio.conf.default b/deploy/iso/minikube-iso/package/crio-bin/crio.conf.default
index 9c22500d0b..25debfab9f 100644
--- a/deploy/iso/minikube-iso/package/crio-bin/crio.conf.default
+++ b/deploy/iso/minikube-iso/package/crio-bin/crio.conf.default
@@ -29,6 +29,7 @@
 # List to pass options to the storage driver. Please refer to
 # containers-storage.conf(5) to see all available storage options.
 #storage_option = [
+#	"overlay.mountopt=nodev,metacopy=on",
 #]
 
 # The default log directory where all logs will go unless directly specified by
@@ -92,11 +93,6 @@ grpc_max_recv_msg_size = 16777216
 #default_ulimits = [
 #]
 
-# default_runtime is the _name_ of the OCI runtime to be used as the default.
-# The name is matched against the runtimes map below. If this value is changed,
-# the corresponding existing entry from the runtimes map below will be ignored.
-default_runtime = "runc"
-
 # If true, the runtime will not use pivot_root, but instead use MS_MOVE.
 no_pivot = false
 
@@ -131,6 +127,12 @@ selinux = false
 # will be used. This option supports live configuration reload.
 seccomp_profile = ""
 
+# Changes the meaning of an empty seccomp profile. By default
+# (and according to CRI spec), an empty profile means unconfined.
+# This option tells CRI-O to treat an empty profile as the default profile,
+# which might increase security.
+seccomp_use_default_when_empty = false
+
 # Used to change the name of the default AppArmor profile of CRI-O. The default
 # profile name is "crio-default". This profile only takes effect if the user
 # does not specify a profile via the Kubernetes Pod's metadata annotation. If
@@ -141,6 +143,9 @@ apparmor_profile = "crio-default"
 # Cgroup management implementation used for the runtime.
 cgroup_manager = "systemd"
 
+# Specify whether the image pull must be performed in a separate cgroup.
+separate_pull_cgroup = ""
+
 # List of default capabilities for containers. If it is empty or commented out,
 # only the capabilities defined in the containers json file by the user/kube
 # will be added.
@@ -174,11 +179,6 @@ hooks_dir = [
 	"/usr/share/containers/oci/hooks.d",
 ]
 
-# List of default mounts for each container. **Deprecated:** this option will
-# be removed in future versions in favor of default_mounts_file.
-default_mounts = [
-]
-
 # Path to the file specifying the defaults mounts for each container. The
 # format of the config is /SRC:/DST, one mount per line. Notice that CRI-O reads
 # its default mounts from the following two files:
@@ -243,7 +243,8 @@ gid_mappings = ""
 ctr_stop_timeout = 30
 
 # manage_ns_lifecycle determines whether we pin and remove namespaces
-# and manage their lifecycle
+# and manage their lifecycle.
+# This option is being deprecated, and will be unconditionally true in the future.
 manage_ns_lifecycle = true
 
 # drop_infra_ctr determines whether CRI-O drops the infra container
@@ -259,6 +260,11 @@ namespaces_dir = "/var/run"
 # pinns_path is the path to find the pinns binary, which is needed to manage namespace lifecycle
 pinns_path = ""
 
+# default_runtime is the _name_ of the OCI runtime to be used as the default.
+# The name is matched against the runtimes map below. If this value is changed,
+# the corresponding existing entry from the runtimes map below will be ignored.
+default_runtime = "runc"
+
 # The "crio.runtime.runtimes" table defines a list of OCI compatible runtimes.
 # The runtime to use is picked based on the runtime_handler provided by the CRI.
 # If no runtime_handler is provided, the runtime will be picked based on the level
@@ -268,7 +274,8 @@ pinns_path = ""
 #  runtime_path = "/path/to/the/executable"
 #  runtime_type = "oci"
 #  runtime_root = "/path/to/the/root"
-#
+#  privileged_without_host_devices = false
+#  allowed_annotations = []
 # Where:
 # - runtime-handler: name used to identify the runtime
 # - runtime_path (optional, string): absolute path to the runtime executable in
@@ -279,6 +286,14 @@ pinns_path = ""
 #   omitted, an "oci" runtime is assumed.
 # - runtime_root (optional, string): root directory for storage of containers
 #   state.
+# - privileged_without_host_devices (optional, bool): an option for restricting
+#   host devices from being passed to privileged containers.
+# - allowed_annotations (optional, array of strings): an option for specifying
+#   a list of experimental annotations that this runtime handler is allowed to process.
+#   The currently recognized values are:
+#   "io.kubernetes.cri-o.userns-mode" for configuring a user namespace for the pod.
+#   "io.kubernetes.cri-o.Devices" for configuring devices for the pod.
+#   "io.kubernetes.cri-o.ShmSize" for configuring the size of /dev/shm.
 
 
 [crio.runtime.runtimes.runc]
@@ -287,6 +302,8 @@ runtime_type = "oci"
 runtime_root = "/run/runc"
 
 
+
+
 # crun is a fast and lightweight fully featured OCI runtime and C library for
 # running containers
 #[crio.runtime.runtimes.crun]

From 77b86ef82dd09cbf72cc3038a62fecfb924665ae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= <anders.f.bjorklund@gmail.com>
Date: Sun, 14 Feb 2021 20:21:21 +0100
Subject: [PATCH 041/114] Make sure to use CRI for docker if configured

The default is still the built-in dockershim,
but make sure to honor any external CRI socket.

Also forward the other configuration such as
the Kubernetes version and Image repository.
---
 pkg/minikube/cruntime/cruntime.go |  9 +++++---
 pkg/minikube/cruntime/docker.go   | 36 ++++++++++++++++++++++++++++---
 pkg/minikube/node/start.go        |  1 +
 3 files changed, 40 insertions(+), 6 deletions(-)

diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go
index ba9075051d..a201aaffb0 100644
--- a/pkg/minikube/cruntime/cruntime.go
+++ b/pkg/minikube/cruntime/cruntime.go
@@ -151,9 +151,12 @@ func New(c Config) (Manager, error) {
 	switch c.Type {
 	case "", "docker":
 		return &Docker{
-			Socket: c.Socket,
-			Runner: c.Runner,
-			Init:   sm,
+			Socket:            c.Socket,
+			Runner:            c.Runner,
+			ImageRepository:   c.ImageRepository,
+			KubernetesVersion: c.KubernetesVersion,
+			Init:              sm,
+			UseCRI:            (c.Socket != ""), // !dockershim
 		}, nil
 	case "crio", "cri-o":
 		return &CRIO{
diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go
index dddb1a0e72..6ce546e9fc 100644
--- a/pkg/minikube/cruntime/docker.go
+++ b/pkg/minikube/cruntime/docker.go
@@ -23,6 +23,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/blang/semver"
 	"github.com/pkg/errors"
 	"k8s.io/klog/v2"
 	"k8s.io/minikube/pkg/minikube/assets"
@@ -56,9 +57,12 @@ func (e *ErrISOFeature) Error() string {
 
 // Docker contains Docker runtime state
 type Docker struct {
-	Socket string
-	Runner CommandRunner
-	Init   sysinit.Manager
+	Socket            string
+	Runner            CommandRunner
+	ImageRepository   string
+	KubernetesVersion semver.Version
+	Init              sysinit.Manager
+	UseCRI            bool
 }
 
 // Name is a human readable name for Docker
@@ -181,6 +185,14 @@ func (r *Docker) CGroupDriver() (string, error) {
 
 // KubeletOptions returns kubelet options for a runtime.
 func (r *Docker) KubeletOptions() map[string]string {
+	if r.UseCRI {
+		return map[string]string{
+			"container-runtime":          "remote",
+			"container-runtime-endpoint": r.SocketPath(),
+			"image-service-endpoint":     r.SocketPath(),
+			"runtime-request-timeout":    "15m",
+		}
+	}
 	return map[string]string{
 		"container-runtime": "docker",
 	}
@@ -188,6 +200,9 @@ func (r *Docker) KubeletOptions() map[string]string {
 
 // ListContainers returns a list of containers
 func (r *Docker) ListContainers(o ListOptions) ([]string, error) {
+	if r.UseCRI {
+		return listCRIContainers(r.Runner, "", o)
+	}
 	args := []string{"ps"}
 	switch o.State {
 	case All:
@@ -220,6 +235,9 @@ func (r *Docker) ListContainers(o ListOptions) ([]string, error) {
 
 // KillContainers forcibly removes a running container based on ID
 func (r *Docker) KillContainers(ids []string) error {
+	if r.UseCRI {
+		return killCRIContainers(r.Runner, ids)
+	}
 	if len(ids) == 0 {
 		return nil
 	}
@@ -234,6 +252,9 @@ func (r *Docker) KillContainers(ids []string) error {
 
 // StopContainers stops a running container based on ID
 func (r *Docker) StopContainers(ids []string) error {
+	if r.UseCRI {
+		return stopCRIContainers(r.Runner, ids)
+	}
 	if len(ids) == 0 {
 		return nil
 	}
@@ -248,6 +269,9 @@ func (r *Docker) StopContainers(ids []string) error {
 
 // PauseContainers pauses a running container based on ID
 func (r *Docker) PauseContainers(ids []string) error {
+	if r.UseCRI {
+		return pauseCRIContainers(r.Runner, "", ids)
+	}
 	if len(ids) == 0 {
 		return nil
 	}
@@ -262,6 +286,9 @@ func (r *Docker) PauseContainers(ids []string) error {
 
 // UnpauseContainers unpauses a container based on ID
 func (r *Docker) UnpauseContainers(ids []string) error {
+	if r.UseCRI {
+		return unpauseCRIContainers(r.Runner, "", ids)
+	}
 	if len(ids) == 0 {
 		return nil
 	}
@@ -276,6 +303,9 @@ func (r *Docker) UnpauseContainers(ids []string) error {
 
 // ContainerLogCmd returns the command to retrieve the log for a container based on ID
 func (r *Docker) ContainerLogCmd(id string, len int, follow bool) string {
+	if r.UseCRI {
+		return criContainerLogCmd(r.Runner, id, len, follow)
+	}
 	var cmd strings.Builder
 	cmd.WriteString("docker logs ")
 	if len > 0 {
diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go
index 42f9395e69..85e1d28488 100644
--- a/pkg/minikube/node/start.go
+++ b/pkg/minikube/node/start.go
@@ -245,6 +245,7 @@ func Provision(cc *config.ClusterConfig, n *config.Node, apiServer bool, delOnFa
 func configureRuntimes(runner cruntime.CommandRunner, cc config.ClusterConfig, kv semver.Version) cruntime.Manager {
 	co := cruntime.Config{
 		Type:              cc.KubernetesConfig.ContainerRuntime,
+		Socket:            cc.KubernetesConfig.CRISocket,
 		Runner:            runner,
 		ImageRepository:   cc.KubernetesConfig.ImageRepository,
 		KubernetesVersion: kv,

From 90cd9c3a6069e6a948c70ebbe0e63e32ecc2b0b8 Mon Sep 17 00:00:00 2001
From: Predrag Rogic <prezha@users.noreply.github.com>
Date: Tue, 16 Feb 2021 15:50:54 +0000
Subject: [PATCH 042/114] wait kubelet stabilise after restart

---
 .../bootstrapper/bsutil/kverify/pod_ready.go  | 41 ++++++++++++++-----
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go  | 26 ++++++++----
 2 files changed, 49 insertions(+), 18 deletions(-)

diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/pod_ready.go b/pkg/minikube/bootstrapper/bsutil/kverify/pod_ready.go
index d9c4c70539..29115b66a5 100644
--- a/pkg/minikube/bootstrapper/bsutil/kverify/pod_ready.go
+++ b/pkg/minikube/bootstrapper/bsutil/kverify/pod_ready.go
@@ -79,6 +79,7 @@ func WaitForPodReadyByLabel(cs *kubernetes.Clientset, label, namespace string, t
 		return fmt.Errorf("pod label %q is malformed", label)
 	}
 
+	lap := time.Now()
 	checkReady := func() (bool, error) {
 		if time.Since(start) > timeout {
 			return false, fmt.Errorf("wait for pod with %q label in %q namespace to be Ready timed out", label, namespace)
@@ -92,7 +93,17 @@ func WaitForPodReadyByLabel(cs *kubernetes.Clientset, label, namespace string, t
 		for _, pod := range pods.Items {
 			for k, v := range pod.ObjectMeta.Labels {
 				if ((lkey == "" && (k == "component" || k == "k8s-app")) || lkey == k) && v == lval {
-					return checkPodStatus(&pod)
+					ready, reason := IsPodReady(&pod)
+					if ready {
+						klog.Info(reason)
+						return true, nil
+					}
+					// reduce log spam
+					if time.Since(lap) > (1 * time.Second) {
+						klog.Info(reason)
+						lap = time.Now()
+					}
+					return false, nil
 				}
 			}
 		}
@@ -120,6 +131,7 @@ func WaitForPodReadyByName(cs *kubernetes.Clientset, name, namespace string, tim
 		namespace = "kube-system"
 	}
 
+	lap := time.Now()
 	checkReady := func() (bool, error) {
 		if time.Since(start) > timeout {
 			return false, fmt.Errorf("wait for pod %q in %q namespace to be Ready timed out", name, namespace)
@@ -130,7 +142,17 @@ func WaitForPodReadyByName(cs *kubernetes.Clientset, name, namespace string, tim
 			klog.Infof("error getting pod %q in %q namespace, will retry: %v", name, namespace, err)
 			return false, nil
 		}
-		return checkPodStatus(pod)
+		ready, reason := IsPodReady(pod)
+		if ready {
+			klog.Info(reason)
+			return true, nil
+		}
+		// reduce log spam
+		if time.Since(lap) > (1 * time.Second) {
+			klog.Info(reason)
+			lap = time.Now()
+		}
+		return false, nil
 	}
 
 	if err := wait.PollImmediate(kconst.APICallRetryInterval, kconst.DefaultControlPlaneTimeout, checkReady); err != nil {
@@ -140,21 +162,18 @@ func WaitForPodReadyByName(cs *kubernetes.Clientset, name, namespace string, tim
 	return nil
 }
 
-// checkPodStatus returns if pod is Ready and any error occurred.
-func checkPodStatus(pod *core.Pod) (bool, error) {
+// IsPodReady returns if pod is Ready and verbose reason.
+func IsPodReady(pod *core.Pod) (ready bool, reason string) {
 	if pod.Status.Phase != core.PodRunning {
-		klog.Infof("pod %q in %q namespace is not Running, will retry: %+v", pod.Name, pod.Namespace, pod.Status)
-		return false, nil
+		return false, fmt.Sprintf("pod %q in %q namespace is not Running: %+v", pod.Name, pod.Namespace, pod.Status)
 	}
 	for _, c := range pod.Status.Conditions {
 		if c.Type == core.PodReady {
 			if c.Status != core.ConditionTrue {
-				klog.Infof("pod %q in %q namespace is not Ready, will retry: %+v", pod.Name, pod.Namespace, c)
-				return false, nil
+				return false, fmt.Sprintf("pod %q in %q namespace is not Ready: %+v", pod.Name, pod.Namespace, c)
 			}
-			klog.Infof("pod %q in %q namespace is Ready ...", pod.Name, pod.Namespace)
-			return true, nil
+			return true, fmt.Sprintf("pod %q in %q namespace is Ready: %+v", pod.Name, pod.Namespace, c)
 		}
 	}
-	return false, fmt.Errorf("pod %q in %q namespace does not have %q status: %+v", pod.Name, pod.Namespace, core.PodReady, pod.Status)
+	return false, fmt.Sprintf("pod %q in %q namespace does not have %q status: %+v", pod.Name, pod.Namespace, core.PodReady, pod.Status)
 }
diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index 6efdd1dbd7..dea2a80b81 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -470,6 +470,12 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time
 		return nil
 	}
 
+	if cfg.VerifyComponents[kverify.OperationalKey] {
+		if err := kverify.WaitOperational(client, kverify.CorePodsList, timeout); err != nil {
+			return errors.Wrap(err, "waiting for operational status")
+		}
+	}
+
 	cr, err := cruntime.New(cruntime.Config{Type: cfg.KubernetesConfig.ContainerRuntime, Runner: k.c})
 	if err != nil {
 		return errors.Wrapf(err, "create runtme-manager %s", cfg.KubernetesConfig.ContainerRuntime)
@@ -503,18 +509,12 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time
 				return errors.Wrap(err, "waiting for apps_running")
 			}
 		}
-
-		if cfg.VerifyComponents[kverify.OperationalKey] {
-			if err := kverify.WaitOperational(client, kverify.CorePodsList, timeout); err != nil {
-				return errors.Wrap(err, "waiting for operational status")
-			}
-		}
 	}
+
 	if cfg.VerifyComponents[kverify.KubeletKey] {
 		if err := kverify.WaitForService(k.c, "kubelet", timeout); err != nil {
 			return errors.Wrap(err, "waiting for kubelet")
 		}
-
 	}
 
 	if cfg.VerifyComponents[kverify.NodeReadyKey] {
@@ -664,6 +664,17 @@ func (k *Bootstrapper) restartControlPlane(cfg config.ClusterConfig) error {
 		}
 	}
 
+	if cfg.VerifyComponents[kverify.OperationalKey] {
+		// after kubelet is restarted (with 'kubeadm init phase kubelet-start' above),
+		// it appears to be immediately Ready as are all kube-system pods
+		// then (after ~10sec) it realises it has some changes to apply, implying also pods restarts
+		// so we wait for kubelet to initialise itself...
+		time.Sleep(10 * time.Second)
+		if err := kverify.WaitOperational(client, kverify.CorePodsList, kconst.DefaultControlPlaneTimeout); err != nil {
+			return errors.Wrap(err, "operational status")
+		}
+	}
+
 	cr, err := cruntime.New(cruntime.Config{Type: cfg.KubernetesConfig.ContainerRuntime, Runner: k.c})
 	if err != nil {
 		return errors.Wrap(err, "runtime")
@@ -704,6 +715,7 @@ func (k *Bootstrapper) restartControlPlane(cfg config.ClusterConfig) error {
 	if err := bsutil.AdjustResourceLimits(k.c); err != nil {
 		klog.Warningf("unable to adjust resource limits: %v", err)
 	}
+
 	return nil
 }
 

From 2c3d91acef1e5c5862af8488fe786706274731bb Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Tue, 16 Feb 2021 10:24:43 -0800
Subject: [PATCH 043/114] reduce time limit on TestStartStop because it is
 timing out on kvm_containerd

---
 test/integration/start_stop_delete_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/integration/start_stop_delete_test.go b/test/integration/start_stop_delete_test.go
index 8154b53449..fec0c8d014 100644
--- a/test/integration/start_stop_delete_test.go
+++ b/test/integration/start_stop_delete_test.go
@@ -79,7 +79,7 @@ func TestStartStop(t *testing.T) {
 			t.Run(tc.name, func(t *testing.T) {
 				MaybeParallel(t)
 				profile := UniqueProfileName(tc.name)
-				ctx, cancel := context.WithTimeout(context.Background(), Minutes(40))
+				ctx, cancel := context.WithTimeout(context.Background(), Minutes(30))
 				defer Cleanup(t, profile, cancel)
 				type validateStartStopFunc func(context.Context, *testing.T, string, string, string, []string)
 				if !strings.Contains(tc.name, "docker") && NoneDriver() {

From 803934a6702b9d349689378475a32717c7dd5d87 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Tue, 16 Feb 2021 14:22:49 -0800
Subject: [PATCH 044/114] add automation script

---
 hack/jenkins/kicbase_auto_build.sh | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)
 create mode 100644 hack/jenkins/kicbase_auto_build.sh

diff --git a/hack/jenkins/kicbase_auto_build.sh b/hack/jenkins/kicbase_auto_build.sh
new file mode 100644
index 0000000000..51ab4932bc
--- /dev/null
+++ b/hack/jenkins/kicbase_auto_build.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+./hack/jenkins/installers/check_install_docker.sh
+yes|gcloud auth configure-docker
+now=$(date +%s)
+export KV=$(egrep "Version =" pkg/drivers/kic/types.go | cut -d \" -f 2 | cut -d "-" -f 1)
+export KIC_VERSION=$KV-$now-$ghprbPullId
+export KICBASE_IMAGE_REGISTRIES=gcr.io/k8s-minikube/kicbase-builds:$KIC_VERSION
+yes|make push-kic-base-image
+
+docker pull $KICBASE_IMAGE_REGISTRIES
+fullsha=$(docker inspect --format='{{index .RepoDigests 0}}' $KICBASE_IMAGE_REGISTRIES)
+sha=$(echo ${fullsha} | cut -d ":" -f 2)
+
+message="Hi ${ghprbPullAuthorLoginMention},
+
+A new kicbase image is available, please update your PR with the new tag and SHA.
+In pkg/drivers/kic/types.go:
+
+	// Version is the current version of kic
+	Version = \"${KICBASE_IMAGE_REGISTRIES}\"
+	// SHA of the kic base image
+	baseImageSHA = \"${sha}\"
+"
+
+curl -s -H "Authorization: token ${access_token}" \
+	 -H "Accept: application/vnd.github.v3+json" \
+	 -X POST -d "{\"body\": \"${message}\"}" "https://api.github.com/repos/kubernetes/minikube/issues/$ghprbPullId/comments"

From 29352b294b4e71facc0e903e4d1199258d14441e Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Tue, 16 Feb 2021 14:23:23 -0800
Subject: [PATCH 045/114] make executable

---
 hack/jenkins/kicbase_auto_build.sh | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 hack/jenkins/kicbase_auto_build.sh

diff --git a/hack/jenkins/kicbase_auto_build.sh b/hack/jenkins/kicbase_auto_build.sh
old mode 100644
new mode 100755

From b8052fe33d2d6daa61f501e52d163e31d88063a5 Mon Sep 17 00:00:00 2001
From: Predrag Rogic <prezha@users.noreply.github.com>
Date: Tue, 16 Feb 2021 22:27:26 +0000
Subject: [PATCH 046/114] replace sleep with retry.Expo()

---
 .../bootstrapper/bsutil/kverify/kverify.go    | 10 ++---
 .../bootstrapper/bsutil/kverify/pod_ready.go  | 14 +++----
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go  | 39 ++++++++++++++-----
 test/integration/functional_test.go           |  1 +
 4 files changed, 40 insertions(+), 24 deletions(-)

diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
index f6abca315f..a9f9ec6638 100644
--- a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
+++ b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
@@ -37,8 +37,8 @@ const (
 	NodeReadyKey = "node_ready"
 	// KubeletKey is the name used in the flags for waiting for the kubelet status to be ready
 	KubeletKey = "kubelet"
-	// OperationalKey is the name used for waiting for pods in CorePodsList to be Ready
-	OperationalKey = "operational"
+	// ExtraKey is the name used for extra waiting for pods in CorePodsList to be Ready
+	ExtraKey = "extra"
 )
 
 //  vars related to the --wait flag
@@ -46,9 +46,9 @@ var (
 	// DefaultComponents is map of the the default components to wait for
 	DefaultComponents = map[string]bool{APIServerWaitKey: true, SystemPodsWaitKey: true}
 	// NoWaitComponents is map of componets to wait for if specified 'none' or 'false'
-	NoComponents = map[string]bool{APIServerWaitKey: false, SystemPodsWaitKey: false, DefaultSAWaitKey: false, AppsRunningKey: false, NodeReadyKey: false, KubeletKey: false, OperationalKey: false}
+	NoComponents = map[string]bool{APIServerWaitKey: false, SystemPodsWaitKey: false, DefaultSAWaitKey: false, AppsRunningKey: false, NodeReadyKey: false, KubeletKey: false, ExtraKey: false}
 	// AllComponents is map for waiting for all components.
-	AllComponents = map[string]bool{APIServerWaitKey: true, SystemPodsWaitKey: true, DefaultSAWaitKey: true, AppsRunningKey: true, NodeReadyKey: true, KubeletKey: true, OperationalKey: true}
+	AllComponents = map[string]bool{APIServerWaitKey: true, SystemPodsWaitKey: true, DefaultSAWaitKey: true, AppsRunningKey: true, NodeReadyKey: true, KubeletKey: true, ExtraKey: true}
 	// DefaultWaitList is list of all default components to wait for. only names to be used for start flags.
 	DefaultWaitList = []string{APIServerWaitKey, SystemPodsWaitKey}
 	// AllComponentsList list of all valid components keys to wait for. only names to be used used for start flags.
@@ -62,7 +62,7 @@ var (
 		"kube-proxy",
 		"kube-scheduler",
 	}
-	// CorePodsList is a list of essential pods for running kurnetes to wait for them to be operational ("Ready")
+	// CorePodsList is a list of essential pods for running kurnetes to extra wait for them to be Ready
 	CorePodsList = []string{
 		"kube-dns", // coredns
 		"etcd",
diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/pod_ready.go b/pkg/minikube/bootstrapper/bsutil/kverify/pod_ready.go
index 29115b66a5..125a2e5a6f 100644
--- a/pkg/minikube/bootstrapper/bsutil/kverify/pod_ready.go
+++ b/pkg/minikube/bootstrapper/bsutil/kverify/pod_ready.go
@@ -31,12 +31,12 @@ import (
 	kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 )
 
-// WaitOperational calls WaitForPodReadyByLabel for each pod in labels list and returns any errors occurred.
-func WaitOperational(cs *kubernetes.Clientset, labels []string, timeout time.Duration) error {
-	klog.Info("waiting for kube-system core pods %s to be Ready ...", labels)
-	pStart := time.Now()
+// WaitExtra calls WaitForPodReadyByLabel for each pod in labels list and returns any errors occurred.
+func WaitExtra(cs *kubernetes.Clientset, labels []string, timeout time.Duration) error {
+	klog.Infof("extra waiting for kube-system core pods %s to be Ready ...", labels)
+	start := time.Now()
 	defer func() {
-		klog.Infof("duration metric: took %s for waiting for kube-system core pods to be Ready ...", time.Since(pStart))
+		klog.Infof("duration metric: took %s for extra waiting for kube-system core pods to be Ready ...", time.Since(start))
 	}()
 
 	var errs []string
@@ -84,7 +84,6 @@ func WaitForPodReadyByLabel(cs *kubernetes.Clientset, label, namespace string, t
 		if time.Since(start) > timeout {
 			return false, fmt.Errorf("wait for pod with %q label in %q namespace to be Ready timed out", label, namespace)
 		}
-
 		pods, err := cs.CoreV1().Pods(namespace).List(meta.ListOptions{})
 		if err != nil {
 			klog.Infof("error listing pods in %q namespace, will retry: %v", namespace, err)
@@ -110,7 +109,6 @@ func WaitForPodReadyByLabel(cs *kubernetes.Clientset, label, namespace string, t
 		klog.Infof("pod with %q label in %q namespace was not found, will retry", label, namespace)
 		return false, nil
 	}
-
 	if err := wait.PollImmediate(kconst.APICallRetryInterval, kconst.DefaultControlPlaneTimeout, checkReady); err != nil {
 		return errors.Wrapf(err, "wait pod Ready")
 	}
@@ -136,7 +134,6 @@ func WaitForPodReadyByName(cs *kubernetes.Clientset, name, namespace string, tim
 		if time.Since(start) > timeout {
 			return false, fmt.Errorf("wait for pod %q in %q namespace to be Ready timed out", name, namespace)
 		}
-
 		pod, err := cs.CoreV1().Pods(namespace).Get(name, meta.GetOptions{})
 		if err != nil {
 			klog.Infof("error getting pod %q in %q namespace, will retry: %v", name, namespace, err)
@@ -154,7 +151,6 @@ func WaitForPodReadyByName(cs *kubernetes.Clientset, name, namespace string, tim
 		}
 		return false, nil
 	}
-
 	if err := wait.PollImmediate(kconst.APICallRetryInterval, kconst.DefaultControlPlaneTimeout, checkReady); err != nil {
 		return errors.Wrapf(err, "wait pod Ready")
 	}
diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index dea2a80b81..34194b9540 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -36,6 +36,7 @@ import (
 	"github.com/docker/machine/libmachine"
 	"github.com/docker/machine/libmachine/state"
 	"github.com/pkg/errors"
+	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/klog/v2"
@@ -470,9 +471,9 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time
 		return nil
 	}
 
-	if cfg.VerifyComponents[kverify.OperationalKey] {
-		if err := kverify.WaitOperational(client, kverify.CorePodsList, timeout); err != nil {
-			return errors.Wrap(err, "waiting for operational status")
+	if cfg.VerifyComponents[kverify.ExtraKey] {
+		if err := kverify.WaitExtra(client, kverify.CorePodsList, timeout); err != nil {
+			return errors.Wrap(err, "extra waiting")
 		}
 	}
 
@@ -664,14 +665,32 @@ func (k *Bootstrapper) restartControlPlane(cfg config.ClusterConfig) error {
 		}
 	}
 
-	if cfg.VerifyComponents[kverify.OperationalKey] {
+	if cfg.VerifyComponents[kverify.ExtraKey] {
 		// after kubelet is restarted (with 'kubeadm init phase kubelet-start' above),
-		// it appears to be immediately Ready as are all kube-system pods
-		// then (after ~10sec) it realises it has some changes to apply, implying also pods restarts
-		// so we wait for kubelet to initialise itself...
-		time.Sleep(10 * time.Second)
-		if err := kverify.WaitOperational(client, kverify.CorePodsList, kconst.DefaultControlPlaneTimeout); err != nil {
-			return errors.Wrap(err, "operational status")
+		// it appears as to be immediately Ready as well as all kube-system pods,
+		// then (after ~10sec) it realises it has some changes to apply, implying also pods restarts,
+		// and by that time we would exit completely, so we wait until kubelet begins restarting pods
+		klog.Info("waiting for restarted kubelet to initialise ...")
+		start := time.Now()
+		wait := func() error {
+			pods, err := client.CoreV1().Pods("kube-system").List(meta.ListOptions{})
+			if err != nil {
+				return err
+			}
+			for _, pod := range pods.Items {
+				if pod.Labels["tier"] == "control-plane" {
+					if ready, _ := kverify.IsPodReady(&pod); !ready {
+						return nil
+					}
+				}
+			}
+			return fmt.Errorf("kubelet not initialised")
+		}
+		_ = retry.Expo(wait, 250*time.Millisecond, 1*time.Minute)
+		klog.Infof("kubelet initialised")
+		klog.Infof("duration metric: took %s waiting for restarted kubelet to initialise ...", time.Since(start))
+		if err := kverify.WaitExtra(client, kverify.CorePodsList, kconst.DefaultControlPlaneTimeout); err != nil {
+			return errors.Wrap(err, "extra")
 		}
 	}
 
diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go
index 0f90a5cbc7..d29a8f7916 100644
--- a/test/integration/functional_test.go
+++ b/test/integration/functional_test.go
@@ -448,6 +448,7 @@ func imageID(image string) string {
 }
 
 // validateComponentHealth asserts that all Kubernetes components are healthy
+// note: it expects all components to be Ready, so it makes sense to run it close after only those tests that include '--wait=all' start flag (ie, with extra wait)
 func validateComponentHealth(ctx context.Context, t *testing.T, profile string) {
 	defer PostMortemLogs(t, profile)
 

From 0147378459e672cd1d5bb1137a8a51bce92a0f73 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Tue, 16 Feb 2021 14:28:58 -0800
Subject: [PATCH 047/114] boilerplate

---
 hack/jenkins/kicbase_auto_build.sh | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/hack/jenkins/kicbase_auto_build.sh b/hack/jenkins/kicbase_auto_build.sh
index 51ab4932bc..18a8bf4d87 100755
--- a/hack/jenkins/kicbase_auto_build.sh
+++ b/hack/jenkins/kicbase_auto_build.sh
@@ -1,5 +1,19 @@
 #!/bin/bash
 
+# Copyright 2021 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 ./hack/jenkins/installers/check_install_docker.sh
 yes|gcloud auth configure-docker
 now=$(date +%s)

From 1c8753d1f861f4db4b6286bd47f58f1b122d9dd1 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Tue, 16 Feb 2021 14:42:30 -0800
Subject: [PATCH 048/114] Set --interactive=false flag on skaffold test

This way we won't be sending metrics from integration tests to skaffold
---
 test/integration/skaffold_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/integration/skaffold_test.go b/test/integration/skaffold_test.go
index 7695f9dc77..0365b33177 100644
--- a/test/integration/skaffold_test.go
+++ b/test/integration/skaffold_test.go
@@ -95,7 +95,7 @@ func TestSkaffold(t *testing.T) {
 	}()
 
 	// make sure "skaffold run" exits without failure
-	cmd := exec.CommandContext(ctx, tf.Name(), "run", "--minikube-profile", profile, "--kube-context", profile, "--status-check=true", "--port-forward=false")
+	cmd := exec.CommandContext(ctx, tf.Name(), "run", "--minikube-profile", profile, "--kube-context", profile, "--status-check=true", "--port-forward=false", "--interactive=false")
 	cmd.Dir = "testdata/skaffold"
 	rr, err = Run(t, cmd)
 	if err != nil {

From 8099f7514aeeda3e9e14ce5790c54018a8cb463e Mon Sep 17 00:00:00 2001
From: Predrag Rogic <prezha@users.noreply.github.com>
Date: Tue, 16 Feb 2021 22:56:04 +0000
Subject: [PATCH 049/114] cleanup: remove unused code, make internal-only func
 private

---
 .../bootstrapper/bsutil/kverify/pod_ready.go  | 48 ++-----------------
 1 file changed, 3 insertions(+), 45 deletions(-)

diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/pod_ready.go b/pkg/minikube/bootstrapper/bsutil/kverify/pod_ready.go
index 125a2e5a6f..982cee5461 100644
--- a/pkg/minikube/bootstrapper/bsutil/kverify/pod_ready.go
+++ b/pkg/minikube/bootstrapper/bsutil/kverify/pod_ready.go
@@ -41,7 +41,7 @@ func WaitExtra(cs *kubernetes.Clientset, labels []string, timeout time.Duration)
 
 	var errs []string
 	for _, label := range labels {
-		if err := WaitForPodReadyByLabel(cs, label, "kube-system", timeout); err != nil {
+		if err := waitForPodReadyByLabel(cs, label, "kube-system", timeout); err != nil {
 			errs = append(errs, fmt.Sprintf("%q: %q", label, err.Error()))
 		}
 	}
@@ -52,10 +52,10 @@ func WaitExtra(cs *kubernetes.Clientset, labels []string, timeout time.Duration)
 	return nil
 }
 
-// WaitForPodReadyByLabel waits for pod with label ([key:]val) in a namespace to be in Ready condition.
+// waitForPodReadyByLabel waits for pod with label ([key:]val) in a namespace to be in Ready condition.
 // If namespace is not provided, it defaults to "kube-system".
 // If label key is not provided, it will try with "component" and "k8s-app".
-func WaitForPodReadyByLabel(cs *kubernetes.Clientset, label, namespace string, timeout time.Duration) error {
+func waitForPodReadyByLabel(cs *kubernetes.Clientset, label, namespace string, timeout time.Duration) error {
 	klog.Infof("waiting %v for pod with %q label in %q namespace to be Ready ...", timeout, label, namespace)
 	start := time.Now()
 	defer func() {
@@ -116,48 +116,6 @@ func WaitForPodReadyByLabel(cs *kubernetes.Clientset, label, namespace string, t
 	return nil
 }
 
-// WaitForPodReadyByName waits for pod with name in a namespace to be in Ready condition.
-// If namespace is not provided, it defaults to "kube-system".
-func WaitForPodReadyByName(cs *kubernetes.Clientset, name, namespace string, timeout time.Duration) error {
-	klog.Infof("waiting %v for pod %q in %q namespace to be Ready ...", timeout, name, namespace)
-	start := time.Now()
-	defer func() {
-		klog.Infof("duration metric: took %v to run WaitForPodReadyByName for pod %q in %q namespace ...", time.Since(start), name, namespace)
-	}()
-
-	if namespace == "" {
-		namespace = "kube-system"
-	}
-
-	lap := time.Now()
-	checkReady := func() (bool, error) {
-		if time.Since(start) > timeout {
-			return false, fmt.Errorf("wait for pod %q in %q namespace to be Ready timed out", name, namespace)
-		}
-		pod, err := cs.CoreV1().Pods(namespace).Get(name, meta.GetOptions{})
-		if err != nil {
-			klog.Infof("error getting pod %q in %q namespace, will retry: %v", name, namespace, err)
-			return false, nil
-		}
-		ready, reason := IsPodReady(pod)
-		if ready {
-			klog.Info(reason)
-			return true, nil
-		}
-		// reduce log spam
-		if time.Since(lap) > (1 * time.Second) {
-			klog.Info(reason)
-			lap = time.Now()
-		}
-		return false, nil
-	}
-	if err := wait.PollImmediate(kconst.APICallRetryInterval, kconst.DefaultControlPlaneTimeout, checkReady); err != nil {
-		return errors.Wrapf(err, "wait pod Ready")
-	}
-
-	return nil
-}
-
 // IsPodReady returns if pod is Ready and verbose reason.
 func IsPodReady(pod *core.Pod) (ready bool, reason string) {
 	if pod.Status.Phase != core.PodRunning {

From e3cd93aa1a01c08fe1a1ad75617faca1b11bc1b3 Mon Sep 17 00:00:00 2001
From: Kent Iso <kentaiso@google.com>
Date: Fri, 12 Feb 2021 22:26:40 +0900
Subject: [PATCH 050/114] Change prepareVolume func to add container name for
 preload side car

---
 pkg/drivers/kic/oci/oci.go     | 2 +-
 pkg/drivers/kic/oci/volumes.go | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go
index 27f7a49a73..ff829caf65 100644
--- a/pkg/drivers/kic/oci/oci.go
+++ b/pkg/drivers/kic/oci/oci.go
@@ -100,7 +100,7 @@ func PrepareContainerNode(p CreateParams) error {
 		return errors.Wrapf(err, "creating volume for %s container", p.Name)
 	}
 	klog.Infof("Successfully created a %s volume %s", p.OCIBinary, p.Name)
-	if err := prepareVolume(p.OCIBinary, p.Image, p.Name); err != nil {
+	if err := prepareVolumeSideCar(p.OCIBinary, p.Image, p.Name); err != nil {
 		return errors.Wrapf(err, "preparing volume for %s container", p.Name)
 	}
 	klog.Infof("Successfully prepared a %s volume %s", p.OCIBinary, p.Name)
diff --git a/pkg/drivers/kic/oci/volumes.go b/pkg/drivers/kic/oci/volumes.go
index 7c5260cd82..d6698f3a92 100644
--- a/pkg/drivers/kic/oci/volumes.go
+++ b/pkg/drivers/kic/oci/volumes.go
@@ -151,9 +151,9 @@ func createVolume(ociBin string, profile string, nodeName string) error {
 	return nil
 }
 
-// prepareVolume will copy the initial content of the mount point by starting a container to check the expected content
-func prepareVolume(ociBin string, imageName string, nodeName string) error {
-	cmdArgs := []string{"run", "--rm", "--entrypoint", "/usr/bin/test"}
+// prepareVolumeSideCar will copy the initial content of the mount point by starting a temporary container to check the expected content
+func prepareVolumeSideCar(ociBin string, imageName string, nodeName string) error {
+	cmdArgs := []string{"run", "--rm", "--name", fmt.Sprintf("%s-preload-sidecar", nodeName), "--entrypoint", "/usr/bin/test"}
 	cmdArgs = append(cmdArgs, "-v", fmt.Sprintf("%s:/var", nodeName), imageName, "-d", "/var/lib")
 	cmd := exec.Command(ociBin, cmdArgs...)
 	if _, err := runCmd(cmd); err != nil {

From fd58d0fc90717eab49000c21791a6db0902c011c Mon Sep 17 00:00:00 2001
From: Kent Iso <kentaiso@google.com>
Date: Sun, 14 Feb 2021 00:15:19 +0900
Subject: [PATCH 051/114] Add CreatedByLabelKey to make sure that preload side
 car will be target of deletion

---
 pkg/drivers/kic/oci/volumes.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/drivers/kic/oci/volumes.go b/pkg/drivers/kic/oci/volumes.go
index d6698f3a92..0a330c1684 100644
--- a/pkg/drivers/kic/oci/volumes.go
+++ b/pkg/drivers/kic/oci/volumes.go
@@ -153,7 +153,7 @@ func createVolume(ociBin string, profile string, nodeName string) error {
 
 // prepareVolumeSideCar will copy the initial content of the mount point by starting a temporary container to check the expected content
 func prepareVolumeSideCar(ociBin string, imageName string, nodeName string) error {
-	cmdArgs := []string{"run", "--rm", "--name", fmt.Sprintf("%s-preload-sidecar", nodeName), "--entrypoint", "/usr/bin/test"}
+	cmdArgs := []string{"run", "--rm", "--name", fmt.Sprintf("%s-preload-sidecar", nodeName), "--entrypoint", "--label", fmt.Sprintf("%s=%s", CreatedByLabelKey, "true"), "/usr/bin/test"}
 	cmdArgs = append(cmdArgs, "-v", fmt.Sprintf("%s:/var", nodeName), imageName, "-d", "/var/lib")
 	cmd := exec.Command(ociBin, cmdArgs...)
 	if _, err := runCmd(cmd); err != nil {

From 5bffb823b25186aa35686c6b8524e938c9319f63 Mon Sep 17 00:00:00 2001
From: Kent Iso <kentaiso@google.com>
Date: Wed, 17 Feb 2021 10:27:54 +0900
Subject: [PATCH 052/114] Add ProfileLableKey to make sure that preload side
 car will be target of deletion. minikube delete doesn't check
 CreatedByLabelKey

---
 pkg/drivers/kic/oci/volumes.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/drivers/kic/oci/volumes.go b/pkg/drivers/kic/oci/volumes.go
index 0a330c1684..bf32155511 100644
--- a/pkg/drivers/kic/oci/volumes.go
+++ b/pkg/drivers/kic/oci/volumes.go
@@ -153,7 +153,7 @@ func createVolume(ociBin string, profile string, nodeName string) error {
 
 // prepareVolumeSideCar will copy the initial content of the mount point by starting a temporary container to check the expected content
 func prepareVolumeSideCar(ociBin string, imageName string, nodeName string) error {
-	cmdArgs := []string{"run", "--rm", "--name", fmt.Sprintf("%s-preload-sidecar", nodeName), "--entrypoint", "--label", fmt.Sprintf("%s=%s", CreatedByLabelKey, "true"), "/usr/bin/test"}
+	cmdArgs := []string{"run", "--rm", "--name", fmt.Sprintf("%s-preload-sidecar", nodeName), "--label", fmt.Sprintf("%s=%s", CreatedByLabelKey, "true"), "--label", fmt.Sprintf("%s=%s", ProfileLabelKey, nodeName), "--entrypoint", "/usr/bin/test"}
 	cmdArgs = append(cmdArgs, "-v", fmt.Sprintf("%s:/var", nodeName), imageName, "-d", "/var/lib")
 	cmd := exec.Command(ociBin, cmdArgs...)
 	if _, err := runCmd(cmd); err != nil {

From 5d642c5311e2d4469e245048de58004470c38fba Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Tue, 16 Feb 2021 17:39:46 -0800
Subject: [PATCH 053/114] fix up escaping nonsense

---
 hack/jenkins/kicbase_auto_build.sh | 13 +++----------
 1 file changed, 3 insertions(+), 10 deletions(-)

diff --git a/hack/jenkins/kicbase_auto_build.sh b/hack/jenkins/kicbase_auto_build.sh
index 18a8bf4d87..2920120639 100755
--- a/hack/jenkins/kicbase_auto_build.sh
+++ b/hack/jenkins/kicbase_auto_build.sh
@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+set -x
+
 ./hack/jenkins/installers/check_install_docker.sh
 yes|gcloud auth configure-docker
 now=$(date +%s)
@@ -26,16 +28,7 @@ docker pull $KICBASE_IMAGE_REGISTRIES
 fullsha=$(docker inspect --format='{{index .RepoDigests 0}}' $KICBASE_IMAGE_REGISTRIES)
 sha=$(echo ${fullsha} | cut -d ":" -f 2)
 
-message="Hi ${ghprbPullAuthorLoginMention},
-
-A new kicbase image is available, please update your PR with the new tag and SHA.
-In pkg/drivers/kic/types.go:
-
-	// Version is the current version of kic
-	Version = \"${KICBASE_IMAGE_REGISTRIES}\"
-	// SHA of the kic base image
-	baseImageSHA = \"${sha}\"
-"
+message="Hi ${ghprbPullAuthorLoginMention},\\n\\nA new kicbase image is available, please update your PR with the new tag and SHA.\\nIn pkg/drivers/kic/types.go:\\n\\n\\t// Version is the current version of kic\\n\\tVersion = \\\"${KICBASE_IMAGE_REGISTRIES}\\\"\\n\\t// SHA of the kic base image\\n\\tbaseImageSHA = \\\"${sha}\\\"\\n"
 
 curl -s -H "Authorization: token ${access_token}" \
 	 -H "Accept: application/vnd.github.v3+json" \

From bba35e25f56378f3d9322ca83391b742a3e70acd Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Tue, 16 Feb 2021 18:08:45 -0800
Subject: [PATCH 054/114] display proper image tag

---
 hack/jenkins/kicbase_auto_build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hack/jenkins/kicbase_auto_build.sh b/hack/jenkins/kicbase_auto_build.sh
index 2920120639..f679482a98 100755
--- a/hack/jenkins/kicbase_auto_build.sh
+++ b/hack/jenkins/kicbase_auto_build.sh
@@ -28,7 +28,7 @@ docker pull $KICBASE_IMAGE_REGISTRIES
 fullsha=$(docker inspect --format='{{index .RepoDigests 0}}' $KICBASE_IMAGE_REGISTRIES)
 sha=$(echo ${fullsha} | cut -d ":" -f 2)
 
-message="Hi ${ghprbPullAuthorLoginMention},\\n\\nA new kicbase image is available, please update your PR with the new tag and SHA.\\nIn pkg/drivers/kic/types.go:\\n\\n\\t// Version is the current version of kic\\n\\tVersion = \\\"${KICBASE_IMAGE_REGISTRIES}\\\"\\n\\t// SHA of the kic base image\\n\\tbaseImageSHA = \\\"${sha}\\\"\\n"
+message="Hi ${ghprbPullAuthorLoginMention},\\n\\nA new kicbase image is available, please update your PR with the new tag and SHA.\\nIn pkg/drivers/kic/types.go:\\n\\n\\t// Version is the current version of kic\\n\\tVersion = \\\"${KIC_VERSION}\\\"\\n\\t// SHA of the kic base image\\n\\tbaseImageSHA = \\\"${sha}\\\"\\n"
 
 curl -s -H "Authorization: token ${access_token}" \
 	 -H "Accept: application/vnd.github.v3+json" \

From 5c8ea8dc5b959edc140909a7847e841b3cf8e16f Mon Sep 17 00:00:00 2001
From: Predrag Rogic <prezha@users.noreply.github.com>
Date: Sun, 14 Feb 2021 17:56:26 +0000
Subject: [PATCH 055/114] improve kvm network delete/cleanup

---
 pkg/drivers/kvm/network.go | 54 ++++++++------------------------------
 1 file changed, 11 insertions(+), 43 deletions(-)

diff --git a/pkg/drivers/kvm/network.go b/pkg/drivers/kvm/network.go
index d60ae20fd3..d7e06707b1 100644
--- a/pkg/drivers/kvm/network.go
+++ b/pkg/drivers/kvm/network.go
@@ -213,58 +213,26 @@ func (d *Driver) deleteNetwork() error {
 
 	// when we reach this point, it means it is safe to delete the network
 
-	// cannot destroy an inactive network - try to activate it first
-	log.Debugf("Trying to reactivate network %s first (if needed)...", d.PrivateNetwork)
-	activate := func() error {
+	log.Debugf("Trying to delete network %s...", d.PrivateNetwork)
+	destroy := func() error {
 		active, err := network.IsActive()
-		if err == nil && active {
-			return nil
-		}
 		if err != nil {
 			return err
 		}
-		// inactive, try to activate
-		if err := network.Create(); err != nil {
-			return err
+		if active {
+			log.Debugf("Destroying active network %s", d.PrivateNetwork)
+			err := network.Destroy()
+			if err != nil {
+				return err
+			}
 		}
-		return errors.Errorf("needs confirmation") // confirm in the next cycle
-	}
-	if err := retry.Local(activate, 10*time.Second); err != nil {
-		log.Debugf("Reactivating network %s failed, will continue anyway...", d.PrivateNetwork)
-	}
-
-	log.Debugf("Trying to destroy network %s...", d.PrivateNetwork)
-	destroy := func() error {
-		if err := network.Destroy(); err != nil {
-			return err
-		}
-		active, err := network.IsActive()
-		if err == nil && !active {
-			return nil
-		}
-		return errors.Errorf("retrying %v", err)
+		log.Debugf("Undefining inactive network %s", d.PrivateNetwork)
+		return network.Undefine()
 	}
 	if err := retry.Local(destroy, 10*time.Second); err != nil {
 		return errors.Wrap(err, "destroying network")
 	}
-
-	log.Debugf("Trying to undefine network %s...", d.PrivateNetwork)
-	undefine := func() error {
-		if err := network.Undefine(); err != nil {
-			return err
-		}
-		netp, err := conn.LookupNetworkByName(d.PrivateNetwork)
-		if netp != nil {
-			_ = netp.Free()
-		}
-		if lvErr(err).Code == libvirt.ERR_NO_NETWORK {
-			return nil
-		}
-		return errors.Errorf("retrying %v", err)
-	}
-	if err := retry.Local(undefine, 10*time.Second); err != nil {
-		return errors.Wrap(err, "undefining network")
-	}
+	log.Debugf("Network %s deleted", d.PrivateNetwork)
 
 	return nil
 }

From 1f50c63adb23f8de0339d32e933419379a7cac31 Mon Sep 17 00:00:00 2001
From: Predrag Rogic <prezha@users.noreply.github.com>
Date: Sun, 14 Feb 2021 18:12:15 +0000
Subject: [PATCH 056/114] fix wording

---
 pkg/drivers/kvm/network.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/drivers/kvm/network.go b/pkg/drivers/kvm/network.go
index d7e06707b1..a0f41b2f0a 100644
--- a/pkg/drivers/kvm/network.go
+++ b/pkg/drivers/kvm/network.go
@@ -230,7 +230,7 @@ func (d *Driver) deleteNetwork() error {
 		return network.Undefine()
 	}
 	if err := retry.Local(destroy, 10*time.Second); err != nil {
-		return errors.Wrap(err, "destroying network")
+		return errors.Wrap(err, "deleting network")
 	}
 	log.Debugf("Network %s deleted", d.PrivateNetwork)
 

From fc07cfd3953fe560909c60b4b7f5d51fb66985f0 Mon Sep 17 00:00:00 2001
From: Predrag Rogic <prezha@users.noreply.github.com>
Date: Sun, 14 Feb 2021 19:06:54 +0000
Subject: [PATCH 057/114] fix wording

---
 pkg/drivers/kvm/network.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkg/drivers/kvm/network.go b/pkg/drivers/kvm/network.go
index a0f41b2f0a..0c056469ea 100644
--- a/pkg/drivers/kvm/network.go
+++ b/pkg/drivers/kvm/network.go
@@ -214,7 +214,7 @@ func (d *Driver) deleteNetwork() error {
 	// when we reach this point, it means it is safe to delete the network
 
 	log.Debugf("Trying to delete network %s...", d.PrivateNetwork)
-	destroy := func() error {
+	delete := func() error {
 		active, err := network.IsActive()
 		if err != nil {
 			return err
@@ -229,7 +229,7 @@ func (d *Driver) deleteNetwork() error {
 		log.Debugf("Undefining inactive network %s", d.PrivateNetwork)
 		return network.Undefine()
 	}
-	if err := retry.Local(destroy, 10*time.Second); err != nil {
+	if err := retry.Local(delete, 10*time.Second); err != nil {
 		return errors.Wrap(err, "deleting network")
 	}
 	log.Debugf("Network %s deleted", d.PrivateNetwork)

From a5f75b4a29b55eef9108e230f7391b57fe8155bd Mon Sep 17 00:00:00 2001
From: Predrag Rogic <prezha@users.noreply.github.com>
Date: Mon, 15 Feb 2021 16:58:40 +0000
Subject: [PATCH 058/114] fix wording

---
 pkg/drivers/kvm/network.go | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/pkg/drivers/kvm/network.go b/pkg/drivers/kvm/network.go
index 0c056469ea..1377945078 100644
--- a/pkg/drivers/kvm/network.go
+++ b/pkg/drivers/kvm/network.go
@@ -201,7 +201,7 @@ func (d *Driver) deleteNetwork() error {
 			log.Warnf("Network %s does not exist. Skipping deletion", d.PrivateNetwork)
 			return nil
 		}
-		return errors.Wrapf(err, "failed looking for network %s", d.PrivateNetwork)
+		return errors.Wrapf(err, "failed looking up network %s", d.PrivateNetwork)
 	}
 	defer func() { _ = network.Free() }()
 	log.Debugf("Network %s exists", d.PrivateNetwork)
@@ -221,8 +221,7 @@ func (d *Driver) deleteNetwork() error {
 		}
 		if active {
 			log.Debugf("Destroying active network %s", d.PrivateNetwork)
-			err := network.Destroy()
-			if err != nil {
+			if err := network.Destroy(); err != nil {
 				return err
 			}
 		}

From c6446e5fbdeb3ade20c281fb9410a49989aa7f22 Mon Sep 17 00:00:00 2001
From: Predrag Rogic <prezha@users.noreply.github.com>
Date: Thu, 11 Feb 2021 01:22:27 +0000
Subject: [PATCH 059/114] prevent oci try to create overlapping network

---
 pkg/drivers/kic/oci/network_create.go | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/pkg/drivers/kic/oci/network_create.go b/pkg/drivers/kic/oci/network_create.go
index d12e56f09b..9386bd14dd 100644
--- a/pkg/drivers/kic/oci/network_create.go
+++ b/pkg/drivers/kic/oci/network_create.go
@@ -75,6 +75,28 @@ func CreateNetwork(ociBin string, networkName string) (net.IP, error) {
 	// Rather than iterate through all of the valid subnets, give up at 20 to avoid a lengthy user delay for something that is unlikely to work.
 	// will be like 192.168.49.0/24 ,...,192.168.239.0/24
 	for attempts < 20 {
+		// check if subnetAddr overlaps with *any* existing local networks
+		free := true
+		ips, err := net.InterfaceAddrs()
+		if err != nil {
+			klog.Errorf("failed to get list of local network addresses: %v", err)
+		} else {
+			for _, ip := range ips {
+				_, lan, err := net.ParseCIDR(ip.String())
+				if err != nil {
+					klog.Errorf("failed to parse local network address %q: %v", ip, err)
+					continue
+				}
+				if lan.Contains(net.ParseIP(subnetAddr)) {
+					free = false
+					break
+				}
+			}
+		}
+		if !free {
+			continue
+		}
+
 		info.gateway, err = tryCreateDockerNetwork(ociBin, subnetAddr, defaultSubnetMask, info.mtu, networkName)
 		if err == nil {
 			return info.gateway, nil

From d48b0a58e8dfbcdb05f9d9114f8f79eba652586f Mon Sep 17 00:00:00 2001
From: Predrag Rogic <prezha@users.noreply.github.com>
Date: Mon, 15 Feb 2021 00:13:20 +0000
Subject: [PATCH 060/114] add ip conflict auto-resolution also to kvm and
 consolidate

---
 pkg/drivers/kic/oci/network_create.go |  33 +---
 pkg/drivers/kvm/network.go            |  28 +++-
 pkg/minikube/cluster/ip.go            |   6 +-
 pkg/util/network.go                   | 210 ++++++++++++++++++++++++++
 4 files changed, 244 insertions(+), 33 deletions(-)
 create mode 100644 pkg/util/network.go

diff --git a/pkg/drivers/kic/oci/network_create.go b/pkg/drivers/kic/oci/network_create.go
index 9386bd14dd..1fd256c115 100644
--- a/pkg/drivers/kic/oci/network_create.go
+++ b/pkg/drivers/kic/oci/network_create.go
@@ -28,6 +28,7 @@ import (
 	"github.com/pkg/errors"
 
 	"k8s.io/klog/v2"
+	"k8s.io/minikube/pkg/util"
 )
 
 // firstSubnetAddr subnet to be used on first kic cluster
@@ -74,28 +75,12 @@ func CreateNetwork(ociBin string, networkName string) (net.IP, error) {
 	subnetAddr := firstSubnetAddr
 	// Rather than iterate through all of the valid subnets, give up at 20 to avoid a lengthy user delay for something that is unlikely to work.
 	// will be like 192.168.49.0/24 ,...,192.168.239.0/24
-	for attempts < 20 {
-		// check if subnetAddr overlaps with *any* existing local networks
-		free := true
-		ips, err := net.InterfaceAddrs()
+	for attempts < 2 {
+		subnet, err := util.GetFreePrivateNetwork(subnetAddr, 10, 10)
 		if err != nil {
-			klog.Errorf("failed to get list of local network addresses: %v", err)
-		} else {
-			for _, ip := range ips {
-				_, lan, err := net.ParseCIDR(ip.String())
-				if err != nil {
-					klog.Errorf("failed to parse local network address %q: %v", ip, err)
-					continue
-				}
-				if lan.Contains(net.ParseIP(subnetAddr)) {
-					free = false
-					break
-				}
-			}
-		}
-		if !free {
-			continue
+			klog.Warningf("failed to find free private network subnet starting with %q, step %d, tries %d: %v", subnetAddr, 10, 10, err)
 		}
+		subnetAddr = subnet.IP
 
 		info.gateway, err = tryCreateDockerNetwork(ociBin, subnetAddr, defaultSubnetMask, info.mtu, networkName)
 		if err == nil {
@@ -108,14 +93,6 @@ func CreateNetwork(ociBin string, networkName string) (net.IP, error) {
 			return nil, errors.Wrap(err, "un-retryable")
 		}
 		attempts++
-		// Find an open subnet by incrementing the 3rd octet by 10 for each try
-		// 13 times adding 10 firstSubnetAddr "192.168.49.0/24"
-		// at most it will add up to 169 which is still less than max allowed 255
-		// this is large enough to try more and not too small to not try enough
-		// can be tuned in the next iterations
-		newSubnet := net.ParseIP(subnetAddr).To4()
-		newSubnet[2] += byte(9 + attempts)
-		subnetAddr = newSubnet.String()
 	}
 	return info.gateway, fmt.Errorf("failed to create network after 20 attempts")
 }
diff --git a/pkg/drivers/kvm/network.go b/pkg/drivers/kvm/network.go
index d60ae20fd3..ebc2e6c648 100644
--- a/pkg/drivers/kvm/network.go
+++ b/pkg/drivers/kvm/network.go
@@ -31,6 +31,7 @@ import (
 	"github.com/docker/machine/libmachine/log"
 	libvirt "github.com/libvirt/libvirt-go"
 	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/util"
 	"k8s.io/minikube/pkg/util/retry"
 )
 
@@ -38,16 +39,25 @@ import (
 // https://play.golang.org/p/m8TNTtygK0
 const networkTmpl = `
 <network>
-  <name>{{.PrivateNetwork}}</name>
+  <name>{{.Name}}</name>
   <dns enable='no'/>
-  <ip address='192.168.39.1' netmask='255.255.255.0'>
+  <ip address='{{.Network.Gateway}}' netmask='{{.Network.Netmask}}'>
     <dhcp>
-      <range start='192.168.39.2' end='192.168.39.254'/>
+      <range start='{{.Network.ClientMin}}' end='{{.Network.ClientMax}}'/>
     </dhcp>
   </ip>
 </network>
 `
 
+type kvmNetwork struct {
+	Name string
+	util.Network
+}
+
+// firstSubnetAddr is starting subnet to try for new KVM cluster,
+// avoiding possible conflict with other local networks by further incrementing it up to 20 times by 10.
+const firstSubnetAddr = "192.168.39.0"
+
 // setupNetwork ensures that the network with `name` is started (active)
 // and has the autostart feature set.
 func setupNetwork(conn *libvirt.Connect, name string) error {
@@ -145,10 +155,19 @@ func (d *Driver) createNetwork() error {
 	// Only create the private network if it does not already exist
 	netp, err := conn.LookupNetworkByName(d.PrivateNetwork)
 	if err != nil {
+		subnet, err := util.GetFreePrivateNetwork(firstSubnetAddr, 10, 20)
+		if err != nil {
+			return errors.Wrapf(err, "failed to find free private network subnet starting with %q, step: %d, tries:%d", firstSubnetAddr, 10, 20)
+		}
+		tryNet := kvmNetwork{
+			Name:    d.PrivateNetwork,
+			Network: *subnet,
+		}
+
 		// create the XML for the private network from our networkTmpl
 		tmpl := template.Must(template.New("network").Parse(networkTmpl))
 		var networkXML bytes.Buffer
-		if err := tmpl.Execute(&networkXML, d); err != nil {
+		if err := tmpl.Execute(&networkXML, tryNet); err != nil {
 			return errors.Wrap(err, "executing network template")
 		}
 
@@ -173,6 +192,7 @@ func (d *Driver) createNetwork() error {
 		if err := retry.Local(create, 10*time.Second); err != nil {
 			return errors.Wrapf(err, "creating network %s", d.PrivateNetwork)
 		}
+		log.Debugf("Network %s created", d.PrivateNetwork)
 	}
 	defer func() {
 		if netp != nil {
diff --git a/pkg/minikube/cluster/ip.go b/pkg/minikube/cluster/ip.go
index 54a715c3bd..fc5adeb267 100644
--- a/pkg/minikube/cluster/ip.go
+++ b/pkg/minikube/cluster/ip.go
@@ -47,7 +47,11 @@ func HostIP(host *host.Host, clusterName string) (net.IP, error) {
 		}
 		return net.ParseIP(ip), nil
 	case driver.KVM2:
-		return net.ParseIP("192.168.39.1"), nil
+		ip, err := host.Driver.GetIP()
+		if err != nil {
+			return []byte{}, errors.Wrap(err, "Error getting VM/Host IP address")
+		}
+		return net.ParseIP(ip), nil
 	case driver.HyperV:
 		v := reflect.ValueOf(host.Driver).Elem()
 		var hypervVirtualSwitch string
diff --git a/pkg/util/network.go b/pkg/util/network.go
new file mode 100644
index 0000000000..8a9c5a98b8
--- /dev/null
+++ b/pkg/util/network.go
@@ -0,0 +1,210 @@
+/*
+Copyright 2021 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"encoding/binary"
+	"fmt"
+	"net"
+
+	"github.com/pkg/errors"
+	"k8s.io/klog/v2"
+)
+
+var (
+	// valid private networks (RFC1918)
+	privateNetworks = []net.IPNet{
+		// 10.0.0.0/8
+		{
+			IP:   []byte{10, 0, 0, 0},
+			Mask: []byte{255, 0, 0, 0},
+		},
+		// 172.16.0.0/12
+		{
+			IP:   []byte{172, 16, 0, 0},
+			Mask: []byte{255, 240, 0, 0},
+		},
+		// 192.168.0.0/16
+		{
+			IP:   []byte{192, 168, 0, 0},
+			Mask: []byte{255, 255, 0, 0},
+		},
+	}
+)
+
+// Network contains main network parameters.
+type Network struct {
+	IP        string // IP address of the network
+	Netmask   string // form: 4-byte ('a.b.c.d')
+	CIDR      string // form: CIDR
+	Gateway   string // first IP address (assumed, not checked !)
+	ClientMin string // second IP address
+	ClientMax string // last IP address before broadcastS
+	Broadcast string // last IP address
+	Interface
+}
+
+// Interface contains main network interface parameters.
+type Interface struct {
+	IfaceName string
+	IfaceIPv4 string
+	IfaceMTU  int
+	IfaceMAC  string
+}
+
+// GetNetworkFrom initialises IPv4 network struct from given address.
+// address can be single address (like "192.168.17.42"), network address (like "192.168.17.0"), or in cidr form (like "192.168.17.42/24 or "192.168.17.0/24").
+// If addr is valid existsing interface address, network struct will also contain info about the respective interface.
+func GetNetworkFrom(addr string) (*Network, error) {
+	n := &Network{}
+
+	// extract ip from addr
+	ip, network, err := net.ParseCIDR(addr)
+	if err != nil {
+		ip = net.ParseIP(addr)
+		if ip == nil {
+			return nil, errors.Wrapf(err, "parsing address %q", addr)
+		}
+	}
+
+	// check local interfaces
+	ifaces, _ := net.Interfaces()
+	for _, iface := range ifaces {
+		ifAddrs, err := iface.Addrs()
+		if err != nil {
+			return nil, errors.Wrapf(err, "listing addresses of network interface %+v", iface)
+		}
+		for _, ifAddr := range ifAddrs {
+			ifip, lan, err := net.ParseCIDR(ifAddr.String())
+			if err != nil {
+				return nil, errors.Wrapf(err, "parsing address of network iface %+v", ifAddr)
+			}
+			if lan.Contains(ip) {
+				n.IfaceName = iface.Name
+				n.IfaceIPv4 = ifip.To4().String()
+				n.IfaceMTU = iface.MTU
+				n.IfaceMAC = iface.HardwareAddr.String()
+				n.Gateway = n.IfaceIPv4
+				network = lan
+				break
+			}
+		}
+	}
+
+	if network == nil {
+		ipnet := &net.IPNet{
+			IP:   ip,
+			Mask: ip.DefaultMask(), // assume default network mask
+		}
+		_, network, err = net.ParseCIDR(ipnet.String())
+		if err != nil {
+			return nil, errors.Wrapf(err, "determining network address from %q", addr)
+		}
+	}
+
+	n.IP = network.IP.String()
+	n.Netmask = net.IP(network.Mask).String() // form: 4-byte ('a.b.c.d')
+	n.CIDR = network.String()
+
+	networkIP := binary.BigEndian.Uint32(network.IP)                      // IP address of the network
+	networkMask := binary.BigEndian.Uint32(network.Mask)                  // network mask
+	broadcastIP := (networkIP & networkMask) | (networkMask ^ 0xffffffff) // last network IP address
+
+	broadcast := make(net.IP, 4)
+	binary.BigEndian.PutUint32(broadcast, broadcastIP)
+	n.Broadcast = broadcast.String()
+
+	gateway := net.ParseIP(n.Gateway).To4() // has to be converted to 4-byte representation!
+	if gateway == nil {
+		gateway = make(net.IP, 4)
+		binary.BigEndian.PutUint32(gateway, networkIP+1) // assume first network IP address
+		n.Gateway = gateway.String()
+	}
+	gatewayIP := binary.BigEndian.Uint32(gateway)
+
+	min := make(net.IP, 4)
+	binary.BigEndian.PutUint32(min, gatewayIP+1) // clients-from: first network IP address after gateway
+	n.ClientMin = min.String()
+
+	max := make(net.IP, 4)
+	binary.BigEndian.PutUint32(max, broadcastIP-1) // clients-from: last network IP address before broadcast
+	n.ClientMax = max.String()
+
+	return n, nil
+}
+
+// IsNetworkTaken returns if local network subnet exists and any error occurred.
+// If will return false in case of an error.
+func IsNetworkTaken(subnet string) (bool, error) {
+	ips, err := net.InterfaceAddrs()
+	if err != nil {
+		return false, errors.Wrap(err, "listing local networks")
+	}
+	for _, ip := range ips {
+		_, lan, err := net.ParseCIDR(ip.String())
+		if err != nil {
+			return false, errors.Wrapf(err, "parsing network iface address %q", ip)
+		}
+		if lan.Contains(net.ParseIP(subnet)) {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+// IsNetworkPrivate returns if subnet is a private network.
+func IsNetworkPrivate(subnet string) bool {
+	for _, ipnet := range privateNetworks {
+		if ipnet.Contains(net.ParseIP(subnet)) {
+			return true
+		}
+	}
+	return false
+}
+
+// GetFreePrivateNetwork will try to find an free private network starting with subnet, incrementing it in steps up to number of tries.
+func GetFreePrivateNetwork(subnet string, step, tries int) (*Network, error) {
+	for try := 0; try < tries; try++ {
+		n, err := GetNetworkFrom(subnet)
+		if err != nil {
+			return nil, err
+		}
+		subnet = n.IP
+		if IsNetworkPrivate(subnet) {
+			taken, err := IsNetworkTaken(subnet)
+			if err != nil {
+				return nil, err
+			}
+			if !taken {
+				klog.Infof("using free private subnet %s: %+v", n.CIDR, n)
+				return n, nil
+			}
+			klog.Infof("skipping subnet %s that is taken: %+v", n.CIDR, n)
+		} else {
+			klog.Infof("skipping subnet %s that is not private", n.CIDR)
+		}
+		ones, _ := net.ParseIP(n.IP).DefaultMask().Size()
+		newSubnet := net.ParseIP(subnet).To4()
+		if ones <= 16 {
+			newSubnet[1] += byte(step)
+		} else {
+			newSubnet[2] += byte(step)
+		}
+		subnet = newSubnet.String()
+	}
+	return nil, fmt.Errorf("no free private network subnets found with given parameters")
+}

From 4018ee3682672c57cd1cf41f3570aa77d08c85bd Mon Sep 17 00:00:00 2001
From: Predrag Rogic <prezha@users.noreply.github.com>
Date: Mon, 15 Feb 2021 16:00:50 +0000
Subject: [PATCH 061/114] move to separate network package and fix naming

---
 pkg/drivers/kic/oci/network_create.go |  4 +--
 pkg/drivers/kvm/network.go            | 16 ++++++----
 pkg/{util => network}/network.go      | 46 +++++++++++++--------------
 3 files changed, 34 insertions(+), 32 deletions(-)
 rename pkg/{util => network}/network.go (82%)

diff --git a/pkg/drivers/kic/oci/network_create.go b/pkg/drivers/kic/oci/network_create.go
index 1fd256c115..3fbb9a6500 100644
--- a/pkg/drivers/kic/oci/network_create.go
+++ b/pkg/drivers/kic/oci/network_create.go
@@ -28,7 +28,7 @@ import (
 	"github.com/pkg/errors"
 
 	"k8s.io/klog/v2"
-	"k8s.io/minikube/pkg/util"
+	"k8s.io/minikube/pkg/network"
 )
 
 // firstSubnetAddr subnet to be used on first kic cluster
@@ -76,7 +76,7 @@ func CreateNetwork(ociBin string, networkName string) (net.IP, error) {
 	// Rather than iterate through all of the valid subnets, give up at 20 to avoid a lengthy user delay for something that is unlikely to work.
 	// will be like 192.168.49.0/24 ,...,192.168.239.0/24
 	for attempts < 2 {
-		subnet, err := util.GetFreePrivateNetwork(subnetAddr, 10, 10)
+		subnet, err := network.FreeSubnet(subnetAddr, 10, 10)
 		if err != nil {
 			klog.Warningf("failed to find free private network subnet starting with %q, step %d, tries %d: %v", subnetAddr, 10, 10, err)
 		}
diff --git a/pkg/drivers/kvm/network.go b/pkg/drivers/kvm/network.go
index ebc2e6c648..00b582e196 100644
--- a/pkg/drivers/kvm/network.go
+++ b/pkg/drivers/kvm/network.go
@@ -31,7 +31,7 @@ import (
 	"github.com/docker/machine/libmachine/log"
 	libvirt "github.com/libvirt/libvirt-go"
 	"github.com/pkg/errors"
-	"k8s.io/minikube/pkg/util"
+	"k8s.io/minikube/pkg/network"
 	"k8s.io/minikube/pkg/util/retry"
 )
 
@@ -41,17 +41,19 @@ const networkTmpl = `
 <network>
   <name>{{.Name}}</name>
   <dns enable='no'/>
-  <ip address='{{.Network.Gateway}}' netmask='{{.Network.Netmask}}'>
+  {{with .Parameters}}
+  <ip address='{{.Gateway}}' netmask='{{.Netmask}}'>
     <dhcp>
-      <range start='{{.Network.ClientMin}}' end='{{.Network.ClientMax}}'/>
+      <range start='{{.ClientMin}}' end='{{.ClientMax}}'/>
     </dhcp>
   </ip>
+  {{end}}
 </network>
 `
 
 type kvmNetwork struct {
 	Name string
-	util.Network
+	network.Parameters
 }
 
 // firstSubnetAddr is starting subnet to try for new KVM cluster,
@@ -155,13 +157,13 @@ func (d *Driver) createNetwork() error {
 	// Only create the private network if it does not already exist
 	netp, err := conn.LookupNetworkByName(d.PrivateNetwork)
 	if err != nil {
-		subnet, err := util.GetFreePrivateNetwork(firstSubnetAddr, 10, 20)
+		subnet, err := network.FreeSubnet(firstSubnetAddr, 10, 20)
 		if err != nil {
 			return errors.Wrapf(err, "failed to find free private network subnet starting with %q, step: %d, tries:%d", firstSubnetAddr, 10, 20)
 		}
 		tryNet := kvmNetwork{
-			Name:    d.PrivateNetwork,
-			Network: *subnet,
+			Name:       d.PrivateNetwork,
+			Parameters: *subnet,
 		}
 
 		// create the XML for the private network from our networkTmpl
diff --git a/pkg/util/network.go b/pkg/network/network.go
similarity index 82%
rename from pkg/util/network.go
rename to pkg/network/network.go
index 8a9c5a98b8..5cd3f1a362 100644
--- a/pkg/util/network.go
+++ b/pkg/network/network.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package util
+package network
 
 import (
 	"encoding/binary"
@@ -26,8 +26,8 @@ import (
 )
 
 var (
-	// valid private networks (RFC1918)
-	privateNetworks = []net.IPNet{
+	// valid private network subnets (RFC1918)
+	privateSubnets = []net.IPNet{
 		// 10.0.0.0/8
 		{
 			IP:   []byte{10, 0, 0, 0},
@@ -46,8 +46,8 @@ var (
 	}
 )
 
-// Network contains main network parameters.
-type Network struct {
+// Parameters contains main network parameters.
+type Parameters struct {
 	IP        string // IP address of the network
 	Netmask   string // form: 4-byte ('a.b.c.d')
 	CIDR      string // form: CIDR
@@ -66,11 +66,11 @@ type Interface struct {
 	IfaceMAC  string
 }
 
-// GetNetworkFrom initialises IPv4 network struct from given address.
+// Inspect initialises IPv4 network parameters struct from given address.
 // address can be single address (like "192.168.17.42"), network address (like "192.168.17.0"), or in cidr form (like "192.168.17.42/24 or "192.168.17.0/24").
 // If addr is valid existsing interface address, network struct will also contain info about the respective interface.
-func GetNetworkFrom(addr string) (*Network, error) {
-	n := &Network{}
+func Inspect(addr string) (*Parameters, error) {
+	n := &Parameters{}
 
 	// extract ip from addr
 	ip, network, err := net.ParseCIDR(addr)
@@ -147,9 +147,9 @@ func GetNetworkFrom(addr string) (*Network, error) {
 	return n, nil
 }
 
-// IsNetworkTaken returns if local network subnet exists and any error occurred.
+// IsSubnetTaken returns if local network subnet exists and any error occurred.
 // If will return false in case of an error.
-func IsNetworkTaken(subnet string) (bool, error) {
+func IsSubnetTaken(subnet string) (bool, error) {
 	ips, err := net.InterfaceAddrs()
 	if err != nil {
 		return false, errors.Wrap(err, "listing local networks")
@@ -166,9 +166,9 @@ func IsNetworkTaken(subnet string) (bool, error) {
 	return false, nil
 }
 
-// IsNetworkPrivate returns if subnet is a private network.
-func IsNetworkPrivate(subnet string) bool {
-	for _, ipnet := range privateNetworks {
+// IsSubnetPrivate returns if subnet is a private network.
+func IsSubnetPrivate(subnet string) bool {
+	for _, ipnet := range privateSubnets {
 		if ipnet.Contains(net.ParseIP(subnet)) {
 			return true
 		}
@@ -176,16 +176,16 @@ func IsNetworkPrivate(subnet string) bool {
 	return false
 }
 
-// GetFreePrivateNetwork will try to find an free private network starting with subnet, incrementing it in steps up to number of tries.
-func GetFreePrivateNetwork(subnet string, step, tries int) (*Network, error) {
+// FreeSubnet will try to find free private network beginning with startSubnet, incrementing it in steps up to number of tries.
+func FreeSubnet(startSubnet string, step, tries int) (*Parameters, error) {
 	for try := 0; try < tries; try++ {
-		n, err := GetNetworkFrom(subnet)
+		n, err := Inspect(startSubnet)
 		if err != nil {
 			return nil, err
 		}
-		subnet = n.IP
-		if IsNetworkPrivate(subnet) {
-			taken, err := IsNetworkTaken(subnet)
+		startSubnet = n.IP
+		if IsSubnetPrivate(startSubnet) {
+			taken, err := IsSubnetTaken(startSubnet)
 			if err != nil {
 				return nil, err
 			}
@@ -198,13 +198,13 @@ func GetFreePrivateNetwork(subnet string, step, tries int) (*Network, error) {
 			klog.Infof("skipping subnet %s that is not private", n.CIDR)
 		}
 		ones, _ := net.ParseIP(n.IP).DefaultMask().Size()
-		newSubnet := net.ParseIP(subnet).To4()
+		nextSubnet := net.ParseIP(startSubnet).To4()
 		if ones <= 16 {
-			newSubnet[1] += byte(step)
+			nextSubnet[1] += byte(step)
 		} else {
-			newSubnet[2] += byte(step)
+			nextSubnet[2] += byte(step)
 		}
-		subnet = newSubnet.String()
+		startSubnet = nextSubnet.String()
 	}
 	return nil, fmt.Errorf("no free private network subnets found with given parameters")
 }

From 6d61795352e530bc67364ea74e9774ef149de03c Mon Sep 17 00:00:00 2001
From: Predrag Rogic <prezha@users.noreply.github.com>
Date: Mon, 15 Feb 2021 16:43:37 +0000
Subject: [PATCH 062/114] simplify free subnet query for oci net create

---
 pkg/drivers/kic/oci/network_create.go | 30 ++++++++-------------------
 pkg/drivers/kvm/network.go            |  3 ++-
 pkg/network/network.go                |  2 +-
 3 files changed, 12 insertions(+), 23 deletions(-)

diff --git a/pkg/drivers/kic/oci/network_create.go b/pkg/drivers/kic/oci/network_create.go
index 3fbb9a6500..2766044e05 100644
--- a/pkg/drivers/kic/oci/network_create.go
+++ b/pkg/drivers/kic/oci/network_create.go
@@ -71,30 +71,18 @@ func CreateNetwork(ociBin string, networkName string) (net.IP, error) {
 	if err != nil {
 		klog.Warningf("failed to get mtu information from the %s's default network %q: %v", ociBin, defaultBridgeName, err)
 	}
-	attempts := 0
-	subnetAddr := firstSubnetAddr
 	// Rather than iterate through all of the valid subnets, give up at 20 to avoid a lengthy user delay for something that is unlikely to work.
 	// will be like 192.168.49.0/24 ,...,192.168.239.0/24
-	for attempts < 2 {
-		subnet, err := network.FreeSubnet(subnetAddr, 10, 10)
-		if err != nil {
-			klog.Warningf("failed to find free private network subnet starting with %q, step %d, tries %d: %v", subnetAddr, 10, 10, err)
-		}
-		subnetAddr = subnet.IP
-
-		info.gateway, err = tryCreateDockerNetwork(ociBin, subnetAddr, defaultSubnetMask, info.mtu, networkName)
-		if err == nil {
-			return info.gateway, nil
-		}
-
-		// don't retry if error is not adddress is taken
-		if !(errors.Is(err, ErrNetworkSubnetTaken) || errors.Is(err, ErrNetworkGatewayTaken)) {
-			klog.Errorf("error while trying to create network %v", err)
-			return nil, errors.Wrap(err, "un-retryable")
-		}
-		attempts++
+	subnet, err := network.FreeSubnet(firstSubnetAddr, 10, 20)
+	if err != nil {
+		klog.Errorf("error while trying to create network: %v", err)
+		return nil, errors.Wrap(err, "un-retryable")
 	}
-	return info.gateway, fmt.Errorf("failed to create network after 20 attempts")
+	info.gateway, err = tryCreateDockerNetwork(ociBin, subnet.IP, defaultSubnetMask, info.mtu, networkName)
+	if err != nil {
+		return info.gateway, fmt.Errorf("failed to create network after 20 attempts")
+	}
+	return info.gateway, nil
 }
 
 func tryCreateDockerNetwork(ociBin string, subnetAddr string, subnetMask int, mtu int, name string) (net.IP, error) {
diff --git a/pkg/drivers/kvm/network.go b/pkg/drivers/kvm/network.go
index 00b582e196..289a9d5fb8 100644
--- a/pkg/drivers/kvm/network.go
+++ b/pkg/drivers/kvm/network.go
@@ -159,7 +159,8 @@ func (d *Driver) createNetwork() error {
 	if err != nil {
 		subnet, err := network.FreeSubnet(firstSubnetAddr, 10, 20)
 		if err != nil {
-			return errors.Wrapf(err, "failed to find free private network subnet starting with %q, step: %d, tries:%d", firstSubnetAddr, 10, 20)
+			log.Debugf("error while trying to create network: %v", err)
+			return errors.Wrap(err, "un-retryable")
 		}
 		tryNet := kvmNetwork{
 			Name:       d.PrivateNetwork,
diff --git a/pkg/network/network.go b/pkg/network/network.go
index 5cd3f1a362..049329e6d8 100644
--- a/pkg/network/network.go
+++ b/pkg/network/network.go
@@ -206,5 +206,5 @@ func FreeSubnet(startSubnet string, step, tries int) (*Parameters, error) {
 		}
 		startSubnet = nextSubnet.String()
 	}
-	return nil, fmt.Errorf("no free private network subnets found with given parameters")
+	return nil, fmt.Errorf("no free private network subnets found with given parameters (start: %q, step: %d, tries: %d)", startSubnet, step, tries)
 }

From 7727b855bf9daf1029a3bed7cad4598f8ed904f6 Mon Sep 17 00:00:00 2001
From: Predrag Rogic <prezha@users.noreply.github.com>
Date: Wed, 17 Feb 2021 03:03:41 +0000
Subject: [PATCH 063/114] cleanup: make internal-only funcs private

---
 pkg/network/network.go | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/pkg/network/network.go b/pkg/network/network.go
index 049329e6d8..80933b072d 100644
--- a/pkg/network/network.go
+++ b/pkg/network/network.go
@@ -66,10 +66,10 @@ type Interface struct {
 	IfaceMAC  string
 }
 
-// Inspect initialises IPv4 network parameters struct from given address.
+// inspect initialises IPv4 network parameters struct from given address.
 // address can be single address (like "192.168.17.42"), network address (like "192.168.17.0"), or in cidr form (like "192.168.17.42/24 or "192.168.17.0/24").
 // If addr is valid existsing interface address, network struct will also contain info about the respective interface.
-func Inspect(addr string) (*Parameters, error) {
+func inspect(addr string) (*Parameters, error) {
 	n := &Parameters{}
 
 	// extract ip from addr
@@ -147,9 +147,9 @@ func Inspect(addr string) (*Parameters, error) {
 	return n, nil
 }
 
-// IsSubnetTaken returns if local network subnet exists and any error occurred.
+// isSubnetTaken returns if local network subnet exists and any error occurred.
 // If will return false in case of an error.
-func IsSubnetTaken(subnet string) (bool, error) {
+func isSubnetTaken(subnet string) (bool, error) {
 	ips, err := net.InterfaceAddrs()
 	if err != nil {
 		return false, errors.Wrap(err, "listing local networks")
@@ -166,8 +166,8 @@ func IsSubnetTaken(subnet string) (bool, error) {
 	return false, nil
 }
 
-// IsSubnetPrivate returns if subnet is a private network.
-func IsSubnetPrivate(subnet string) bool {
+// isSubnetPrivate returns if subnet is a private network.
+func isSubnetPrivate(subnet string) bool {
 	for _, ipnet := range privateSubnets {
 		if ipnet.Contains(net.ParseIP(subnet)) {
 			return true
@@ -179,13 +179,13 @@ func IsSubnetPrivate(subnet string) bool {
 // FreeSubnet will try to find free private network beginning with startSubnet, incrementing it in steps up to number of tries.
 func FreeSubnet(startSubnet string, step, tries int) (*Parameters, error) {
 	for try := 0; try < tries; try++ {
-		n, err := Inspect(startSubnet)
+		n, err := inspect(startSubnet)
 		if err != nil {
 			return nil, err
 		}
 		startSubnet = n.IP
-		if IsSubnetPrivate(startSubnet) {
-			taken, err := IsSubnetTaken(startSubnet)
+		if isSubnetPrivate(startSubnet) {
+			taken, err := isSubnetTaken(startSubnet)
 			if err != nil {
 				return nil, err
 			}

From de750b73e2cec591d8fe45c44a7ce6b86a57ebd3 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Tue, 16 Feb 2021 19:25:22 -0800
Subject: [PATCH 064/114] timestamp checks

---
 hack/jenkins/kicbase_auto_build.sh | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/hack/jenkins/kicbase_auto_build.sh b/hack/jenkins/kicbase_auto_build.sh
index f679482a98..3587aff14a 100755
--- a/hack/jenkins/kicbase_auto_build.sh
+++ b/hack/jenkins/kicbase_auto_build.sh
@@ -19,9 +19,23 @@ set -x
 ./hack/jenkins/installers/check_install_docker.sh
 yes|gcloud auth configure-docker
 now=$(date +%s)
-export KV=$(egrep "Version =" pkg/drivers/kic/types.go | cut -d \" -f 2 | cut -d "-" -f 1)
-export KIC_VERSION=$KV-$now-$ghprbPullId
-export KICBASE_IMAGE_REGISTRIES=gcr.io/k8s-minikube/kicbase-builds:$KIC_VERSION
+KV=$(egrep "Version =" pkg/drivers/kic/types.go | cut -d \" -f 2 | cut -d "-" -f 1)
+KIC_VERSION=$KV-$now-$ghprbPullId
+KICBASE_IMAGE_REGISTRIES=gcr.io/k8s-minikube/kicbase-builds:$KIC_VERSION
+
+curl -L https://github.com/kubernetes/minikube/raw/master/pkg/drivers/kic/types.go --output types-head.go
+HEAD_KIC_TIMESTAMP=$(egrep "Version =" types-head.go | cut -d \" -f 2 | cut -d "-" -f 2)
+CURRENT_KIC_TS=$(egrep "Version =" pkg/drivers/kic/types.go | cut -d \" -f 2 | cut -d "-" -f 2)
+if [[ $HEAD_KIC_TIMESTAMP != v* ]] && [[ $CURRENT_KIC_TS != v* ]]; then
+	diff=$((CURRENT_KIC_TS-HEAD_KIC_TIMESTAMP))
+	if [ $diff -lt 0 ]; then
+		curl -s -H "Authorization: token ${access_token}" \
+		-H "Accept: application/vnd.github.v3+json" \
+		-X POST -d "{\"body\": \"Hi ${ghprbPullAuthorLoginMention}, your kicbase info is out of date. Please rebase.\"}" "https://api.github.com/repos/kubernetes/minikube/issues/$ghprbPullId/comments"
+		exit 0
+	fi
+fi
+rm types-head.go
 yes|make push-kic-base-image
 
 docker pull $KICBASE_IMAGE_REGISTRIES

From 6934c854513dd66298d1ec8f79b8c0e8488bafcd Mon Sep 17 00:00:00 2001
From: Pablo Caderno <kaderno@gmail.com>
Date: Wed, 17 Feb 2021 18:06:00 +1100
Subject: [PATCH 065/114] Improved insecure registry validation function

---
 cmd/minikube/cmd/start.go | 34 ++++++++++++++++++++++------------
 1 file changed, 22 insertions(+), 12 deletions(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 7810dd0ce7..9fa69225cc 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -27,6 +27,7 @@ import (
 	"os/user"
 	"regexp"
 	"runtime"
+	"strconv"
 	"strings"
 
 	"github.com/blang/semver"
@@ -72,7 +73,7 @@ var (
 	insecureRegistry []string
 	apiServerNames   []string
 	apiServerIPs     []net.IP
-	hostRe           = regexp.MustCompile(`[\w\.-]+`)
+	hostRe           = regexp.MustCompile(`^[^-][\w\.-]+$`)
 )
 
 func init() {
@@ -1066,28 +1067,37 @@ func validateRegistryMirror() {
 }
 
 // This function validates that the --insecure-registry follows one of the following formats:
-// "<ip>:<port>" "<hostname>:<port>" "<network>/<netmask>"
+// "<ip>[:<port>]" "<hostname>[:<port>]" "<network>/<netmask>"
 func validateInsecureRegistry() {
 	if len(insecureRegistry) > 0 {
 		for _, addr := range insecureRegistry {
+			// Remove http or https from registryMirror
+			if strings.HasPrefix(strings.ToLower(addr), "http://") || strings.HasPrefix(strings.ToLower(addr), "https://") {
+				i := strings.Index(addr, "//")
+				addr = addr[i+2:]
+			} else if strings.Contains(addr, "://") || strings.HasSuffix(addr, ":") {
+				exit.Message(reason.Usage, "Sorry, the address provided with the --insecure-registry flag is invalid: {{.addr}}. Expected formtas are: <ip>[:<port>], <hostname>[:<port>] or <network>/<netmask>", out.V{"addr": addr})
+			}
 			hostnameOrIP, port, err := net.SplitHostPort(addr)
 			if err != nil {
 				_, _, err := net.ParseCIDR(addr)
 				if err == nil {
 					continue
 				}
+				hostnameOrIP = addr
 			}
-			if port == "" {
-				exit.Message(reason.Usage, "Sorry, the address provided with the --insecure-registry flag is invalid: {{.addr}}. Expected formtas are: <ip>:<port>, <hostname>:<port> or <network>/<netmask>", out.V{"addr": addr})
+			if !hostRe.MatchString(hostnameOrIP) && net.ParseIP(hostnameOrIP) == nil {
+				//		fmt.Printf("This is not hostname or ip %s", hostnameOrIP)
+				exit.Message(reason.Usage, "Sorry, the address provided with the --insecure-registry flag is invalid: {{.addr}}. Expected formtas are: <ip>[:<port>], <hostname>[:<port>] or <network>/<netmask>", out.V{"addr": addr})
 			}
-			// checks both IPv4 and IPv6
-			ipAddr := net.ParseIP(hostnameOrIP)
-			if ipAddr != nil {
-				continue
-			}
-			isValidHost := hostRe.MatchString(hostnameOrIP)
-			if err != nil || !isValidHost {
-				exit.Message(reason.Usage, "Sorry, the address provided with the --insecure-registry flag is invalid: {{.addr}}. Expected formtas are: <ip>:<port>, <hostname>:<port> or <network>/<netmask>", out.V{"addr": addr})
+			if port != "" {
+				v, err := strconv.Atoi(port)
+				if err != nil {
+					exit.Message(reason.Usage, "Sorry, the address provided with the --insecure-registry flag is invalid: {{.addr}}. Expected formtas are: <ip>[:<port>], <hostname>[:<port>] or <network>/<netmask>", out.V{"addr": addr})
+				}
+				if v < 0 || v > 65535 {
+					exit.Message(reason.Usage, "Sorry, the address provided with the --insecure-registry flag is invalid: {{.addr}}. Expected formtas are: <ip>[:<port>], <hostname>[:<port>] or <network>/<netmask>", out.V{"addr": addr})
+				}
 			}
 		}
 	}

From e8f00d726f68e3e4785215add7f4812bfce57c43 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Wed, 17 Feb 2021 09:23:30 -0800
Subject: [PATCH 066/114] export the correct variables

---
 hack/jenkins/kicbase_auto_build.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/hack/jenkins/kicbase_auto_build.sh b/hack/jenkins/kicbase_auto_build.sh
index 3587aff14a..11eb4dc6d9 100755
--- a/hack/jenkins/kicbase_auto_build.sh
+++ b/hack/jenkins/kicbase_auto_build.sh
@@ -20,8 +20,8 @@ set -x
 yes|gcloud auth configure-docker
 now=$(date +%s)
 KV=$(egrep "Version =" pkg/drivers/kic/types.go | cut -d \" -f 2 | cut -d "-" -f 1)
-KIC_VERSION=$KV-$now-$ghprbPullId
-KICBASE_IMAGE_REGISTRIES=gcr.io/k8s-minikube/kicbase-builds:$KIC_VERSION
+export KIC_VERSION=$KV-$now-$ghprbPullId
+export KICBASE_IMAGE_REGISTRIES=gcr.io/k8s-minikube/kicbase-builds:$KIC_VERSION
 
 curl -L https://github.com/kubernetes/minikube/raw/master/pkg/drivers/kic/types.go --output types-head.go
 HEAD_KIC_TIMESTAMP=$(egrep "Version =" types-head.go | cut -d \" -f 2 | cut -d "-" -f 2)

From 9aeb4a75fade8082e8c5c0a7d93c3bd32ffbddee Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Wed, 17 Feb 2021 09:35:32 -0800
Subject: [PATCH 067/114] update kicbase image

---
 pkg/drivers/kic/types.go | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go
index 419945a92e..5955b33dae 100644
--- a/pkg/drivers/kic/types.go
+++ b/pkg/drivers/kic/types.go
@@ -24,10 +24,9 @@ import (
 
 const (
 	// Version is the current version of kic
-	Version = "v0.0.17"
+	Version = "v0.0.17-1613582775-10408"
 	// SHA of the kic base image
-	baseImageSHA = "1cd2e039ec9d418e6380b2fa0280503a72e5b282adea674ee67882f59f4f546e"
-)
+	baseImageSHA = "1537fe47d39640aa11d6c819fe39cbba6e250872c8fe2bd7701c5171a32fbc4e"
 
 var (
 	// BaseImage is the base image is used to spin up kic containers. it uses same base-image as kind.

From ea16737271be3e77ddce44cbb386af8bbbfbed8a Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Wed, 17 Feb 2021 09:37:34 -0800
Subject: [PATCH 068/114] syntax

---
 pkg/drivers/kic/types.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go
index 5955b33dae..03b5149b6e 100644
--- a/pkg/drivers/kic/types.go
+++ b/pkg/drivers/kic/types.go
@@ -27,6 +27,7 @@ const (
 	Version = "v0.0.17-1613582775-10408"
 	// SHA of the kic base image
 	baseImageSHA = "1537fe47d39640aa11d6c819fe39cbba6e250872c8fe2bd7701c5171a32fbc4e"
+)
 
 var (
 	// BaseImage is the base image is used to spin up kic containers. it uses same base-image as kind.

From 1233508e3ce4190dca52b6c71d258c51cf01aede Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Wed, 17 Feb 2021 09:59:56 -0800
Subject: [PATCH 069/114] generate-docs

---
 site/content/en/docs/commands/start.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/site/content/en/docs/commands/start.md b/site/content/en/docs/commands/start.md
index 9f13828d44..7bd751e454 100644
--- a/site/content/en/docs/commands/start.md
+++ b/site/content/en/docs/commands/start.md
@@ -26,7 +26,7 @@ minikube start [flags]
       --apiserver-names strings           A set of apiserver names which are used in the generated certificate for kubernetes.  This can be used if you want to make the apiserver available from outside the machine
       --apiserver-port int                The apiserver listening port (default 8443)
       --auto-update-drivers               If set, automatically updates drivers to the latest version. Defaults to true. (default true)
-      --base-image string                 The base image to use for docker/podman drivers. Intended for local development. (default "gcr.io/k8s-minikube/kicbase:v0.0.17@sha256:1cd2e039ec9d418e6380b2fa0280503a72e5b282adea674ee67882f59f4f546e")
+      --base-image string                 The base image to use for docker/podman drivers. Intended for local development. (default "gcr.io/k8s-minikube/kicbase:v0.0.17-1613582775-10408@sha256:1537fe47d39640aa11d6c819fe39cbba6e250872c8fe2bd7701c5171a32fbc4e")
       --cache-images                      If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --driver=none. (default true)
       --cni string                        CNI plug-in to use. Valid options: auto, bridge, calico, cilium, flannel, kindnet, or path to a CNI manifest (default: auto)
       --container-runtime string          The container runtime to be used (docker, cri-o, containerd). (default "docker")

From 9aee2bd46a12d59d58abe37435c8923447a98655 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Wed, 17 Feb 2021 10:04:02 -0800
Subject: [PATCH 070/114] add generate-docs to message

---
 hack/jenkins/kicbase_auto_build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hack/jenkins/kicbase_auto_build.sh b/hack/jenkins/kicbase_auto_build.sh
index 11eb4dc6d9..f547d6c985 100755
--- a/hack/jenkins/kicbase_auto_build.sh
+++ b/hack/jenkins/kicbase_auto_build.sh
@@ -42,7 +42,7 @@ docker pull $KICBASE_IMAGE_REGISTRIES
 fullsha=$(docker inspect --format='{{index .RepoDigests 0}}' $KICBASE_IMAGE_REGISTRIES)
 sha=$(echo ${fullsha} | cut -d ":" -f 2)
 
-message="Hi ${ghprbPullAuthorLoginMention},\\n\\nA new kicbase image is available, please update your PR with the new tag and SHA.\\nIn pkg/drivers/kic/types.go:\\n\\n\\t// Version is the current version of kic\\n\\tVersion = \\\"${KIC_VERSION}\\\"\\n\\t// SHA of the kic base image\\n\\tbaseImageSHA = \\\"${sha}\\\"\\n"
+message="Hi ${ghprbPullAuthorLoginMention},\\n\\nA new kicbase image is available, please update your PR with the new tag and SHA.\\nIn pkg/drivers/kic/types.go:\\n\\n\\t// Version is the current version of kic\\n\\tVersion = \\\"${KIC_VERSION}\\\"\\n\\t// SHA of the kic base image\\n\\tbaseImageSHA = \\\"${sha}\\\"\\nThen run \`make generate-docs\` to update our documentation to reference the new image."
 
 curl -s -H "Authorization: token ${access_token}" \
 	 -H "Accept: application/vnd.github.v3+json" \

From fd897f4f23c4deffebb62ef82ed733b67da15897 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Wed, 17 Feb 2021 10:32:48 -0800
Subject: [PATCH 071/114] fix up repo stuff

---
 pkg/drivers/kic/types.go               | 4 +++-
 site/content/en/docs/commands/start.md | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go
index 03b5149b6e..d653970d24 100644
--- a/pkg/drivers/kic/types.go
+++ b/pkg/drivers/kic/types.go
@@ -27,11 +27,13 @@ const (
 	Version = "v0.0.17-1613582775-10408"
 	// SHA of the kic base image
 	baseImageSHA = "1537fe47d39640aa11d6c819fe39cbba6e250872c8fe2bd7701c5171a32fbc4e"
+	// The name of the GCR kicbase repository
+	gcrRepo = "gcr.io/k8s-minikube/kicbase-builds"
 )
 
 var (
 	// BaseImage is the base image is used to spin up kic containers. it uses same base-image as kind.
-	BaseImage = fmt.Sprintf("gcr.io/k8s-minikube/kicbase:%s@sha256:%s", Version, baseImageSHA)
+	BaseImage = fmt.Sprintf("%s:%s@sha256:%s", gcrRepo, Version, baseImageSHA)
 
 	// FallbackImages are backup base images in case gcr isn't available
 	FallbackImages = []string{
diff --git a/site/content/en/docs/commands/start.md b/site/content/en/docs/commands/start.md
index 7bd751e454..b9f7c81804 100644
--- a/site/content/en/docs/commands/start.md
+++ b/site/content/en/docs/commands/start.md
@@ -26,7 +26,7 @@ minikube start [flags]
       --apiserver-names strings           A set of apiserver names which are used in the generated certificate for kubernetes.  This can be used if you want to make the apiserver available from outside the machine
       --apiserver-port int                The apiserver listening port (default 8443)
       --auto-update-drivers               If set, automatically updates drivers to the latest version. Defaults to true. (default true)
-      --base-image string                 The base image to use for docker/podman drivers. Intended for local development. (default "gcr.io/k8s-minikube/kicbase:v0.0.17-1613582775-10408@sha256:1537fe47d39640aa11d6c819fe39cbba6e250872c8fe2bd7701c5171a32fbc4e")
+      --base-image string                 The base image to use for docker/podman drivers. Intended for local development. (default "gcr.io/k8s-minikube/kicbase-builds:v0.0.17-1613582775-10408@sha256:1537fe47d39640aa11d6c819fe39cbba6e250872c8fe2bd7701c5171a32fbc4e")
       --cache-images                      If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --driver=none. (default true)
       --cni string                        CNI plug-in to use. Valid options: auto, bridge, calico, cilium, flannel, kindnet, or path to a CNI manifest (default: auto)
       --container-runtime string          The container runtime to be used (docker, cri-o, containerd). (default "docker")

From 68691821959d5ce040bd79e91f409d11ab4dc3f7 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Wed, 17 Feb 2021 10:39:24 -0800
Subject: [PATCH 072/114] add repo to instructions

---
 hack/jenkins/kicbase_auto_build.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/hack/jenkins/kicbase_auto_build.sh b/hack/jenkins/kicbase_auto_build.sh
index f547d6c985..d611ea593d 100755
--- a/hack/jenkins/kicbase_auto_build.sh
+++ b/hack/jenkins/kicbase_auto_build.sh
@@ -20,8 +20,9 @@ set -x
 yes|gcloud auth configure-docker
 now=$(date +%s)
 KV=$(egrep "Version =" pkg/drivers/kic/types.go | cut -d \" -f 2 | cut -d "-" -f 1)
+KIC_REPO=gcr.io/k8s-minikube/kicbase-builds
 export KIC_VERSION=$KV-$now-$ghprbPullId
-export KICBASE_IMAGE_REGISTRIES=gcr.io/k8s-minikube/kicbase-builds:$KIC_VERSION
+export KICBASE_IMAGE_REGISTRIES=$KIC_REPO:$KIC_VERSION
 
 curl -L https://github.com/kubernetes/minikube/raw/master/pkg/drivers/kic/types.go --output types-head.go
 HEAD_KIC_TIMESTAMP=$(egrep "Version =" types-head.go | cut -d \" -f 2 | cut -d "-" -f 2)
@@ -42,7 +43,7 @@ docker pull $KICBASE_IMAGE_REGISTRIES
 fullsha=$(docker inspect --format='{{index .RepoDigests 0}}' $KICBASE_IMAGE_REGISTRIES)
 sha=$(echo ${fullsha} | cut -d ":" -f 2)
 
-message="Hi ${ghprbPullAuthorLoginMention},\\n\\nA new kicbase image is available, please update your PR with the new tag and SHA.\\nIn pkg/drivers/kic/types.go:\\n\\n\\t// Version is the current version of kic\\n\\tVersion = \\\"${KIC_VERSION}\\\"\\n\\t// SHA of the kic base image\\n\\tbaseImageSHA = \\\"${sha}\\\"\\nThen run \`make generate-docs\` to update our documentation to reference the new image."
+message="Hi ${ghprbPullAuthorLoginMention},\\n\\nA new kicbase image is available, please update your PR with the new tag and SHA.\\nIn pkg/drivers/kic/types.go:\\n\\n\\t// Version is the current version of kic\\n\\tVersion = \\\"${KIC_VERSION}\\\"\\n\\t// SHA of the kic base image\\n\\tbaseImageSHA = \\\"${sha}\\\"\\n\\t// The name of the GCR kicbase repository\\n\\tgcrRepo = \\\"${KIC_REPO}\\\"\\nThen run \`make generate-docs\` to update our documentation to reference the new image."
 
 curl -s -H "Authorization: token ${access_token}" \
 	 -H "Accept: application/vnd.github.v3+json" \

From e042e2c5c6b1c125ba600dd197bc5b72fe018525 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Wed, 17 Feb 2021 10:51:16 -0800
Subject: [PATCH 073/114] check for failure

---
 hack/jenkins/kicbase_auto_build.sh | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/hack/jenkins/kicbase_auto_build.sh b/hack/jenkins/kicbase_auto_build.sh
index d611ea593d..fd7393b679 100755
--- a/hack/jenkins/kicbase_auto_build.sh
+++ b/hack/jenkins/kicbase_auto_build.sh
@@ -39,6 +39,14 @@ fi
 rm types-head.go
 yes|make push-kic-base-image
 
+# Abort with error message if above command failed
+if [ $? -gt 0 ]; then
+	curl -s -H "Authorization: token ${access_token}" \
+                -H "Accept: application/vnd.github.v3+json" \
+                -X POST -d "{\"body\": \"Hi ${ghprbPullAuthorLoginMention}, building a new kicbase image failed, please try again.\"}" "https://api.github.com/repos/kubernetes/minikube/issues/$ghprbPullId/comments"
+	exit $?
+fi
+
 docker pull $KICBASE_IMAGE_REGISTRIES
 fullsha=$(docker inspect --format='{{index .RepoDigests 0}}' $KICBASE_IMAGE_REGISTRIES)
 sha=$(echo ${fullsha} | cut -d ":" -f 2)

From 05df740f12586cedc90b301445f996d9f94c9372 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Wed, 17 Feb 2021 10:57:25 -0800
Subject: [PATCH 074/114] fail if timestamps are off

---
 hack/jenkins/kicbase_auto_build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hack/jenkins/kicbase_auto_build.sh b/hack/jenkins/kicbase_auto_build.sh
index fd7393b679..9b7f58d08f 100755
--- a/hack/jenkins/kicbase_auto_build.sh
+++ b/hack/jenkins/kicbase_auto_build.sh
@@ -33,7 +33,7 @@ if [[ $HEAD_KIC_TIMESTAMP != v* ]] && [[ $CURRENT_KIC_TS != v* ]]; then
 		curl -s -H "Authorization: token ${access_token}" \
 		-H "Accept: application/vnd.github.v3+json" \
 		-X POST -d "{\"body\": \"Hi ${ghprbPullAuthorLoginMention}, your kicbase info is out of date. Please rebase.\"}" "https://api.github.com/repos/kubernetes/minikube/issues/$ghprbPullId/comments"
-		exit 0
+		exit 1
 	fi
 fi
 rm types-head.go

From 6a44f6e1b0dc5900fe5e8615ca45621f0c88d820 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Wed, 17 Feb 2021 11:44:13 -0800
Subject: [PATCH 075/114] changing step to infof

---
 pkg/minikube/assets/addons.go | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go
index 41050358ab..88b44d6d4d 100644
--- a/pkg/minikube/assets/addons.go
+++ b/pkg/minikube/assets/addons.go
@@ -25,7 +25,6 @@ import (
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/out"
-	"k8s.io/minikube/pkg/minikube/style"
 	"k8s.io/minikube/pkg/minikube/vmpath"
 	"k8s.io/minikube/pkg/version"
 )
@@ -690,17 +689,17 @@ func GenerateTemplateData(addon *Addon, cfg config.KubernetesConfig) interface{}
 		}
 
 		if override, ok := opts.CustomRegistries[name]; ok {
-			out.Step(style.Option, "Using image {{.registry}}{{.image}}", out.V{
+			out.Infof("Using image {{.registry}}{{.image}}", out.V{
 				"registry": override,
 				"image":    image,
 			})
 		} else if opts.ImageRepository != "" {
-			out.Step(style.Option, "Using image {{.registry}}{{.image}} (global image repository)", out.V{
+			out.Infof("Using image {{.registry}}{{.image}} (global image repository)", out.V{
 				"registry": opts.ImageRepository,
 				"image":    image,
 			})
 		} else {
-			out.Step(style.Option, "Using image {{.registry}}{{.image}}", out.V{
+			out.Infof("Using image {{.registry}}{{.image}}", out.V{
 				"registry": opts.Registries[name],
 				"image":    image,
 			})

From b94aaf3639fbfa46e70a7b83062ad4b38946d3c4 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Wed, 17 Feb 2021 11:57:12 -0800
Subject: [PATCH 076/114] comments

---
 hack/jenkins/kicbase_auto_build.sh | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/hack/jenkins/kicbase_auto_build.sh b/hack/jenkins/kicbase_auto_build.sh
index 9b7f58d08f..3819d1f9fe 100755
--- a/hack/jenkins/kicbase_auto_build.sh
+++ b/hack/jenkins/kicbase_auto_build.sh
@@ -16,14 +16,18 @@
 
 set -x
 
+# Make sure docker is installed and configured
 ./hack/jenkins/installers/check_install_docker.sh
 yes|gcloud auth configure-docker
+
+# Setup variables
 now=$(date +%s)
 KV=$(egrep "Version =" pkg/drivers/kic/types.go | cut -d \" -f 2 | cut -d "-" -f 1)
 KIC_REPO=gcr.io/k8s-minikube/kicbase-builds
 export KIC_VERSION=$KV-$now-$ghprbPullId
 export KICBASE_IMAGE_REGISTRIES=$KIC_REPO:$KIC_VERSION
 
+# Let's make sure we have the newest kicbase reference
 curl -L https://github.com/kubernetes/minikube/raw/master/pkg/drivers/kic/types.go --output types-head.go
 HEAD_KIC_TIMESTAMP=$(egrep "Version =" types-head.go | cut -d \" -f 2 | cut -d "-" -f 2)
 CURRENT_KIC_TS=$(egrep "Version =" pkg/drivers/kic/types.go | cut -d \" -f 2 | cut -d "-" -f 2)
@@ -37,6 +41,8 @@ if [[ $HEAD_KIC_TIMESTAMP != v* ]] && [[ $CURRENT_KIC_TS != v* ]]; then
 	fi
 fi
 rm types-head.go
+
+# Build a new kicbase image
 yes|make push-kic-base-image
 
 # Abort with error message if above command failed
@@ -47,10 +53,12 @@ if [ $? -gt 0 ]; then
 	exit $?
 fi
 
+# Retrieve the sha from the new imnage
 docker pull $KICBASE_IMAGE_REGISTRIES
 fullsha=$(docker inspect --format='{{index .RepoDigests 0}}' $KICBASE_IMAGE_REGISTRIES)
 sha=$(echo ${fullsha} | cut -d ":" -f 2)
 
+# Display the message to the user
 message="Hi ${ghprbPullAuthorLoginMention},\\n\\nA new kicbase image is available, please update your PR with the new tag and SHA.\\nIn pkg/drivers/kic/types.go:\\n\\n\\t// Version is the current version of kic\\n\\tVersion = \\\"${KIC_VERSION}\\\"\\n\\t// SHA of the kic base image\\n\\tbaseImageSHA = \\\"${sha}\\\"\\n\\t// The name of the GCR kicbase repository\\n\\tgcrRepo = \\\"${KIC_REPO}\\\"\\nThen run \`make generate-docs\` to update our documentation to reference the new image."
 
 curl -s -H "Authorization: token ${access_token}" \

From 1c585e2885d0538cd3c492463407b111067639e5 Mon Sep 17 00:00:00 2001
From: Steven Powell <spowellgrr@gmail.com>
Date: Wed, 17 Feb 2021 13:49:22 -0700
Subject: [PATCH 077/114] renamed funcs and vars

---
 pkg/minikube/audit/audit.go                   |  4 +-
 pkg/minikube/audit/logFile.go                 |  6 +--
 pkg/minikube/audit/logFile_test.go            |  4 +-
 pkg/minikube/audit/report.go                  | 20 ++++----
 pkg/minikube/audit/report_test.go             |  4 +-
 pkg/minikube/audit/{entry.go => row.go}       | 46 ++++++++---------
 .../audit/{entry_test.go => row_test.go}      | 50 +++++++++----------
 pkg/minikube/logs/logs.go                     |  2 +-
 8 files changed, 67 insertions(+), 69 deletions(-)
 rename pkg/minikube/audit/{entry.go => row.go} (70%)
 rename pkg/minikube/audit/{entry_test.go => row_test.go} (75%)

diff --git a/pkg/minikube/audit/audit.go b/pkg/minikube/audit/audit.go
index 7b7e9d2d95..d1c5d07d73 100644
--- a/pkg/minikube/audit/audit.go
+++ b/pkg/minikube/audit/audit.go
@@ -55,8 +55,8 @@ func Log(startTime time.Time) {
 	if !shouldLog() {
 		return
 	}
-	e := newEntry(os.Args[1], args(), userName(), version.GetVersion(), startTime, time.Now())
-	if err := appendToLog(e); err != nil {
+	r := newRow(os.Args[1], args(), userName(), version.GetVersion(), startTime, time.Now())
+	if err := appendToLog(r); err != nil {
 		klog.Error(err)
 	}
 }
diff --git a/pkg/minikube/audit/logFile.go b/pkg/minikube/audit/logFile.go
index af6e2741a9..0cadd67df3 100644
--- a/pkg/minikube/audit/logFile.go
+++ b/pkg/minikube/audit/logFile.go
@@ -39,14 +39,14 @@ func setLogFile() error {
 }
 
 // appendToLog appends the audit entry to the log file.
-func appendToLog(entry *singleEntry) error {
+func appendToLog(row *row) error {
 	if currentLogFile == nil {
 		if err := setLogFile(); err != nil {
 			return err
 		}
 	}
-	e := register.CloudEvent(entry, entry.toMap())
-	bs, err := e.MarshalJSON()
+	ce := register.CloudEvent(row, row.toMap())
+	bs, err := ce.MarshalJSON()
 	if err != nil {
 		return fmt.Errorf("error marshalling event: %v", err)
 	}
diff --git a/pkg/minikube/audit/logFile_test.go b/pkg/minikube/audit/logFile_test.go
index 90c8faba4f..1d83a67281 100644
--- a/pkg/minikube/audit/logFile_test.go
+++ b/pkg/minikube/audit/logFile_test.go
@@ -42,8 +42,8 @@ func TestLogFile(t *testing.T) {
 		defer func() { currentLogFile = &oldLogFile }()
 		currentLogFile = f
 
-		e := newEntry("start", "-v", "user1", "v0.17.1", time.Now(), time.Now())
-		if err := appendToLog(e); err != nil {
+		r := newRow("start", "-v", "user1", "v0.17.1", time.Now(), time.Now())
+		if err := appendToLog(r); err != nil {
 			t.Fatalf("Error appendingToLog: %v", err)
 		}
 
diff --git a/pkg/minikube/audit/report.go b/pkg/minikube/audit/report.go
index c4e8a211e9..4b07ed8eb6 100644
--- a/pkg/minikube/audit/report.go
+++ b/pkg/minikube/audit/report.go
@@ -21,13 +21,13 @@ import (
 	"fmt"
 )
 
-type Data struct {
+type RawReport struct {
 	headers []string
-	entries []singleEntry
+	rows    []row
 }
 
 // Report is created from the log file.
-func Report(lines int) (*Data, error) {
+func Report(lines int) (*RawReport, error) {
 	if lines <= 0 {
 		return nil, fmt.Errorf("number of lines must be 1 or greater")
 	}
@@ -48,18 +48,18 @@ func Report(lines int) (*Data, error) {
 	if err := s.Err(); err != nil {
 		return nil, fmt.Errorf("failed to read from audit file: %v", err)
 	}
-	e, err := logsToEntries(logs)
+	rows, err := logsToRows(logs)
 	if err != nil {
-		return nil, fmt.Errorf("failed to convert logs to entries: %v", err)
+		return nil, fmt.Errorf("failed to convert logs to rows: %v", err)
 	}
-	r := &Data{
+	r := &RawReport{
 		[]string{"Command", "Args", "Profile", "User", "Version", "Start Time", "End Time"},
-		e,
+		rows,
 	}
 	return r, nil
 }
 
-// Table creates a formatted table using entries from the report.
-func (r *Data) Table() string {
-	return entriesToTable(r.entries, r.headers)
+// ASCIITable creates a formatted table using the headers and rows from the report.
+func (rr *RawReport) ASCIITable() string {
+	return rowsToTable(rr.rows, rr.headers)
 }
diff --git a/pkg/minikube/audit/report_test.go b/pkg/minikube/audit/report_test.go
index a6547b0d2b..2d77ad0145 100644
--- a/pkg/minikube/audit/report_test.go
+++ b/pkg/minikube/audit/report_test.go
@@ -50,7 +50,7 @@ func TestAuditReport(t *testing.T) {
 		t.Fatalf("failed to create report: %v", err)
 	}
 
-	if len(r.entries) != wantedLines {
-		t.Fatalf("report has %d lines of logs, want %d", len(r.entries), wantedLines)
+	if len(r.rows) != wantedLines {
+		t.Fatalf("report has %d lines of logs, want %d", len(r.rows), wantedLines)
 	}
 }
diff --git a/pkg/minikube/audit/entry.go b/pkg/minikube/audit/row.go
similarity index 70%
rename from pkg/minikube/audit/entry.go
rename to pkg/minikube/audit/row.go
index 58503be55c..4c6154aa20 100644
--- a/pkg/minikube/audit/entry.go
+++ b/pkg/minikube/audit/row.go
@@ -24,13 +24,12 @@ import (
 
 	"github.com/olekukonko/tablewriter"
 	"github.com/spf13/viper"
-	"k8s.io/klog"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
 )
 
-// singleEntry is the log entry of a single command.
-type singleEntry struct {
+// row is the log of a single command.
+type row struct {
 	args      string
 	command   string
 	endTime   string
@@ -42,12 +41,12 @@ type singleEntry struct {
 }
 
 // Type returns the cloud events compatible type of this struct.
-func (e *singleEntry) Type() string {
+func (e *row) Type() string {
 	return "io.k8s.sigs.minikube.audit"
 }
 
 // assignFields converts the map values to their proper fields
-func (e *singleEntry) assignFields() {
+func (e *row) assignFields() {
 	e.args = e.Data["args"]
 	e.command = e.Data["command"]
 	e.endTime = e.Data["endTime"]
@@ -58,7 +57,7 @@ func (e *singleEntry) assignFields() {
 }
 
 // toMap combines fields into a string map
-func (e *singleEntry) toMap() map[string]string {
+func (e *row) toMap() map[string]string {
 	return map[string]string{
 		"args":      e.args,
 		"command":   e.command,
@@ -70,13 +69,13 @@ func (e *singleEntry) toMap() map[string]string {
 	}
 }
 
-// newEntry returns a new audit type.
-func newEntry(command string, args string, user string, version string, startTime time.Time, endTime time.Time, profile ...string) *singleEntry {
+// newRow returns a new audit type.
+func newRow(command string, args string, user string, version string, startTime time.Time, endTime time.Time, profile ...string) *row {
 	p := viper.GetString(config.ProfileName)
 	if len(profile) > 0 {
 		p = profile[0]
 	}
-	return &singleEntry{
+	return &row{
 		args:      args,
 		command:   command,
 		endTime:   endTime.Format(constants.TimeFormat),
@@ -87,32 +86,31 @@ func newEntry(command string, args string, user string, version string, startTim
 	}
 }
 
-// toFields converts an entry to an array of fields.
-func (e *singleEntry) toFields() []string {
+// toFields converts a row to an array of fields.
+func (e *row) toFields() []string {
 	return []string{e.command, e.args, e.profile, e.user, e.version, e.startTime, e.endTime}
 }
 
-// logsToEntries converts audit logs into arrays of entries.
-func logsToEntries(logs []string) ([]singleEntry, error) {
-	c := []singleEntry{}
+// logsToRows converts audit logs into arrays of rows.
+func logsToRows(logs []string) ([]row, error) {
+	rows := []row{}
 	for _, l := range logs {
-		e := singleEntry{}
-		if err := json.Unmarshal([]byte(l), &e); err != nil {
+		r := row{}
+		if err := json.Unmarshal([]byte(l), &r); err != nil {
 			return nil, fmt.Errorf("failed to unmarshal %q: %v", l, err)
 		}
-		e.assignFields()
-		c = append(c, e)
+		r.assignFields()
+		rows = append(rows, r)
 	}
-	return c, nil
+	return rows, nil
 }
 
-// entriesToTable converts audit lines into a formatted table.
-func entriesToTable(entries []singleEntry, headers []string) string {
+// rowsToTable converts audit rows into a formatted table.
+func rowsToTable(rows []row, headers []string) string {
 	c := [][]string{}
-	for _, e := range entries {
-		c = append(c, e.toFields())
+	for _, r := range rows {
+		c = append(c, r.toFields())
 	}
-	klog.Info(c)
 	b := new(bytes.Buffer)
 	t := tablewriter.NewWriter(b)
 	t.SetHeader(headers)
diff --git a/pkg/minikube/audit/entry_test.go b/pkg/minikube/audit/row_test.go
similarity index 75%
rename from pkg/minikube/audit/entry_test.go
rename to pkg/minikube/audit/row_test.go
index c8d8b6c562..50fe990396 100644
--- a/pkg/minikube/audit/entry_test.go
+++ b/pkg/minikube/audit/row_test.go
@@ -26,7 +26,7 @@ import (
 	"k8s.io/minikube/pkg/minikube/constants"
 )
 
-func TestEntry(t *testing.T) {
+func TestRow(t *testing.T) {
 	c := "start"
 	a := "--alsologtostderr"
 	p := "profile1"
@@ -37,32 +37,32 @@ func TestEntry(t *testing.T) {
 	et := time.Now()
 	etFormatted := et.Format(constants.TimeFormat)
 
-	e := newEntry(c, a, u, v, st, et, p)
+	r := newRow(c, a, u, v, st, et, p)
 
-	t.Run("TestNewEntry", func(t *testing.T) {
+	t.Run("TestNewRow", func(t *testing.T) {
 		tests := []struct {
 			key  string
 			got  string
 			want string
 		}{
-			{"command", e.command, c},
-			{"args", e.args, a},
-			{"profile", e.profile, p},
-			{"user", e.user, u},
-			{"version", e.version, v},
-			{"startTime", e.startTime, stFormatted},
-			{"endTime", e.endTime, etFormatted},
+			{"command", r.command, c},
+			{"args", r.args, a},
+			{"profile", r.profile, p},
+			{"user", r.user, u},
+			{"version", r.version, v},
+			{"startTime", r.startTime, stFormatted},
+			{"endTime", r.endTime, etFormatted},
 		}
 
 		for _, tt := range tests {
 			if tt.got != tt.want {
-				t.Errorf("singleEntry.%s = %s; want %s", tt.key, tt.got, tt.want)
+				t.Errorf("row.%s = %s; want %s", tt.key, tt.got, tt.want)
 			}
 		}
 	})
 
 	t.Run("TestType", func(t *testing.T) {
-		got := e.Type()
+		got := r.Type()
 		want := "io.k8s.sigs.minikube.audit"
 
 		if got != want {
@@ -71,7 +71,7 @@ func TestEntry(t *testing.T) {
 	})
 
 	t.Run("TestToMap", func(t *testing.T) {
-		m := e.toMap()
+		m := r.toMap()
 
 		tests := []struct {
 			key  string
@@ -95,7 +95,7 @@ func TestEntry(t *testing.T) {
 	})
 
 	t.Run("TestToField", func(t *testing.T) {
-		got := e.toFields()
+		got := r.toFields()
 		gotString := strings.Join(got, ",")
 		want := []string{c, a, p, u, v, stFormatted, etFormatted}
 		wantString := strings.Join(want, ",")
@@ -108,25 +108,25 @@ func TestEntry(t *testing.T) {
 	t.Run("TestAssignFields", func(t *testing.T) {
 		l := fmt.Sprintf(`{"data":{"args":"%s","command":"%s","endTime":"%s","profile":"%s","startTime":"%s","user":"%s","version":"v0.17.1"},"datacontenttype":"application/json","id":"bc6ec9d4-0d08-4b57-ac3b-db8d67774768","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.audit"}`, a, c, etFormatted, p, stFormatted, u)
 
-		e := &singleEntry{}
-		if err := json.Unmarshal([]byte(l), e); err != nil {
-			t.Fatalf("failed to unmarshal log:: %v", err)
+		r := &row{}
+		if err := json.Unmarshal([]byte(l), r); err != nil {
+			t.Fatalf("failed to unmarshal log: %v", err)
 		}
 
-		e.assignFields()
+		r.assignFields()
 
 		tests := []struct {
 			key  string
 			got  string
 			want string
 		}{
-			{"command", e.command, c},
-			{"args", e.args, a},
-			{"profile", e.profile, p},
-			{"user", e.user, u},
-			{"version", e.version, v},
-			{"startTime", e.startTime, stFormatted},
-			{"endTime", e.endTime, etFormatted},
+			{"command", r.command, c},
+			{"args", r.args, a},
+			{"profile", r.profile, p},
+			{"user", r.user, u},
+			{"version", r.version, v},
+			{"startTime", r.startTime, stFormatted},
+			{"endTime", r.endTime, etFormatted},
 		}
 
 		for _, tt := range tests {
diff --git a/pkg/minikube/logs/logs.go b/pkg/minikube/logs/logs.go
index 568f26ee7a..d4a528681f 100644
--- a/pkg/minikube/logs/logs.go
+++ b/pkg/minikube/logs/logs.go
@@ -208,7 +208,7 @@ func outputAudit(lines int) error {
 	if err != nil {
 		return fmt.Errorf("failed to create audit report with error: %v", err)
 	}
-	out.Step(style.Empty, r.Table())
+	out.Step(style.Empty, r.ASCIITable())
 	return nil
 }
 

From 04b03439cdaa0b6d059ec31d6ad873045bc2ccfc Mon Sep 17 00:00:00 2001
From: Steven Powell <spowellgrr@gmail.com>
Date: Wed, 17 Feb 2021 13:52:49 -0700
Subject: [PATCH 078/114] updated test names

---
 pkg/minikube/audit/report_test.go |  2 +-
 pkg/minikube/audit/row_test.go    | 10 +++++-----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/pkg/minikube/audit/report_test.go b/pkg/minikube/audit/report_test.go
index 2d77ad0145..f5f3e8012a 100644
--- a/pkg/minikube/audit/report_test.go
+++ b/pkg/minikube/audit/report_test.go
@@ -22,7 +22,7 @@ import (
 	"testing"
 )
 
-func TestAuditReport(t *testing.T) {
+func TestReport(t *testing.T) {
 	f, err := ioutil.TempFile("", "audit.json")
 	if err != nil {
 		t.Fatalf("failed creating temporary file: %v", err)
diff --git a/pkg/minikube/audit/row_test.go b/pkg/minikube/audit/row_test.go
index 50fe990396..7e80a5febd 100644
--- a/pkg/minikube/audit/row_test.go
+++ b/pkg/minikube/audit/row_test.go
@@ -39,7 +39,7 @@ func TestRow(t *testing.T) {
 
 	r := newRow(c, a, u, v, st, et, p)
 
-	t.Run("TestNewRow", func(t *testing.T) {
+	t.Run("NewRow", func(t *testing.T) {
 		tests := []struct {
 			key  string
 			got  string
@@ -61,7 +61,7 @@ func TestRow(t *testing.T) {
 		}
 	})
 
-	t.Run("TestType", func(t *testing.T) {
+	t.Run("Type", func(t *testing.T) {
 		got := r.Type()
 		want := "io.k8s.sigs.minikube.audit"
 
@@ -70,7 +70,7 @@ func TestRow(t *testing.T) {
 		}
 	})
 
-	t.Run("TestToMap", func(t *testing.T) {
+	t.Run("ToMap", func(t *testing.T) {
 		m := r.toMap()
 
 		tests := []struct {
@@ -94,7 +94,7 @@ func TestRow(t *testing.T) {
 		}
 	})
 
-	t.Run("TestToField", func(t *testing.T) {
+	t.Run("ToFields", func(t *testing.T) {
 		got := r.toFields()
 		gotString := strings.Join(got, ",")
 		want := []string{c, a, p, u, v, stFormatted, etFormatted}
@@ -105,7 +105,7 @@ func TestRow(t *testing.T) {
 		}
 	})
 
-	t.Run("TestAssignFields", func(t *testing.T) {
+	t.Run("AssignFields", func(t *testing.T) {
 		l := fmt.Sprintf(`{"data":{"args":"%s","command":"%s","endTime":"%s","profile":"%s","startTime":"%s","user":"%s","version":"v0.17.1"},"datacontenttype":"application/json","id":"bc6ec9d4-0d08-4b57-ac3b-db8d67774768","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.audit"}`, a, c, etFormatted, p, stFormatted, u)
 
 		r := &row{}

From 0f8f40b340101ac69ae155dc52e8eb90d95e6fa7 Mon Sep 17 00:00:00 2001
From: Steven Powell <spowellgrr@gmail.com>
Date: Wed, 17 Feb 2021 15:02:00 -0700
Subject: [PATCH 079/114] added integration tests

---
 test/integration/functional_test.go      | 2 +-
 test/integration/version_upgrade_test.go | 7 +++++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go
index d52ae2f864..e279ae4a43 100644
--- a/test/integration/functional_test.go
+++ b/test/integration/functional_test.go
@@ -722,7 +722,7 @@ func validateLogsCmd(ctx context.Context, t *testing.T, profile string) {
 	if err != nil {
 		t.Errorf("%s failed: %v", rr.Command(), err)
 	}
-	expectedWords := []string{"apiserver", "Linux", "kubelet"}
+	expectedWords := []string{"apiserver", "Linux", "kubelet", "Audit"}
 	switch ContainerRuntime() {
 	case "docker":
 		expectedWords = append(expectedWords, "Docker")
diff --git a/test/integration/version_upgrade_test.go b/test/integration/version_upgrade_test.go
index 564803e87d..055fc59da5 100644
--- a/test/integration/version_upgrade_test.go
+++ b/test/integration/version_upgrade_test.go
@@ -196,6 +196,13 @@ func TestStoppedBinaryUpgrade(t *testing.T) {
 	if err != nil {
 		t.Fatalf("upgrade from %s to HEAD failed: %s: %v", legacyVersion, rr.Command(), err)
 	}
+
+	t.Run("MinikubeLogs", func(t *testing.T) {
+		rr, err = Run(t, exec.CommandContext(ctx, Target(), "logs"))
+		if err != nil {
+			t.Fatalf("minikube logs after upgrade to HEAD failed: %v", err)
+		}
+	})
 }
 
 // TestKubernetesUpgrade upgrades Kubernetes from oldest to newest

From e6ca0268685a9f70100495bf575266614968e06e Mon Sep 17 00:00:00 2001
From: Ilya Zuyev <izuyev@google.com>
Date: Wed, 17 Feb 2021 14:20:31 -0800
Subject: [PATCH 080/114] Use golang:1.16.0-buster to build kicbase

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index f6d00b7ca5..5e80d390aa 100644
--- a/Makefile
+++ b/Makefile
@@ -51,7 +51,7 @@ HYPERKIT_BUILD_IMAGE 	?= karalabe/xgo-1.12.x
 #
 # TODO: See https://github.com/kubernetes/minikube/issues/10276
 #BUILD_IMAGE 	?= us.gcr.io/k8s-artifacts-prod/build-image/kube-cross:v$(GO_VERSION)-1
-BUILD_IMAGE 	?= golang:1.16-rc-buster
+BUILD_IMAGE 	?= golang:1.16.0-buster
 #
 
 ISO_BUILD_IMAGE ?= $(REGISTRY)/buildroot-image

From 090da08ad73ab06a401f975f12643536cdabba63 Mon Sep 17 00:00:00 2001
From: Ilya Zuyev <izuyev@google.com>
Date: Wed, 17 Feb 2021 14:27:11 -0800
Subject: [PATCH 081/114] fix typo

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 1ab694eb0b..95a58056fc 100644
--- a/Makefile
+++ b/Makefile
@@ -643,7 +643,7 @@ docker-multi-arch-builder:
 KICBASE_ARCH = linux/arm64,linux/amd64
 KICBASE_IMAGE_REGISTRIES ?= $(REGISTRY)/kicbase:$(KIC_VERSION) $(REGISTRY_GH)/kicbase:$(KIC_VERSION) kicbase/stable:$(KIC_VERSION)
 
-.PHONY: ppush-kic-base-image 
+.PHONY: push-kic-base-image 
 push-kic-base-image: docker-multi-arch-builder ## Push multi-arch local/kicbase:latest to all remote registries
 ifdef AUTOPUSH
 	docker login gcr.io/k8s-minikube

From 687cb22f2201bcbb009a289ff608f1829a7cb25d Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Wed, 17 Feb 2021 16:18:53 -0800
Subject: [PATCH 082/114] add single sed command to message for simplicity

---
 Makefile                           | 2 +-
 hack/jenkins/kicbase_auto_build.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 593401e55f..95a58056fc 100644
--- a/Makefile
+++ b/Makefile
@@ -631,7 +631,7 @@ storage-provisioner-image-%: out/storage-provisioner-%
 	docker build -t $(REGISTRY)/storage-provisioner-$*:$(STORAGE_PROVISIONER_TAG) -f deploy/storage-provisioner/Dockerfile  --build-arg arch=$* .
 
 
-X_DOCKER_BUILDER ?= kicbase-builder
+X_DOCKER_BUILDER ?= minikube-builder
 X_BUILD_ENV ?= DOCKER_CLI_EXPERIMENTAL=enabled
 
 .PHONY: docker-multi-arch-builder
diff --git a/hack/jenkins/kicbase_auto_build.sh b/hack/jenkins/kicbase_auto_build.sh
index 3819d1f9fe..40eac1528d 100755
--- a/hack/jenkins/kicbase_auto_build.sh
+++ b/hack/jenkins/kicbase_auto_build.sh
@@ -59,7 +59,7 @@ fullsha=$(docker inspect --format='{{index .RepoDigests 0}}' $KICBASE_IMAGE_REGI
 sha=$(echo ${fullsha} | cut -d ":" -f 2)
 
 # Display the message to the user
-message="Hi ${ghprbPullAuthorLoginMention},\\n\\nA new kicbase image is available, please update your PR with the new tag and SHA.\\nIn pkg/drivers/kic/types.go:\\n\\n\\t// Version is the current version of kic\\n\\tVersion = \\\"${KIC_VERSION}\\\"\\n\\t// SHA of the kic base image\\n\\tbaseImageSHA = \\\"${sha}\\\"\\n\\t// The name of the GCR kicbase repository\\n\\tgcrRepo = \\\"${KIC_REPO}\\\"\\nThen run \`make generate-docs\` to update our documentation to reference the new image."
+message="Hi ${ghprbPullAuthorLoginMention},\\n\\nA new kicbase image is available, please update your PR with the new tag and SHA.\\nIn pkg/drivers/kic/types.go:\\n\\n\\t// Version is the current version of kic\\n\\tVersion = \\\"${KIC_VERSION}\\\"\\n\\t// SHA of the kic base image\\n\\tbaseImageSHA = \\\"${sha}\\\"\\n\\t// The name of the GCR kicbase repository\\n\\tgcrRepo = \\\"${KIC_REPO}\\\"\\nThen run \`make generate-docs\` to update our documentation to reference the new image.\n\nAlternatively, run the following command and commit the changes: \`sed -i 's/Version = .*/Version = ${KIC_VERSION}/;s/baseImageSHA = .*/baseImageSHA = ${sha}/;s/gcrRepo = .*/gcrRepo = ${KIC_REPO}/' pkg/drivers/kic/types.go; make generate-docs\`"
 
 curl -s -H "Authorization: token ${access_token}" \
 	 -H "Accept: application/vnd.github.v3+json" \

From c3f7e8c7b20636e7af4b479403fc74532b4b344c Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Wed, 17 Feb 2021 16:42:15 -0800
Subject: [PATCH 083/114] fix up sed command

---
 hack/jenkins/kicbase_auto_build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hack/jenkins/kicbase_auto_build.sh b/hack/jenkins/kicbase_auto_build.sh
index 40eac1528d..2f65865618 100755
--- a/hack/jenkins/kicbase_auto_build.sh
+++ b/hack/jenkins/kicbase_auto_build.sh
@@ -59,7 +59,7 @@ fullsha=$(docker inspect --format='{{index .RepoDigests 0}}' $KICBASE_IMAGE_REGI
 sha=$(echo ${fullsha} | cut -d ":" -f 2)
 
 # Display the message to the user
-message="Hi ${ghprbPullAuthorLoginMention},\\n\\nA new kicbase image is available, please update your PR with the new tag and SHA.\\nIn pkg/drivers/kic/types.go:\\n\\n\\t// Version is the current version of kic\\n\\tVersion = \\\"${KIC_VERSION}\\\"\\n\\t// SHA of the kic base image\\n\\tbaseImageSHA = \\\"${sha}\\\"\\n\\t// The name of the GCR kicbase repository\\n\\tgcrRepo = \\\"${KIC_REPO}\\\"\\nThen run \`make generate-docs\` to update our documentation to reference the new image.\n\nAlternatively, run the following command and commit the changes: \`sed -i 's/Version = .*/Version = ${KIC_VERSION}/;s/baseImageSHA = .*/baseImageSHA = ${sha}/;s/gcrRepo = .*/gcrRepo = ${KIC_REPO}/' pkg/drivers/kic/types.go; make generate-docs\`"
+message="Hi ${ghprbPullAuthorLoginMention},\\n\\nA new kicbase image is available, please update your PR with the new tag and SHA.\\nIn pkg/drivers/kic/types.go:\\n\\n\\t// Version is the current version of kic\\n\\tVersion = \\\"${KIC_VERSION}\\\"\\n\\t// SHA of the kic base image\\n\\tbaseImageSHA = \\\"${sha}\\\"\\n\\t// The name of the GCR kicbase repository\\n\\tgcrRepo = \\\"${KIC_REPO}\\\"\\nThen run \`make generate-docs\` to update our documentation to reference the new image.\n\nAlternatively, run the following command and commit the changes:\\n\\t sed -i 's|Version = .*|Version = \\\"${KIC_VERSION}\\\"|;s|baseImageSHA = .*|baseImageSHA = \\\"${sha}\\\"|;s|gcrRepo = .*|gcrRepo = \\\"${KIC_REPO}\\\"|' pkg/drivers/kic/types.go; make generate-docs"
 
 curl -s -H "Authorization: token ${access_token}" \
 	 -H "Accept: application/vnd.github.v3+json" \

From e6c1a682b512030b24f6377e5a2ae590bdc6a560 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Wed, 17 Feb 2021 16:50:48 -0800
Subject: [PATCH 084/114] fix up sed command some more

---
 hack/jenkins/kicbase_auto_build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hack/jenkins/kicbase_auto_build.sh b/hack/jenkins/kicbase_auto_build.sh
index 2f65865618..58609f49e9 100755
--- a/hack/jenkins/kicbase_auto_build.sh
+++ b/hack/jenkins/kicbase_auto_build.sh
@@ -59,7 +59,7 @@ fullsha=$(docker inspect --format='{{index .RepoDigests 0}}' $KICBASE_IMAGE_REGI
 sha=$(echo ${fullsha} | cut -d ":" -f 2)
 
 # Display the message to the user
-message="Hi ${ghprbPullAuthorLoginMention},\\n\\nA new kicbase image is available, please update your PR with the new tag and SHA.\\nIn pkg/drivers/kic/types.go:\\n\\n\\t// Version is the current version of kic\\n\\tVersion = \\\"${KIC_VERSION}\\\"\\n\\t// SHA of the kic base image\\n\\tbaseImageSHA = \\\"${sha}\\\"\\n\\t// The name of the GCR kicbase repository\\n\\tgcrRepo = \\\"${KIC_REPO}\\\"\\nThen run \`make generate-docs\` to update our documentation to reference the new image.\n\nAlternatively, run the following command and commit the changes:\\n\\t sed -i 's|Version = .*|Version = \\\"${KIC_VERSION}\\\"|;s|baseImageSHA = .*|baseImageSHA = \\\"${sha}\\\"|;s|gcrRepo = .*|gcrRepo = \\\"${KIC_REPO}\\\"|' pkg/drivers/kic/types.go; make generate-docs"
+message="Hi ${ghprbPullAuthorLoginMention},\\n\\nA new kicbase image is available, please update your PR with the new tag and SHA.\\nIn pkg/drivers/kic/types.go:\\n\\n\\t// Version is the current version of kic\\n\\tVersion = \\\"${KIC_VERSION}\\\"\\n\\t// SHA of the kic base image\\n\\tbaseImageSHA = \\\"${sha}\\\"\\n\\t// The name of the GCR kicbase repository\\n\\tgcrRepo = \\\"${KIC_REPO}\\\"\\nThen run \`make generate-docs\` to update our documentation to reference the new image.\n\nAlternatively, run the following command and commit the changes:\\n\\t sed 's|Version = .*|Version = \\\"${KIC_VERSION}\\\"|;s|baseImageSHA = .*|baseImageSHA = \\\"${sha}\\\"|;s|gcrRepo = .*|gcrRepo = \\\"${KIC_REPO}\\\"|' pkg/drivers/kic/types.go > new-types.go; mv new-types.go pkg/drivers/kic/types.go; make generate-docs"
 
 curl -s -H "Authorization: token ${access_token}" \
 	 -H "Accept: application/vnd.github.v3+json" \

From 710b680e163945a01b968a24edb86225be225533 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Wed, 17 Feb 2021 17:03:49 -0800
Subject: [PATCH 085/114] ok now we're just down to formatting

---
 hack/jenkins/kicbase_auto_build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hack/jenkins/kicbase_auto_build.sh b/hack/jenkins/kicbase_auto_build.sh
index 58609f49e9..38c28989bd 100755
--- a/hack/jenkins/kicbase_auto_build.sh
+++ b/hack/jenkins/kicbase_auto_build.sh
@@ -59,7 +59,7 @@ fullsha=$(docker inspect --format='{{index .RepoDigests 0}}' $KICBASE_IMAGE_REGI
 sha=$(echo ${fullsha} | cut -d ":" -f 2)
 
 # Display the message to the user
-message="Hi ${ghprbPullAuthorLoginMention},\\n\\nA new kicbase image is available, please update your PR with the new tag and SHA.\\nIn pkg/drivers/kic/types.go:\\n\\n\\t// Version is the current version of kic\\n\\tVersion = \\\"${KIC_VERSION}\\\"\\n\\t// SHA of the kic base image\\n\\tbaseImageSHA = \\\"${sha}\\\"\\n\\t// The name of the GCR kicbase repository\\n\\tgcrRepo = \\\"${KIC_REPO}\\\"\\nThen run \`make generate-docs\` to update our documentation to reference the new image.\n\nAlternatively, run the following command and commit the changes:\\n\\t sed 's|Version = .*|Version = \\\"${KIC_VERSION}\\\"|;s|baseImageSHA = .*|baseImageSHA = \\\"${sha}\\\"|;s|gcrRepo = .*|gcrRepo = \\\"${KIC_REPO}\\\"|' pkg/drivers/kic/types.go > new-types.go; mv new-types.go pkg/drivers/kic/types.go; make generate-docs"
+message="Hi ${ghprbPullAuthorLoginMention},\\n\\nA new kicbase image is available, please update your PR with the new tag and SHA.\\nIn pkg/drivers/kic/types.go:\\n\\n\\t// Version is the current version of kic\\n\\tVersion = \\\"${KIC_VERSION}\\\"\\n\\t// SHA of the kic base image\\n\\tbaseImageSHA = \\\"${sha}\\\"\\n\\t// The name of the GCR kicbase repository\\n\\tgcrRepo = \\\"${KIC_REPO}\\\"\\nThen run \`make generate-docs\` to update our documentation to reference the new image.\n\nAlternatively, run the following command and commit the changes:\\n\'\'\'\\n sed 's|Version = .*|Version = \\\"${KIC_VERSION}\\\"|;s|baseImageSHA = .*|baseImageSHA = \\\"${sha}\\\"|;s|gcrRepo = .*|gcrRepo = \\\"${KIC_REPO}\\\"|' pkg/drivers/kic/types.go > new-types.go; mv new-types.go pkg/drivers/kic/types.go; make generate-docs\\n\`\`\`"
 
 curl -s -H "Authorization: token ${access_token}" \
 	 -H "Accept: application/vnd.github.v3+json" \

From 29ac10b51474fc9a7c8a410797743934f624e394 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Wed, 17 Feb 2021 17:19:45 -0800
Subject: [PATCH 086/114] formatting is hard

---
 hack/jenkins/kicbase_auto_build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hack/jenkins/kicbase_auto_build.sh b/hack/jenkins/kicbase_auto_build.sh
index 38c28989bd..f164c9ab3e 100755
--- a/hack/jenkins/kicbase_auto_build.sh
+++ b/hack/jenkins/kicbase_auto_build.sh
@@ -59,7 +59,7 @@ fullsha=$(docker inspect --format='{{index .RepoDigests 0}}' $KICBASE_IMAGE_REGI
 sha=$(echo ${fullsha} | cut -d ":" -f 2)
 
 # Display the message to the user
-message="Hi ${ghprbPullAuthorLoginMention},\\n\\nA new kicbase image is available, please update your PR with the new tag and SHA.\\nIn pkg/drivers/kic/types.go:\\n\\n\\t// Version is the current version of kic\\n\\tVersion = \\\"${KIC_VERSION}\\\"\\n\\t// SHA of the kic base image\\n\\tbaseImageSHA = \\\"${sha}\\\"\\n\\t// The name of the GCR kicbase repository\\n\\tgcrRepo = \\\"${KIC_REPO}\\\"\\nThen run \`make generate-docs\` to update our documentation to reference the new image.\n\nAlternatively, run the following command and commit the changes:\\n\'\'\'\\n sed 's|Version = .*|Version = \\\"${KIC_VERSION}\\\"|;s|baseImageSHA = .*|baseImageSHA = \\\"${sha}\\\"|;s|gcrRepo = .*|gcrRepo = \\\"${KIC_REPO}\\\"|' pkg/drivers/kic/types.go > new-types.go; mv new-types.go pkg/drivers/kic/types.go; make generate-docs\\n\`\`\`"
+message="Hi ${ghprbPullAuthorLoginMention},\\n\\nA new kicbase image is available, please update your PR with the new tag and SHA.\\nIn pkg/drivers/kic/types.go:\\n\\n\\t// Version is the current version of kic\\n\\tVersion = \\\"${KIC_VERSION}\\\"\\n\\t// SHA of the kic base image\\n\\tbaseImageSHA = \\\"${sha}\\\"\\n\\t// The name of the GCR kicbase repository\\n\\tgcrRepo = \\\"${KIC_REPO}\\\"\\nThen run \`make generate-docs\` to update our documentation to reference the new image.\n\nAlternatively, run the following command and commit the changes:\\n\`\`\`\\n sed 's|Version = .*|Version = \\\"${KIC_VERSION}\\\"|;s|baseImageSHA = .*|baseImageSHA = \\\"${sha}\\\"|;s|gcrRepo = .*|gcrRepo = \\\"${KIC_REPO}\\\"|' pkg/drivers/kic/types.go > new-types.go; mv new-types.go pkg/drivers/kic/types.go; make generate-docs\\n\`\`\`"
 
 curl -s -H "Authorization: token ${access_token}" \
 	 -H "Accept: application/vnd.github.v3+json" \

From eabc02d3c4058be63b44fdce7cd3fe3fa87465d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= <anders.f.bjorklund@gmail.com>
Date: Thu, 18 Feb 2021 18:48:39 +0100
Subject: [PATCH 087/114] Stop using --memory-swap if it is not available

With Debian and Ubuntu kernels, it needs to be configured.

And now with cgroups v2, it starts throwing errors at run.
---
 pkg/drivers/kic/oci/oci.go | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go
index 27f7a49a73..e88f5e635e 100644
--- a/pkg/drivers/kic/oci/oci.go
+++ b/pkg/drivers/kic/oci/oci.go
@@ -181,8 +181,10 @@ func CreateContainerNode(p CreateParams) error {
 		runArgs = append(runArgs, "--security-opt", "apparmor=unconfined")
 
 		runArgs = append(runArgs, fmt.Sprintf("--memory=%s", p.Memory))
-		// Disable swap by setting the value to match
-		runArgs = append(runArgs, fmt.Sprintf("--memory-swap=%s", p.Memory))
+		if memcgSwap {
+			// Disable swap by setting the value to match
+			runArgs = append(runArgs, fmt.Sprintf("--memory-swap=%s", p.Memory))
+		}
 
 		virtualization = "docker" // VIRTUALIZATION_DOCKER
 	}

From 3f045350456ad95c57eecae54065373c77507b27 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Thu, 18 Feb 2021 11:47:59 -0800
Subject: [PATCH 088/114] add dockerhub to autopush

---
 hack/jenkins/kicbase_auto_build.sh | 9 ++++++---
 pkg/drivers/kic/types.go           | 6 ++++--
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/hack/jenkins/kicbase_auto_build.sh b/hack/jenkins/kicbase_auto_build.sh
index f164c9ab3e..336c4ef7f2 100755
--- a/hack/jenkins/kicbase_auto_build.sh
+++ b/hack/jenkins/kicbase_auto_build.sh
@@ -23,12 +23,15 @@ yes|gcloud auth configure-docker
 # Setup variables
 now=$(date +%s)
 KV=$(egrep "Version =" pkg/drivers/kic/types.go | cut -d \" -f 2 | cut -d "-" -f 1)
-KIC_REPO=gcr.io/k8s-minikube/kicbase-builds
+GCR_REPO=gcr.io/k8s-minikube/kicbase-builds
+DH_REPO=kicbase/build
 export KIC_VERSION=$KV-$now-$ghprbPullId
-export KICBASE_IMAGE_REGISTRIES=$KIC_REPO:$KIC_VERSION
+export KICBASE_IMAGE_REGISTRIES="${KIC_REPO}:${KIC_VERSION} ${DH_REPO}:${KIC_VERSION}"
 
 # Let's make sure we have the newest kicbase reference
 curl -L https://github.com/kubernetes/minikube/raw/master/pkg/drivers/kic/types.go --output types-head.go
+# kicbase tags are of the form VERSION-TIMESTAMP-PR, so this grep finds that TIMESTAMP in the middle
+# if it doesn't exist, it will just return VERSION, which is covered in the if statement below
 HEAD_KIC_TIMESTAMP=$(egrep "Version =" types-head.go | cut -d \" -f 2 | cut -d "-" -f 2)
 CURRENT_KIC_TS=$(egrep "Version =" pkg/drivers/kic/types.go | cut -d \" -f 2 | cut -d "-" -f 2)
 if [[ $HEAD_KIC_TIMESTAMP != v* ]] && [[ $CURRENT_KIC_TS != v* ]]; then
@@ -59,7 +62,7 @@ fullsha=$(docker inspect --format='{{index .RepoDigests 0}}' $KICBASE_IMAGE_REGI
 sha=$(echo ${fullsha} | cut -d ":" -f 2)
 
 # Display the message to the user
-message="Hi ${ghprbPullAuthorLoginMention},\\n\\nA new kicbase image is available, please update your PR with the new tag and SHA.\\nIn pkg/drivers/kic/types.go:\\n\\n\\t// Version is the current version of kic\\n\\tVersion = \\\"${KIC_VERSION}\\\"\\n\\t// SHA of the kic base image\\n\\tbaseImageSHA = \\\"${sha}\\\"\\n\\t// The name of the GCR kicbase repository\\n\\tgcrRepo = \\\"${KIC_REPO}\\\"\\nThen run \`make generate-docs\` to update our documentation to reference the new image.\n\nAlternatively, run the following command and commit the changes:\\n\`\`\`\\n sed 's|Version = .*|Version = \\\"${KIC_VERSION}\\\"|;s|baseImageSHA = .*|baseImageSHA = \\\"${sha}\\\"|;s|gcrRepo = .*|gcrRepo = \\\"${KIC_REPO}\\\"|' pkg/drivers/kic/types.go > new-types.go; mv new-types.go pkg/drivers/kic/types.go; make generate-docs\\n\`\`\`"
+message="Hi ${ghprbPullAuthorLoginMention},\\n\\nA new kicbase image is available, please update your PR with the new tag and SHA.\\nIn pkg/drivers/kic/types.go:\\n\\n\\t// Version is the current version of kic\\n\\tVersion = \\\"${KIC_VERSION}\\\"\\n\\t// SHA of the kic base image\\n\\tbaseImageSHA = \\\"${sha}\\\"\\n\\t// The name of the GCR kicbase repository\\n\\tgcrRepo = \\\"${KIC_REPO}\\\"\\n\\t// The name of the Dockerhub kicbase repository\\n\\tdockerhubRepo = \\\"${DH_REPO}\\\"\\nThen run \`make generate-docs\` to update our documentation to reference the new image.\n\nAlternatively, run the following command and commit the changes:\\n\`\`\`\\n sed 's|Version = .*|Version = \\\"${KIC_VERSION}\\\"|;s|baseImageSHA = .*|baseImageSHA = \\\"${sha}\\\"|;s|gcrRepo = .*|gcrRepo = \\\"${KIC_REPO}\\\"|;s|dockerhubRepo = .*|dockerhubRepo = ${DH_REPO}|' pkg/drivers/kic/types.go > new-types.go; mv new-types.go pkg/drivers/kic/types.go; make generate-docs\\n\`\`\`"
 
 curl -s -H "Authorization: token ${access_token}" \
 	 -H "Accept: application/vnd.github.v3+json" \
diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go
index d653970d24..27357a7473 100644
--- a/pkg/drivers/kic/types.go
+++ b/pkg/drivers/kic/types.go
@@ -1,7 +1,7 @@
 /*
 Copyright 2019 The Kubernetes Authors All rights reserved.
 
-Licensed under the Apache License, Version 2.0 (the "License");
+Licensed under theApache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 
@@ -29,6 +29,8 @@ const (
 	baseImageSHA = "1537fe47d39640aa11d6c819fe39cbba6e250872c8fe2bd7701c5171a32fbc4e"
 	// The name of the GCR kicbase repository
 	gcrRepo = "gcr.io/k8s-minikube/kicbase-builds"
+	// The name of the Dockerhub kicbase repository
+	dockerhubRepo = "kicbase/stable"
 )
 
 var (
@@ -39,7 +41,7 @@ var (
 	FallbackImages = []string{
 		// the fallback of BaseImage in case gcr.io is not available. stored in docker hub
 		// same image is push to https://github.com/kicbase/stable
-		fmt.Sprintf("kicbase/stable:%s@sha256:%s", Version, baseImageSHA),
+		fmt.Sprintf("%s:%s@sha256:%s", dockerhubRepo, Version, baseImageSHA),
 
 		// the fallback of BaseImage in case gcr.io is not available. stored in github packages https://github.com/kubernetes/minikube/packages/206071
 		// github packages docker does _NOT_ support pulling by sha as mentioned in the docs:

From f11498046b95130ec81676f8b5e4af210903aaa9 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Thu, 18 Feb 2021 11:52:38 -0800
Subject: [PATCH 089/114] messed up boilerplate

---
 pkg/drivers/kic/types.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go
index 27357a7473..e26ef3f58f 100644
--- a/pkg/drivers/kic/types.go
+++ b/pkg/drivers/kic/types.go
@@ -1,7 +1,7 @@
 /*
 Copyright 2019 The Kubernetes Authors All rights reserved.
 
-Licensed under theApache License, Version 2.0 (the "License");
+Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 

From af4364ba6f5d6d25fb24585ead6c28cdd4b38623 Mon Sep 17 00:00:00 2001
From: Steven Powell <spowellgrr@gmail.com>
Date: Thu, 18 Feb 2021 13:34:13 -0700
Subject: [PATCH 090/114] Updated dashboard URL detection logic for
 DashboardCmd test

---
 test/integration/functional_test.go | 25 ++++++++++++++++++++-----
 test/integration/util_test.go       | 25 -------------------------
 2 files changed, 20 insertions(+), 30 deletions(-)

diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go
index d29a8f7916..cc7f19cd89 100644
--- a/test/integration/functional_test.go
+++ b/test/integration/functional_test.go
@@ -19,6 +19,7 @@ limitations under the License.
 package integration
 
 import (
+	"bufio"
 	"bytes"
 	"context"
 	"encoding/json"
@@ -546,8 +547,11 @@ func validateStatusCmd(ctx context.Context, t *testing.T, profile string) {
 func validateDashboardCmd(ctx context.Context, t *testing.T, profile string) {
 	defer PostMortemLogs(t, profile)
 
+	mctx, cancel := context.WithTimeout(ctx, Seconds(300))
+	defer cancel()
+
 	args := []string{"dashboard", "--url", "-p", profile, "--alsologtostderr", "-v=1"}
-	ss, err := Start(t, exec.CommandContext(ctx, Target(), args...))
+	ss, err := Start(t, exec.CommandContext(mctx, Target(), args...))
 	if err != nil {
 		t.Errorf("failed to run minikube dashboard. args %q : %v", args, err)
 	}
@@ -555,13 +559,12 @@ func validateDashboardCmd(ctx context.Context, t *testing.T, profile string) {
 		ss.Stop(t)
 	}()
 
-	start := time.Now()
-	s, err := ReadLineWithTimeout(ss.Stdout, Seconds(300))
+	s, err := dashboardURL(ss.Stdout)
 	if err != nil {
 		if runtime.GOOS == "windows" {
-			t.Skipf("failed to read url within %s: %v\noutput: %q\n", time.Since(start), err, s)
+			t.Skip(err)
 		}
-		t.Fatalf("failed to read url within %s: %v\noutput: %q\n", time.Since(start), err, s)
+		t.Fatal(err)
 	}
 
 	u, err := url.Parse(strings.TrimSpace(s))
@@ -583,6 +586,18 @@ func validateDashboardCmd(ctx context.Context, t *testing.T, profile string) {
 	}
 }
 
+// dashboardURL gets the dashboard URL from the command stdout.
+func dashboardURL(b *bufio.Reader) (string, error) {
+	s := bufio.NewScanner(b)
+	for s.Scan() {
+		l := s.Text()
+		if strings.HasPrefix(l, "http") {
+			return l, nil
+		}
+	}
+	return "", fmt.Errorf("output didn't produce a URL")
+}
+
 // validateDryRun asserts that the dry-run mode quickly exits with the right code
 func validateDryRun(ctx context.Context, t *testing.T, profile string) {
 	// dry-run mode should always be able to finish quickly (<5s)
diff --git a/test/integration/util_test.go b/test/integration/util_test.go
index c924b78c2e..c5d3094a36 100644
--- a/test/integration/util_test.go
+++ b/test/integration/util_test.go
@@ -26,31 +26,6 @@ import (
 	"k8s.io/minikube/pkg/minikube/localpath"
 )
 
-// ReadLineWithTimeout reads a line of text from a buffer with a timeout
-func ReadLineWithTimeout(b *bufio.Reader, timeout time.Duration) (string, error) {
-	s := make(chan string)
-	e := make(chan error)
-	go func() {
-		read, err := b.ReadString('\n')
-		if err != nil {
-			e <- err
-		} else {
-			s <- read
-		}
-		close(s)
-		close(e)
-	}()
-
-	select {
-	case line := <-s:
-		return line, nil
-	case err := <-e:
-		return "", err
-	case <-time.After(timeout):
-		return "", fmt.Errorf("timeout after %s", timeout)
-	}
-}
-
 // UniqueProfileName returns a reasonably unique profile name
 func UniqueProfileName(prefix string) string {
 	if *forceProfile != "" {

From 969a52b38cd743b2557df9bf7cd3a635cfa891dc Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <t+github@stromberg.org>
Date: Thu, 18 Feb 2021 13:16:17 -0800
Subject: [PATCH 091/114] Loosen docker_engine pipe regexp

---
 pkg/minikube/registry/drvs/docker/docker.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/minikube/registry/drvs/docker/docker.go b/pkg/minikube/registry/drvs/docker/docker.go
index 4a16ded746..f1ee471a82 100644
--- a/pkg/minikube/registry/drvs/docker/docker.go
+++ b/pkg/minikube/registry/drvs/docker/docker.go
@@ -166,7 +166,7 @@ func suggestFix(stderr string, err error) registry.State {
 		return registry.State{Error: err, Installed: true, Running: true, Healthy: false, Fix: "Add your user to the 'docker' group: 'sudo usermod -aG docker $USER && newgrp docker'", Doc: "https://docs.docker.com/engine/install/linux-postinstall/"}
 	}
 
-	if strings.Contains(stderr, "/pipe/docker_engine: The system cannot find the file specified.") && runtime.GOOS == "windows" {
+	if strings.Contains(stderr, "pipe.*docker_engine.*: The system cannot find the file specified.") && runtime.GOOS == "windows" {
 		return registry.State{Error: err, Installed: true, Running: false, Healthy: false, Fix: "Start the Docker service. If Docker is already running, you may need to reset Docker to factory settings with: Settings > Reset.", Doc: "https://github.com/docker/for-win/issues/1825#issuecomment-450501157"}
 	}
 

From e0a74c23969122eb8fdf4301eb06f2c813602930 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Thu, 18 Feb 2021 14:20:08 -0800
Subject: [PATCH 092/114] fix variable names

---
 hack/jenkins/kicbase_auto_build.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/hack/jenkins/kicbase_auto_build.sh b/hack/jenkins/kicbase_auto_build.sh
index 336c4ef7f2..dd847f0338 100755
--- a/hack/jenkins/kicbase_auto_build.sh
+++ b/hack/jenkins/kicbase_auto_build.sh
@@ -26,7 +26,7 @@ KV=$(egrep "Version =" pkg/drivers/kic/types.go | cut -d \" -f 2 | cut -d "-" -f
 GCR_REPO=gcr.io/k8s-minikube/kicbase-builds
 DH_REPO=kicbase/build
 export KIC_VERSION=$KV-$now-$ghprbPullId
-export KICBASE_IMAGE_REGISTRIES="${KIC_REPO}:${KIC_VERSION} ${DH_REPO}:${KIC_VERSION}"
+export KICBASE_IMAGE_REGISTRIES="${GCR_REPO}:${KIC_VERSION} ${DH_REPO}:${KIC_VERSION}"
 
 # Let's make sure we have the newest kicbase reference
 curl -L https://github.com/kubernetes/minikube/raw/master/pkg/drivers/kic/types.go --output types-head.go
@@ -62,7 +62,7 @@ fullsha=$(docker inspect --format='{{index .RepoDigests 0}}' $KICBASE_IMAGE_REGI
 sha=$(echo ${fullsha} | cut -d ":" -f 2)
 
 # Display the message to the user
-message="Hi ${ghprbPullAuthorLoginMention},\\n\\nA new kicbase image is available, please update your PR with the new tag and SHA.\\nIn pkg/drivers/kic/types.go:\\n\\n\\t// Version is the current version of kic\\n\\tVersion = \\\"${KIC_VERSION}\\\"\\n\\t// SHA of the kic base image\\n\\tbaseImageSHA = \\\"${sha}\\\"\\n\\t// The name of the GCR kicbase repository\\n\\tgcrRepo = \\\"${KIC_REPO}\\\"\\n\\t// The name of the Dockerhub kicbase repository\\n\\tdockerhubRepo = \\\"${DH_REPO}\\\"\\nThen run \`make generate-docs\` to update our documentation to reference the new image.\n\nAlternatively, run the following command and commit the changes:\\n\`\`\`\\n sed 's|Version = .*|Version = \\\"${KIC_VERSION}\\\"|;s|baseImageSHA = .*|baseImageSHA = \\\"${sha}\\\"|;s|gcrRepo = .*|gcrRepo = \\\"${KIC_REPO}\\\"|;s|dockerhubRepo = .*|dockerhubRepo = ${DH_REPO}|' pkg/drivers/kic/types.go > new-types.go; mv new-types.go pkg/drivers/kic/types.go; make generate-docs\\n\`\`\`"
+message="Hi ${ghprbPullAuthorLoginMention},\\n\\nA new kicbase image is available, please update your PR with the new tag and SHA.\\nIn pkg/drivers/kic/types.go:\\n\\n\\t// Version is the current version of kic\\n\\tVersion = \\\"${KIC_VERSION}\\\"\\n\\t// SHA of the kic base image\\n\\tbaseImageSHA = \\\"${sha}\\\"\\n\\t// The name of the GCR kicbase repository\\n\\tgcrRepo = \\\"${GCR_REPO}\\\"\\n\\t// The name of the Dockerhub kicbase repository\\n\\tdockerhubRepo = \\\"${DH_REPO}\\\"\\nThen run \`make generate-docs\` to update our documentation to reference the new image.\n\nAlternatively, run the following command and commit the changes:\\n\`\`\`\\n sed 's|Version = .*|Version = \\\"${KIC_VERSION}\\\"|;s|baseImageSHA = .*|baseImageSHA = \\\"${sha}\\\"|;s|gcrRepo = .*|gcrRepo = \\\"${GCR_REPO}\\\"|;s|dockerhubRepo = .*|dockerhubRepo = ${DH_REPO}|' pkg/drivers/kic/types.go > new-types.go; mv new-types.go pkg/drivers/kic/types.go; make generate-docs\\n\`\`\`"
 
 curl -s -H "Authorization: token ${access_token}" \
 	 -H "Accept: application/vnd.github.v3+json" \

From df036e1e006f09ba280306a4ffea888e55ed2a90 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Thu, 18 Feb 2021 14:40:09 -0800
Subject: [PATCH 093/114] retrieve sha properly

---
 hack/jenkins/kicbase_auto_build.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/hack/jenkins/kicbase_auto_build.sh b/hack/jenkins/kicbase_auto_build.sh
index dd847f0338..a6be4e85ba 100755
--- a/hack/jenkins/kicbase_auto_build.sh
+++ b/hack/jenkins/kicbase_auto_build.sh
@@ -26,7 +26,9 @@ KV=$(egrep "Version =" pkg/drivers/kic/types.go | cut -d \" -f 2 | cut -d "-" -f
 GCR_REPO=gcr.io/k8s-minikube/kicbase-builds
 DH_REPO=kicbase/build
 export KIC_VERSION=$KV-$now-$ghprbPullId
-export KICBASE_IMAGE_REGISTRIES="${GCR_REPO}:${KIC_VERSION} ${DH_REPO}:${KIC_VERSION}"
+GCR_IMG=${GCR_REPO}:${KIC_VERSION}
+DH_IMG=${DH_REPO}:${KIC_VERSION}
+export KICBASE_IMAGE_REGISTRIES="${GCR_IMG} ${DH_IMG}"
 
 # Let's make sure we have the newest kicbase reference
 curl -L https://github.com/kubernetes/minikube/raw/master/pkg/drivers/kic/types.go --output types-head.go
@@ -57,7 +59,7 @@ if [ $? -gt 0 ]; then
 fi
 
 # Retrieve the sha from the new imnage
-docker pull $KICBASE_IMAGE_REGISTRIES
+docker pull $GCR_IMG
 fullsha=$(docker inspect --format='{{index .RepoDigests 0}}' $KICBASE_IMAGE_REGISTRIES)
 sha=$(echo ${fullsha} | cut -d ":" -f 2)
 

From dfc2e854655d8a6665fa4ac5bbc0043a123fc676 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Thu, 18 Feb 2021 14:48:46 -0800
Subject: [PATCH 094/114] proper exit codes

---
 hack/jenkins/kicbase_auto_build.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/hack/jenkins/kicbase_auto_build.sh b/hack/jenkins/kicbase_auto_build.sh
index a6be4e85ba..24a102f137 100755
--- a/hack/jenkins/kicbase_auto_build.sh
+++ b/hack/jenkins/kicbase_auto_build.sh
@@ -52,13 +52,14 @@ yes|make push-kic-base-image
 
 # Abort with error message if above command failed
 if [ $? -gt 0 ]; then
+	ec=$?
 	curl -s -H "Authorization: token ${access_token}" \
                 -H "Accept: application/vnd.github.v3+json" \
                 -X POST -d "{\"body\": \"Hi ${ghprbPullAuthorLoginMention}, building a new kicbase image failed, please try again.\"}" "https://api.github.com/repos/kubernetes/minikube/issues/$ghprbPullId/comments"
-	exit $?
+	exit $ec
 fi
 
-# Retrieve the sha from the new imnage
+# Retrieve the sha from the new image
 docker pull $GCR_IMG
 fullsha=$(docker inspect --format='{{index .RepoDigests 0}}' $KICBASE_IMAGE_REGISTRIES)
 sha=$(echo ${fullsha} | cut -d ":" -f 2)

From 31ea98ca504b68ed617a9d1457b2ad1092958582 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Thu, 18 Feb 2021 15:04:34 -0800
Subject: [PATCH 095/114] update kicbase to newest build

---
 hack/jenkins/kicbase_auto_build.sh     | 2 +-
 pkg/drivers/kic/types.go               | 6 +++---
 site/content/en/docs/commands/start.md | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/hack/jenkins/kicbase_auto_build.sh b/hack/jenkins/kicbase_auto_build.sh
index 24a102f137..9df71b5ec7 100755
--- a/hack/jenkins/kicbase_auto_build.sh
+++ b/hack/jenkins/kicbase_auto_build.sh
@@ -65,7 +65,7 @@ fullsha=$(docker inspect --format='{{index .RepoDigests 0}}' $KICBASE_IMAGE_REGI
 sha=$(echo ${fullsha} | cut -d ":" -f 2)
 
 # Display the message to the user
-message="Hi ${ghprbPullAuthorLoginMention},\\n\\nA new kicbase image is available, please update your PR with the new tag and SHA.\\nIn pkg/drivers/kic/types.go:\\n\\n\\t// Version is the current version of kic\\n\\tVersion = \\\"${KIC_VERSION}\\\"\\n\\t// SHA of the kic base image\\n\\tbaseImageSHA = \\\"${sha}\\\"\\n\\t// The name of the GCR kicbase repository\\n\\tgcrRepo = \\\"${GCR_REPO}\\\"\\n\\t// The name of the Dockerhub kicbase repository\\n\\tdockerhubRepo = \\\"${DH_REPO}\\\"\\nThen run \`make generate-docs\` to update our documentation to reference the new image.\n\nAlternatively, run the following command and commit the changes:\\n\`\`\`\\n sed 's|Version = .*|Version = \\\"${KIC_VERSION}\\\"|;s|baseImageSHA = .*|baseImageSHA = \\\"${sha}\\\"|;s|gcrRepo = .*|gcrRepo = \\\"${GCR_REPO}\\\"|;s|dockerhubRepo = .*|dockerhubRepo = ${DH_REPO}|' pkg/drivers/kic/types.go > new-types.go; mv new-types.go pkg/drivers/kic/types.go; make generate-docs\\n\`\`\`"
+message="Hi ${ghprbPullAuthorLoginMention},\\n\\nA new kicbase image is available, please update your PR with the new tag and SHA.\\nIn pkg/drivers/kic/types.go:\\n\\n\\t// Version is the current version of kic\\n\\tVersion = \\\"${KIC_VERSION}\\\"\\n\\t// SHA of the kic base image\\n\\tbaseImageSHA = \\\"${sha}\\\"\\n\\t// The name of the GCR kicbase repository\\n\\tgcrRepo = \\\"${GCR_REPO}\\\"\\n\\t// The name of the Dockerhub kicbase repository\\n\\tdockerhubRepo = \\\"${DH_REPO}\\\"\\nThen run \`make generate-docs\` to update our documentation to reference the new image.\n\nAlternatively, run the following command and commit the changes:\\n\`\`\`\\n sed 's|Version = .*|Version = \\\"${KIC_VERSION}\\\"|;s|baseImageSHA = .*|baseImageSHA = \\\"${sha}\\\"|;s|gcrRepo = .*|gcrRepo = \\\"${GCR_REPO}\\\"|;s|dockerhubRepo = .*|dockerhubRepo = \\\"${DH_REPO}\\\"|' pkg/drivers/kic/types.go > new-types.go; mv new-types.go pkg/drivers/kic/types.go; make generate-docs\\n\`\`\`"
 
 curl -s -H "Authorization: token ${access_token}" \
 	 -H "Accept: application/vnd.github.v3+json" \
diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go
index e26ef3f58f..2df051c66e 100644
--- a/pkg/drivers/kic/types.go
+++ b/pkg/drivers/kic/types.go
@@ -24,13 +24,13 @@ import (
 
 const (
 	// Version is the current version of kic
-	Version = "v0.0.17-1613582775-10408"
+	Version = "v0.0.17-1613688791-10408"
 	// SHA of the kic base image
-	baseImageSHA = "1537fe47d39640aa11d6c819fe39cbba6e250872c8fe2bd7701c5171a32fbc4e"
+	baseImageSHA = "b8cea5f43f362e3c8e15a3731df2af7dcd08b82b5cec91feaceab9ad70fbddaf"
 	// The name of the GCR kicbase repository
 	gcrRepo = "gcr.io/k8s-minikube/kicbase-builds"
 	// The name of the Dockerhub kicbase repository
-	dockerhubRepo = "kicbase/stable"
+	dockerhubRepo = "kicbase/build"
 )
 
 var (
diff --git a/site/content/en/docs/commands/start.md b/site/content/en/docs/commands/start.md
index b9f7c81804..57ddd3c361 100644
--- a/site/content/en/docs/commands/start.md
+++ b/site/content/en/docs/commands/start.md
@@ -26,7 +26,7 @@ minikube start [flags]
       --apiserver-names strings           A set of apiserver names which are used in the generated certificate for kubernetes.  This can be used if you want to make the apiserver available from outside the machine
       --apiserver-port int                The apiserver listening port (default 8443)
       --auto-update-drivers               If set, automatically updates drivers to the latest version. Defaults to true. (default true)
-      --base-image string                 The base image to use for docker/podman drivers. Intended for local development. (default "gcr.io/k8s-minikube/kicbase-builds:v0.0.17-1613582775-10408@sha256:1537fe47d39640aa11d6c819fe39cbba6e250872c8fe2bd7701c5171a32fbc4e")
+      --base-image string                 The base image to use for docker/podman drivers. Intended for local development. (default "gcr.io/k8s-minikube/kicbase-builds:v0.0.17-1613688791-10408@sha256:b8cea5f43f362e3c8e15a3731df2af7dcd08b82b5cec91feaceab9ad70fbddaf")
       --cache-images                      If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --driver=none. (default true)
       --cni string                        CNI plug-in to use. Valid options: auto, bridge, calico, cilium, flannel, kindnet, or path to a CNI manifest (default: auto)
       --container-runtime string          The container runtime to be used (docker, cri-o, containerd). (default "docker")

From 7b0bf57f4c9b87a7b0c9362bdfbb5c0add2094d7 Mon Sep 17 00:00:00 2001
From: hetong07 <hetong07@gmail.com>
Date: Thu, 18 Feb 2021 15:43:26 -0800
Subject: [PATCH 096/114] Stop using --memory for cgroup v2.

---
 pkg/drivers/kic/oci/oci.go | 28 +++++++++++++++++++++++-----
 1 file changed, 23 insertions(+), 5 deletions(-)

diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go
index 4993ddcd4c..7007e50a24 100644
--- a/pkg/drivers/kic/oci/oci.go
+++ b/pkg/drivers/kic/oci/oci.go
@@ -107,14 +107,27 @@ func PrepareContainerNode(p CreateParams) error {
 	return nil
 }
 
+func hasMemoryCgroup() bool {
+	memcg := true
+	if runtime.GOOS == "linux" {
+		var memory string
+		if cgroup2, err := IsCgroup2UnifiedMode(); err == nil && cgroup2 {
+			memory = "/sys/fs/cgroup/memory/memsw.limit_in_bytes"
+		}
+		if _, err := os.Stat(memory); os.IsNotExist(err) {
+			klog.Warning("Your kernel does not support memory limit capabilities or the cgroup is not mounted.")
+			memcg = false
+		}
+	}
+	return memcg
+}
+
 func hasMemorySwapCgroup() bool {
 	memcgSwap := true
 	if runtime.GOOS == "linux" {
 		var memoryswap string
 		if cgroup2, err := IsCgroup2UnifiedMode(); err == nil && cgroup2 {
 			memoryswap = "/sys/fs/cgroup/memory/memory.swap.max"
-		} else {
-			memoryswap = "/sys/fs/cgroup/memory/memsw.limit_in_bytes"
 		}
 		if _, err := os.Stat(memoryswap); os.IsNotExist(err) {
 			// requires CONFIG_MEMCG_SWAP_ENABLED or cgroup_enable=memory in grub
@@ -171,6 +184,7 @@ func CreateContainerNode(p CreateParams) error {
 	}
 
 	memcgSwap := hasMemorySwapCgroup()
+	memcg := hasMemoryCgroup()
 
 	// https://www.freedesktop.org/wiki/Software/systemd/ContainerInterface/
 	var virtualization string
@@ -179,11 +193,13 @@ func CreateContainerNode(p CreateParams) error {
 		runArgs = append(runArgs, "--volume", fmt.Sprintf("%s:/var:exec", p.Name))
 
 		if memcgSwap {
-			runArgs = append(runArgs, fmt.Sprintf("--memory=%s", p.Memory))
-			// Disable swap by setting the value to match
 			runArgs = append(runArgs, fmt.Sprintf("--memory-swap=%s", p.Memory))
 		}
 
+		if memcg {
+			runArgs = append(runArgs, fmt.Sprintf("--memory=%s", p.Memory))
+		}
+
 		virtualization = "podman" // VIRTUALIZATION_PODMAN
 	}
 	if p.OCIBinary == Docker {
@@ -191,7 +207,9 @@ func CreateContainerNode(p CreateParams) error {
 		// ignore apparmore github actions docker: https://github.com/kubernetes/minikube/issues/7624
 		runArgs = append(runArgs, "--security-opt", "apparmor=unconfined")
 
-		runArgs = append(runArgs, fmt.Sprintf("--memory=%s", p.Memory))
+		if memcg {
+			runArgs = append(runArgs, fmt.Sprintf("--memory=%s", p.Memory))
+		}
 		if memcgSwap {
 			// Disable swap by setting the value to match
 			runArgs = append(runArgs, fmt.Sprintf("--memory-swap=%s", p.Memory))

From e8c2a233982215ca4f73bd4fb990dc5770070586 Mon Sep 17 00:00:00 2001
From: hetong07 <hetong07@gmail.com>
Date: Thu, 18 Feb 2021 16:09:51 -0800
Subject: [PATCH 097/114] Provide an advice for users on how to modify Grub
 setting.

---
 pkg/drivers/kic/oci/oci.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go
index 7007e50a24..cbf3188ea6 100644
--- a/pkg/drivers/kic/oci/oci.go
+++ b/pkg/drivers/kic/oci/oci.go
@@ -186,6 +186,10 @@ func CreateContainerNode(p CreateParams) error {
 	memcgSwap := hasMemorySwapCgroup()
 	memcg := hasMemoryCgroup()
 
+	if !memcgSwap || !memcg {
+		out.WarningT("Cgroup v2 does not allow setting memory, if you want to set memory, please modify your Grub as instructed in https://docs.docker.com/engine/install/linux-postinstall/#your-kernel-does-not-support-cgroup-swap-limit-capabilities")
+	}
+
 	// https://www.freedesktop.org/wiki/Software/systemd/ContainerInterface/
 	var virtualization string
 	if p.OCIBinary == Podman { // enable execing in /var

From 114364409d9a1c0c91291cbd0f914affb9f40b7b Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Thu, 18 Feb 2021 16:17:09 -0800
Subject: [PATCH 098/114] add github packages to script

---
 hack/jenkins/kicbase_auto_build.sh | 7 +++++--
 pkg/drivers/kic/types.go           | 4 +++-
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/hack/jenkins/kicbase_auto_build.sh b/hack/jenkins/kicbase_auto_build.sh
index 9df71b5ec7..5ec4886096 100755
--- a/hack/jenkins/kicbase_auto_build.sh
+++ b/hack/jenkins/kicbase_auto_build.sh
@@ -19,16 +19,19 @@ set -x
 # Make sure docker is installed and configured
 ./hack/jenkins/installers/check_install_docker.sh
 yes|gcloud auth configure-docker
+docker login -u ${DOCKERHUB_USER} -p ${DOCKERHUB_PASS}
 
 # Setup variables
 now=$(date +%s)
 KV=$(egrep "Version =" pkg/drivers/kic/types.go | cut -d \" -f 2 | cut -d "-" -f 1)
 GCR_REPO=gcr.io/k8s-minikube/kicbase-builds
 DH_REPO=kicbase/build
+GH_REPO=docker.pkg.github.com/kubernetes/minikube/kicbase-build
 export KIC_VERSION=$KV-$now-$ghprbPullId
 GCR_IMG=${GCR_REPO}:${KIC_VERSION}
 DH_IMG=${DH_REPO}:${KIC_VERSION}
-export KICBASE_IMAGE_REGISTRIES="${GCR_IMG} ${DH_IMG}"
+GH_IMG=${GH_REPO}:${KIC_VERSION}
+export KICBASE_IMAGE_REGISTRIES="${GCR_IMG} ${DH_IMG} ${GH_IMG}"
 
 # Let's make sure we have the newest kicbase reference
 curl -L https://github.com/kubernetes/minikube/raw/master/pkg/drivers/kic/types.go --output types-head.go
@@ -65,7 +68,7 @@ fullsha=$(docker inspect --format='{{index .RepoDigests 0}}' $KICBASE_IMAGE_REGI
 sha=$(echo ${fullsha} | cut -d ":" -f 2)
 
 # Display the message to the user
-message="Hi ${ghprbPullAuthorLoginMention},\\n\\nA new kicbase image is available, please update your PR with the new tag and SHA.\\nIn pkg/drivers/kic/types.go:\\n\\n\\t// Version is the current version of kic\\n\\tVersion = \\\"${KIC_VERSION}\\\"\\n\\t// SHA of the kic base image\\n\\tbaseImageSHA = \\\"${sha}\\\"\\n\\t// The name of the GCR kicbase repository\\n\\tgcrRepo = \\\"${GCR_REPO}\\\"\\n\\t// The name of the Dockerhub kicbase repository\\n\\tdockerhubRepo = \\\"${DH_REPO}\\\"\\nThen run \`make generate-docs\` to update our documentation to reference the new image.\n\nAlternatively, run the following command and commit the changes:\\n\`\`\`\\n sed 's|Version = .*|Version = \\\"${KIC_VERSION}\\\"|;s|baseImageSHA = .*|baseImageSHA = \\\"${sha}\\\"|;s|gcrRepo = .*|gcrRepo = \\\"${GCR_REPO}\\\"|;s|dockerhubRepo = .*|dockerhubRepo = \\\"${DH_REPO}\\\"|' pkg/drivers/kic/types.go > new-types.go; mv new-types.go pkg/drivers/kic/types.go; make generate-docs\\n\`\`\`"
+message="Hi ${ghprbPullAuthorLoginMention},\\n\\nA new kicbase image is available, please update your PR with the new tag and SHA.\\nIn pkg/drivers/kic/types.go:\\n\\n\\t// Version is the current version of kic\\n\\tVersion = \\\"${KIC_VERSION}\\\"\\n\\t// SHA of the kic base image\\n\\tbaseImageSHA = \\\"${sha}\\\"\\n\\t// The name of the GCR kicbase repository\\n\\tgcrRepo = \\\"${GCR_REPO}\\\"\\n\\t// The name of the Dockerhub kicbase repository\\n\\tdockerhubRepo = \\\"${DH_REPO}\\\"\\n\\t// The name of the Github Packages repository\\n\\tghRepo = \\\"${GH_REPO}\\\"\\nThen run \`make generate-docs\` to update our documentation to reference the new image.\n\nAlternatively, run the following command and commit the changes:\\n\`\`\`\\n sed 's|Version = .*|Version = \\\"${KIC_VERSION}\\\"|;s|baseImageSHA = .*|baseImageSHA = \\\"${sha}\\\"|;s|gcrRepo = .*|gcrRepo = \\\"${GCR_REPO}\\\"|;s|dockerhubRepo = .*|dockerhubRepo = \\\"${DH_REPO}\\\"|;s|ghRepo = .*|ghRepo = \\\"${GH_REPO}\\\"' pkg/drivers/kic/types.go > new-types.go; mv new-types.go pkg/drivers/kic/types.go; make generate-docs;\\n\`\`\`"
 
 curl -s -H "Authorization: token ${access_token}" \
 	 -H "Accept: application/vnd.github.v3+json" \
diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go
index 2df051c66e..49fc0a2171 100644
--- a/pkg/drivers/kic/types.go
+++ b/pkg/drivers/kic/types.go
@@ -31,6 +31,8 @@ const (
 	gcrRepo = "gcr.io/k8s-minikube/kicbase-builds"
 	// The name of the Dockerhub kicbase repository
 	dockerhubRepo = "kicbase/build"
+	// The name of the Github Packages repository
+	ghRepo = "kicbase"
 )
 
 var (
@@ -46,7 +48,7 @@ var (
 		// the fallback of BaseImage in case gcr.io is not available. stored in github packages https://github.com/kubernetes/minikube/packages/206071
 		// github packages docker does _NOT_ support pulling by sha as mentioned in the docs:
 		// https://help.github.com/en/packages/using-github-packages-with-your-projects-ecosystem/configuring-docker-for-use-with-github-packages
-		fmt.Sprintf("docker.pkg.github.com/kubernetes/minikube/kicbase:%s", Version),
+		fmt.Sprintf("docker.pkg.github.com/kubernetes/minikube/%s:%s", ghRepo, Version),
 	}
 )
 

From d9aca3d0c9fafe1b640f8251fc247d9217ff6f6a Mon Sep 17 00:00:00 2001
From: Steven Powell <spowellgrr@gmail.com>
Date: Thu, 18 Feb 2021 17:34:22 -0700
Subject: [PATCH 099/114] make dashboard URL check more strict

---
 test/integration/functional_test.go | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go
index cc7f19cd89..9cf735c840 100644
--- a/test/integration/functional_test.go
+++ b/test/integration/functional_test.go
@@ -559,7 +559,7 @@ func validateDashboardCmd(ctx context.Context, t *testing.T, profile string) {
 		ss.Stop(t)
 	}()
 
-	s, err := dashboardURL(ss.Stdout)
+	s, err := dashboardURL(t, ss.Stdout)
 	if err != nil {
 		if runtime.GOOS == "windows" {
 			t.Skip(err)
@@ -587,11 +587,18 @@ func validateDashboardCmd(ctx context.Context, t *testing.T, profile string) {
 }
 
 // dashboardURL gets the dashboard URL from the command stdout.
-func dashboardURL(b *bufio.Reader) (string, error) {
+func dashboardURL(t *testing.T, b *bufio.Reader) (string, error) {
 	s := bufio.NewScanner(b)
+
+	// match http://127.0.0.1:XXXXX/api/v1/namespaces/kubernetes-dashboard/services/http:kubernetes-dashboard:/proxy/
+	dashURLRegexp, err := regexp.Compile(`^http:\/\/127\.0\.0\.1:[0-9]{5}\/api\/v1\/namespaces\/kubernetes-dashboard\/services\/http:kubernetes-dashboard:\/proxy\/$`)
+	if err != nil {
+		t.Fatalf("dashboard URL regex is invalid: %v", err)
+	}
+
 	for s.Scan() {
 		l := s.Text()
-		if strings.HasPrefix(l, "http") {
+		if dashURLRegexp.MatchString(l) {
 			return l, nil
 		}
 	}

From 635f89b079d8807ccf2f71c32ff4663ad21d3479 Mon Sep 17 00:00:00 2001
From: Steven Powell <spowellgrr@gmail.com>
Date: Thu, 18 Feb 2021 17:39:44 -0700
Subject: [PATCH 100/114] variable rename

---
 test/integration/functional_test.go | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go
index 9cf735c840..4fbd0304c5 100644
--- a/test/integration/functional_test.go
+++ b/test/integration/functional_test.go
@@ -588,7 +588,7 @@ func validateDashboardCmd(ctx context.Context, t *testing.T, profile string) {
 
 // dashboardURL gets the dashboard URL from the command stdout.
 func dashboardURL(t *testing.T, b *bufio.Reader) (string, error) {
-	s := bufio.NewScanner(b)
+	scanner := bufio.NewScanner(b)
 
 	// match http://127.0.0.1:XXXXX/api/v1/namespaces/kubernetes-dashboard/services/http:kubernetes-dashboard:/proxy/
 	dashURLRegexp, err := regexp.Compile(`^http:\/\/127\.0\.0\.1:[0-9]{5}\/api\/v1\/namespaces\/kubernetes-dashboard\/services\/http:kubernetes-dashboard:\/proxy\/$`)
@@ -596,10 +596,10 @@ func dashboardURL(t *testing.T, b *bufio.Reader) (string, error) {
 		t.Fatalf("dashboard URL regex is invalid: %v", err)
 	}
 
-	for s.Scan() {
-		l := s.Text()
-		if dashURLRegexp.MatchString(l) {
-			return l, nil
+	for scanner.Scan() {
+		s := scanner.Text()
+		if dashURLRegexp.MatchString(s) {
+			return s, nil
 		}
 	}
 	return "", fmt.Errorf("output didn't produce a URL")

From 2982b7db4ca82349eb28a49288112c709325d0bc Mon Sep 17 00:00:00 2001
From: Steven Powell <spowellgrr@gmail.com>
Date: Thu, 18 Feb 2021 17:42:06 -0700
Subject: [PATCH 101/114] use MustCompile instead of Compile as per Go lint

---
 test/integration/functional_test.go | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go
index 4fbd0304c5..53f0a632b9 100644
--- a/test/integration/functional_test.go
+++ b/test/integration/functional_test.go
@@ -559,7 +559,7 @@ func validateDashboardCmd(ctx context.Context, t *testing.T, profile string) {
 		ss.Stop(t)
 	}()
 
-	s, err := dashboardURL(t, ss.Stdout)
+	s, err := dashboardURL(ss.Stdout)
 	if err != nil {
 		if runtime.GOOS == "windows" {
 			t.Skip(err)
@@ -587,14 +587,11 @@ func validateDashboardCmd(ctx context.Context, t *testing.T, profile string) {
 }
 
 // dashboardURL gets the dashboard URL from the command stdout.
-func dashboardURL(t *testing.T, b *bufio.Reader) (string, error) {
+func dashboardURL(b *bufio.Reader) (string, error) {
 	scanner := bufio.NewScanner(b)
 
 	// match http://127.0.0.1:XXXXX/api/v1/namespaces/kubernetes-dashboard/services/http:kubernetes-dashboard:/proxy/
-	dashURLRegexp, err := regexp.Compile(`^http:\/\/127\.0\.0\.1:[0-9]{5}\/api\/v1\/namespaces\/kubernetes-dashboard\/services\/http:kubernetes-dashboard:\/proxy\/$`)
-	if err != nil {
-		t.Fatalf("dashboard URL regex is invalid: %v", err)
-	}
+	dashURLRegexp := regexp.MustCompile(`^http:\/\/127\.0\.0\.1:[0-9]{5}\/api\/v1\/namespaces\/kubernetes-dashboard\/services\/http:kubernetes-dashboard:\/proxy\/$`)
 
 	for scanner.Scan() {
 		s := scanner.Text()

From 556a8d793040f97927e1ea177ca92d1402ce8250 Mon Sep 17 00:00:00 2001
From: hetong07 <hetong07@gmail.com>
Date: Thu, 18 Feb 2021 16:43:19 -0800
Subject: [PATCH 102/114] Move the user facing warning inside
 hasMemoryCgroup().

---
 pkg/drivers/kic/oci/oci.go | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go
index cbf3188ea6..4fa94113b9 100644
--- a/pkg/drivers/kic/oci/oci.go
+++ b/pkg/drivers/kic/oci/oci.go
@@ -116,6 +116,7 @@ func hasMemoryCgroup() bool {
 		}
 		if _, err := os.Stat(memory); os.IsNotExist(err) {
 			klog.Warning("Your kernel does not support memory limit capabilities or the cgroup is not mounted.")
+			out.WarningT("Cgroup v2 does not allow setting memory, if you want to set memory, please modify your Grub as instructed in https://docs.docker.com/engine/install/linux-postinstall/#your-kernel-does-not-support-cgroup-swap-limit-capabilities")
 			memcg = false
 		}
 	}
@@ -186,10 +187,6 @@ func CreateContainerNode(p CreateParams) error {
 	memcgSwap := hasMemorySwapCgroup()
 	memcg := hasMemoryCgroup()
 
-	if !memcgSwap || !memcg {
-		out.WarningT("Cgroup v2 does not allow setting memory, if you want to set memory, please modify your Grub as instructed in https://docs.docker.com/engine/install/linux-postinstall/#your-kernel-does-not-support-cgroup-swap-limit-capabilities")
-	}
-
 	// https://www.freedesktop.org/wiki/Software/systemd/ContainerInterface/
 	var virtualization string
 	if p.OCIBinary == Podman { // enable execing in /var

From 1a5c1f8219a021d3ebe77349d589e1e49ad7e76e Mon Sep 17 00:00:00 2001
From: Steven Powell <spowellgrr@gmail.com>
Date: Thu, 18 Feb 2021 17:44:23 -0700
Subject: [PATCH 103/114] refactored to make a bit cleaner

---
 test/integration/functional_test.go | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go
index 53f0a632b9..4f78a2281e 100644
--- a/test/integration/functional_test.go
+++ b/test/integration/functional_test.go
@@ -588,15 +588,14 @@ func validateDashboardCmd(ctx context.Context, t *testing.T, profile string) {
 
 // dashboardURL gets the dashboard URL from the command stdout.
 func dashboardURL(b *bufio.Reader) (string, error) {
-	scanner := bufio.NewScanner(b)
-
 	// match http://127.0.0.1:XXXXX/api/v1/namespaces/kubernetes-dashboard/services/http:kubernetes-dashboard:/proxy/
 	dashURLRegexp := regexp.MustCompile(`^http:\/\/127\.0\.0\.1:[0-9]{5}\/api\/v1\/namespaces\/kubernetes-dashboard\/services\/http:kubernetes-dashboard:\/proxy\/$`)
 
-	for scanner.Scan() {
-		s := scanner.Text()
-		if dashURLRegexp.MatchString(s) {
-			return s, nil
+	s := bufio.NewScanner(b)
+	for s.Scan() {
+		t := s.Text()
+		if dashURLRegexp.MatchString(t) {
+			return t, nil
 		}
 	}
 	return "", fmt.Errorf("output didn't produce a URL")

From c5c1ee90138fd900b303fca6fbbbbe1bdb849de4 Mon Sep 17 00:00:00 2001
From: Steven Powell <spowellgrr@gmail.com>
Date: Thu, 18 Feb 2021 18:13:17 -0700
Subject: [PATCH 104/114] improved comments, var names & error messages

---
 pkg/minikube/audit/logFile.go            |  2 +-
 pkg/minikube/audit/report.go             | 13 +++++++------
 pkg/minikube/audit/report_test.go        |  2 +-
 pkg/minikube/audit/row.go                | 15 +++++++++------
 pkg/minikube/audit/row_test.go           |  6 +++---
 pkg/minikube/logs/logs.go                |  2 +-
 test/integration/version_upgrade_test.go |  2 +-
 7 files changed, 23 insertions(+), 19 deletions(-)

diff --git a/pkg/minikube/audit/logFile.go b/pkg/minikube/audit/logFile.go
index 0cadd67df3..1762070f1e 100644
--- a/pkg/minikube/audit/logFile.go
+++ b/pkg/minikube/audit/logFile.go
@@ -38,7 +38,7 @@ func setLogFile() error {
 	return nil
 }
 
-// appendToLog appends the audit entry to the log file.
+// appendToLog appends the row to the log file.
 func appendToLog(row *row) error {
 	if currentLogFile == nil {
 		if err := setLogFile(); err != nil {
diff --git a/pkg/minikube/audit/report.go b/pkg/minikube/audit/report.go
index 4b07ed8eb6..6816757126 100644
--- a/pkg/minikube/audit/report.go
+++ b/pkg/minikube/audit/report.go
@@ -21,15 +21,16 @@ import (
 	"fmt"
 )
 
+// RawReport contains the information required to generate formatted reports.
 type RawReport struct {
 	headers []string
 	rows    []row
 }
 
-// Report is created from the log file.
-func Report(lines int) (*RawReport, error) {
-	if lines <= 0 {
-		return nil, fmt.Errorf("number of lines must be 1 or greater")
+// Report is created using the last n lines from the log file.
+func Report(lastNLines int) (*RawReport, error) {
+	if lastNLines <= 0 {
+		return nil, fmt.Errorf("last n lines must be 1 or greater")
 	}
 	if currentLogFile == nil {
 		if err := setLogFile(); err != nil {
@@ -40,7 +41,7 @@ func Report(lines int) (*RawReport, error) {
 	s := bufio.NewScanner(currentLogFile)
 	for s.Scan() {
 		// pop off the earliest line if already at desired log length
-		if len(logs) == lines {
+		if len(logs) == lastNLines {
 			logs = logs[1:]
 		}
 		logs = append(logs, s.Text())
@@ -61,5 +62,5 @@ func Report(lines int) (*RawReport, error) {
 
 // ASCIITable creates a formatted table using the headers and rows from the report.
 func (rr *RawReport) ASCIITable() string {
-	return rowsToTable(rr.rows, rr.headers)
+	return rowsToASCIITable(rr.rows, rr.headers)
 }
diff --git a/pkg/minikube/audit/report_test.go b/pkg/minikube/audit/report_test.go
index f5f3e8012a..cb5cbfe0af 100644
--- a/pkg/minikube/audit/report_test.go
+++ b/pkg/minikube/audit/report_test.go
@@ -51,6 +51,6 @@ func TestReport(t *testing.T) {
 	}
 
 	if len(r.rows) != wantedLines {
-		t.Fatalf("report has %d lines of logs, want %d", len(r.rows), wantedLines)
+		t.Errorf("report has %d lines of logs, want %d", len(r.rows), wantedLines)
 	}
 }
diff --git a/pkg/minikube/audit/row.go b/pkg/minikube/audit/row.go
index 4c6154aa20..fb5991a008 100644
--- a/pkg/minikube/audit/row.go
+++ b/pkg/minikube/audit/row.go
@@ -45,7 +45,8 @@ func (e *row) Type() string {
 	return "io.k8s.sigs.minikube.audit"
 }
 
-// assignFields converts the map values to their proper fields
+// assignFields converts the map values to their proper fields,
+// to be used when converting from JSON Cloud Event format.
 func (e *row) assignFields() {
 	e.args = e.Data["args"]
 	e.command = e.Data["command"]
@@ -56,7 +57,8 @@ func (e *row) assignFields() {
 	e.version = e.Data["version"]
 }
 
-// toMap combines fields into a string map
+// toMap combines fields into a string map,
+// to be used when converting to JSON Cloud Event format.
 func (e *row) toMap() map[string]string {
 	return map[string]string{
 		"args":      e.args,
@@ -69,7 +71,7 @@ func (e *row) toMap() map[string]string {
 	}
 }
 
-// newRow returns a new audit type.
+// newRow creates a new audit row.
 func newRow(command string, args string, user string, version string, startTime time.Time, endTime time.Time, profile ...string) *row {
 	p := viper.GetString(config.ProfileName)
 	if len(profile) > 0 {
@@ -86,7 +88,8 @@ func newRow(command string, args string, user string, version string, startTime
 	}
 }
 
-// toFields converts a row to an array of fields.
+// toFields converts a row to an array of fields,
+// to be used when converting to a table.
 func (e *row) toFields() []string {
 	return []string{e.command, e.args, e.profile, e.user, e.version, e.startTime, e.endTime}
 }
@@ -105,8 +108,8 @@ func logsToRows(logs []string) ([]row, error) {
 	return rows, nil
 }
 
-// rowsToTable converts audit rows into a formatted table.
-func rowsToTable(rows []row, headers []string) string {
+// rowsToASCIITable converts rows into a formatted ASCII table.
+func rowsToASCIITable(rows []row, headers []string) string {
 	c := [][]string{}
 	for _, r := range rows {
 		c = append(c, r.toFields())
diff --git a/pkg/minikube/audit/row_test.go b/pkg/minikube/audit/row_test.go
index 7e80a5febd..88b54174df 100644
--- a/pkg/minikube/audit/row_test.go
+++ b/pkg/minikube/audit/row_test.go
@@ -70,7 +70,7 @@ func TestRow(t *testing.T) {
 		}
 	})
 
-	t.Run("ToMap", func(t *testing.T) {
+	t.Run("toMap", func(t *testing.T) {
 		m := r.toMap()
 
 		tests := []struct {
@@ -94,7 +94,7 @@ func TestRow(t *testing.T) {
 		}
 	})
 
-	t.Run("ToFields", func(t *testing.T) {
+	t.Run("toFields", func(t *testing.T) {
 		got := r.toFields()
 		gotString := strings.Join(got, ",")
 		want := []string{c, a, p, u, v, stFormatted, etFormatted}
@@ -105,7 +105,7 @@ func TestRow(t *testing.T) {
 		}
 	})
 
-	t.Run("AssignFields", func(t *testing.T) {
+	t.Run("assignFields", func(t *testing.T) {
 		l := fmt.Sprintf(`{"data":{"args":"%s","command":"%s","endTime":"%s","profile":"%s","startTime":"%s","user":"%s","version":"v0.17.1"},"datacontenttype":"application/json","id":"bc6ec9d4-0d08-4b57-ac3b-db8d67774768","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.audit"}`, a, c, etFormatted, p, stFormatted, u)
 
 		r := &row{}
diff --git a/pkg/minikube/logs/logs.go b/pkg/minikube/logs/logs.go
index d4a528681f..8f20856249 100644
--- a/pkg/minikube/logs/logs.go
+++ b/pkg/minikube/logs/logs.go
@@ -206,7 +206,7 @@ func outputAudit(lines int) error {
 	out.Step(style.Empty, "==> Audit <==")
 	r, err := audit.Report(lines)
 	if err != nil {
-		return fmt.Errorf("failed to create audit report with error: %v", err)
+		return fmt.Errorf("failed to create audit report: %v", err)
 	}
 	out.Step(style.Empty, r.ASCIITable())
 	return nil
diff --git a/test/integration/version_upgrade_test.go b/test/integration/version_upgrade_test.go
index 055fc59da5..9e12e18fb0 100644
--- a/test/integration/version_upgrade_test.go
+++ b/test/integration/version_upgrade_test.go
@@ -200,7 +200,7 @@ func TestStoppedBinaryUpgrade(t *testing.T) {
 	t.Run("MinikubeLogs", func(t *testing.T) {
 		rr, err = Run(t, exec.CommandContext(ctx, Target(), "logs"))
 		if err != nil {
-			t.Fatalf("minikube logs after upgrade to HEAD failed: %v", err)
+			t.Fatalf("`minikube logs` after upgrade to HEAD from %s failed: %v", legacyVersion, err)
 		}
 	})
 }

From b14a1b1188a05a2f4084fabdf3b099750848ea6c Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Thu, 18 Feb 2021 17:22:17 -0800
Subject: [PATCH 105/114] authenticate with gh packages

---
 hack/jenkins/kicbase_auto_build.sh | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/hack/jenkins/kicbase_auto_build.sh b/hack/jenkins/kicbase_auto_build.sh
index 5ec4886096..f7acff0744 100755
--- a/hack/jenkins/kicbase_auto_build.sh
+++ b/hack/jenkins/kicbase_auto_build.sh
@@ -20,6 +20,7 @@ set -x
 ./hack/jenkins/installers/check_install_docker.sh
 yes|gcloud auth configure-docker
 docker login -u ${DOCKERHUB_USER} -p ${DOCKERHUB_PASS}
+docker login https://docker.pkg.github.com -u minikube-bot -p ${access_token}
 
 # Setup variables
 now=$(date +%s)
@@ -39,9 +40,9 @@ curl -L https://github.com/kubernetes/minikube/raw/master/pkg/drivers/kic/types.
 # if it doesn't exist, it will just return VERSION, which is covered in the if statement below
 HEAD_KIC_TIMESTAMP=$(egrep "Version =" types-head.go | cut -d \" -f 2 | cut -d "-" -f 2)
 CURRENT_KIC_TS=$(egrep "Version =" pkg/drivers/kic/types.go | cut -d \" -f 2 | cut -d "-" -f 2)
-if [[ $HEAD_KIC_TIMESTAMP != v* ]] && [[ $CURRENT_KIC_TS != v* ]]; then
+if [[ $HEAD_KIC_TIMESTAMP != v* ]]; then
 	diff=$((CURRENT_KIC_TS-HEAD_KIC_TIMESTAMP))
-	if [ $diff -lt 0 ]; then
+	if [[ $CURRENT_KIC_TS == v* ]] || [ $diff -lt 0 ]; then
 		curl -s -H "Authorization: token ${access_token}" \
 		-H "Accept: application/vnd.github.v3+json" \
 		-X POST -d "{\"body\": \"Hi ${ghprbPullAuthorLoginMention}, your kicbase info is out of date. Please rebase.\"}" "https://api.github.com/repos/kubernetes/minikube/issues/$ghprbPullId/comments"
@@ -54,8 +55,8 @@ rm types-head.go
 yes|make push-kic-base-image
 
 # Abort with error message if above command failed
-if [ $? -gt 0 ]; then
-	ec=$?
+ec=$?
+if [ $ec -gt 0 ]; then
 	curl -s -H "Authorization: token ${access_token}" \
                 -H "Accept: application/vnd.github.v3+json" \
                 -X POST -d "{\"body\": \"Hi ${ghprbPullAuthorLoginMention}, building a new kicbase image failed, please try again.\"}" "https://api.github.com/repos/kubernetes/minikube/issues/$ghprbPullId/comments"

From 3a0e3bd3008f06b4a1219095fde3313399140d7e Mon Sep 17 00:00:00 2001
From: Steven Powell <spowellgrr@gmail.com>
Date: Thu, 18 Feb 2021 18:27:55 -0700
Subject: [PATCH 106/114] added missing error check to scanner

---
 test/integration/functional_test.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go
index 4f78a2281e..791dffb898 100644
--- a/test/integration/functional_test.go
+++ b/test/integration/functional_test.go
@@ -598,6 +598,9 @@ func dashboardURL(b *bufio.Reader) (string, error) {
 			return t, nil
 		}
 	}
+	if err := s.Err(); err != nil {
+		return "", fmt.Errorf("failed reading input: %v", err)
+	}
 	return "", fmt.Errorf("output didn't produce a URL")
 }
 

From e5ed6659afb2395d9255613d16346d450bdb98ce Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Thu, 18 Feb 2021 17:46:35 -0800
Subject: [PATCH 107/114] fix up gh package info

---
 hack/jenkins/kicbase_auto_build.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/hack/jenkins/kicbase_auto_build.sh b/hack/jenkins/kicbase_auto_build.sh
index f7acff0744..afc7911e7f 100755
--- a/hack/jenkins/kicbase_auto_build.sh
+++ b/hack/jenkins/kicbase_auto_build.sh
@@ -27,11 +27,11 @@ now=$(date +%s)
 KV=$(egrep "Version =" pkg/drivers/kic/types.go | cut -d \" -f 2 | cut -d "-" -f 1)
 GCR_REPO=gcr.io/k8s-minikube/kicbase-builds
 DH_REPO=kicbase/build
-GH_REPO=docker.pkg.github.com/kubernetes/minikube/kicbase-build
+GH_REPO=kicbase-build
 export KIC_VERSION=$KV-$now-$ghprbPullId
 GCR_IMG=${GCR_REPO}:${KIC_VERSION}
 DH_IMG=${DH_REPO}:${KIC_VERSION}
-GH_IMG=${GH_REPO}:${KIC_VERSION}
+GH_IMG=docker.pkg.github.com/kubernetes/minikube/${GH_REPO}:${KIC_VERSION}
 export KICBASE_IMAGE_REGISTRIES="${GCR_IMG} ${DH_IMG} ${GH_IMG}"
 
 # Let's make sure we have the newest kicbase reference

From 568cae60cc0432477ea5bc91b209925317f38324 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Thu, 18 Feb 2021 18:06:37 -0800
Subject: [PATCH 108/114] remove gh package stuff for now

---
 hack/jenkins/kicbase_auto_build.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/hack/jenkins/kicbase_auto_build.sh b/hack/jenkins/kicbase_auto_build.sh
index afc7911e7f..e47d3d3534 100755
--- a/hack/jenkins/kicbase_auto_build.sh
+++ b/hack/jenkins/kicbase_auto_build.sh
@@ -32,7 +32,7 @@ export KIC_VERSION=$KV-$now-$ghprbPullId
 GCR_IMG=${GCR_REPO}:${KIC_VERSION}
 DH_IMG=${DH_REPO}:${KIC_VERSION}
 GH_IMG=docker.pkg.github.com/kubernetes/minikube/${GH_REPO}:${KIC_VERSION}
-export KICBASE_IMAGE_REGISTRIES="${GCR_IMG} ${DH_IMG} ${GH_IMG}"
+export KICBASE_IMAGE_REGISTRIES="${GCR_IMG} ${DH_IMG}"
 
 # Let's make sure we have the newest kicbase reference
 curl -L https://github.com/kubernetes/minikube/raw/master/pkg/drivers/kic/types.go --output types-head.go
@@ -69,7 +69,7 @@ fullsha=$(docker inspect --format='{{index .RepoDigests 0}}' $KICBASE_IMAGE_REGI
 sha=$(echo ${fullsha} | cut -d ":" -f 2)
 
 # Display the message to the user
-message="Hi ${ghprbPullAuthorLoginMention},\\n\\nA new kicbase image is available, please update your PR with the new tag and SHA.\\nIn pkg/drivers/kic/types.go:\\n\\n\\t// Version is the current version of kic\\n\\tVersion = \\\"${KIC_VERSION}\\\"\\n\\t// SHA of the kic base image\\n\\tbaseImageSHA = \\\"${sha}\\\"\\n\\t// The name of the GCR kicbase repository\\n\\tgcrRepo = \\\"${GCR_REPO}\\\"\\n\\t// The name of the Dockerhub kicbase repository\\n\\tdockerhubRepo = \\\"${DH_REPO}\\\"\\n\\t// The name of the Github Packages repository\\n\\tghRepo = \\\"${GH_REPO}\\\"\\nThen run \`make generate-docs\` to update our documentation to reference the new image.\n\nAlternatively, run the following command and commit the changes:\\n\`\`\`\\n sed 's|Version = .*|Version = \\\"${KIC_VERSION}\\\"|;s|baseImageSHA = .*|baseImageSHA = \\\"${sha}\\\"|;s|gcrRepo = .*|gcrRepo = \\\"${GCR_REPO}\\\"|;s|dockerhubRepo = .*|dockerhubRepo = \\\"${DH_REPO}\\\"|;s|ghRepo = .*|ghRepo = \\\"${GH_REPO}\\\"' pkg/drivers/kic/types.go > new-types.go; mv new-types.go pkg/drivers/kic/types.go; make generate-docs;\\n\`\`\`"
+message="Hi ${ghprbPullAuthorLoginMention},\\n\\nA new kicbase image is available, please update your PR with the new tag and SHA.\\nIn pkg/drivers/kic/types.go:\\n\\n\\t// Version is the current version of kic\\n\\tVersion = \\\"${KIC_VERSION}\\\"\\n\\t// SHA of the kic base image\\n\\tbaseImageSHA = \\\"${sha}\\\"\\n\\t// The name of the GCR kicbase repository\\n\\tgcrRepo = \\\"${GCR_REPO}\\\"\\n\\t// The name of the Dockerhub kicbase repository\\n\\tdockerhubRepo = \\\"${DH_REPO}\\\"\\nThen run \`make generate-docs\` to update our documentation to reference the new image.\n\nAlternatively, run the following command and commit the changes:\\n\`\`\`\\n sed 's|Version = .*|Version = \\\"${KIC_VERSION}\\\"|;s|baseImageSHA = .*|baseImageSHA = \\\"${sha}\\\"|;s|gcrRepo = .*|gcrRepo = \\\"${GCR_REPO}\\\"|;s|dockerhubRepo = .*|dockerhubRepo = \\\"${DH_REPO}\\\"|' pkg/drivers/kic/types.go > new-types.go; mv new-types.go pkg/drivers/kic/types.go; make generate-docs;\\n\`\`\`"
 
 curl -s -H "Authorization: token ${access_token}" \
 	 -H "Accept: application/vnd.github.v3+json" \

From 74e2127a8792386e845deb4cf2fb930c9726af2b Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Thu, 18 Feb 2021 18:28:02 -0800
Subject: [PATCH 109/114] update kicbase image for the last time

---
 pkg/drivers/kic/types.go               | 4 ++--
 site/content/en/docs/commands/start.md | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go
index 49fc0a2171..2c85735148 100644
--- a/pkg/drivers/kic/types.go
+++ b/pkg/drivers/kic/types.go
@@ -24,9 +24,9 @@ import (
 
 const (
 	// Version is the current version of kic
-	Version = "v0.0.17-1613688791-10408"
+	Version = "v0.0.17-1613701030-10408"
 	// SHA of the kic base image
-	baseImageSHA = "b8cea5f43f362e3c8e15a3731df2af7dcd08b82b5cec91feaceab9ad70fbddaf"
+	baseImageSHA = "d9d270eb0af8fd98fa3f4ca5363ad026dc7119d213df933434b7cd273554971e"
 	// The name of the GCR kicbase repository
 	gcrRepo = "gcr.io/k8s-minikube/kicbase-builds"
 	// The name of the Dockerhub kicbase repository
diff --git a/site/content/en/docs/commands/start.md b/site/content/en/docs/commands/start.md
index 57ddd3c361..5b1c7d5115 100644
--- a/site/content/en/docs/commands/start.md
+++ b/site/content/en/docs/commands/start.md
@@ -26,7 +26,7 @@ minikube start [flags]
       --apiserver-names strings           A set of apiserver names which are used in the generated certificate for kubernetes.  This can be used if you want to make the apiserver available from outside the machine
       --apiserver-port int                The apiserver listening port (default 8443)
       --auto-update-drivers               If set, automatically updates drivers to the latest version. Defaults to true. (default true)
-      --base-image string                 The base image to use for docker/podman drivers. Intended for local development. (default "gcr.io/k8s-minikube/kicbase-builds:v0.0.17-1613688791-10408@sha256:b8cea5f43f362e3c8e15a3731df2af7dcd08b82b5cec91feaceab9ad70fbddaf")
+      --base-image string                 The base image to use for docker/podman drivers. Intended for local development. (default "gcr.io/k8s-minikube/kicbase-builds:v0.0.17-1613701030-10408@sha256:d9d270eb0af8fd98fa3f4ca5363ad026dc7119d213df933434b7cd273554971e")
       --cache-images                      If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --driver=none. (default true)
       --cni string                        CNI plug-in to use. Valid options: auto, bridge, calico, cilium, flannel, kindnet, or path to a CNI manifest (default: auto)
       --container-runtime string          The container runtime to be used (docker, cri-o, containerd). (default "docker")

From 17af0f999bbf79acb87788adfabe9bd78262c071 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Thu, 18 Feb 2021 19:20:17 -0800
Subject: [PATCH 110/114] update kicbase image for testing

---
 pkg/drivers/kic/types.go               | 4 ++--
 site/content/en/docs/commands/start.md | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go
index 2c85735148..482f57aaed 100644
--- a/pkg/drivers/kic/types.go
+++ b/pkg/drivers/kic/types.go
@@ -24,9 +24,9 @@ import (
 
 const (
 	// Version is the current version of kic
-	Version = "v0.0.17-1613701030-10408"
+	Version = "v0.0.17-1613704090-10418"
 	// SHA of the kic base image
-	baseImageSHA = "d9d270eb0af8fd98fa3f4ca5363ad026dc7119d213df933434b7cd273554971e"
+	baseImageSHA = "876f620cdc40b4616e4e11db64524c520e252ede006357eaa963488ae852b6ed"
 	// The name of the GCR kicbase repository
 	gcrRepo = "gcr.io/k8s-minikube/kicbase-builds"
 	// The name of the Dockerhub kicbase repository
diff --git a/site/content/en/docs/commands/start.md b/site/content/en/docs/commands/start.md
index 5b1c7d5115..0e95fffc1a 100644
--- a/site/content/en/docs/commands/start.md
+++ b/site/content/en/docs/commands/start.md
@@ -26,7 +26,7 @@ minikube start [flags]
       --apiserver-names strings           A set of apiserver names which are used in the generated certificate for kubernetes.  This can be used if you want to make the apiserver available from outside the machine
       --apiserver-port int                The apiserver listening port (default 8443)
       --auto-update-drivers               If set, automatically updates drivers to the latest version. Defaults to true. (default true)
-      --base-image string                 The base image to use for docker/podman drivers. Intended for local development. (default "gcr.io/k8s-minikube/kicbase-builds:v0.0.17-1613701030-10408@sha256:d9d270eb0af8fd98fa3f4ca5363ad026dc7119d213df933434b7cd273554971e")
+      --base-image string                 The base image to use for docker/podman drivers. Intended for local development. (default "gcr.io/k8s-minikube/kicbase-builds:v0.0.17-1613704090-10418@sha256:876f620cdc40b4616e4e11db64524c520e252ede006357eaa963488ae852b6ed")
       --cache-images                      If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --driver=none. (default true)
       --cni string                        CNI plug-in to use. Valid options: auto, bridge, calico, cilium, flannel, kindnet, or path to a CNI manifest (default: auto)
       --container-runtime string          The container runtime to be used (docker, cri-o, containerd). (default "docker")

From 00a480f06c087abc0240769feb775b671ab980f9 Mon Sep 17 00:00:00 2001
From: Steven Powell <spowellgrr@gmail.com>
Date: Fri, 19 Feb 2021 09:56:35 -0700
Subject: [PATCH 111/114] set profile for test

---
 test/integration/version_upgrade_test.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/test/integration/version_upgrade_test.go b/test/integration/version_upgrade_test.go
index 9e12e18fb0..01f02e122c 100644
--- a/test/integration/version_upgrade_test.go
+++ b/test/integration/version_upgrade_test.go
@@ -198,7 +198,8 @@ func TestStoppedBinaryUpgrade(t *testing.T) {
 	}
 
 	t.Run("MinikubeLogs", func(t *testing.T) {
-		rr, err = Run(t, exec.CommandContext(ctx, Target(), "logs"))
+		args := []string{"logs", "-p", profile}
+		rr, err = Run(t, exec.CommandContext(ctx, Target(), args...))
 		if err != nil {
 			t.Fatalf("`minikube logs` after upgrade to HEAD from %s failed: %v", legacyVersion, err)
 		}

From cfd0dbdfc7687c0a09b593b78e43391206babf07 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Fri, 19 Feb 2021 11:41:04 -0800
Subject: [PATCH 112/114] bump gopogh version v0.6.0

---
 .github/workflows/iso.yml    |  2 +-
 .github/workflows/master.yml | 24 ++++++++++++------------
 .github/workflows/pr.yml     | 24 ++++++++++++------------
 hack/jenkins/common.sh       |  4 ++--
 4 files changed, 27 insertions(+), 27 deletions(-)

diff --git a/.github/workflows/iso.yml b/.github/workflows/iso.yml
index 2eac6cc640..cd3649967c 100644
--- a/.github/workflows/iso.yml
+++ b/.github/workflows/iso.yml
@@ -82,7 +82,7 @@ jobs:
       - name: Install gopogh
         shell: bash
         run: |
-          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-linux-amd64
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.6.0/gopogh-linux-amd64
           sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
           sudo apt-get install -y jq
       - name: Run Integration Test
diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index 7af4c3bcdf..6434d9f755 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -120,7 +120,7 @@ jobs:
 
         shell: bash
         run: |
-          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-linux-amd64
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.6.0/gopogh-linux-amd64
           sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
       - name: Download Binaries
         uses: actions/download-artifact@v1
@@ -205,7 +205,7 @@ jobs:
 
         shell: bash
         run: |
-          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-darwin-amd64
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.6.0/gopogh-darwin-amd64
           sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh
       - name: Install docker
         shell: bash
@@ -350,7 +350,7 @@ jobs:
         continue-on-error: true
         shell: powershell
         run: |
-          (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe")
+          (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.6.0/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe")
           choco install -y kubernetes-cli
           choco install -y jq
           choco install -y caffeine
@@ -487,7 +487,7 @@ jobs:
         shell: powershell
         run: |
           $ErrorActionPreference = "SilentlyContinue"
-          (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe")
+          (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.6.0/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe")
           choco install -y kubernetes-cli
           choco install -y jq
           choco install -y caffeine
@@ -592,7 +592,7 @@ jobs:
 
         shell: bash
         run: |
-          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-linux-amd64
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.6.0/gopogh-linux-amd64
           sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
       - name: Download Binaries
         uses: actions/download-artifact@v1
@@ -862,7 +862,7 @@ jobs:
       - name: Install gopogh
         shell: bash
         run: |
-          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-linux-arm64
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.6.0/gopogh-linux-arm64
           sudo install gopogh-linux-arm64 /usr/local/bin/gopogh
       
       - name: Install tools
@@ -996,7 +996,7 @@ jobs:
 
         shell: bash
         run: |
-          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-linux-amd64
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.6.0/gopogh-linux-amd64
           sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
       - name: Download Binaries
         uses: actions/download-artifact@v1
@@ -1078,7 +1078,7 @@ jobs:
 
         shell: bash
         run: |
-          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-darwin-amd64
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.6.0/gopogh-darwin-amd64
           sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh
       - name: Install docker
         shell: bash
@@ -1190,7 +1190,7 @@ jobs:
 
         shell: bash
         run: |
-          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-linux-amd64
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.6.0/gopogh-linux-amd64
           sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
       - name: Download Binaries
         uses: actions/download-artifact@v1
@@ -1274,7 +1274,7 @@ jobs:
 
         shell: bash
         run: |
-          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-darwin-amd64
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.6.0/gopogh-darwin-amd64
           sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh
       - name: Download Binaries
         uses: actions/download-artifact@v1
@@ -1381,7 +1381,7 @@ jobs:
 
         shell: bash
         run: |
-          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-linux-amd64
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.6.0/gopogh-linux-amd64
           sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
       - name: Download Binaries
         uses: actions/download-artifact@v1
@@ -1463,7 +1463,7 @@ jobs:
 
         shell: bash
         run: |
-          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-darwin-amd64
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.6.0/gopogh-darwin-amd64
           sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh
       - name: Download Binaries
         uses: actions/download-artifact@v1
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index f04f074a69..d6545bd308 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -118,7 +118,7 @@ jobs:
 
         shell: bash
         run: |
-          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-linux-amd64
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.6.0/gopogh-linux-amd64
           sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
       - name: Download Binaries
         uses: actions/download-artifact@v1
@@ -203,7 +203,7 @@ jobs:
 
         shell: bash
         run: |
-          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-darwin-amd64
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.6.0/gopogh-darwin-amd64
           sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh
       - name: Install docker
         shell: bash
@@ -348,7 +348,7 @@ jobs:
         continue-on-error: true
         shell: powershell
         run: |
-          (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe")
+          (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.6.0/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe")
           choco install -y kubernetes-cli
           choco install -y jq
           choco install -y caffeine
@@ -485,7 +485,7 @@ jobs:
         shell: powershell
         run: |
           $ErrorActionPreference = "SilentlyContinue"
-          (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe")
+          (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.6.0/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe")
           choco install -y kubernetes-cli
           choco install -y jq
           choco install -y caffeine
@@ -782,7 +782,7 @@ jobs:
 
         shell: bash
         run: |
-          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-linux-amd64
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.6.0/gopogh-linux-amd64
           sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
       - name: Download Binaries
         uses: actions/download-artifact@v1
@@ -861,7 +861,7 @@ jobs:
       - name: Install gopogh
         shell: bash
         run: |
-          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-linux-arm64
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.6.0/gopogh-linux-arm64
           sudo install gopogh-linux-arm64 /usr/local/bin/gopogh
 
       - name: Install tools
@@ -993,7 +993,7 @@ jobs:
       - name: Install gopogh
         shell: bash
         run: |
-          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-linux-amd64
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.6.0/gopogh-linux-amd64
           sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
       - name: Download Binaries
         uses: actions/download-artifact@v1
@@ -1075,7 +1075,7 @@ jobs:
 
         shell: bash
         run: |
-          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-darwin-amd64
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.6.0/gopogh-darwin-amd64
           sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh
       - name: Install docker
         shell: bash
@@ -1187,7 +1187,7 @@ jobs:
       - name: Install gopogh
         shell: bash
         run: |
-          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-linux-amd64
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.6.0/gopogh-linux-amd64
           sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
       - name: Download Binaries
         uses: actions/download-artifact@v1
@@ -1271,7 +1271,7 @@ jobs:
 
         shell: bash
         run: |
-          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-darwin-amd64
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.6.0/gopogh-darwin-amd64
           sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh
       - name: Download Binaries
         uses: actions/download-artifact@v1
@@ -1378,7 +1378,7 @@ jobs:
 
         shell: bash
         run: |
-          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-linux-amd64
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.6.0/gopogh-linux-amd64
           sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
       - name: Download Binaries
         uses: actions/download-artifact@v1
@@ -1460,7 +1460,7 @@ jobs:
 
         shell: bash
         run: |
-          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-darwin-amd64
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.6.0/gopogh-darwin-amd64
           sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh
       - name: Download Binaries
         uses: actions/download-artifact@v1
diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh
index 0e4f4e4e58..9818c5002a 100755
--- a/hack/jenkins/common.sh
+++ b/hack/jenkins/common.sh
@@ -360,9 +360,9 @@ fi
 
 echo ">> Installing gopogh"
 if [ "$(uname)" != "Darwin" ]; then
-  curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-linux-amd64 && sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
+  curl -LO https://github.com/medyagh/gopogh/releases/download/v0.6.0/gopogh-linux-amd64 && sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
 else
-  curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-darwin-amd64 && sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh
+  curl -LO https://github.com/medyagh/gopogh/releases/download/v0.6.0/gopogh-darwin-amd64 && sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh
 fi
 
 echo ">> Running gopogh"

From 0fce9d47268a1218cba90c8198fba39801c7ff49 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Fri, 19 Feb 2021 12:05:34 -0800
Subject: [PATCH 113/114] add json summary report to gopogh

---
 .github/workflows/iso.yml                     |  2 +-
 .github/workflows/master.yml                  | 20 +++++++++----------
 .github/workflows/pr.yml                      | 20 +++++++++----------
 hack/jenkins/common.sh                        |  7 ++++++-
 .../windows_integration_test_docker.ps1       |  4 +++-
 5 files changed, 30 insertions(+), 23 deletions(-)

diff --git a/.github/workflows/iso.yml b/.github/workflows/iso.yml
index cd3649967c..c1acfb57ba 100644
--- a/.github/workflows/iso.yml
+++ b/.github/workflows/iso.yml
@@ -110,7 +110,7 @@ jobs:
           cd out
           export PATH=${PATH}:`go env GOPATH`/bin
           go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
-          STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
+          STAT=$(gopogh -in ./report/testout.json -out_html ./report/testout.html -out_summary ./report/testout_summary.json -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
           echo status: ${STAT}
           FailNum=$(echo $STAT | jq '.NumberOfFail')
           TestsNum=$(echo $STAT | jq '.NumberOfTests')
diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index 6434d9f755..8b74d018e4 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -152,7 +152,7 @@ jobs:
           cd minikube_binaries
           export PATH=${PATH}:`go env GOPATH`/bin
           go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
-          STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
+          STAT=$(gopogh -in ./report/testout.json -out_html ./report/testout.html -out_summary ./report/testout_summary.json -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
           echo status: ${STAT}
           FailNum=$(echo $STAT | jq '.NumberOfFail')
           TestsNum=$(echo $STAT | jq '.NumberOfTests')
@@ -250,7 +250,7 @@ jobs:
           cd minikube_binaries
           export PATH=${PATH}:`go env GOPATH`/bin
           go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
-          STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
+          STAT=$(gopogh -in ./report/testout.json -out_html ./report/testout.html -out_summary ./report/testout_summary.json -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
           echo status: ${STAT}
           FailNum=$(echo $STAT | jq '.NumberOfFail')
           TestsNum=$(echo $STAT | jq '.NumberOfTests')
@@ -622,7 +622,7 @@ jobs:
           cd minikube_binaries
           export PATH=${PATH}:`go env GOPATH`/bin
           go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
-          STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
+          STAT=$(gopogh -in ./report/testout.json -out_html ./report/testout.html -out_summary ./report/testout_summary.json -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
           echo status: ${STAT}
           FailNum=$(echo $STAT | jq '.NumberOfFail')
           TestsNum=$(echo $STAT | jq '.NumberOfTests')
@@ -925,7 +925,7 @@ jobs:
           cd minikube_binaries
           export PATH=${PATH}:`go env GOPATH`/bin
           go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
-          STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
+          STAT=$(gopogh -in ./report/testout.json -out_html ./report/testout.html -out_summary ./report/testout_summary.json -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
           echo status: ${STAT}
           FailNum=$(echo $STAT | jq '.NumberOfFail')
           TestsNum=$(echo $STAT | jq '.NumberOfTests')
@@ -1026,7 +1026,7 @@ jobs:
           cd minikube_binaries
           export PATH=${PATH}:`go env GOPATH`/bin
           go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
-          STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
+          STAT=$(gopogh -in ./report/testout.json -out_html ./report/testout.html -out_summary ./report/testout_summary.json -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
           echo status: ${STAT}
           FailNum=$(echo $STAT | jq '.NumberOfFail')
           TestsNum=$(echo $STAT | jq '.NumberOfTests')
@@ -1124,7 +1124,7 @@ jobs:
           cd minikube_binaries
           export PATH=${PATH}:`go env GOPATH`/bin
           go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
-          STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
+          STAT=$(gopogh -in ./report/testout.json -out_html ./report/testout.html -out_summary ./report/testout_summary.json -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
           echo status: ${STAT}
           FailNum=$(echo $STAT | jq '.NumberOfFail')
           TestsNum=$(echo $STAT | jq '.NumberOfTests')
@@ -1222,7 +1222,7 @@ jobs:
           cd minikube_binaries
           export PATH=${PATH}:`go env GOPATH`/bin
           go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
-          STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
+          STAT=$(gopogh -in ./report/testout.json -out_html ./report/testout.html -out_summary ./report/testout_summary.json -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
           echo status: ${STAT}
           FailNum=$(echo $STAT | jq '.NumberOfFail')
           TestsNum=$(echo $STAT | jq '.NumberOfTests')
@@ -1314,7 +1314,7 @@ jobs:
           cd minikube_binaries
           export PATH=${PATH}:`go env GOPATH`/bin
           go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
-          STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
+          STAT=$(gopogh -in ./report/testout.json -out_html ./report/testout.html -out_summary ./report/testout_summary.json -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
           echo status: ${STAT}
           FailNum=$(echo $STAT | jq '.NumberOfFail')
           TestsNum=$(echo $STAT | jq '.NumberOfTests')
@@ -1411,7 +1411,7 @@ jobs:
           cd minikube_binaries
           export PATH=${PATH}:`go env GOPATH`/bin
           go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
-          STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
+          STAT=$(gopogh -in ./report/testout.json -out_html ./report/testout.html -out_summary ./report/testout_summary.json -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
           echo status: ${STAT}
           FailNum=$(echo $STAT | jq '.NumberOfFail')
           TestsNum=$(echo $STAT | jq '.NumberOfTests')
@@ -1503,7 +1503,7 @@ jobs:
           cd minikube_binaries
           export PATH=${PATH}:`go env GOPATH`/bin
           go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
-          STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
+          STAT=$(gopogh -in ./report/testout.json -out_html ./report/testout.html -out_summary ./report/testout_summary.json -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
           echo status: ${STAT}
           FailNum=$(echo $STAT | jq '.NumberOfFail')
           TestsNum=$(echo $STAT | jq '.NumberOfTests')
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index d6545bd308..f9dbe579cf 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -150,7 +150,7 @@ jobs:
           cd minikube_binaries
           export PATH=${PATH}:`go env GOPATH`/bin
           go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
-          STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
+          STAT=$(gopogh -in ./report/testout.json -out_html ./report/testout.html -out_summary ./report/testout_summary.json -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
           echo status: ${STAT}
           FailNum=$(echo $STAT | jq '.NumberOfFail')
           TestsNum=$(echo $STAT | jq '.NumberOfTests')
@@ -248,7 +248,7 @@ jobs:
           cd minikube_binaries
           export PATH=${PATH}:`go env GOPATH`/bin
           go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
-          STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
+          STAT=$(gopogh -in ./report/testout.json -out_html ./report/testout.html -out_summary ./report/testout_summary.json -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
           echo status: ${STAT}
           FailNum=$(echo $STAT | jq '.NumberOfFail')
           TestsNum=$(echo $STAT | jq '.NumberOfTests')
@@ -812,7 +812,7 @@ jobs:
           cd minikube_binaries
           export PATH=${PATH}:`go env GOPATH`/bin
           go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
-          STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
+          STAT=$(gopogh -in ./report/testout.json -out_html ./report/testout.html -out_summary ./report/testout_summary.json -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
           echo status: ${STAT}
           FailNum=$(echo $STAT | jq '.NumberOfFail')
           TestsNum=$(echo $STAT | jq '.NumberOfTests')
@@ -924,7 +924,7 @@ jobs:
           cd minikube_binaries
           export PATH=${PATH}:`go env GOPATH`/bin
           go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
-          STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
+          STAT=$(gopogh -in ./report/testout.json -out_html ./report/testout.html -out_summary ./report/testout_summary.json -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
           echo status: ${STAT}
           FailNum=$(echo $STAT | jq '.NumberOfFail')
           TestsNum=$(echo $STAT | jq '.NumberOfTests')
@@ -1023,7 +1023,7 @@ jobs:
           cd minikube_binaries
           export PATH=${PATH}:`go env GOPATH`/bin
           go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
-          STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
+          STAT=$(gopogh -in ./report/testout.json -out_html ./report/testout.html -out_summary ./report/testout_summary.json -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
           echo status: ${STAT}
           FailNum=$(echo $STAT | jq '.NumberOfFail')
           TestsNum=$(echo $STAT | jq '.NumberOfTests')
@@ -1121,7 +1121,7 @@ jobs:
           cd minikube_binaries
           export PATH=${PATH}:`go env GOPATH`/bin
           go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
-          STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
+          STAT=$(gopogh -in ./report/testout.json -out_html ./report/testout.html -out_summary ./report/testout_summary.json -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
           echo status: ${STAT}
           FailNum=$(echo $STAT | jq '.NumberOfFail')
           TestsNum=$(echo $STAT | jq '.NumberOfTests')
@@ -1219,7 +1219,7 @@ jobs:
           cd minikube_binaries
           export PATH=${PATH}:`go env GOPATH`/bin
           go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
-          STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
+          STAT=$(gopogh -in ./report/testout.json -out_html ./report/testout.html -out_summary ./report/testout_summary.json -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
           echo status: ${STAT}
           FailNum=$(echo $STAT | jq '.NumberOfFail')
           TestsNum=$(echo $STAT | jq '.NumberOfTests')
@@ -1311,7 +1311,7 @@ jobs:
           cd minikube_binaries
           export PATH=${PATH}:`go env GOPATH`/bin
           go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
-          STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
+          STAT=$(gopogh -in ./report/testout.json -out_html ./report/testout.html -out_summary ./report/testout_summary.json -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
           echo status: ${STAT}
           FailNum=$(echo $STAT | jq '.NumberOfFail')
           TestsNum=$(echo $STAT | jq '.NumberOfTests')
@@ -1408,7 +1408,7 @@ jobs:
           cd minikube_binaries
           export PATH=${PATH}:`go env GOPATH`/bin
           go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
-          STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
+          STAT=$(gopogh -in ./report/testout.json -out_html ./report/testout.html -out_summary ./report/testout_summary.json -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
           echo status: ${STAT}
           FailNum=$(echo $STAT | jq '.NumberOfFail')
           TestsNum=$(echo $STAT | jq '.NumberOfTests')
@@ -1500,7 +1500,7 @@ jobs:
           cd minikube_binaries
           export PATH=${PATH}:`go env GOPATH`/bin
           go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
-          STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
+          STAT=$(gopogh -in ./report/testout.json -out_html ./report/testout.html -out_summary ./report/testout_summary.json -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
           echo status: ${STAT}
           FailNum=$(echo $STAT | jq '.NumberOfFail')
           TestsNum=$(echo $STAT | jq '.NumberOfTests')
diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh
index 9818c5002a..16e73e4a7d 100755
--- a/hack/jenkins/common.sh
+++ b/hack/jenkins/common.sh
@@ -290,6 +290,7 @@ fi
 readonly TEST_OUT="${TEST_HOME}/testout.txt"
 readonly JSON_OUT="${TEST_HOME}/test.json"
 readonly HTML_OUT="${TEST_HOME}/test.html"
+readonly SUMMARY_OUT="${TEST_HOME}/test_summary.json"
 
 e2e_start_time="$(date -u +%s)"
 echo ""
@@ -371,7 +372,8 @@ if test -f "${HTML_OUT}"; then
 fi
 
 touch "${HTML_OUT}"
-gopogh_status=$(gopogh -in "${JSON_OUT}" -out "${HTML_OUT}" -name "${JOB_NAME}" -pr "${MINIKUBE_LOCATION}" -repo github.com/kubernetes/minikube/  -details "${COMMIT}") || true
+touch "${SUMMARY_OUT}"
+gopogh_status=$(gopogh -in "${JSON_OUT}" -out_html "${HTML_OUT}" -out_summary "${SUMMARY_OUT}" -name "${JOB_NAME}" -pr "${MINIKUBE_LOCATION}" -repo github.com/kubernetes/minikube/  -details "${COMMIT}") || true
 fail_num=$(echo $gopogh_status | jq '.NumberOfFail')
 test_num=$(echo $gopogh_status | jq '.NumberOfTests')       
 pessimistic_status="${fail_num} / ${test_num} failures"
@@ -385,6 +387,9 @@ echo ">> uploading ${JSON_OUT}"
 gsutil -qm cp "${JSON_OUT}" "gs://${JOB_GCS_BUCKET}.json" || true
 echo ">> uploading ${HTML_OUT}"
 gsutil -qm cp "${HTML_OUT}" "gs://${JOB_GCS_BUCKET}.html" || true
+echo ">> uploading ${SUMMARY_OUT}"
+gsutil -qm cp "${SUMMARY_OUT}" "gs://${JOB_GCS_BUCKET}_summary.json" || true
+
 
 
 public_log_url="https://storage.googleapis.com/${JOB_GCS_BUCKET}.txt"
diff --git a/hack/jenkins/windows_integration_test_docker.ps1 b/hack/jenkins/windows_integration_test_docker.ps1
index 2265c1d0b9..a6a2a73973 100644
--- a/hack/jenkins/windows_integration_test_docker.ps1
+++ b/hack/jenkins/windows_integration_test_docker.ps1
@@ -49,7 +49,7 @@ $elapsed=[math]::Round($elapsed, 2)
 
 Get-Content testout.txt -Encoding ASCII | go tool test2json -t | Out-File -FilePath testout.json -Encoding ASCII
 
-$gopogh_status=gopogh --in testout.json --out testout.html --name "Docker_Windows" -pr $env:MINIKUBE_LOCATION --repo github.com/kubernetes/minikube/ --details $env:COMMIT
+$gopogh_status=gopogh --in testout.json --out_html testout.html --out_summary testout_summary.json --name "Docker_Windows" -pr $env:MINIKUBE_LOCATION --repo github.com/kubernetes/minikube/ --details $env:COMMIT
 
 $failures=echo $gopogh_status | jq '.NumberOfFail'
 $tests=echo $gopogh_status | jq '.NumberOfTests'
@@ -69,6 +69,8 @@ $env:target_url="https://storage.googleapis.com/$gcs_bucket/Docker_Windows.html"
 gsutil -qm cp testout.txt gs://$gcs_bucket/Docker_Windowsout.txt
 gsutil -qm cp testout.json gs://$gcs_bucket/Docker_Windows.json
 gsutil -qm cp testout.html gs://$gcs_bucket/Docker_Windows.html
+gsutil -qm cp testout_summary.json gs://$gcs_bucket/Docker_Windows_summary.json
+
 
 # Update the PR with the new info
 $json = "{`"state`": `"$env:status`", `"description`": `"Jenkins: $description`", `"target_url`": `"$env:target_url`", `"context`": `"Docker_Windows`"}"

From 7feabad12d813b73c5090f8482afca85cab85d4b Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Fri, 19 Feb 2021 14:53:06 -0800
Subject: [PATCH 114/114] install gopogh on windows

---
 hack/jenkins/windows_integration_test_docker.ps1     | 3 +++
 hack/jenkins/windows_integration_test_hyperv.ps1     | 2 ++
 hack/jenkins/windows_integration_test_virtualbox.ps1 | 3 +++
 3 files changed, 8 insertions(+)

diff --git a/hack/jenkins/windows_integration_test_docker.ps1 b/hack/jenkins/windows_integration_test_docker.ps1
index a6a2a73973..44d31d8fec 100644
--- a/hack/jenkins/windows_integration_test_docker.ps1
+++ b/hack/jenkins/windows_integration_test_docker.ps1
@@ -13,6 +13,9 @@
 # limitations under the License.
 
 mkdir -p out
+
+(New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.6.0/gopogh.exe", "C:\Go\bin\gopogh.exe")
+
 gsutil.cmd -m cp gs://minikube-builds/$env:MINIKUBE_LOCATION/minikube-windows-amd64.exe out/
 gsutil.cmd -m cp gs://minikube-builds/$env:MINIKUBE_LOCATION/e2e-windows-amd64.exe out/
 gsutil.cmd -m cp -r gs://minikube-builds/$env:MINIKUBE_LOCATION/testdata .
diff --git a/hack/jenkins/windows_integration_test_hyperv.ps1 b/hack/jenkins/windows_integration_test_hyperv.ps1
index 45cd4f92d0..395d0524b0 100644
--- a/hack/jenkins/windows_integration_test_hyperv.ps1
+++ b/hack/jenkins/windows_integration_test_hyperv.ps1
@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+(New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.6.0/gopogh.exe", "C:\Go\bin\gopogh.exe")
+
 mkdir -p out
 gsutil.cmd -m cp gs://minikube-builds/$env:MINIKUBE_LOCATION/minikube-windows-amd64.exe out/
 gsutil.cmd -m cp gs://minikube-builds/$env:MINIKUBE_LOCATION/e2e-windows-amd64.exe out/
diff --git a/hack/jenkins/windows_integration_test_virtualbox.ps1 b/hack/jenkins/windows_integration_test_virtualbox.ps1
index 2f5957301f..a94a6e835b 100644
--- a/hack/jenkins/windows_integration_test_virtualbox.ps1
+++ b/hack/jenkins/windows_integration_test_virtualbox.ps1
@@ -12,6 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+(New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.6.0/gopogh.exe", "C:\Go\bin\gopogh.exe")
+
+
 mkdir -p out
 gsutil.cmd -m cp gs://minikube-builds/$env:MINIKUBE_LOCATION/minikube-windows-amd64.exe out/
 gsutil.cmd -m cp gs://minikube-builds/$env:MINIKUBE_LOCATION/e2e-windows-amd64.exe out/