From 8cb2a491d6e0fe7dac0cf6f9da3f366000d6a01d Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Mon, 21 Oct 2019 11:08:21 -0700
Subject: [PATCH 001/250] Add binary type to hold all necessary info about each
 minikube binary

---
 pkg/minikube/performance/binary.go | 58 ++++++++++++++++++++++++++++++
 1 file changed, 58 insertions(+)
 create mode 100644 pkg/minikube/performance/binary.go

diff --git a/pkg/minikube/performance/binary.go b/pkg/minikube/performance/binary.go
new file mode 100644
index 0000000000..3e0817b3e7
--- /dev/null
+++ b/pkg/minikube/performance/binary.go
@@ -0,0 +1,58 @@
+/*
+Copyright 2017 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package performance
+
+import (
+	"os"
+	"path/filepath"
+)
+
+// Binary holds all necessary information
+// to call a minikube binary
+type Binary struct {
+	path string
+}
+
+// Binaries returns the type *Binary for each provided path
+func Binaries(paths []string) ([]*Binary, error) {
+	var binaries []*Binary
+	for _, path := range paths {
+		b, err := newBinary(path)
+		if err != nil {
+			return nil, err
+		}
+		binaries = append(binaries, b)
+	}
+	return binaries, nil
+}
+
+func newBinary(path string) (*Binary, error) {
+	var err error
+	if !filepath.IsAbs(path) {
+		path, err = filepath.Abs(path)
+		if err != nil {
+			return nil, err
+		}
+	}
+	// make sure binary exists
+	if _, err := os.Stat(path); err != nil {
+		return nil, err
+	}
+	return &Binary{
+		path: path,
+	}, nil
+}

From 4a57b8864da6b86b5e60ccd6ae251aea31f478ed Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Mon, 21 Oct 2019 11:08:31 -0700
Subject: [PATCH 002/250] Add code to time 'minikube start'

---
 pkg/minikube/performance/start.go | 83 +++++++++++++++++++++++++++++++
 1 file changed, 83 insertions(+)
 create mode 100644 pkg/minikube/performance/start.go

diff --git a/pkg/minikube/performance/start.go b/pkg/minikube/performance/start.go
new file mode 100644
index 0000000000..1fdb7cab37
--- /dev/null
+++ b/pkg/minikube/performance/start.go
@@ -0,0 +1,83 @@
+/*
+Copyright 2017 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package performance
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"time"
+
+	"github.com/pkg/errors"
+)
+
+const (
+	runs = 1
+)
+
+// CompareMinikubeStart compares the time to run `minikube start` between two minikube binaries
+func CompareMinikubeStart(ctx context.Context, binaries []*Binary) error {
+	durations := make([][]float64, len(binaries))
+	for i := range durations {
+		durations[i] = make([]float64, runs)
+	}
+
+	for r := 0; r < runs; r++ {
+		log.Printf("Executing run %d...", r)
+		for index, binary := range binaries {
+			duration, err := timeMinikubeStart(ctx, binary)
+			if err != nil {
+				return errors.Wrapf(err, "timing run %d with %s", r, binary.path)
+			}
+			durations[index][r] = duration
+		}
+	}
+
+	fmt.Printf("Old binary: %v\nNew binary: %v\nAverage Old: %f\nAverage New: %f\n", durations[0], durations[1], average(durations[0]), average(durations[1]))
+
+	return nil
+}
+
+func average(array []float64) float64 {
+	total := float64(0)
+	for _, a := range array {
+		total += a
+	}
+	return total / float64(len(array))
+}
+
+// timeMinikubeStart returns the time it takes to execute `minikube start`
+// It deletes the VM after `minikube start`.
+func timeMinikubeStart(ctx context.Context, binary *Binary) (float64, error) {
+	startCmd := exec.CommandContext(ctx, binary.path, "start")
+	startCmd.Stdout = os.Stdout
+	startCmd.Stderr = os.Stderr
+
+	deleteCmd := exec.CommandContext(ctx, binary.path, "delete")
+	defer deleteCmd.Run()
+
+	log.Printf("Running: %v...", startCmd.Args)
+	start := time.Now()
+	if err := startCmd.Run(); err != nil {
+		return 0, errors.Wrap(err, "starting minikube")
+	}
+
+	startDuration := time.Since(start).Seconds()
+	return startDuration, nil
+}

From 5242dcb0b8353004c28c556b803c42285a79b85a Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Mon, 21 Oct 2019 11:10:49 -0700
Subject: [PATCH 003/250] hook up mkcmp command to code to time 'minikube
 start'

---
 cmd/performance/cmd/mkcmp.go | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/cmd/performance/cmd/mkcmp.go b/cmd/performance/cmd/mkcmp.go
index f1a26fe953..91e887907c 100644
--- a/cmd/performance/cmd/mkcmp.go
+++ b/cmd/performance/cmd/mkcmp.go
@@ -17,20 +17,30 @@ limitations under the License.
 package cmd
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"os"
 
 	"github.com/spf13/cobra"
+	"k8s.io/minikube/pkg/minikube/performance"
 )
 
 var rootCmd = &cobra.Command{
-	Use:   "mkcmp [path to first binary] [path to second binary]",
-	Short: "mkcmp is used to compare performance of two minikube binaries",
+	Use:           "mkcmp [path to first binary] [path to second binary]",
+	Short:         "mkcmp is used to compare performance of two minikube binaries",
+	SilenceUsage:  true,
+	SilenceErrors: true,
 	PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
 		return validateArgs(args)
 	},
-	Run: func(cmd *cobra.Command, args []string) {},
+	RunE: func(cmd *cobra.Command, args []string) error {
+		binaries, err := performance.Binaries(args)
+		if err != nil {
+			return err
+		}
+		return performance.CompareMinikubeStart(context.Background(), binaries)
+	},
 }
 
 func validateArgs(args []string) error {
@@ -43,7 +53,7 @@ func validateArgs(args []string) error {
 // Execute runs the mkcmp command
 func Execute() {
 	if err := rootCmd.Execute(); err != nil {
-		fmt.Println(err)
+		fmt.Println("Error:", err)
 		os.Exit(1)
 	}
 }

From 9856a04487a34b040b70e0def57402dec242d9fa Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Mon, 21 Oct 2019 11:11:26 -0700
Subject: [PATCH 004/250] Add compare Make rule to compare current branch with
 master branch

---
 Makefile | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/Makefile b/Makefile
index 0b159ac3ee..0e0ab93c19 100755
--- a/Makefile
+++ b/Makefile
@@ -70,6 +70,7 @@ GOARCH ?= $(shell go env GOARCH)
 GOPATH ?= $(shell go env GOPATH)
 BUILD_DIR ?= ./out
 $(shell mkdir -p $(BUILD_DIR))
+CURRENT_GIT_BRANCH ?= $(shell git branch | grep \* | cut -d ' ' -f2)
 
 # Use system python if it exists, otherwise use Docker.
 PYTHON := $(shell command -v python || echo "docker run --rm -it -v $(shell pwd):/minikube -w /minikube python python")
@@ -571,3 +572,12 @@ site: site/themes/docsy/assets/vendor/bootstrap/package.js out/hugo/hugo
 .PHONY: out/mkcmp
 out/mkcmp:
 	GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o $@ cmd/performance/main.go
+
+.PHONY: compare
+compare: out/mkcmp out/minikube
+	cp out/minikube out/$(CURRENT_GIT_BRANCH).minikube
+	git checkout master
+	make out/minikube
+	cp out/minikube out/master.minikube
+	git checkout $(CURRENT_GIT_BRANCH)
+	out/mkcmp out/master.minikube out/$(CURRENT_GIT_BRANCH).minikube

From 80940538bd666d9190a2446a6eff42e152855c16 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Mon, 21 Oct 2019 11:19:13 -0700
Subject: [PATCH 005/250] remove 'make compare' rule as it won't work on
 windows

---
 Makefile | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/Makefile b/Makefile
index 0e0ab93c19..0b159ac3ee 100755
--- a/Makefile
+++ b/Makefile
@@ -70,7 +70,6 @@ GOARCH ?= $(shell go env GOARCH)
 GOPATH ?= $(shell go env GOPATH)
 BUILD_DIR ?= ./out
 $(shell mkdir -p $(BUILD_DIR))
-CURRENT_GIT_BRANCH ?= $(shell git branch | grep \* | cut -d ' ' -f2)
 
 # Use system python if it exists, otherwise use Docker.
 PYTHON := $(shell command -v python || echo "docker run --rm -it -v $(shell pwd):/minikube -w /minikube python python")
@@ -572,12 +571,3 @@ site: site/themes/docsy/assets/vendor/bootstrap/package.js out/hugo/hugo
 .PHONY: out/mkcmp
 out/mkcmp:
 	GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o $@ cmd/performance/main.go
-
-.PHONY: compare
-compare: out/mkcmp out/minikube
-	cp out/minikube out/$(CURRENT_GIT_BRANCH).minikube
-	git checkout master
-	make out/minikube
-	cp out/minikube out/master.minikube
-	git checkout $(CURRENT_GIT_BRANCH)
-	out/mkcmp out/master.minikube out/$(CURRENT_GIT_BRANCH).minikube

From f4e59d51bee7b2f522be361e8a937c9eeaae6d66 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Mon, 21 Oct 2019 11:55:43 -0700
Subject: [PATCH 006/250] Add unit tests

---
 cmd/performance/cmd/mkcmp.go           |   2 +-
 pkg/minikube/performance/start.go      |  25 +++--
 pkg/minikube/performance/start_test.go | 130 +++++++++++++++++++++++++
 3 files changed, 149 insertions(+), 8 deletions(-)
 create mode 100644 pkg/minikube/performance/start_test.go

diff --git a/cmd/performance/cmd/mkcmp.go b/cmd/performance/cmd/mkcmp.go
index 91e887907c..d146bb9bed 100644
--- a/cmd/performance/cmd/mkcmp.go
+++ b/cmd/performance/cmd/mkcmp.go
@@ -39,7 +39,7 @@ var rootCmd = &cobra.Command{
 		if err != nil {
 			return err
 		}
-		return performance.CompareMinikubeStart(context.Background(), binaries)
+		return performance.CompareMinikubeStart(context.Background(), os.Stdout, binaries)
 	},
 }
 
diff --git a/pkg/minikube/performance/start.go b/pkg/minikube/performance/start.go
index 1fdb7cab37..7cd8ece6b5 100644
--- a/pkg/minikube/performance/start.go
+++ b/pkg/minikube/performance/start.go
@@ -19,6 +19,7 @@ package performance
 import (
 	"context"
 	"fmt"
+	"io"
 	"log"
 	"os"
 	"os/exec"
@@ -27,12 +28,24 @@ import (
 	"github.com/pkg/errors"
 )
 
-const (
+var (
 	runs = 1
+	// For testing
+	collectTimeMinikubeStart = timeMinikubeStart
 )
 
 // CompareMinikubeStart compares the time to run `minikube start` between two minikube binaries
-func CompareMinikubeStart(ctx context.Context, binaries []*Binary) error {
+func CompareMinikubeStart(ctx context.Context, out io.Writer, binaries []*Binary) error {
+	durations, err := collectTimes(ctx, binaries)
+	if err != nil {
+		return err
+	}
+
+	fmt.Fprintf(out, "Old binary: %v\nNew binary: %v\nAverage Old: %f\nAverage New: %f\n", durations[0], durations[1], average(durations[0]), average(durations[1]))
+	return nil
+}
+
+func collectTimes(ctx context.Context, binaries []*Binary) ([][]float64, error) {
 	durations := make([][]float64, len(binaries))
 	for i := range durations {
 		durations[i] = make([]float64, runs)
@@ -41,17 +54,15 @@ func CompareMinikubeStart(ctx context.Context, binaries []*Binary) error {
 	for r := 0; r < runs; r++ {
 		log.Printf("Executing run %d...", r)
 		for index, binary := range binaries {
-			duration, err := timeMinikubeStart(ctx, binary)
+			duration, err := collectTimeMinikubeStart(ctx, binary)
 			if err != nil {
-				return errors.Wrapf(err, "timing run %d with %s", r, binary.path)
+				return nil, errors.Wrapf(err, "timing run %d with %s", r, binary.path)
 			}
 			durations[index][r] = duration
 		}
 	}
 
-	fmt.Printf("Old binary: %v\nNew binary: %v\nAverage Old: %f\nAverage New: %f\n", durations[0], durations[1], average(durations[0]), average(durations[1]))
-
-	return nil
+	return durations, nil
 }
 
 func average(array []float64) float64 {
diff --git a/pkg/minikube/performance/start_test.go b/pkg/minikube/performance/start_test.go
new file mode 100644
index 0000000000..1b36bb85d0
--- /dev/null
+++ b/pkg/minikube/performance/start_test.go
@@ -0,0 +1,130 @@
+/*
+Copyright 2017 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package performance
+
+import (
+	"bytes"
+	"context"
+	"reflect"
+	"testing"
+)
+
+func mockCollectTimeMinikubeStart(durations []float64) func(ctx context.Context, binary *Binary) (float64, error) {
+	index := 0
+	return func(context.Context, *Binary) (float64, error) {
+		duration := durations[index]
+		index = index + 1
+		return duration, nil
+	}
+}
+
+func TestCompareMinikubeStartOutput(t *testing.T) {
+	tests := []struct {
+		description string
+		durations   []float64
+		expected    string
+	}{
+		{
+			description: "standard run",
+			durations:   []float64{4.5, 6},
+			expected:    "Old binary: [4.5]\nNew binary: [6]\nAverage Old: 4.500000\nAverage New: 6.000000\n",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.description, func(t *testing.T) {
+			originalCollectTimes := collectTimeMinikubeStart
+			collectTimeMinikubeStart = mockCollectTimeMinikubeStart(test.durations)
+			defer func() { collectTimeMinikubeStart = originalCollectTimes }()
+
+			buf := bytes.NewBuffer([]byte{})
+			err := CompareMinikubeStart(context.Background(), buf, []*Binary{{}, {}})
+			if err != nil {
+				t.Fatalf("error comparing minikube start: %v", err)
+			}
+
+			actual := buf.String()
+			if test.expected != actual {
+				t.Fatalf("actual output does not match expected output\nActual: %v\nExpected: %v", actual, test.expected)
+			}
+		})
+	}
+}
+
+func TestCollectTimes(t *testing.T) {
+	tests := []struct {
+		description string
+		durations   []float64
+		runs        int
+		expected    [][]float64
+	}{
+		{
+			description: "two runs",
+			durations:   []float64{1, 2, 3, 4},
+			runs:        2,
+			expected: [][]float64{
+				{1, 3},
+				{2, 4},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.description, func(t *testing.T) {
+			originalCollectTimes := collectTimeMinikubeStart
+			collectTimeMinikubeStart = mockCollectTimeMinikubeStart(test.durations)
+			defer func() { collectTimeMinikubeStart = originalCollectTimes }()
+
+			runs = test.runs
+			actual, err := collectTimes(context.Background(), []*Binary{{}, {}})
+			if err != nil {
+				t.Fatalf("error collecting times: %v", err)
+			}
+
+			if !reflect.DeepEqual(actual, test.expected) {
+				t.Fatalf("actual output does not match expected output\nActual: %v\nExpected: %v", actual, test.expected)
+			}
+		})
+	}
+}
+
+func TestAverage(t *testing.T) {
+	tests := []struct {
+		description string
+		nums        []float64
+		expected    float64
+	}{
+		{
+			description: "one number",
+			nums:        []float64{4},
+			expected:    4,
+		}, {
+			description: "multiple numbers",
+			nums:        []float64{1, 4},
+			expected:    2.5,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.description, func(t *testing.T) {
+			actual := average(test.nums)
+			if actual != test.expected {
+				t.Fatalf("actual output does not match expected output\nActual: %v\nExpected: %v", actual, test.expected)
+			}
+		})
+	}
+}

From 42304d8533357ff19ee64b8d4075fec7aefaeef4 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Mon, 21 Oct 2019 12:00:53 -0700
Subject: [PATCH 007/250] Fixed linting errors

---
 go.sum                                 | 1 +
 pkg/minikube/performance/start.go      | 6 +++++-
 pkg/minikube/performance/start_test.go | 2 +-
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/go.sum b/go.sum
index 03c9eb6c9c..6ebce314fd 100644
--- a/go.sum
+++ b/go.sum
@@ -383,6 +383,7 @@ github.com/opencontainers/selinux v0.0.0-20170621221121-4a2974bf1ee9/go.mod h1:+
 github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
 github.com/otiai10/copy v1.0.2 h1:DDNipYy6RkIkjMwy+AWzgKiNTyj2RUI9yEMeETEpVyc=
 github.com/otiai10/copy v1.0.2/go.mod h1:c7RpqBkwMom4bYTSkLSym4VSJz/XtncWRAj/J4PEIMY=
+github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95 h1:+OLn68pqasWca0z5ryit9KGfp3sUsW4Lqg32iRMJyzs=
 github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE=
 github.com/otiai10/mint v1.3.0 h1:Ady6MKVezQwHBkGzLFbrsywyp09Ah7rkmfjV3Bcr5uc=
 github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
diff --git a/pkg/minikube/performance/start.go b/pkg/minikube/performance/start.go
index 7cd8ece6b5..c50828ea10 100644
--- a/pkg/minikube/performance/start.go
+++ b/pkg/minikube/performance/start.go
@@ -81,7 +81,11 @@ func timeMinikubeStart(ctx context.Context, binary *Binary) (float64, error) {
 	startCmd.Stderr = os.Stderr
 
 	deleteCmd := exec.CommandContext(ctx, binary.path, "delete")
-	defer deleteCmd.Run()
+	defer func() {
+		if err := deleteCmd.Run(); err != nil {
+			log.Printf("error deleting minikube: %v", err)
+		}
+	}()
 
 	log.Printf("Running: %v...", startCmd.Args)
 	start := time.Now()
diff --git a/pkg/minikube/performance/start_test.go b/pkg/minikube/performance/start_test.go
index 1b36bb85d0..ecabbd9617 100644
--- a/pkg/minikube/performance/start_test.go
+++ b/pkg/minikube/performance/start_test.go
@@ -27,7 +27,7 @@ func mockCollectTimeMinikubeStart(durations []float64) func(ctx context.Context,
 	index := 0
 	return func(context.Context, *Binary) (float64, error) {
 		duration := durations[index]
-		index = index + 1
+		index++
 		return duration, nil
 	}
 }

From 76bfafb5377974df0fc05b5e68cefc66e68b65e0 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Mon, 21 Oct 2019 13:11:30 -0700
Subject: [PATCH 008/250] add 'make compare' Makefile rule

---
 Makefile | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/Makefile b/Makefile
index 0b159ac3ee..cbc9013971 100755
--- a/Makefile
+++ b/Makefile
@@ -70,6 +70,7 @@ GOARCH ?= $(shell go env GOARCH)
 GOPATH ?= $(shell go env GOPATH)
 BUILD_DIR ?= ./out
 $(shell mkdir -p $(BUILD_DIR))
+CURRENT_GIT_BRANCH ?= $(shell git branch | grep \* | cut -d ' ' -f2)
 
 # Use system python if it exists, otherwise use Docker.
 PYTHON := $(shell command -v python || echo "docker run --rm -it -v $(shell pwd):/minikube -w /minikube python python")
@@ -571,3 +572,12 @@ site: site/themes/docsy/assets/vendor/bootstrap/package.js out/hugo/hugo
 .PHONY: out/mkcmp
 out/mkcmp:
 	GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o $@ cmd/performance/main.go
+
+.PHONY: compare
+compare: out/mkcmp out/minikube
+	mv out/minikube out/$(CURRENT_GIT_BRANCH).minikube
+	git checkout master
+	make out/minikube
+	mv out/minikube out/master.minikube
+	git checkout $(CURRENT_GIT_BRANCH)
+	out/mkcmp out/master.minikube out/$(CURRENT_GIT_BRANCH).minikube

From 337db1707daa3c110b079f6d1baa64a3aa1f7207 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Fri, 22 Nov 2019 15:11:51 -0800
Subject: [PATCH 009/250] First pass at multi-runtime pause/unpause

---
 cmd/minikube/cmd/mount.go           |   1 -
 cmd/minikube/cmd/pause.go           |  67 +++++
 cmd/minikube/cmd/root.go            |   2 +
 cmd/minikube/cmd/unpause.go         |  64 ++++
 go.sum                              |   3 +
 pkg/drivers/none/none.go            |  58 +---
 pkg/minikube/config/profile.go.orig | 201 +++++++++++++
 pkg/minikube/cruntime/containerd.go |  15 +-
 pkg/minikube/cruntime/cri.go        | 436 ++++++----------------------
 pkg/minikube/cruntime/crio.go       | 340 +++++++++++++++++++++-
 pkg/minikube/cruntime/cruntime.go   |  25 +-
 pkg/minikube/cruntime/docker.go     |  45 ++-
 pkg/minikube/kubelet/kubelet.go     | 105 +++++++
 pkg/minikube/logs/logs.go           |   2 +-
 pkg/minikube/out/style.go           |   2 +
 pkg/minikube/out/style_enum.go      |   2 +
 pkg/minikube/pause/pause.go         |  69 +++++
 17 files changed, 1031 insertions(+), 406 deletions(-)
 create mode 100644 cmd/minikube/cmd/pause.go
 create mode 100644 cmd/minikube/cmd/unpause.go
 create mode 100644 pkg/minikube/config/profile.go.orig
 create mode 100644 pkg/minikube/kubelet/kubelet.go
 create mode 100644 pkg/minikube/pause/pause.go

diff --git a/cmd/minikube/cmd/mount.go b/cmd/minikube/cmd/mount.go
index d5cdaf418a..e5f6abf453 100644
--- a/cmd/minikube/cmd/mount.go
+++ b/cmd/minikube/cmd/mount.go
@@ -107,7 +107,6 @@ var mountCmd = &cobra.Command{
 			exit.WithError("Error getting config", err)
 		}
 		host, err := api.Load(cc.Name)
-
 		if err != nil {
 			exit.WithError("Error loading api", err)
 		}
diff --git a/cmd/minikube/cmd/pause.go b/cmd/minikube/cmd/pause.go
new file mode 100644
index 0000000000..731c5f87d0
--- /dev/null
+++ b/cmd/minikube/cmd/pause.go
@@ -0,0 +1,67 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cmd
+
+import (
+	"github.com/spf13/cobra"
+
+	"k8s.io/minikube/pkg/minikube/cluster"
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/cruntime"
+	"k8s.io/minikube/pkg/minikube/exit"
+	"k8s.io/minikube/pkg/minikube/machine"
+	"k8s.io/minikube/pkg/minikube/out"
+	"k8s.io/minikube/pkg/minikube/pause"
+)
+
+// pauseCmd represents the docker-pause command
+var pauseCmd = &cobra.Command{
+	Use:   "pause",
+	Short: "pause containers",
+	Run: func(cmd *cobra.Command, args []string) {
+		api, err := machine.NewAPIClient()
+		if err != nil {
+			exit.WithError("Error getting client", err)
+		}
+		defer api.Close()
+		cc, err := config.Load()
+		if err != nil {
+			exit.WithError("Error getting config", err)
+		}
+		host, err := cluster.CheckIfHostExistsAndLoad(api, cc.Name)
+		if err != nil {
+			exit.WithError("Error getting host", err)
+		}
+
+		r, err := machine.CommandRunner(host)
+		if err != nil {
+			exit.WithError("Failed to get command runner", err)
+		}
+
+		config := cruntime.Config{Type: cc.ContainerRuntime}
+		cr, err := cruntime.New(config)
+		if err != nil {
+			exit.WithError("Failed runtime", err)
+		}
+
+		err = pause.Pause(cr, r)
+		if err != nil {
+			exit.WithError("Pause", err)
+		}
+		out.T(out.Pause, "The '{{.name}}' cluster is now paused", out.V{"name": cc.Name})
+	},
+}
diff --git a/cmd/minikube/cmd/root.go b/cmd/minikube/cmd/root.go
index 38f84246d9..6c09694ebf 100644
--- a/cmd/minikube/cmd/root.go
+++ b/cmd/minikube/cmd/root.go
@@ -172,6 +172,8 @@ func init() {
 				stopCmd,
 				deleteCmd,
 				dashboardCmd,
+				pauseCmd,
+				unpauseCmd,
 			},
 		},
 		{
diff --git a/cmd/minikube/cmd/unpause.go b/cmd/minikube/cmd/unpause.go
new file mode 100644
index 0000000000..3e45eaf711
--- /dev/null
+++ b/cmd/minikube/cmd/unpause.go
@@ -0,0 +1,64 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cmd
+
+import (
+	"github.com/spf13/cobra"
+
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/cruntime"
+	"k8s.io/minikube/pkg/minikube/exit"
+	"k8s.io/minikube/pkg/minikube/machine"
+	"k8s.io/minikube/pkg/minikube/out"
+	"k8s.io/minikube/pkg/minikube/pause"
+)
+
+// unpauseCmd represents the docker-pause command
+var unpauseCmd = &cobra.Command{
+	Use:   "unpause",
+	Short: "unpause Kubernetes",
+	Run: func(cmd *cobra.Command, args []string) {
+		api, err := machine.NewAPIClient()
+		if err != nil {
+			exit.WithError("Error getting client", err)
+		}
+		defer api.Close()
+
+		cc, err := config.Load()
+		if err != nil {
+			exit.WithError("Error getting config", err)
+		}
+		host, err := api.Load(cc.Name)
+
+		config := cruntime.Config{Type: cc.ContainerRuntime}
+		cr, err := cruntime.New(config)
+		if err != nil {
+			exit.WithError("Failed runtime", err)
+		}
+
+		r, err := machine.CommandRunner(host)
+		if err != nil {
+			exit.WithError("Failed to get command runner", err)
+		}
+
+		err = pause.Pause(cr, r)
+		if err != nil {
+			exit.WithError("Pause", err)
+		}
+		out.T(out.Pause, "The '{{.name}}' cluster is now paused", out.V{"name": cc.Name})
+	},
+}
diff --git a/go.sum b/go.sum
index 1971629b40..f8c771f026 100644
--- a/go.sum
+++ b/go.sum
@@ -720,6 +720,7 @@ k8s.io/kubernetes v1.15.2 h1:RO9EuRw5vlN3oa/lnmPxmywOoJRtg9o40KcklHXNIAQ=
 k8s.io/kubernetes v1.15.2/go.mod h1:3RE5ikMc73WK+dSxk4pQuQ6ZaJcPXiZX2dj98RcdCuM=
 k8s.io/kubernetes/staging/src/k8s.io/api v0.0.0-20190623232353-8c3b7d7679cc h1:vZ5+77WP1yImZo23wc75vV5b5zCGq9gv484q8Yw5sBw=
 k8s.io/kubernetes/staging/src/k8s.io/api v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:pU9hbGZc8Z6+6HlNLEFY1GiNGzcCykU1Glsd4vEea2U=
+k8s.io/kubernetes/staging/src/k8s.io/apiextensions-apiserver v0.0.0-20190623232353-8c3b7d7679cc h1:8L3YgoEmmOxIGWNv9Hj6WhJuUspT+sw4gJs2nEc0qI0=
 k8s.io/kubernetes/staging/src/k8s.io/apiextensions-apiserver v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:Q49J/iUBV6A9nn8loyV72DK2EXhN8sqCR8FyfxIFDA4=
 k8s.io/kubernetes/staging/src/k8s.io/apimachinery v0.0.0-20190623232353-8c3b7d7679cc h1:SHxaBZWgNouwsZCVg2+iffu0Um1ExSLPKgvO1drWShs=
 k8s.io/kubernetes/staging/src/k8s.io/apimachinery v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:rRBYbORqofLsn4/tsQWkeXkdKUoGrTfUwbI9s7NhU0Q=
@@ -738,10 +739,12 @@ k8s.io/kubernetes/staging/src/k8s.io/cri-api v0.0.0-20190623232353-8c3b7d7679cc/
 k8s.io/kubernetes/staging/src/k8s.io/csi-translation-lib v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:5RWpGgZKzUcW9gCtmSVRq8maZkOetGv87HrohpTrnLI=
 k8s.io/kubernetes/staging/src/k8s.io/kube-aggregator v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:ogOX4l9UCMfFGIF+FZqmsln4NtCGPqf9zTMCIlm2YX4=
 k8s.io/kubernetes/staging/src/k8s.io/kube-controller-manager v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:o6aAFW1lCnr5CJm1riWnhQskrAHhyt8btyv5UHhgZ6c=
+k8s.io/kubernetes/staging/src/k8s.io/kube-proxy v0.0.0-20190623232353-8c3b7d7679cc h1:j30roBbl6b5Mom66efcNOHyjdYXU2RD8UWYnL0Adb8I=
 k8s.io/kubernetes/staging/src/k8s.io/kube-proxy v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:y0hpsQGN8h3HcNqYbpSZEH4yC1ohi45N35c8ma9yg6M=
 k8s.io/kubernetes/staging/src/k8s.io/kube-scheduler v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:/hbCTKdfutEO2iTQv8NuYcnAxd8Tuu4mMEymYv/EZHk=
 k8s.io/kubernetes/staging/src/k8s.io/kubectl v0.0.0-20190623232353-8c3b7d7679cc h1:Bsljf/3UDy91qqLkevAiq6y+wl0qJrkLjWfBCQs9rss=
 k8s.io/kubernetes/staging/src/k8s.io/kubectl v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:Vg6Q3IDU3hfYMICKyb43lClOXWtCtOBh2o1FfuQw8mQ=
+k8s.io/kubernetes/staging/src/k8s.io/kubelet v0.0.0-20190623232353-8c3b7d7679cc h1:ZUouIndlzPLGsRpeNAswxcs//fyODrNZOYybP6JZ9mM=
 k8s.io/kubernetes/staging/src/k8s.io/kubelet v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:9UInPlSttlDwZBFNMAsqhTtl7zH00dE2M88B9Z0Ennc=
 k8s.io/kubernetes/staging/src/k8s.io/legacy-cloud-providers v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:xlTRb77uaXbuT6evILwFescWPMENFKYGYj3a/kOjYQE=
 k8s.io/kubernetes/staging/src/k8s.io/metrics v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:6Cs3k9ccbWbJo3CQOrGDu2QEVLwsWbBlu9HitjPhuSk=
diff --git a/pkg/drivers/none/none.go b/pkg/drivers/none/none.go
index 4d99fd8b8e..9546cee2d6 100644
--- a/pkg/drivers/none/none.go
+++ b/pkg/drivers/none/none.go
@@ -19,19 +19,17 @@ package none
 import (
 	"fmt"
 	"os/exec"
-	"strings"
-	"time"
 
 	"github.com/docker/machine/libmachine/drivers"
 	"github.com/docker/machine/libmachine/state"
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/util/net"
+	"k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet"
 	pkgdrivers "k8s.io/minikube/pkg/drivers"
 	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/minikube/cruntime"
 	"k8s.io/minikube/pkg/minikube/vmpath"
-	"k8s.io/minikube/pkg/util/retry"
 )
 
 // cleanupPaths are paths to be removed by cleanup, and are used by both kubeadm and minikube.
@@ -123,7 +121,7 @@ func (d *Driver) GetURL() (string, error) {
 
 // GetState returns the state that the host is in (running, stopped, etc)
 func (d *Driver) GetState() (state.State, error) {
-	if err := checkKubelet(d.exec); err != nil {
+	if err := kubelet.Check(d.exec); err != nil {
 		glog.Infof("kubelet not running: %v", err)
 		return state.Stopped, nil
 	}
@@ -132,7 +130,7 @@ func (d *Driver) GetState() (state.State, error) {
 
 // Kill stops a host forcefully, including any containers that we are managing.
 func (d *Driver) Kill() error {
-	if err := stopKubelet(d.exec); err != nil {
+	if err := kubelet.TryStopKubelet(d.exec); err != nil {
 		return errors.Wrap(err, "kubelet")
 	}
 
@@ -177,7 +175,7 @@ func (d *Driver) Remove() error {
 
 // Restart a host
 func (d *Driver) Restart() error {
-	return restartKubelet(d.exec)
+	return kubelet.Restart(d.exec)
 }
 
 // Start a host
@@ -196,7 +194,7 @@ func (d *Driver) Start() error {
 
 // Stop a host gracefully, including any containers that we are managing.
 func (d *Driver) Stop() error {
-	if err := stopKubelet(d.exec); err != nil {
+	if err := kubelet.Stop(d.exec); err != nil {
 		return err
 	}
 	containers, err := d.runtime.ListContainers("")
@@ -215,49 +213,3 @@ func (d *Driver) Stop() error {
 func (d *Driver) RunSSHCommandFromDriver() error {
 	return fmt.Errorf("driver does not support ssh commands")
 }
-
-// stopKubelet idempotently stops the kubelet
-func stopKubelet(cr command.Runner) error {
-	glog.Infof("stopping kubelet.service ...")
-	stop := func() error {
-		cmd := exec.Command("sudo", "systemctl", "stop", "kubelet.service")
-		if rr, err := cr.RunCmd(cmd); err != nil {
-			glog.Errorf("temporary error for %q : %v", rr.Command(), err)
-		}
-		cmd = exec.Command("sudo", "systemctl", "show", "-p", "SubState", "kubelet")
-		rr, err := cr.RunCmd(cmd)
-		if err != nil {
-			glog.Errorf("temporary error: for %q : %v", rr.Command(), err)
-		}
-		if !strings.Contains(rr.Stdout.String(), "dead") && !strings.Contains(rr.Stdout.String(), "failed") {
-			return fmt.Errorf("unexpected kubelet state: %q", rr.Stdout.String())
-		}
-		return nil
-	}
-
-	if err := retry.Expo(stop, 2*time.Second, time.Minute*3, 5); err != nil {
-		return errors.Wrapf(err, "error stopping kubelet")
-	}
-
-	return nil
-}
-
-// restartKubelet restarts the kubelet
-func restartKubelet(cr command.Runner) error {
-	glog.Infof("restarting kubelet.service ...")
-	c := exec.Command("sudo", "systemctl", "restart", "kubelet.service")
-	if _, err := cr.RunCmd(c); err != nil {
-		return err
-	}
-	return nil
-}
-
-// checkKubelet returns an error if the kubelet is not running.
-func checkKubelet(cr command.Runner) error {
-	glog.Infof("checking for running kubelet ...")
-	c := exec.Command("systemctl", "is-active", "--quiet", "service", "kubelet")
-	if _, err := cr.RunCmd(c); err != nil {
-		return errors.Wrap(err, "check kubelet")
-	}
-	return nil
-}
diff --git a/pkg/minikube/config/profile.go.orig b/pkg/minikube/config/profile.go.orig
new file mode 100644
index 0000000000..3546ec2e93
--- /dev/null
+++ b/pkg/minikube/config/profile.go.orig
@@ -0,0 +1,201 @@
+/*
+Copyright 2019 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package config
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/golang/glog"
+	"k8s.io/minikube/pkg/minikube/localpath"
+	"k8s.io/minikube/pkg/util/lock"
+)
+
+var keywords = []string{"start", "stop", "status", "delete", "config", "open", "profile", "addons", "cache", "logs"}
+
+// IsValid checks if the profile has the essential info needed for a profile
+func (p *Profile) IsValid() bool {
+	if p.Config == nil {
+		return false
+	}
+	if len(p.Config) == 0 {
+		return false
+	}
+	// This will become a loop for multinode
+	if p.Config[0] == nil {
+		return false
+	}
+	if p.Config[0].VMDriver == "" {
+		return false
+	}
+	if p.Config[0].KubernetesConfig.KubernetesVersion == "" {
+		return false
+	}
+	return true
+}
+
+// ProfileNameInReservedKeywords checks if the profile is an internal keywords
+func ProfileNameInReservedKeywords(name string) bool {
+	for _, v := range keywords {
+		if strings.EqualFold(v, name) {
+			return true
+		}
+	}
+	return false
+}
+
+// ProfileExists returns true if there is a profile config (regardless of being valid)
+func ProfileExists(name string, miniHome ...string) bool {
+	miniPath := localpath.MiniPath()
+	if len(miniHome) > 0 {
+		miniPath = miniHome[0]
+	}
+
+	p := profileFilePath(name, miniPath)
+	_, err := os.Stat(p)
+	return err == nil
+}
+
+// CreateEmptyProfile creates an empty profile stores in $MINIKUBE_HOME/profiles/<profilename>/config.json
+func CreateEmptyProfile(name string, miniHome ...string) error {
+	cfg := &MachineConfig{}
+	return CreateProfile(name, cfg, miniHome...)
+}
+
+// CreateProfile creates an profile out of the cfg and stores in $MINIKUBE_HOME/profiles/<profilename>/config.json
+func CreateProfile(name string, cfg *MachineConfig, miniHome ...string) error {
+	data, err := json.MarshalIndent(cfg, "", "    ")
+	if err != nil {
+		return err
+	}
+	path := profileFilePath(name, miniHome...)
+	glog.Infof("Saving config to %s ...", path)
+	if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
+		return err
+	}
+
+	// If no config file exists, don't worry about swapping paths
+	if _, err := os.Stat(path); os.IsNotExist(err) {
+		if err := lock.WriteFile(path, data, 0600); err != nil {
+			return err
+		}
+		return nil
+	}
+
+	tf, err := ioutil.TempFile(filepath.Dir(path), "config.json.tmp")
+	if err != nil {
+		return err
+	}
+	defer os.Remove(tf.Name())
+
+	if err = lock.WriteFile(tf.Name(), data, 0600); err != nil {
+		return err
+	}
+
+	if err = tf.Close(); err != nil {
+		return err
+	}
+
+	if err = os.Remove(path); err != nil {
+		return err
+	}
+
+	if err = os.Rename(tf.Name(), path); err != nil {
+		return err
+	}
+	return nil
+}
+
+// DeleteProfile deletes a profile and removes the profile dir
+func DeleteProfile(profile string, miniHome ...string) error {
+	miniPath := localpath.MiniPath()
+	if len(miniHome) > 0 {
+		miniPath = miniHome[0]
+	}
+	return os.RemoveAll(ProfileFolderPath(profile, miniPath))
+}
+
+// ListProfiles returns all valid and invalid (if any) minikube profiles
+// invalidPs are the profiles that have a directory or config file but not usable
+// invalidPs would be suggested to be deleted
+func ListProfiles(miniHome ...string) (validPs []*Profile, inValidPs []*Profile, err error) {
+	pDirs, err := profileDirs(miniHome...)
+	if err != nil {
+		return nil, nil, err
+	}
+	for _, n := range pDirs {
+		p, err := LoadProfile(n, miniHome...)
+		if err != nil {
+			inValidPs = append(inValidPs, p)
+			continue
+		}
+		if !p.IsValid() {
+			inValidPs = append(inValidPs, p)
+			continue
+		}
+		validPs = append(validPs, p)
+	}
+	return validPs, inValidPs, nil
+}
+
+// LoadProfile loads type Profile based on its name
+func LoadProfile(name string, miniHome ...string) (*Profile, error) {
+	cfg, err := DefaultLoader.LoadConfigFromFile(name, miniHome...)
+	p := &Profile{
+		Name:   name,
+		Config: []*MachineConfig{cfg},
+	}
+	return p, err
+}
+
+// profileDirs gets all the folders in the user's profiles folder regardless of valid or invalid config
+func profileDirs(miniHome ...string) (dirs []string, err error) {
+	miniPath := localpath.MiniPath()
+	if len(miniHome) > 0 {
+		miniPath = miniHome[0]
+	}
+	pRootDir := filepath.Join(miniPath, "profiles")
+	items, err := ioutil.ReadDir(pRootDir)
+	for _, f := range items {
+		if f.IsDir() {
+			dirs = append(dirs, f.Name())
+		}
+	}
+	return dirs, err
+}
+
+// profileFilePath returns the Minikube profile config file
+func profileFilePath(profile string, miniHome ...string) string {
+	miniPath := localpath.MiniPath()
+	if len(miniHome) > 0 {
+		miniPath = miniHome[0]
+	}
+
+	return filepath.Join(miniPath, "profiles", profile, "config.json")
+}
+
+// ProfileFolderPath returns path of profile folder
+func ProfileFolderPath(profile string, miniHome ...string) string {
+	miniPath := localpath.MiniPath()
+	if len(miniHome) > 0 {
+		miniPath = miniHome[0]
+	}
+	return filepath.Join(miniPath, "profiles", profile)
+}
diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go
index 934f65c416..e11d9f8ec5 100644
--- a/pkg/minikube/cruntime/containerd.go
+++ b/pkg/minikube/cruntime/containerd.go
@@ -32,6 +32,7 @@ import (
 )
 
 const (
+	containerdNamespaceRoot = "/run/containerd/runc/k8s.io"
 	// ContainerdConfFile is the path to the containerd configuration
 	containerdConfigFile     = "/etc/containerd/config.toml"
 	containerdConfigTemplate = `root = "/var/lib/containerd"
@@ -242,8 +243,18 @@ func (r *Containerd) KubeletOptions() map[string]string {
 }
 
 // ListContainers returns a list of managed by this container runtime
-func (r *Containerd) ListContainers(filter string) ([]string, error) {
-	return listCRIContainers(r.Runner, filter)
+func (r *Containerd) ListContainers(o ListOptions) ([]string, error) {
+	return listCRIContainers(r.Runner, containerdNamespaceRoot, o)
+}
+
+// PauseContainers pauses a running container based on ID
+func (r *Containerd) PauseContainers(ids []string) error {
+	return pauseCRIContainers(r.Runner, containerdNamespaceRoot, ids)
+}
+
+// PauseContainers pauses a running container based on ID
+func (r *Containerd) UnpauseContainers(ids []string) error {
+	return unpauseCRIContainers(r.Runner, containerdNamespaceRoot, ids)
 }
 
 // KillContainers removes containers based on ID
diff --git a/pkg/minikube/cruntime/cri.go b/pkg/minikube/cruntime/cri.go
index 173e652301..ca77a58dab 100644
--- a/pkg/minikube/cruntime/cri.go
+++ b/pkg/minikube/cruntime/cri.go
@@ -18,7 +18,7 @@ package cruntime
 
 import (
 	"bytes"
-	"encoding/base64"
+	"encoding/json"
 	"fmt"
 	"html/template"
 	"os/exec"
@@ -27,331 +27,107 @@ import (
 
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
-	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
-	"k8s.io/minikube/pkg/minikube/command"
 )
 
-const (
-	// CRIOConfFile is the path to the CRI-O configuration
-	crioConfigFile     = "/etc/crio/crio.conf"
-	crioConfigTemplate = `# The CRI-O configuration file specifies all of the available configuration
-# options and command-line flags for the crio(8) OCI Kubernetes Container Runtime
-# daemon, but in a TOML format that can be more easily modified and versioned.
-#
-# Please refer to crio.conf(5) for details of all configuration options.
-
-# CRI-O supports partial configuration reload during runtime, which can be
-# done by sending SIGHUP to the running process. Currently supported options
-# are explicitly mentioned with: 'This option supports live configuration
-# reload'.
-
-# CRI-O reads its storage defaults from the containers-storage.conf(5) file
-# located at /etc/containers/storage.conf. Modify this storage configuration if
-# you want to change the system's defaults. If you want to modify storage just
-# for CRI-O, you can change the storage configuration options here.
-[crio]
-
-# Path to the "root directory". CRI-O stores all of its data, including
-# containers images, in this directory.
-root = "/var/lib/containers/storage"
-
-# Path to the "run directory". CRI-O stores all of its state in this directory.
-runroot = "/var/run/containers/storage"
-
-# Storage driver used to manage the storage of images and containers. Please
-# refer to containers-storage.conf(5) to see all available storage drivers.
-storage_driver = "overlay"
-
-# List to pass options to the storage driver. Please refer to
-# containers-storage.conf(5) to see all available storage options.
-#storage_option = [
-#]
-
-# If set to false, in-memory locking will be used instead of file-based locking.
-# **Deprecated** this option will be removed in the future.
-file_locking = false
-
-# Path to the lock file.
-# **Deprecated** this option will be removed in the future.
-file_locking_path = "/run/crio.lock"
-
-
-# The crio.api table contains settings for the kubelet/gRPC interface.
-[crio.api]
-
-# Path to AF_LOCAL socket on which CRI-O will listen.
-listen = "/var/run/crio/crio.sock"
-
-# IP address on which the stream server will listen.
-stream_address = "127.0.0.1"
-
-# The port on which the stream server will listen.
-stream_port = "0"
-
-# Enable encrypted TLS transport of the stream server.
-stream_enable_tls = false
-
-# Path to the x509 certificate file used to serve the encrypted stream. This
-# file can change, and CRI-O will automatically pick up the changes within 5
-# minutes.
-stream_tls_cert = ""
-
-# Path to the key file used to serve the encrypted stream. This file can
-# change, and CRI-O will automatically pick up the changes within 5 minutes.
-stream_tls_key = ""
-
-# Path to the x509 CA(s) file used to verify and authenticate client
-# communication with the encrypted stream. This file can change, and CRI-O will
-# automatically pick up the changes within 5 minutes.
-stream_tls_ca = ""
-
-# Maximum grpc send message size in bytes. If not set or <=0, then CRI-O will default to 16 * 1024 * 1024.
-grpc_max_send_msg_size = 16777216
-
-# Maximum grpc receive message size. If not set or <= 0, then CRI-O will default to 16 * 1024 * 1024.
-grpc_max_recv_msg_size = 16777216
-
-# The crio.runtime table contains settings pertaining to the OCI runtime used
-# and options for how to set up and manage the OCI runtime.
-[crio.runtime]
-
-# A list of ulimits to be set in containers by default, specified as
-# "<ulimit name>=<soft limit>:<hard limit>", for example:
-# "nofile=1024:2048"
-# If nothing is set here, settings will be inherited from the CRI-O daemon
-#default_ulimits = [
-#]
-
-# default_runtime is the _name_ of the OCI runtime to be used as the default.
-# The name is matched against the runtimes map below.
-default_runtime = "runc"
-
-# If true, the runtime will not use pivot_root, but instead use MS_MOVE.
-no_pivot = true
-
-# Path to the conmon binary, used for monitoring the OCI runtime.
-conmon = "/usr/libexec/crio/conmon"
-
-# Cgroup setting for conmon
-conmon_cgroup = "pod"
-
-# Environment variable list for the conmon process, used for passing necessary
-# environment variables to conmon or the runtime.
-conmon_env = [
-	"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
-]
-
-# If true, SELinux will be used for pod separation on the host.
-selinux = false
-
-# Path to the seccomp.json profile which is used as the default seccomp profile
-# for the runtime. If not specified, then the internal default seccomp profile
-# will be used.
-seccomp_profile = ""
-
-# Used to change the name of the default AppArmor profile of CRI-O. The default
-# profile name is "crio-default-" followed by the version string of CRI-O.
-apparmor_profile = "crio-default"
-
-# Cgroup management implementation used for the runtime.
-cgroup_manager = "cgroupfs"
-
-# List of default capabilities for containers. If it is empty or commented out,
-# only the capabilities defined in the containers json file by the user/kube
-# will be added.
-default_capabilities = [
-	"CHOWN",
-	"DAC_OVERRIDE",
-	"FSETID",
-	"FOWNER",
-	"NET_RAW",
-	"SETGID",
-	"SETUID",
-	"SETPCAP",
-	"NET_BIND_SERVICE",
-	"SYS_CHROOT",
-	"KILL",
-]
-
-# List of default sysctls. If it is empty or commented out, only the sysctls
-# defined in the container json file by the user/kube will be added.
-default_sysctls = [
-]
-
-# List of additional devices. specified as
-# "<device-on-host>:<device-on-container>:<permissions>", for example: "--device=/dev/sdc:/dev/xvdc:rwm".
-#If it is empty or commented out, only the devices
-# defined in the container json file by the user/kube will be added.
-additional_devices = [
-]
-
-# Path to OCI hooks directories for automatically executed hooks.
-hooks_dir = [
-]
-
-# List of default mounts for each container. **Deprecated:** this option will
-# be removed in future versions in favor of default_mounts_file.
-default_mounts = [
-]
-
-# Path to the file specifying the defaults mounts for each container. The
-# format of the config is /SRC:/DST, one mount per line. Notice that CRI-O reads
-# its default mounts from the following two files:
-#
-#   1) /etc/containers/mounts.conf (i.e., default_mounts_file): This is the
-#      override file, where users can either add in their own default mounts, or
-#      override the default mounts shipped with the package.
-#
-#   2) /usr/share/containers/mounts.conf: This is the default file read for
-#      mounts. If you want CRI-O to read from a different, specific mounts file,
-#      you can change the default_mounts_file. Note, if this is done, CRI-O will
-#      only add mounts it finds in this file.
-#
-#default_mounts_file = ""
-
-# Maximum number of processes allowed in a container.
-pids_limit = 1024
-
-# Maximum sized allowed for the container log file. Negative numbers indicate
-# that no size limit is imposed. If it is positive, it must be >= 8192 to
-# match/exceed conmon's read buffer. The file is truncated and re-opened so the
-# limit is never exceeded.
-log_size_max = -1
-
-# Whether container output should be logged to journald in addition to the kuberentes log file
-log_to_journald = false
-
-# Path to directory in which container exit files are written to by conmon.
-container_exits_dir = "/var/run/crio/exits"
-
-# Path to directory for container attach sockets.
-container_attach_socket_dir = "/var/run/crio"
-
-# If set to true, all containers will run in read-only mode.
-read_only = false
-
-# Changes the verbosity of the logs based on the level it is set to. Options
-# are fatal, panic, error, warn, info, and debug. This option supports live
-# configuration reload.
-log_level = "error"
-
-# The default log directory where all logs will go unless directly specified by the kubelet
-log_dir = "/var/log/crio/pods"
-
-# The UID mappings for the user namespace of each container. A range is
-# specified in the form containerUID:HostUID:Size. Multiple ranges must be
-# separated by comma.
-uid_mappings = ""
-
-# The GID mappings for the user namespace of each container. A range is
-# specified in the form containerGID:HostGID:Size. Multiple ranges must be
-# separated by comma.
-gid_mappings = ""
-
-# The minimal amount of time in seconds to wait before issuing a timeout
-# regarding the proper termination of the container.
-ctr_stop_timeout = 0
-
-# ManageNetworkNSLifecycle determines whether we pin and remove network namespace
-# and manage its lifecycle.
-manage_network_ns_lifecycle = false
-
-# The "crio.runtime.runtimes" table defines a list of OCI compatible runtimes.
-# The runtime to use is picked based on the runtime_handler provided by the CRI.
-# If no runtime_handler is provided, the runtime will be picked based on the level
-# of trust of the workload.
-
-[crio.runtime.runtimes.runc]
-runtime_path = "/usr/bin/runc"
-runtime_type = "oci"
-runtime_root = "/run/runc"
-
-
-# The crio.image table contains settings pertaining to the management of OCI images.
-#
-# CRI-O reads its configured registries defaults from the system wide
-# containers-registries.conf(5) located in /etc/containers/registries.conf. If
-# you want to modify just CRI-O, you can change the registries configuration in
-# this file. Otherwise, leave insecure_registries and registries commented out to
-# use the system's defaults from /etc/containers/registries.conf.
-[crio.image]
-
-# Default transport for pulling images from a remote container storage.
-default_transport = "docker://"
-
-# The path to a file containing credentials necessary for pulling images from
-# secure registries. The file is similar to that of /var/lib/kubelet/config.json
-global_auth_file = ""
-
-# The image used to instantiate infra containers.
-# This option supports live configuration reload.
-pause_image = "{{ .PodInfraContainerImage }}"
-
-# The path to a file containing credentials specific for pulling the pause_image from
-# above. The file is similar to that of /var/lib/kubelet/config.json
-# This option supports live configuration reload.
-pause_image_auth_file = ""
-
-# The command to run to have a container stay in the paused state.
-# This option supports live configuration reload.
-pause_command = "/pause"
-
-# Path to the file which decides what sort of policy we use when deciding
-# whether or not to trust an image that we've pulled. It is not recommended that
-# this option be used, as the default behavior of using the system-wide default
-# policy (i.e., /etc/containers/policy.json) is most often preferred. Please
-# refer to containers-policy.json(5) for more details.
-signature_policy = ""
-
-# Controls how image volumes are handled. The valid values are mkdir, bind and
-# ignore; the latter will ignore volumes entirely.
-image_volumes = "mkdir"
-
-# List of registries to be used when pulling an unqualified image (e.g.,
-# "alpine:latest"). By default, registries is set to "docker.io" for
-# compatibility reasons. Depending on your workload and usecase you may add more
-# registries (e.g., "quay.io", "registry.fedoraproject.org",
-# "registry.opensuse.org", etc.).
-registries = [
-	"docker.io"
-]
-
-
-# The crio.network table containers settings pertaining to the management of
-# CNI plugins.
-[crio.network]
-
-# Path to the directory where CNI configuration files are located.
-network_dir = "/etc/cni/net.d/"
-
-# Paths to directories where CNI plugin binaries are located.
-plugin_dirs = [
-	"/opt/cni/bin/",
-]
-`
-)
-
-// listCRIContainers returns a list of containers using crictl
-func listCRIContainers(cr CommandRunner, filter string) ([]string, error) {
-	var err error
-	var rr *command.RunResult
-	state := "Running"
-	if filter != "" {
-		c := exec.Command("sudo", "crictl", "ps", "-a", fmt.Sprintf("--name=%s", filter), fmt.Sprintf("--state=%s", state), "--quiet")
-		rr, err = cr.RunCmd(c)
-	} else {
-		rr, err = cr.RunCmd(exec.Command("sudo", "crictl", "ps", "-a", fmt.Sprintf("--state=%s", state), "--quiet"))
+// container maps to 'runc list -f json'
+type container struct {
+	pid         int
+	id          string
+	status      string
+	annotations map[string]string
+}
+
+// listCRIContainers returns a list of containers
+func listCRIContainers(cr CommandRunner, root string, o ListOptions) ([]string, error) {
+	// First use crictl, because it reliably matches names
+	args := []string{"crictl", "ps", "--quiet"}
+	if o.State == All {
+		args = append(args, "-a")
 	}
+	if o.Name != "" {
+		args = append(args, fmt.Sprintf("--name=%s", o.Name))
+	}
+	rr, err := cr.RunCmd(exec.Command("sudo", args...))
 	if err != nil {
 		return nil, err
 	}
+
+	// Avoid an id named ""
 	var ids []string
-	for _, line := range strings.Split(rr.Stderr.String(), "\n") {
-		if line != "" {
-			ids = append(ids, line)
+	seen := map[string]bool{}
+	for _, id := range strings.Split(rr.Stderr.String(), "\n") {
+		if id != "" && !seen[id] {
+			ids = append(ids, id)
+			seen[id] = true
 		}
 	}
-	return ids, nil
+
+	if len(ids) == 0 {
+		return nil, nil
+	}
+	if o.State == All {
+		return ids, nil
+	}
+
+	// crictl does not understand paused pods
+	cs := []container{}
+	args = []string{"runc", "list", "-f", "json"}
+	if root != "" {
+		args = append(args, "--root", root)
+	}
+
+	if _, err := cr.RunCmd(exec.Command("sudo", args...)); err != nil {
+		return nil, errors.Wrap(err, "runc")
+	}
+
+	d := json.NewDecoder(bytes.NewReader(rr.Stdout.Bytes()))
+	if err := d.Decode(&cs); err != nil {
+		return nil, err
+	}
+
+	var fids []string
+	for _, c := range cs {
+		if !seen[c.id] {
+			continue
+		}
+		if o.State.String() != c.status {
+			continue
+		}
+		fids = append(fids, c.id)
+	}
+	return fids, nil
+}
+
+// pauseCRIContainers pauses a list of containers
+func pauseCRIContainers(cr CommandRunner, root string, ids []string) error {
+	args := []string{"runc", "pause"}
+	if root != "" {
+		args = append(args, "--root", root)
+	}
+
+	for _, id := range ids {
+		cargs := append(args, id)
+		if _, err := cr.RunCmd(exec.Command("sudo", cargs...)); err != nil {
+			return errors.Wrap(err, "runc")
+		}
+	}
+	return nil
+}
+
+// unpauseCRIContainers pauses a list of containers
+func unpauseCRIContainers(cr CommandRunner, root string, ids []string) error {
+	args := []string{"runc", "unpause"}
+	if root != "" {
+		args = append(args, "--root", root)
+	}
+
+	for _, id := range ids {
+		cargs := append(args, id)
+		if _, err := cr.RunCmd(exec.Command("sudo", cargs...)); err != nil {
+			return errors.Wrap(err, "runc")
+		}
+	}
+	return nil
 }
 
 // criCRIContainers kills a list of containers using crictl
@@ -364,7 +140,7 @@ func killCRIContainers(cr CommandRunner, ids []string) error {
 	args := append([]string{"crictl", "rm"}, ids...)
 	c := exec.Command("sudo", args...)
 	if _, err := cr.RunCmd(c); err != nil {
-		return errors.Wrap(err, "kill cri containers.")
+		return errors.Wrap(err, "crictl")
 	}
 	return nil
 }
@@ -378,10 +154,9 @@ func stopCRIContainers(cr CommandRunner, ids []string) error {
 	args := append([]string{"crictl", "rm"}, ids...)
 	c := exec.Command("sudo", args...)
 	if _, err := cr.RunCmd(c); err != nil {
-		return errors.Wrap(err, "stop cri containers")
+		return errors.Wrap(err, "crictl")
 	}
 	return nil
-
 }
 
 // populateCRIConfig sets up /etc/crictl.yaml
@@ -406,27 +181,6 @@ image-endpoint: unix://{{.Socket}}
 	return nil
 }
 
-// generateCRIOConfig sets up /etc/crio/crio.conf
-func generateCRIOConfig(cr CommandRunner, imageRepository string, k8sVersion string) error {
-	cPath := crioConfigFile
-	t, err := template.New("crio.conf").Parse(crioConfigTemplate)
-	if err != nil {
-		return err
-	}
-	pauseImage := images.PauseImage(imageRepository, k8sVersion)
-	opts := struct{ PodInfraContainerImage string }{PodInfraContainerImage: pauseImage}
-	var b bytes.Buffer
-	if err := t.Execute(&b, opts); err != nil {
-		return err
-	}
-
-	c := exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo mkdir -p %s && printf %%s \"%s\" | base64 -d | sudo tee %s", path.Dir(cPath), base64.StdEncoding.EncodeToString(b.Bytes()), cPath))
-	if _, err := cr.RunCmd(c); err != nil {
-		return errors.Wrap(err, "generateCRIOConfig.")
-	}
-	return nil
-}
-
 // criContainerLogCmd returns the command to retrieve the log for a container based on ID
 func criContainerLogCmd(id string, len int, follow bool) string {
 	var cmd strings.Builder
diff --git a/pkg/minikube/cruntime/crio.go b/pkg/minikube/cruntime/crio.go
index 763e52ac73..e658494aaf 100644
--- a/pkg/minikube/cruntime/crio.go
+++ b/pkg/minikube/cruntime/crio.go
@@ -17,15 +17,320 @@ limitations under the License.
 package cruntime
 
 import (
+	"bytes"
+	"encoding/base64"
 	"fmt"
 	"os/exec"
+	"path"
 	"strings"
+	"text/template"
 
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
 	"k8s.io/minikube/pkg/minikube/out"
 )
 
+const (
+	// CRIOConfFile is the path to the CRI-O configuration
+	crioConfigFile     = "/etc/crio/crio.conf"
+	crioConfigTemplate = `# The CRI-O configuration file specifies all of the available configuration
+# options and command-line flags for the crio(8) OCI Kubernetes Container Runtime
+# daemon, but in a TOML format that can be more easily modified and versioned.
+#
+# Please refer to crio.conf(5) for details of all configuration options.
+
+# CRI-O supports partial configuration reload during runtime, which can be
+# done by sending SIGHUP to the running process. Currently supported options
+# are explicitly mentioned with: 'This option supports live configuration
+# reload'.
+
+# CRI-O reads its storage defaults from the containers-storage.conf(5) file
+# located at /etc/containers/storage.conf. Modify this storage configuration if
+# you want to change the system's defaults. If you want to modify storage just
+# for CRI-O, you can change the storage configuration options here.
+[crio]
+
+# Path to the "root directory". CRI-O stores all of its data, including
+# containers images, in this directory.
+root = "/var/lib/containers/storage"
+
+# Path to the "run directory". CRI-O stores all of its state in this directory.
+runroot = "/var/run/containers/storage"
+
+# Storage driver used to manage the storage of images and containers. Please
+# refer to containers-storage.conf(5) to see all available storage drivers.
+storage_driver = "overlay"
+
+# List to pass options to the storage driver. Please refer to
+# containers-storage.conf(5) to see all available storage options.
+#storage_option = [
+#]
+
+# If set to false, in-memory locking will be used instead of file-based locking.
+# **Deprecated** this option will be removed in the future.
+file_locking = false
+
+# Path to the lock file.
+# **Deprecated** this option will be removed in the future.
+file_locking_path = "/run/crio.lock"
+
+
+# The crio.api table contains settings for the kubelet/gRPC interface.
+[crio.api]
+
+# Path to AF_LOCAL socket on which CRI-O will listen.
+listen = "/var/run/crio/crio.sock"
+
+# IP address on which the stream server will listen.
+stream_address = "127.0.0.1"
+
+# The port on which the stream server will listen.
+stream_port = "0"
+
+# Enable encrypted TLS transport of the stream server.
+stream_enable_tls = false
+
+# Path to the x509 certificate file used to serve the encrypted stream. This
+# file can change, and CRI-O will automatically pick up the changes within 5
+# minutes.
+stream_tls_cert = ""
+
+# Path to the key file used to serve the encrypted stream. This file can
+# change, and CRI-O will automatically pick up the changes within 5 minutes.
+stream_tls_key = ""
+
+# Path to the x509 CA(s) file used to verify and authenticate client
+# communication with the encrypted stream. This file can change, and CRI-O will
+# automatically pick up the changes within 5 minutes.
+stream_tls_ca = ""
+
+# Maximum grpc send message size in bytes. If not set or <=0, then CRI-O will default to 16 * 1024 * 1024.
+grpc_max_send_msg_size = 16777216
+
+# Maximum grpc receive message size. If not set or <= 0, then CRI-O will default to 16 * 1024 * 1024.
+grpc_max_recv_msg_size = 16777216
+
+# The crio.runtime table contains settings pertaining to the OCI runtime used
+# and options for how to set up and manage the OCI runtime.
+[crio.runtime]
+
+# A list of ulimits to be set in containers by default, specified as
+# "<ulimit name>=<soft limit>:<hard limit>", for example:
+# "nofile=1024:2048"
+# If nothing is set here, settings will be inherited from the CRI-O daemon
+#default_ulimits = [
+#]
+
+# default_runtime is the _name_ of the OCI runtime to be used as the default.
+# The name is matched against the runtimes map below.
+default_runtime = "runc"
+
+# If true, the runtime will not use pivot_root, but instead use MS_MOVE.
+no_pivot = true
+
+# Path to the conmon binary, used for monitoring the OCI runtime.
+conmon = "/usr/libexec/crio/conmon"
+
+# Cgroup setting for conmon
+conmon_cgroup = "pod"
+
+# Environment variable list for the conmon process, used for passing necessary
+# environment variables to conmon or the runtime.
+conmon_env = [
+	"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+]
+
+# If true, SELinux will be used for pod separation on the host.
+selinux = false
+
+# Path to the seccomp.json profile which is used as the default seccomp profile
+# for the runtime. If not specified, then the internal default seccomp profile
+# will be used.
+seccomp_profile = ""
+
+# Used to change the name of the default AppArmor profile of CRI-O. The default
+# profile name is "crio-default-" followed by the version string of CRI-O.
+apparmor_profile = "crio-default"
+
+# Cgroup management implementation used for the runtime.
+cgroup_manager = "cgroupfs"
+
+# List of default capabilities for containers. If it is empty or commented out,
+# only the capabilities defined in the containers json file by the user/kube
+# will be added.
+default_capabilities = [
+	"CHOWN",
+	"DAC_OVERRIDE",
+	"FSETID",
+	"FOWNER",
+	"NET_RAW",
+	"SETGID",
+	"SETUID",
+	"SETPCAP",
+	"NET_BIND_SERVICE",
+	"SYS_CHROOT",
+	"KILL",
+]
+
+# List of default sysctls. If it is empty or commented out, only the sysctls
+# defined in the container json file by the user/kube will be added.
+default_sysctls = [
+]
+
+# List of additional devices. specified as
+# "<device-on-host>:<device-on-container>:<permissions>", for example: "--device=/dev/sdc:/dev/xvdc:rwm".
+#If it is empty or commented out, only the devices
+# defined in the container json file by the user/kube will be added.
+additional_devices = [
+]
+
+# Path to OCI hooks directories for automatically executed hooks.
+hooks_dir = [
+]
+
+# List of default mounts for each container. **Deprecated:** this option will
+# be removed in future versions in favor of default_mounts_file.
+default_mounts = [
+]
+
+# Path to the file specifying the defaults mounts for each container. The
+# format of the config is /SRC:/DST, one mount per line. Notice that CRI-O reads
+# its default mounts from the following two files:
+#
+#   1) /etc/containers/mounts.conf (i.e., default_mounts_file): This is the
+#      override file, where users can either add in their own default mounts, or
+#      override the default mounts shipped with the package.
+#
+#   2) /usr/share/containers/mounts.conf: This is the default file read for
+#      mounts. If you want CRI-O to read from a different, specific mounts file,
+#      you can change the default_mounts_file. Note, if this is done, CRI-O will
+#      only add mounts it finds in this file.
+#
+#default_mounts_file = ""
+
+# Maximum number of processes allowed in a container.
+pids_limit = 1024
+
+# Maximum sized allowed for the container log file. Negative numbers indicate
+# that no size limit is imposed. If it is positive, it must be >= 8192 to
+# match/exceed conmon's read buffer. The file is truncated and re-opened so the
+# limit is never exceeded.
+log_size_max = -1
+
+# Whether container output should be logged to journald in addition to the kuberentes log file
+log_to_journald = false
+
+# Path to directory in which container exit files are written to by conmon.
+container_exits_dir = "/var/run/crio/exits"
+
+# Path to directory for container attach sockets.
+container_attach_socket_dir = "/var/run/crio"
+
+# If set to true, all containers will run in read-only mode.
+read_only = false
+
+# Changes the verbosity of the logs based on the level it is set to. Options
+# are fatal, panic, error, warn, info, and debug. This option supports live
+# configuration reload.
+log_level = "error"
+
+# The default log directory where all logs will go unless directly specified by the kubelet
+log_dir = "/var/log/crio/pods"
+
+# The UID mappings for the user namespace of each container. A range is
+# specified in the form containerUID:HostUID:Size. Multiple ranges must be
+# separated by comma.
+uid_mappings = ""
+
+# The GID mappings for the user namespace of each container. A range is
+# specified in the form containerGID:HostGID:Size. Multiple ranges must be
+# separated by comma.
+gid_mappings = ""
+
+# The minimal amount of time in seconds to wait before issuing a timeout
+# regarding the proper termination of the container.
+ctr_stop_timeout = 0
+
+# ManageNetworkNSLifecycle determines whether we pin and remove network namespace
+# and manage its lifecycle.
+manage_network_ns_lifecycle = false
+
+# The "crio.runtime.runtimes" table defines a list of OCI compatible runtimes.
+# The runtime to use is picked based on the runtime_handler provided by the CRI.
+# If no runtime_handler is provided, the runtime will be picked based on the level
+# of trust of the workload.
+
+[crio.runtime.runtimes.runc]
+runtime_path = "/usr/bin/runc"
+runtime_type = "oci"
+runtime_root = "/run/runc"
+
+
+# The crio.image table contains settings pertaining to the management of OCI images.
+#
+# CRI-O reads its configured registries defaults from the system wide
+# containers-registries.conf(5) located in /etc/containers/registries.conf. If
+# you want to modify just CRI-O, you can change the registries configuration in
+# this file. Otherwise, leave insecure_registries and registries commented out to
+# use the system's defaults from /etc/containers/registries.conf.
+[crio.image]
+
+# Default transport for pulling images from a remote container storage.
+default_transport = "docker://"
+
+# The path to a file containing credentials necessary for pulling images from
+# secure registries. The file is similar to that of /var/lib/kubelet/config.json
+global_auth_file = ""
+
+# The image used to instantiate infra containers.
+# This option supports live configuration reload.
+pause_image = "{{ .PodInfraContainerImage }}"
+
+# The path to a file containing credentials specific for pulling the pause_image from
+# above. The file is similar to that of /var/lib/kubelet/config.json
+# This option supports live configuration reload.
+pause_image_auth_file = ""
+
+# The command to run to have a container stay in the paused state.
+# This option supports live configuration reload.
+pause_command = "/pause"
+
+# Path to the file which decides what sort of policy we use when deciding
+# whether or not to trust an image that we've pulled. It is not recommended that
+# this option be used, as the default behavior of using the system-wide default
+# policy (i.e., /etc/containers/policy.json) is most often preferred. Please
+# refer to containers-policy.json(5) for more details.
+signature_policy = ""
+
+# Controls how image volumes are handled. The valid values are mkdir, bind and
+# ignore; the latter will ignore volumes entirely.
+image_volumes = "mkdir"
+
+# List of registries to be used when pulling an unqualified image (e.g.,
+# "alpine:latest"). By default, registries is set to "docker.io" for
+# compatibility reasons. Depending on your workload and usecase you may add more
+# registries (e.g., "quay.io", "registry.fedoraproject.org",
+# "registry.opensuse.org", etc.).
+registries = [
+	"docker.io"
+]
+
+
+# The crio.network table containers settings pertaining to the management of
+# CNI plugins.
+[crio.network]
+
+# Path to the directory where CNI configuration files are located.
+network_dir = "/etc/cni/net.d/"
+
+# Paths to directories where CNI plugin binaries are located.
+plugin_dirs = [
+	"/opt/cni/bin/",
+]
+`
+)
+
 // CRIO contains CRIO runtime state
 type CRIO struct {
 	Socket            string
@@ -140,8 +445,18 @@ func (r *CRIO) KubeletOptions() map[string]string {
 }
 
 // ListContainers returns a list of managed by this container runtime
-func (r *CRIO) ListContainers(filter string) ([]string, error) {
-	return listCRIContainers(r.Runner, filter)
+func (r *CRIO) ListContainers(o ListOptions) ([]string, error) {
+	return listCRIContainers(r.Runner, "", o)
+}
+
+// PauseContainers pauses a running container based on ID
+func (r *CRIO) PauseContainers(ids []string) error {
+	return pauseCRIContainers(r.Runner, "", ids)
+}
+
+// PauseContainers pauses a running container based on ID
+func (r *CRIO) UnpauseContainers(ids []string) error {
+	return unpauseCRIContainers(r.Runner, "", ids)
 }
 
 // KillContainers removes containers based on ID
@@ -163,3 +478,24 @@ func (r *CRIO) ContainerLogCmd(id string, len int, follow bool) string {
 func (r *CRIO) SystemLogCmd(len int) string {
 	return fmt.Sprintf("sudo journalctl -u crio -n %d", len)
 }
+
+// generateCRIOConfig sets up /etc/crio/crio.conf
+func generateCRIOConfig(cr CommandRunner, imageRepository string, k8sVersion string) error {
+	cPath := crioConfigFile
+	t, err := template.New("crio.conf").Parse(crioConfigTemplate)
+	if err != nil {
+		return err
+	}
+	pauseImage := images.PauseImage(imageRepository, k8sVersion)
+	opts := struct{ PodInfraContainerImage string }{PodInfraContainerImage: pauseImage}
+	var b bytes.Buffer
+	if err := t.Execute(&b, opts); err != nil {
+		return err
+	}
+
+	c := exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo mkdir -p %s && printf %%s \"%s\" | base64 -d | sudo tee %s", path.Dir(cPath), base64.StdEncoding.EncodeToString(b.Bytes()), cPath))
+	if _, err := cr.RunCmd(c); err != nil {
+		return errors.Wrap(err, "generateCRIOConfig.")
+	}
+	return nil
+}
diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go
index 6566a8d8cc..9ca3e078bf 100644
--- a/pkg/minikube/cruntime/cruntime.go
+++ b/pkg/minikube/cruntime/cruntime.go
@@ -27,6 +27,18 @@ import (
 	"k8s.io/minikube/pkg/minikube/out"
 )
 
+type ContainerState int
+
+const (
+	All ContainerState = iota
+	Running
+	Paused
+)
+
+func (cs ContainerState) String() string {
+	return [...]string{"All", "Running", "Paused"}[cs]
+}
+
 // CommandRunner is the subset of command.Runner this package consumes
 type CommandRunner interface {
 	RunCmd(cmd *exec.Cmd) (*command.RunResult, error)
@@ -60,11 +72,15 @@ type Manager interface {
 	LoadImage(string) error
 
 	// ListContainers returns a list of managed by this container runtime
-	ListContainers(string) ([]string, error)
+	ListContainers(ListOptions) ([]string, error)
 	// KillContainers removes containers based on ID
 	KillContainers([]string) error
 	// StopContainers stops containers based on ID
 	StopContainers([]string) error
+	// PauseContainers pauses containers based on ID
+	PauseContainers([]string) error
+	// UnpauseContainers unpauses containers based on ID
+	UnpauseContainers([]string) error
 	// ContainerLogCmd returns the command to retrieve the log for a container based on ID
 	ContainerLogCmd(string, int, bool) string
 	// SystemLogCmd returns the command to return the system logs
@@ -85,6 +101,13 @@ type Config struct {
 	KubernetesVersion string
 }
 
+type ListOptions struct {
+	// State is the container state to filter by (All, Running, Paused)
+	State ContainerState
+	// Name is a name filter
+	Name string
+}
+
 // New returns an appropriately configured runtime
 func New(c Config) (Manager, error) {
 	switch c.Type {
diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go
index 3d80fca459..1bc2ab6920 100644
--- a/pkg/minikube/cruntime/docker.go
+++ b/pkg/minikube/cruntime/docker.go
@@ -121,11 +121,18 @@ func (r *Docker) KubeletOptions() map[string]string {
 }
 
 // ListContainers returns a list of containers
-func (r *Docker) ListContainers(filter string) ([]string, error) {
-	filter = KubernetesContainerPrefix + filter
-	rr, err := r.Runner.RunCmd(exec.Command("docker", "ps", "-a", fmt.Sprintf("--filter=name=%s", filter), "--format=\"{{.ID}}\""))
+func (r *Docker) ListContainers(o ListOptions) ([]string, error) {
+	args := []string{"ps"}
+	switch o.State {
+	case All:
+		args = append(args, "-a")
+	case Paused:
+		args = append(args, "--filter status=paused")
+	}
+	args = append(args, fmt.Sprintf("--filter=name=%s", KubernetesContainerPrefix+o.Name), "--format=\"{{.ID}}\"")
+	rr, err := r.Runner.RunCmd(exec.Command("docker", args...))
 	if err != nil {
-		return nil, errors.Wrapf(err, "docker ListContainers. ")
+		return nil, errors.Wrapf(err, "docker")
 	}
 	var ids []string
 	for _, line := range strings.Split(rr.Stdout.String(), "\n") {
@@ -159,7 +166,35 @@ func (r *Docker) StopContainers(ids []string) error {
 	args := append([]string{"stop"}, ids...)
 	c := exec.Command("docker", args...)
 	if _, err := r.Runner.RunCmd(c); err != nil {
-		return errors.Wrap(err, "stopping containers docker.")
+		return errors.Wrap(err, "docker")
+	}
+	return nil
+}
+
+// PauseContainers pauses a running container based on ID
+func (r *Docker) PauseContainers(ids []string) error {
+	if len(ids) == 0 {
+		return nil
+	}
+	glog.Infof("Pausing containers: %s", ids)
+	args := append([]string{"pause"}, ids...)
+	c := exec.Command("docker", args...)
+	if _, err := r.Runner.RunCmd(c); err != nil {
+		return errors.Wrap(err, "docker")
+	}
+	return nil
+}
+
+// UnpauseContainers unpauses a container based on ID
+func (r *Docker) UnpauseContainers(ids []string) error {
+	if len(ids) == 0 {
+		return nil
+	}
+	glog.Infof("Unpausing containers: %s", ids)
+	args := append([]string{"unpause"}, ids...)
+	c := exec.Command("docker", args...)
+	if _, err := r.Runner.RunCmd(c); err != nil {
+		return errors.Wrap(err, "docker")
 	}
 	return nil
 }
diff --git a/pkg/minikube/kubelet/kubelet.go b/pkg/minikube/kubelet/kubelet.go
new file mode 100644
index 0000000000..c8e0436a2f
--- /dev/null
+++ b/pkg/minikube/kubelet/kubelet.go
@@ -0,0 +1,105 @@
+/*
+Copyright 2019 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubelet
+
+import (
+	"fmt"
+	"os/exec"
+	"strings"
+	"time"
+
+	"github.com/golang/glog"
+	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/minikube/command"
+	"k8s.io/minikube/pkg/util/retry"
+)
+
+// Stop idempotently stops the kubelet
+func Stop(cr command.Runner) error {
+	glog.Infof("stopping kubelet ...")
+	stop := func() error {
+		cmd := exec.Command("sudo", "systemctl", "stop", "kubelet.service")
+		if rr, err := cr.RunCmd(cmd); err != nil {
+			glog.Errorf("temporary error for %q : %v", rr.Command(), err)
+		}
+		cmd = exec.Command("sudo", "systemctl", "show", "-p", "SubState", "kubelet")
+		rr, err := cr.RunCmd(cmd)
+		if err != nil {
+			glog.Errorf("temporary error: for %q : %v", rr.Command(), err)
+		}
+		if !strings.Contains(rr.Stdout.String(), "dead") && !strings.Contains(rr.Stdout.String(), "failed") {
+			return fmt.Errorf("unexpected kubelet state: %q", rr.Stdout.String())
+		}
+		return nil
+	}
+
+	if err := retry.Expo(stop, 2*time.Second, time.Minute*3, 5); err != nil {
+		return errors.Wrapf(err, "error stopping kubelet")
+	}
+
+	return nil
+}
+
+// Start starts the kubelet
+func Start(cr command.Runner) error {
+	glog.Infof("restarting kubelet.service ...")
+	c := exec.Command("sudo", "systemctl", "start", "kubelet")
+	if _, err := cr.RunCmd(c); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Restart restarts the kubelet
+func Restart(cr command.Runner) error {
+	glog.Infof("restarting kubelet.service ...")
+	c := exec.Command("sudo", "systemctl", "restart", "kubelet.service")
+	if _, err := cr.RunCmd(c); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Check checks on the status of the kubelet
+func Check(cr command.Runner) error {
+	glog.Infof("checking for running kubelet ...")
+	c := exec.Command("systemctl", "is-active", "--quiet", "service", "kubelet")
+	if _, err := cr.RunCmd(c); err != nil {
+		return errors.Wrap(err, "check kubelet")
+	}
+	return nil
+}
+
+// Disable disables the Kubelet
+func Disable(cr command.Runner) error {
+	glog.Infof("disabling kubelet ...")
+	c := exec.Command("systemctl", "disable", "kubelet")
+	if _, err := cr.RunCmd(c); err != nil {
+		return errors.Wrap(err, "disable")
+	}
+	return nil
+}
+
+// Enable enables the Kubelet
+func Enable(cr command.Runner) error {
+	glog.Infof("enabling kubelet ...")
+	c := exec.Command("systemctl", "enable", "kubelet")
+	if _, err := cr.RunCmd(c); err != nil {
+		return errors.Wrap(err, "enable")
+	}
+	return nil
+}
diff --git a/pkg/minikube/logs/logs.go b/pkg/minikube/logs/logs.go
index f562302619..ff5dcdacf0 100644
--- a/pkg/minikube/logs/logs.go
+++ b/pkg/minikube/logs/logs.go
@@ -170,7 +170,7 @@ func Output(r cruntime.Manager, bs bootstrapper.Bootstrapper, runner command.Run
 func logCommands(r cruntime.Manager, bs bootstrapper.Bootstrapper, length int, follow bool) map[string]string {
 	cmds := bs.LogCommands(bootstrapper.LogOptions{Lines: length, Follow: follow})
 	for _, pod := range importantPods {
-		ids, err := r.ListContainers(pod)
+		ids, err := r.ListContainers(cruntime.ListOptions{Name: pod})
 		if err != nil {
 			glog.Errorf("Failed to list containers for %q: %v", pod, err)
 			continue
diff --git a/pkg/minikube/out/style.go b/pkg/minikube/out/style.go
index b14c9d5883..946ee16f01 100644
--- a/pkg/minikube/out/style.go
+++ b/pkg/minikube/out/style.go
@@ -81,6 +81,8 @@ var styles = map[StyleEnum]style{
 	Celebration:   {Prefix: "🎉  "},
 	Workaround:    {Prefix: "👉  ", LowPrefix: lowIndent},
 	Sparkle:       {Prefix: "✨  "},
+	Pause:         {Prefix: "⏸️  "},
+	Unpause:       {Prefix: "⏯️  "},
 
 	// Specialized purpose styles
 	ISODownload:      {Prefix: "💿  "},
diff --git a/pkg/minikube/out/style_enum.go b/pkg/minikube/out/style_enum.go
index 0bc8ac1822..3787dbe904 100644
--- a/pkg/minikube/out/style_enum.go
+++ b/pkg/minikube/out/style_enum.go
@@ -84,4 +84,6 @@ const (
 	Empty
 	Workaround
 	Sparkle
+	Pause
+	Unpause
 )
diff --git a/pkg/minikube/pause/pause.go b/pkg/minikube/pause/pause.go
new file mode 100644
index 0000000000..8800fabfef
--- /dev/null
+++ b/pkg/minikube/pause/pause.go
@@ -0,0 +1,69 @@
+/*
+Copyright 2019 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package pause
+
+import (
+	"github.com/golang/glog"
+	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/minikube/command"
+	"k8s.io/minikube/pkg/minikube/cruntime"
+	"k8s.io/minikube/pkg/minikube/kubelet"
+)
+
+// Pause pauses a Kubernetes cluster
+func Pause(cr cruntime.Manager, r command.Runner) error {
+	if err := kubelet.Disable(r); err != nil {
+		return errors.Wrap(err, "kubelet disable")
+	}
+	if err := kubelet.Stop(r); err != nil {
+		return errors.Wrap(err, "kubelet stop")
+	}
+	ids, err := cr.ListContainers(cruntime.ListOptions{State: cruntime.Running})
+	if err != nil {
+		return errors.Wrap(err, "list running")
+	}
+	if len(ids) == 0 {
+		glog.Warningf("no running containers to pause")
+		return nil
+	}
+	return cr.PauseContainers(ids)
+
+}
+
+// Unpause unpauses a Kubernetes cluster
+func Unpause(cr cruntime.Manager, r command.Runner) error {
+	ids, err := cr.ListContainers(cruntime.ListOptions{State: cruntime.Paused})
+	if err != nil {
+		return errors.Wrap(err, "list paused")
+	}
+
+	if len(ids) == 0 {
+		glog.Warningf("no paused containers found")
+	} else {
+		if err := cr.UnpauseContainers(ids); err != nil {
+			return errors.Wrap(err, "unpause")
+		}
+	}
+
+	if err := kubelet.Enable(r); err != nil {
+		return errors.Wrap(err, "kubelet enable")
+	}
+	if err := kubelet.Start(r); err != nil {
+		return errors.Wrap(err, "kubelet start")
+	}
+	return nil
+}

From 64ee657c49a125330e2edacfb12c554eccd85862 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Fri, 22 Nov 2019 16:12:24 -0800
Subject: [PATCH 010/250] Fix some bugs, add some logging, have a turkey

---
 cmd/minikube/cmd/pause.go         |  8 +++++--
 cmd/minikube/cmd/unpause.go       | 22 ++++++++++-------
 pkg/minikube/config/config.go     |  2 ++
 pkg/minikube/cruntime/cri.go      | 40 ++++++++++++++++++++-----------
 pkg/minikube/cruntime/cruntime.go |  2 +-
 pkg/minikube/cruntime/docker.go   |  4 ++--
 pkg/minikube/kubelet/kubelet.go   |  4 ++--
 7 files changed, 53 insertions(+), 29 deletions(-)

diff --git a/cmd/minikube/cmd/pause.go b/cmd/minikube/cmd/pause.go
index 731c5f87d0..dcfb53f82a 100644
--- a/cmd/minikube/cmd/pause.go
+++ b/cmd/minikube/cmd/pause.go
@@ -17,7 +17,9 @@ limitations under the License.
 package cmd
 
 import (
+	"github.com/golang/glog"
 	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
 
 	"k8s.io/minikube/pkg/minikube/cluster"
 	"k8s.io/minikube/pkg/minikube/config"
@@ -33,6 +35,7 @@ var pauseCmd = &cobra.Command{
 	Use:   "pause",
 	Short: "pause containers",
 	Run: func(cmd *cobra.Command, args []string) {
+		cname := viper.GetString(config.MachineProfile)
 		api, err := machine.NewAPIClient()
 		if err != nil {
 			exit.WithError("Error getting client", err)
@@ -42,7 +45,8 @@ var pauseCmd = &cobra.Command{
 		if err != nil {
 			exit.WithError("Error getting config", err)
 		}
-		host, err := cluster.CheckIfHostExistsAndLoad(api, cc.Name)
+		glog.Infof("config: %+v", cc)
+		host, err := cluster.CheckIfHostExistsAndLoad(api, cname)
 		if err != nil {
 			exit.WithError("Error getting host", err)
 		}
@@ -52,7 +56,7 @@ var pauseCmd = &cobra.Command{
 			exit.WithError("Failed to get command runner", err)
 		}
 
-		config := cruntime.Config{Type: cc.ContainerRuntime}
+		config := cruntime.Config{Type: cc.ContainerRuntime, Runner: r}
 		cr, err := cruntime.New(config)
 		if err != nil {
 			exit.WithError("Failed runtime", err)
diff --git a/cmd/minikube/cmd/unpause.go b/cmd/minikube/cmd/unpause.go
index 3e45eaf711..64d4eab3df 100644
--- a/cmd/minikube/cmd/unpause.go
+++ b/cmd/minikube/cmd/unpause.go
@@ -17,8 +17,11 @@ limitations under the License.
 package cmd
 
 import (
+	"github.com/golang/glog"
 	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
 
+	"k8s.io/minikube/pkg/minikube/cluster"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/cruntime"
 	"k8s.io/minikube/pkg/minikube/exit"
@@ -32,22 +35,20 @@ var unpauseCmd = &cobra.Command{
 	Use:   "unpause",
 	Short: "unpause Kubernetes",
 	Run: func(cmd *cobra.Command, args []string) {
+		cname := viper.GetString(config.MachineProfile)
 		api, err := machine.NewAPIClient()
 		if err != nil {
 			exit.WithError("Error getting client", err)
 		}
 		defer api.Close()
-
 		cc, err := config.Load()
 		if err != nil {
 			exit.WithError("Error getting config", err)
 		}
-		host, err := api.Load(cc.Name)
-
-		config := cruntime.Config{Type: cc.ContainerRuntime}
-		cr, err := cruntime.New(config)
+		glog.Infof("config: %+v", cc)
+		host, err := cluster.CheckIfHostExistsAndLoad(api, cname)
 		if err != nil {
-			exit.WithError("Failed runtime", err)
+			exit.WithError("Error getting host", err)
 		}
 
 		r, err := machine.CommandRunner(host)
@@ -55,10 +56,15 @@ var unpauseCmd = &cobra.Command{
 			exit.WithError("Failed to get command runner", err)
 		}
 
-		err = pause.Pause(cr, r)
+		config := cruntime.Config{Type: cc.ContainerRuntime, Runner: r}
+		cr, err := cruntime.New(config)
+		if err != nil {
+			exit.WithError("Failed runtime", err)
+		}
+		err = pause.Unpause(cr, r)
 		if err != nil {
 			exit.WithError("Pause", err)
 		}
-		out.T(out.Pause, "The '{{.name}}' cluster is now paused", out.V{"name": cc.Name})
+		out.T(out.Unpause, "The '{{.name}}' cluster is now unpaused", out.V{"name": cc.Name})
 	},
 }
diff --git a/pkg/minikube/config/config.go b/pkg/minikube/config/config.go
index 7b7501ad15..4c67669e93 100644
--- a/pkg/minikube/config/config.go
+++ b/pkg/minikube/config/config.go
@@ -24,6 +24,7 @@ import (
 	"io/ioutil"
 	"os"
 
+	"github.com/golang/glog"
 	"github.com/spf13/viper"
 	"k8s.io/minikube/pkg/minikube/localpath"
 )
@@ -126,6 +127,7 @@ func encode(w io.Writer, m MinikubeConfig) error {
 // Load loads the kubernetes and machine config for the current machine
 func Load() (*MachineConfig, error) {
 	machine := viper.GetString(MachineProfile)
+	glog.Infof("Load machine name = %q", machine)
 	return DefaultLoader.LoadConfigFromFile(machine)
 }
 
diff --git a/pkg/minikube/cruntime/cri.go b/pkg/minikube/cruntime/cri.go
index ca77a58dab..0d543d276f 100644
--- a/pkg/minikube/cruntime/cri.go
+++ b/pkg/minikube/cruntime/cri.go
@@ -31,10 +31,8 @@ import (
 
 // container maps to 'runc list -f json'
 type container struct {
-	pid         int
-	id          string
-	status      string
-	annotations map[string]string
+	ID     string
+	Status string
 }
 
 // listCRIContainers returns a list of containers
@@ -55,7 +53,8 @@ func listCRIContainers(cr CommandRunner, root string, o ListOptions) ([]string,
 	// Avoid an id named ""
 	var ids []string
 	seen := map[string]bool{}
-	for _, id := range strings.Split(rr.Stderr.String(), "\n") {
+	for _, id := range strings.Split(rr.Stdout.String(), "\n") {
+		glog.Infof("found id: %q", id)
 		if id != "" && !seen[id] {
 			ids = append(ids, id)
 			seen[id] = true
@@ -71,39 +70,51 @@ func listCRIContainers(cr CommandRunner, root string, o ListOptions) ([]string,
 
 	// crictl does not understand paused pods
 	cs := []container{}
-	args = []string{"runc", "list", "-f", "json"}
+	args = []string{"runc"}
 	if root != "" {
 		args = append(args, "--root", root)
 	}
 
-	if _, err := cr.RunCmd(exec.Command("sudo", args...)); err != nil {
+	args = append(args, "list", "-f", "json")
+	rr, err = cr.RunCmd(exec.Command("sudo", args...))
+	if err != nil {
 		return nil, errors.Wrap(err, "runc")
 	}
-
-	d := json.NewDecoder(bytes.NewReader(rr.Stdout.Bytes()))
+	content := rr.Stdout.Bytes()
+	glog.Infof("JSON = %s", content)
+	d := json.NewDecoder(bytes.NewReader(content))
 	if err := d.Decode(&cs); err != nil {
 		return nil, err
 	}
 
+	if len(cs) == 0 {
+		return nil, fmt.Errorf("list returned 0 containers, but ps returned %d", len(ids))
+	}
+
+	glog.Infof("list returned %d containers", len(cs))
 	var fids []string
 	for _, c := range cs {
-		if !seen[c.id] {
+		glog.Infof("container: %+v", c)
+		if !seen[c.ID] {
+			glog.Infof("skipping %s - not in ps", c.ID)
 			continue
 		}
-		if o.State.String() != c.status {
+		if o.State != All && o.State.String() != c.Status {
+			glog.Infof("skipping %s: state = %q, want %q", c, c.Status, o.State)
 			continue
 		}
-		fids = append(fids, c.id)
+		fids = append(fids, c.ID)
 	}
 	return fids, nil
 }
 
 // pauseCRIContainers pauses a list of containers
 func pauseCRIContainers(cr CommandRunner, root string, ids []string) error {
-	args := []string{"runc", "pause"}
+	args := []string{"runc"}
 	if root != "" {
 		args = append(args, "--root", root)
 	}
+	args = append(args, "pause")
 
 	for _, id := range ids {
 		cargs := append(args, id)
@@ -116,10 +127,11 @@ func pauseCRIContainers(cr CommandRunner, root string, ids []string) error {
 
 // unpauseCRIContainers pauses a list of containers
 func unpauseCRIContainers(cr CommandRunner, root string, ids []string) error {
-	args := []string{"runc", "unpause"}
+	args := []string{"runc"}
 	if root != "" {
 		args = append(args, "--root", root)
 	}
+	args = append(args, "resume")
 
 	for _, id := range ids {
 		cargs := append(args, id)
diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go
index 9ca3e078bf..e07ba61081 100644
--- a/pkg/minikube/cruntime/cruntime.go
+++ b/pkg/minikube/cruntime/cruntime.go
@@ -36,7 +36,7 @@ const (
 )
 
 func (cs ContainerState) String() string {
-	return [...]string{"All", "Running", "Paused"}[cs]
+	return [...]string{"all", "running", "paused"}[cs]
 }
 
 // CommandRunner is the subset of command.Runner this package consumes
diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go
index 1bc2ab6920..e72b3b7f97 100644
--- a/pkg/minikube/cruntime/docker.go
+++ b/pkg/minikube/cruntime/docker.go
@@ -127,9 +127,9 @@ func (r *Docker) ListContainers(o ListOptions) ([]string, error) {
 	case All:
 		args = append(args, "-a")
 	case Paused:
-		args = append(args, "--filter status=paused")
+		args = append(args, "--filter", "status=paused")
 	}
-	args = append(args, fmt.Sprintf("--filter=name=%s", KubernetesContainerPrefix+o.Name), "--format=\"{{.ID}}\"")
+	args = append(args, fmt.Sprintf("--filter=name=%s", KubernetesContainerPrefix+o.Name), "--format={{.ID}}")
 	rr, err := r.Runner.RunCmd(exec.Command("docker", args...))
 	if err != nil {
 		return nil, errors.Wrapf(err, "docker")
diff --git a/pkg/minikube/kubelet/kubelet.go b/pkg/minikube/kubelet/kubelet.go
index c8e0436a2f..315f492878 100644
--- a/pkg/minikube/kubelet/kubelet.go
+++ b/pkg/minikube/kubelet/kubelet.go
@@ -87,7 +87,7 @@ func Check(cr command.Runner) error {
 // Disable disables the Kubelet
 func Disable(cr command.Runner) error {
 	glog.Infof("disabling kubelet ...")
-	c := exec.Command("systemctl", "disable", "kubelet")
+	c := exec.Command("sudo", "systemctl", "disable", "kubelet")
 	if _, err := cr.RunCmd(c); err != nil {
 		return errors.Wrap(err, "disable")
 	}
@@ -97,7 +97,7 @@ func Disable(cr command.Runner) error {
 // Enable enables the Kubelet
 func Enable(cr command.Runner) error {
 	glog.Infof("enabling kubelet ...")
-	c := exec.Command("systemctl", "enable", "kubelet")
+	c := exec.Command("sudo", "systemctl", "enable", "kubelet")
 	if _, err := cr.RunCmd(c); err != nil {
 		return errors.Wrap(err, "enable")
 	}

From 0cb99d5fef4da01ff7ef672e29d6cb64c00411b9 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Tue, 26 Nov 2019 10:31:53 -0800
Subject: [PATCH 011/250] update boilerplate

---
 pkg/minikube/performance/binary.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/minikube/performance/binary.go b/pkg/minikube/performance/binary.go
index 3e0817b3e7..91e2a82d90 100644
--- a/pkg/minikube/performance/binary.go
+++ b/pkg/minikube/performance/binary.go
@@ -1,5 +1,5 @@
 /*
-Copyright 2017 The Kubernetes Authors All rights reserved.
+Copyright 2019 The Kubernetes Authors All rights reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

From 7477050192d7265094bb16baffd8e1bf802c291f Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Thu, 5 Dec 2019 10:46:28 -0800
Subject: [PATCH 012/250] Turn addon manager off by default

Add run kubectl apply command directly if addons are changed by the user if the addon manager is disabled.
---
 cmd/minikube/cmd/config/util.go  | 18 +++++++++-
 go.mod                           |  3 --
 pkg/minikube/addons/reconcile.go | 62 ++++++++++++++++++++++++++++++++
 pkg/minikube/assets/addons.go    |  2 +-
 4 files changed, 80 insertions(+), 5 deletions(-)
 create mode 100644 pkg/minikube/addons/reconcile.go

diff --git a/cmd/minikube/cmd/config/util.go b/cmd/minikube/cmd/config/util.go
index c48226e40f..9e601a25b0 100644
--- a/cmd/minikube/cmd/config/util.go
+++ b/cmd/minikube/cmd/config/util.go
@@ -23,6 +23,7 @@ import (
 	"strings"
 
 	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/minikube/addons"
 	"k8s.io/minikube/pkg/minikube/assets"
 	"k8s.io/minikube/pkg/minikube/cluster"
 	"k8s.io/minikube/pkg/minikube/command"
@@ -172,6 +173,11 @@ func isAddonAlreadySet(addon *assets.Addon, enable bool) (bool, error) {
 	return false, nil
 }
 
+func isAddonManagerEnabled() (bool, error) {
+	addonManager := assets.Addons["addon-manager"]
+	return addonManager.IsEnabled()
+}
+
 func enableOrDisableAddonInternal(addon *assets.Addon, cmd command.Runner, data interface{}, enable bool) error {
 	var err error
 
@@ -208,7 +214,17 @@ func enableOrDisableAddonInternal(addon *assets.Addon, cmd command.Runner, data
 			}
 		}
 	}
-	return nil
+	// If addon manager is enabled, return as it will handle this.
+	// If not, reconcile addons ourselves.
+
+	enabled, err := isAddonManagerEnabled()
+	if err != nil {
+		return errors.Wrapf(err, "checking if addon manager is enabled")
+	}
+	if enabled {
+		return nil
+	}
+	return addons.ReconcileAddons(cmd)
 }
 
 // EnableOrDisableStorageClasses enables or disables storage classes
diff --git a/go.mod b/go.mod
index 46cf856a68..8976fb1bec 100644
--- a/go.mod
+++ b/go.mod
@@ -21,14 +21,11 @@ require (
 	github.com/docker/machine v0.7.1-0.20190718054102-a555e4f7a8f5 // version is 0.7.1 to pin to a555e4f7a8f5
 	github.com/elazarl/goproxy v0.0.0-20190421051319-9d40249d3c2f
 	github.com/elazarl/goproxy/ext v0.0.0-20190421051319-9d40249d3c2f // indirect
-	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/go-ole/go-ole v1.2.4 // indirect
 	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
-	github.com/google/btree v1.0.0 // indirect
 	github.com/google/go-cmp v0.3.0
 	github.com/gorilla/mux v1.7.1 // indirect
-	github.com/grpc-ecosystem/grpc-gateway v1.5.0 // indirect
 	github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce // indirect
 	github.com/hashicorp/go-getter v1.4.0
 	github.com/hashicorp/go-multierror v0.0.0-20160811015721-8c5f0ad93604 // indirect
diff --git a/pkg/minikube/addons/reconcile.go b/pkg/minikube/addons/reconcile.go
new file mode 100644
index 0000000000..d336149e7a
--- /dev/null
+++ b/pkg/minikube/addons/reconcile.go
@@ -0,0 +1,62 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package addons
+
+import (
+	"os/exec"
+
+	"github.com/golang/glog"
+	"k8s.io/minikube/pkg/minikube/command"
+)
+
+var kubectlPruneWhitelist = []string{
+	"core/v1/ConfigMap",
+	"core/v1/Endpoints",
+	"core/v1/Namespace",
+	"core/v1/PersistentVolumeClaim",
+	"core/v1/PersistentVolume",
+	"core/v1/Pod",
+	"core/v1/ReplicationController",
+	"core/v1/Secret",
+	"core/v1/Service",
+	"batch/v1/Job",
+	"batch/v1beta1/CronJob",
+	"apps/v1/DaemonSet",
+	"apps/v1/Deployment",
+	"apps/v1/ReplicaSet",
+	"apps/v1/StatefulSet",
+	"extensions/v1beta1/Ingress",
+}
+
+// ReconcileAddons runs kubectl apply -f on the addons directory
+// to reconcile addons state
+func ReconcileAddons(cmd command.Runner) error {
+	if _, err := cmd.RunCmd(kubectlCommand()); err != nil {
+		glog.Warningf("reconciling addons failed: %v", err)
+	}
+	return nil
+}
+
+func kubectlCommand() *exec.Cmd {
+	args := []string{"apply", "-f", "/etc/kubernetes/addons", "-l", "kubernetes.io/cluster-service!=true,addonmanager.kubernetes.io/mode=Reconcile", "--prune=true"}
+	for _, k := range kubectlPruneWhitelist {
+		args = append(args, []string{"--prune-whitelist", k}...)
+	}
+	args = append(args, "--recursive")
+	cmd := exec.Command("kubectl", args...)
+	return cmd
+}
diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go
index 9184ec1d6b..e629d954e4 100644
--- a/pkg/minikube/assets/addons.go
+++ b/pkg/minikube/assets/addons.go
@@ -76,7 +76,7 @@ var Addons = map[string]*Addon{
 			vmpath.GuestManifestsDir,
 			"addon-manager.yaml.tmpl",
 			"0640",
-			true),
+			false),
 	}, true, "addon-manager"),
 	"dashboard": NewAddon([]*BinAsset{
 		MustBinAsset("deploy/addons/dashboard/dashboard-clusterrole.yaml", vmpath.GuestAddonsDir, "dashboard-clusterrole.yaml", "0640", false),

From 4a45956178a75d198acfcf5be381a0be35932580 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Tue, 10 Dec 2019 10:08:23 -0800
Subject: [PATCH 013/250] Add unit tests

---
 cmd/minikube/cmd/config/util.go       |  3 +-
 pkg/minikube/addons/reconcile.go      | 42 +++++++++++++++++++++++----
 pkg/minikube/addons/reconcile_test.go | 25 ++++++++++++++++
 pkg/minikube/constants/constants.go   |  2 +-
 4 files changed, 64 insertions(+), 8 deletions(-)
 create mode 100644 pkg/minikube/addons/reconcile_test.go

diff --git a/cmd/minikube/cmd/config/util.go b/cmd/minikube/cmd/config/util.go
index 9e601a25b0..6016bab0ea 100644
--- a/cmd/minikube/cmd/config/util.go
+++ b/cmd/minikube/cmd/config/util.go
@@ -193,6 +193,7 @@ func enableOrDisableAddonInternal(addon *assets.Addon, cmd command.Runner, data
 			} else {
 				addonFile = addon
 			}
+			fmt.Println("trying to copy", addonFile)
 			if err := cmd.Copy(addonFile); err != nil {
 				return errors.Wrapf(err, "enabling addon %s", addon.AssetName)
 			}
@@ -209,6 +210,7 @@ func enableOrDisableAddonInternal(addon *assets.Addon, cmd command.Runner, data
 			} else {
 				addonFile = addon
 			}
+			fmt.Println("trying to remove", addonFile)
 			if err := cmd.Remove(addonFile); err != nil {
 				return errors.Wrapf(err, "disabling addon %s", addon.AssetName)
 			}
@@ -216,7 +218,6 @@ func enableOrDisableAddonInternal(addon *assets.Addon, cmd command.Runner, data
 	}
 	// If addon manager is enabled, return as it will handle this.
 	// If not, reconcile addons ourselves.
-
 	enabled, err := isAddonManagerEnabled()
 	if err != nil {
 		return errors.Wrapf(err, "checking if addon manager is enabled")
diff --git a/pkg/minikube/addons/reconcile.go b/pkg/minikube/addons/reconcile.go
index d336149e7a..44a4a95117 100644
--- a/pkg/minikube/addons/reconcile.go
+++ b/pkg/minikube/addons/reconcile.go
@@ -17,10 +17,14 @@ limitations under the License.
 package addons
 
 import (
+	"fmt"
+	"os"
 	"os/exec"
 
 	"github.com/golang/glog"
 	"k8s.io/minikube/pkg/minikube/command"
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/constants"
 )
 
 var kubectlPruneWhitelist = []string{
@@ -45,18 +49,44 @@ var kubectlPruneWhitelist = []string{
 // ReconcileAddons runs kubectl apply -f on the addons directory
 // to reconcile addons state
 func ReconcileAddons(cmd command.Runner) error {
-	if _, err := cmd.RunCmd(kubectlCommand()); err != nil {
-		glog.Warningf("reconciling addons failed: %v", err)
+	reconcileCmd, err := kubectlCommand()
+	if err != nil {
+		return err
 	}
+	fmt.Println("running", reconcileCmd)
+	rr, err := cmd.RunCmd(reconcileCmd)
+	if err != nil {
+		glog.Warningf("reconciling addons failed: %v", err)
+		return err
+	}
+	fmt.Println(rr.Stdout.String())
 	return nil
 }
 
-func kubectlCommand() *exec.Cmd {
-	args := []string{"apply", "-f", "/etc/kubernetes/addons", "-l", "kubernetes.io/cluster-service!=true,addonmanager.kubernetes.io/mode=Reconcile", "--prune=true"}
+func kubectlCommand() (*exec.Cmd, error) {
+	kubectlBinary, err := kubectlPath()
+	if err != nil {
+		return nil, err
+	}
+	args := []string{"KUBECONFIG=/var/lib/minikube/kubeconfig", kubectlBinary, "apply", "-f", "/etc/kubernetes/addons", "-l", "kubernetes.io/cluster-service!=true,addonmanager.kubernetes.io/mode=Reconcile", "--prune=true"}
 	for _, k := range kubectlPruneWhitelist {
 		args = append(args, []string{"--prune-whitelist", k}...)
 	}
 	args = append(args, "--recursive")
-	cmd := exec.Command("kubectl", args...)
-	return cmd
+	cmd := exec.Command("sudo", args...)
+	cmd.Env = append(os.Environ(), "KUBECONFIG=/var/lib/minikube/kubeconfig")
+	return cmd, nil
+}
+
+func kubectlPath() (string, error) {
+	cc, err := config.Load()
+	if err != nil && !os.IsNotExist(err) {
+		return "", err
+	}
+	version := constants.DefaultKubernetesVersion
+	if cc != nil {
+		version = cc.KubernetesConfig.KubernetesVersion
+	}
+	path := fmt.Sprintf("/var/lib/minikube/binaries/%s/kubectl", version)
+	return path, nil
 }
diff --git a/pkg/minikube/addons/reconcile_test.go b/pkg/minikube/addons/reconcile_test.go
new file mode 100644
index 0000000000..b8a8d1035d
--- /dev/null
+++ b/pkg/minikube/addons/reconcile_test.go
@@ -0,0 +1,25 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package addons
+
+import (
+	"testing"
+)
+
+func TestKubectlCommand(t *testing.T) {
+
+}
diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go
index 1c8e915f16..cef21e1c97 100644
--- a/pkg/minikube/constants/constants.go
+++ b/pkg/minikube/constants/constants.go
@@ -85,7 +85,7 @@ var ImageRepositories = map[string][]string{
 }
 
 // KubeadmBinaries are Kubernetes release binaries required for kubeadm
-var KubeadmBinaries = []string{"kubelet", "kubeadm"}
+var KubeadmBinaries = []string{"kubelet", "kubeadm", "kubectl"}
 
 // ImageCacheDir is the path to the image cache directory
 var ImageCacheDir = localpath.MakeMiniPath("cache", "images")

From 686b0a68fa33328ba56b3496e7867a7a3271087e Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Tue, 10 Dec 2019 12:34:43 -0800
Subject: [PATCH 014/250] Use kubectl v1.13.2 as in the addon manager image for
 correct functionality

---
 cmd/minikube/cmd/config/util.go              |  2 --
 pkg/minikube/addons/reconcile.go             | 23 ++------------------
 pkg/minikube/assets/addons.go                |  4 ++--
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 11 +++++++---
 pkg/minikube/constants/constants.go          |  9 +++++++-
 5 files changed, 20 insertions(+), 29 deletions(-)

diff --git a/cmd/minikube/cmd/config/util.go b/cmd/minikube/cmd/config/util.go
index 6016bab0ea..6325d5f1b4 100644
--- a/cmd/minikube/cmd/config/util.go
+++ b/cmd/minikube/cmd/config/util.go
@@ -193,7 +193,6 @@ func enableOrDisableAddonInternal(addon *assets.Addon, cmd command.Runner, data
 			} else {
 				addonFile = addon
 			}
-			fmt.Println("trying to copy", addonFile)
 			if err := cmd.Copy(addonFile); err != nil {
 				return errors.Wrapf(err, "enabling addon %s", addon.AssetName)
 			}
@@ -210,7 +209,6 @@ func enableOrDisableAddonInternal(addon *assets.Addon, cmd command.Runner, data
 			} else {
 				addonFile = addon
 			}
-			fmt.Println("trying to remove", addonFile)
 			if err := cmd.Remove(addonFile); err != nil {
 				return errors.Wrapf(err, "disabling addon %s", addon.AssetName)
 			}
diff --git a/pkg/minikube/addons/reconcile.go b/pkg/minikube/addons/reconcile.go
index 44a4a95117..e526b4a587 100644
--- a/pkg/minikube/addons/reconcile.go
+++ b/pkg/minikube/addons/reconcile.go
@@ -23,7 +23,6 @@ import (
 
 	"github.com/golang/glog"
 	"k8s.io/minikube/pkg/minikube/command"
-	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
 )
 
@@ -53,21 +52,16 @@ func ReconcileAddons(cmd command.Runner) error {
 	if err != nil {
 		return err
 	}
-	fmt.Println("running", reconcileCmd)
-	rr, err := cmd.RunCmd(reconcileCmd)
+	_, err = cmd.RunCmd(reconcileCmd)
 	if err != nil {
 		glog.Warningf("reconciling addons failed: %v", err)
 		return err
 	}
-	fmt.Println(rr.Stdout.String())
 	return nil
 }
 
 func kubectlCommand() (*exec.Cmd, error) {
-	kubectlBinary, err := kubectlPath()
-	if err != nil {
-		return nil, err
-	}
+	kubectlBinary := fmt.Sprintf("/var/lib/minikube/binaries/%s/%s", constants.KubectlBinaryVersion, constants.KubectlBinary)
 	args := []string{"KUBECONFIG=/var/lib/minikube/kubeconfig", kubectlBinary, "apply", "-f", "/etc/kubernetes/addons", "-l", "kubernetes.io/cluster-service!=true,addonmanager.kubernetes.io/mode=Reconcile", "--prune=true"}
 	for _, k := range kubectlPruneWhitelist {
 		args = append(args, []string{"--prune-whitelist", k}...)
@@ -77,16 +71,3 @@ func kubectlCommand() (*exec.Cmd, error) {
 	cmd.Env = append(os.Environ(), "KUBECONFIG=/var/lib/minikube/kubeconfig")
 	return cmd, nil
 }
-
-func kubectlPath() (string, error) {
-	cc, err := config.Load()
-	if err != nil && !os.IsNotExist(err) {
-		return "", err
-	}
-	version := constants.DefaultKubernetesVersion
-	if cc != nil {
-		version = cc.KubernetesConfig.KubernetesVersion
-	}
-	path := fmt.Sprintf("/var/lib/minikube/binaries/%s/kubectl", version)
-	return path, nil
-}
diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go
index e629d954e4..e713f21cfd 100644
--- a/pkg/minikube/assets/addons.go
+++ b/pkg/minikube/assets/addons.go
@@ -76,8 +76,8 @@ var Addons = map[string]*Addon{
 			vmpath.GuestManifestsDir,
 			"addon-manager.yaml.tmpl",
 			"0640",
-			false),
-	}, true, "addon-manager"),
+			true),
+	}, false, "addon-manager"),
 	"dashboard": NewAddon([]*BinAsset{
 		MustBinAsset("deploy/addons/dashboard/dashboard-clusterrole.yaml", vmpath.GuestAddonsDir, "dashboard-clusterrole.yaml", "0640", false),
 		MustBinAsset("deploy/addons/dashboard/dashboard-clusterrolebinding.yaml", vmpath.GuestAddonsDir, "dashboard-clusterrolebinding.yaml", "0640", false),
diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index 2643ec9d3c..2edf37a214 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -817,15 +817,20 @@ func invokeKubeadm(version string) string {
 // transferBinaries transfers all required Kubernetes binaries
 func transferBinaries(cfg config.KubernetesConfig, c command.Runner) error {
 	var g errgroup.Group
-	for _, name := range constants.KubeadmBinaries {
+	kubernetesBinaries := append(constants.KubeadmBinaries, constants.KubectlBinary)
+	for _, name := range kubernetesBinaries {
 		name := name
 		g.Go(func() error {
-			src, err := machine.CacheBinary(name, cfg.KubernetesVersion, "linux", runtime.GOARCH)
+			version := cfg.KubernetesVersion
+			if name == constants.KubectlBinary {
+				version = constants.KubectlBinaryVersion
+			}
+			src, err := machine.CacheBinary(name, version, "linux", runtime.GOARCH)
 			if err != nil {
 				return errors.Wrapf(err, "downloading %s", name)
 			}
 
-			dst := path.Join(binRoot(cfg.KubernetesVersion), name)
+			dst := path.Join(binRoot(version), name)
 			if err := machine.CopyBinary(c, src, dst); err != nil {
 				return errors.Wrapf(err, "copybinary %s -> %s", src, dst)
 			}
diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go
index cef21e1c97..42e4019610 100644
--- a/pkg/minikube/constants/constants.go
+++ b/pkg/minikube/constants/constants.go
@@ -85,7 +85,14 @@ var ImageRepositories = map[string][]string{
 }
 
 // KubeadmBinaries are Kubernetes release binaries required for kubeadm
-var KubeadmBinaries = []string{"kubelet", "kubeadm", "kubectl"}
+var KubeadmBinaries = []string{"kubelet", "kubeadm"}
+
+const (
+	// KubectlBinary is the name of the kubectl binary installed in minikube
+	KubectlBinary = "kubectl"
+	// KubectlBinaryVersion is the version of kubectl installed in minikube
+	KubectlBinaryVersion = "v1.13.2"
+)
 
 // ImageCacheDir is the path to the image cache directory
 var ImageCacheDir = localpath.MakeMiniPath("cache", "images")

From 3a20cd75bf2ce846ff974b92c9cc0ea4e95d1a91 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Tue, 10 Dec 2019 13:06:22 -0800
Subject: [PATCH 015/250] Add unit tests

---
 cmd/minikube/cmd/config/util.go       |  2 +-
 pkg/minikube/addons/reconcile.go      | 16 ++++++----------
 pkg/minikube/addons/reconcile_test.go |  7 +++++++
 pkg/minikube/constants/constants.go   |  1 +
 4 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/cmd/minikube/cmd/config/util.go b/cmd/minikube/cmd/config/util.go
index 6325d5f1b4..db215c0568 100644
--- a/cmd/minikube/cmd/config/util.go
+++ b/cmd/minikube/cmd/config/util.go
@@ -214,7 +214,7 @@ func enableOrDisableAddonInternal(addon *assets.Addon, cmd command.Runner, data
 			}
 		}
 	}
-	// If addon manager is enabled, return as it will handle this.
+	// If addon manager is enabled, return as it will handle whatever changes have been made.
 	// If not, reconcile addons ourselves.
 	enabled, err := isAddonManagerEnabled()
 	if err != nil {
diff --git a/pkg/minikube/addons/reconcile.go b/pkg/minikube/addons/reconcile.go
index e526b4a587..6647a6191b 100644
--- a/pkg/minikube/addons/reconcile.go
+++ b/pkg/minikube/addons/reconcile.go
@@ -18,7 +18,6 @@ package addons
 
 import (
 	"fmt"
-	"os"
 	"os/exec"
 
 	"github.com/golang/glog"
@@ -26,6 +25,7 @@ import (
 	"k8s.io/minikube/pkg/minikube/constants"
 )
 
+// taken from https://github.com/kubernetes/kubernetes/blob/master/cluster/addons/addon-manager/kube-addons.sh
 var kubectlPruneWhitelist = []string{
 	"core/v1/ConfigMap",
 	"core/v1/Endpoints",
@@ -48,26 +48,22 @@ var kubectlPruneWhitelist = []string{
 // ReconcileAddons runs kubectl apply -f on the addons directory
 // to reconcile addons state
 func ReconcileAddons(cmd command.Runner) error {
-	reconcileCmd, err := kubectlCommand()
-	if err != nil {
-		return err
-	}
-	_, err = cmd.RunCmd(reconcileCmd)
-	if err != nil {
+	if _, err := cmd.RunCmd(kubectlCommand()); err != nil {
 		glog.Warningf("reconciling addons failed: %v", err)
 		return err
 	}
 	return nil
 }
 
-func kubectlCommand() (*exec.Cmd, error) {
+func kubectlCommand() *exec.Cmd {
 	kubectlBinary := fmt.Sprintf("/var/lib/minikube/binaries/%s/%s", constants.KubectlBinaryVersion, constants.KubectlBinary)
+
 	args := []string{"KUBECONFIG=/var/lib/minikube/kubeconfig", kubectlBinary, "apply", "-f", "/etc/kubernetes/addons", "-l", "kubernetes.io/cluster-service!=true,addonmanager.kubernetes.io/mode=Reconcile", "--prune=true"}
 	for _, k := range kubectlPruneWhitelist {
 		args = append(args, []string{"--prune-whitelist", k}...)
 	}
 	args = append(args, "--recursive")
+
 	cmd := exec.Command("sudo", args...)
-	cmd.Env = append(os.Environ(), "KUBECONFIG=/var/lib/minikube/kubeconfig")
-	return cmd, nil
+	return cmd
 }
diff --git a/pkg/minikube/addons/reconcile_test.go b/pkg/minikube/addons/reconcile_test.go
index b8a8d1035d..ffd61b739d 100644
--- a/pkg/minikube/addons/reconcile_test.go
+++ b/pkg/minikube/addons/reconcile_test.go
@@ -17,9 +17,16 @@ limitations under the License.
 package addons
 
 import (
+	"strings"
 	"testing"
 )
 
 func TestKubectlCommand(t *testing.T) {
+	expected := "sudo KUBECONFIG=/var/lib/minikube/kubeconfig /var/lib/minikube/binaries/v1.13.2/kubectl apply -f /etc/kubernetes/addons -l kubernetes.io/cluster-service!=true,addonmanager.kubernetes.io/mode=Reconcile --prune=true --prune-whitelist core/v1/ConfigMap --prune-whitelist core/v1/Endpoints --prune-whitelist core/v1/Namespace --prune-whitelist core/v1/PersistentVolumeClaim --prune-whitelist core/v1/PersistentVolume --prune-whitelist core/v1/Pod --prune-whitelist core/v1/ReplicationController --prune-whitelist core/v1/Secret --prune-whitelist core/v1/Service --prune-whitelist batch/v1/Job --prune-whitelist batch/v1beta1/CronJob --prune-whitelist apps/v1/DaemonSet --prune-whitelist apps/v1/Deployment --prune-whitelist apps/v1/ReplicaSet --prune-whitelist apps/v1/StatefulSet --prune-whitelist extensions/v1beta1/Ingress --recursive"
+	kc := kubectlCommand()
+	actual := strings.Join(kc.Args, " ")
 
+	if actual != expected {
+		t.Fatalf("expected does not match actual\nExpected: %s\nActual: %s", expected, actual)
+	}
 }
diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go
index 42e4019610..c5eefc3a27 100644
--- a/pkg/minikube/constants/constants.go
+++ b/pkg/minikube/constants/constants.go
@@ -91,6 +91,7 @@ const (
 	// KubectlBinary is the name of the kubectl binary installed in minikube
 	KubectlBinary = "kubectl"
 	// KubectlBinaryVersion is the version of kubectl installed in minikube
+	// it's set to v1.13.2 to match the kubectl version in the addon manager image
 	KubectlBinaryVersion = "v1.13.2"
 )
 

From e0b6a1eb4950bb2a6097a39d54269ba850c3261f Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Tue, 10 Dec 2019 14:16:19 -0800
Subject: [PATCH 016/250] Address review comments

Address nits and remove integration test to make sure addon manager pod
is up and running since we will no longer enable it by default.
---
 cmd/minikube/cmd/config/util.go     |  2 +-
 pkg/minikube/addons/reconcile.go    | 10 ++++++----
 test/integration/functional_test.go |  9 ---------
 3 files changed, 7 insertions(+), 14 deletions(-)

diff --git a/cmd/minikube/cmd/config/util.go b/cmd/minikube/cmd/config/util.go
index db215c0568..1916b28666 100644
--- a/cmd/minikube/cmd/config/util.go
+++ b/cmd/minikube/cmd/config/util.go
@@ -223,7 +223,7 @@ func enableOrDisableAddonInternal(addon *assets.Addon, cmd command.Runner, data
 	if enabled {
 		return nil
 	}
-	return addons.ReconcileAddons(cmd)
+	return addons.Reconcile(cmd)
 }
 
 // EnableOrDisableStorageClasses enables or disables storage classes
diff --git a/pkg/minikube/addons/reconcile.go b/pkg/minikube/addons/reconcile.go
index 6647a6191b..cacaca2d8c 100644
--- a/pkg/minikube/addons/reconcile.go
+++ b/pkg/minikube/addons/reconcile.go
@@ -17,8 +17,8 @@ limitations under the License.
 package addons
 
 import (
-	"fmt"
 	"os/exec"
+	"path"
 
 	"github.com/golang/glog"
 	"k8s.io/minikube/pkg/minikube/command"
@@ -45,9 +45,9 @@ var kubectlPruneWhitelist = []string{
 	"extensions/v1beta1/Ingress",
 }
 
-// ReconcileAddons runs kubectl apply -f on the addons directory
+// Reconcile runs kubectl apply -f on the addons directory
 // to reconcile addons state
-func ReconcileAddons(cmd command.Runner) error {
+func Reconcile(cmd command.Runner) error {
 	if _, err := cmd.RunCmd(kubectlCommand()); err != nil {
 		glog.Warningf("reconciling addons failed: %v", err)
 		return err
@@ -56,8 +56,10 @@ func ReconcileAddons(cmd command.Runner) error {
 }
 
 func kubectlCommand() *exec.Cmd {
-	kubectlBinary := fmt.Sprintf("/var/lib/minikube/binaries/%s/%s", constants.KubectlBinaryVersion, constants.KubectlBinary)
+	kubectlBinary := path.Join("/var/lib/minikube/binaries", constants.KubectlBinaryVersion, constants.KubectlBinary)
 
+	// prune will delete any existing objects with the label specified by "-l" which don't appear in /etc/kubernetes/addons
+	// this is how we delete disabled addons
 	args := []string{"KUBECONFIG=/var/lib/minikube/kubeconfig", kubectlBinary, "apply", "-f", "/etc/kubernetes/addons", "-l", "kubernetes.io/cluster-service!=true,addonmanager.kubernetes.io/mode=Reconcile", "--prune=true"}
 	for _, k := range kubectlPruneWhitelist {
 		args = append(args, []string{"--prune-whitelist", k}...)
diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go
index eab7b6a50a..b82890f3af 100644
--- a/test/integration/functional_test.go
+++ b/test/integration/functional_test.go
@@ -87,7 +87,6 @@ func TestFunctional(t *testing.T) {
 			name      string
 			validator validateFunc
 		}{
-			{"AddonManager", validateAddonManager},
 			{"ComponentHealth", validateComponentHealth},
 			{"ConfigCmd", validateConfigCmd},
 			{"DashboardCmd", validateDashboardCmd},
@@ -168,14 +167,6 @@ func validateKubectlGetPods(ctx context.Context, t *testing.T, profile string) {
 	}
 }
 
-// validateAddonManager asserts that the kube-addon-manager pod is deployed properly
-func validateAddonManager(ctx context.Context, t *testing.T, profile string) {
-	// If --wait=false, this may take a couple of minutes
-	if _, err := PodWait(ctx, t, profile, "kube-system", "component=kube-addon-manager", 5*time.Minute); err != nil {
-		t.Fatalf("wait: %v", err)
-	}
-}
-
 // validateComponentHealth asserts that all Kubernetes components are healthy
 func validateComponentHealth(ctx context.Context, t *testing.T, profile string) {
 	rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "get", "cs", "-o=json"))

From de0fbca8b396177ab29c0ac50458a2f3e3a3fdfd Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Wed, 11 Dec 2019 12:21:31 -0800
Subject: [PATCH 017/250] initial commit

---
 pkg/minikube/config/profile.go | 17 ++++++++++++-
 pkg/minikube/config/types.go   | 45 ++++++++++++++++++++++------------
 2 files changed, 45 insertions(+), 17 deletions(-)

diff --git a/pkg/minikube/config/profile.go b/pkg/minikube/config/profile.go
index 22ad511d51..3889b930f4 100644
--- a/pkg/minikube/config/profile.go
+++ b/pkg/minikube/config/profile.go
@@ -18,6 +18,7 @@ package config
 
 import (
 	"encoding/json"
+	"errors"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -45,12 +46,26 @@ func (p *Profile) IsValid() bool {
 	if p.Config[0].VMDriver == "" {
 		return false
 	}
-	if p.Config[0].KubernetesConfig.KubernetesVersion == "" {
+	n, err := p.Config[0].GetMasterNode()
+	if err != nil {
+		return false
+	}
+	if n.KubernetesVersion == "" {
 		return false
 	}
 	return true
 }
 
+func GetMasterNode(cc MachineConfig) (Node, error) {
+	for _, n := range cc.Nodes {
+		if n.Type == Master {
+			return n, nil
+		}
+	}
+
+	return Node{}, errors.New("could not find master node")
+}
+
 // ProfileNameInReservedKeywords checks if the profile is an internal keywords
 func ProfileNameInReservedKeywords(name string) bool {
 	for _, v := range keywords {
diff --git a/pkg/minikube/config/types.go b/pkg/minikube/config/types.go
index 698f1542b4..e688b2cd6c 100644
--- a/pkg/minikube/config/types.go
+++ b/pkg/minikube/config/types.go
@@ -61,33 +61,46 @@ type MachineConfig struct {
 	NoVTXCheck          bool   // Only used by virtualbox
 	DNSProxy            bool   // Only used by virtualbox
 	HostDNSResolver     bool   // Only used by virtualbox
-	KubernetesConfig    KubernetesConfig
 	HostOnlyNicType     string // Only used by virtualbox
 	NatNicType          string // Only used by virtualbox
+	KubernetesConfig    KubernetesConfig
+	Nodes               []Node
 }
 
 // KubernetesConfig contains the parameters used to configure the VM Kubernetes.
 type KubernetesConfig struct {
-	KubernetesVersion string
-	NodeIP            string
-	NodePort          int
-	NodeName          string
-	APIServerName     string
-	APIServerNames    []string
-	APIServerIPs      []net.IP
-	DNSDomain         string
-	ContainerRuntime  string
-	CRISocket         string
-	NetworkPlugin     string
-	FeatureGates      string
-	ServiceCIDR       string
-	ImageRepository   string
-	ExtraOptions      ExtraOptionSlice
+	NodePort         int
+	APIServerName    string
+	APIServerNames   []string
+	APIServerIPs     []net.IP
+	DNSDomain        string
+	ContainerRuntime string
+	CRISocket        string
+	NetworkPlugin    string
+	FeatureGates     string
+	ServiceCIDR      string
+	ImageRepository  string
+	ExtraOptions     ExtraOptionSlice
 
 	ShouldLoadCachedImages bool
 	EnableDefaultCNI       bool
 }
 
+// Node contains information about specific nodes in a cluster
+type Node struct {
+	Name              string
+	IP                string
+	KubernetesVersion string
+	Type              NodeType
+}
+
+type NodeType int
+
+const (
+	Master NodeType = iota
+	Worker
+)
+
 // VersionedExtraOption holds information on flags to apply to a specific range
 // of versions
 type VersionedExtraOption struct {

From 47d918f1b3527fd78d2668338fdbfef35594214e Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Thu, 12 Dec 2019 10:35:33 -0800
Subject: [PATCH 018/250] making more arbitrary node changes

---
 cmd/minikube/cmd/dashboard.go  |  8 +++++---
 pkg/minikube/config/profile.go | 19 +++++++------------
 pkg/minikube/config/types.go   |  2 +-
 3 files changed, 13 insertions(+), 16 deletions(-)

diff --git a/cmd/minikube/cmd/dashboard.go b/cmd/minikube/cmd/dashboard.go
index f1eaefda34..aed6b015d7 100644
--- a/cmd/minikube/cmd/dashboard.go
+++ b/cmd/minikube/cmd/dashboard.go
@@ -85,9 +85,11 @@ var dashboardCmd = &cobra.Command{
 			}
 		}
 
-		err = proxy.ExcludeIP(cc.KubernetesConfig.NodeIP) // to be used for http get calls
-		if err != nil {
-			glog.Errorf("Error excluding IP from proxy: %s", err)
+		for _, n := range cc.Nodes {
+			err = proxy.ExcludeIP(n.IP) // to be used for http get calls
+			if err != nil {
+				glog.Errorf("Error excluding IP from proxy: %s", err)
+			}
 		}
 
 		kubectl, err := exec.LookPath("kubectl")
diff --git a/pkg/minikube/config/profile.go b/pkg/minikube/config/profile.go
index 3889b930f4..694cd07a10 100644
--- a/pkg/minikube/config/profile.go
+++ b/pkg/minikube/config/profile.go
@@ -36,26 +36,21 @@ func (p *Profile) IsValid() bool {
 	if p.Config == nil {
 		return false
 	}
-	if len(p.Config) == 0 {
+	if p.Config == nil {
 		return false
 	}
-	// This will become a loop for multinode
-	if p.Config[0] == nil {
+	if p.Config.VMDriver == "" {
 		return false
 	}
-	if p.Config[0].VMDriver == "" {
-		return false
-	}
-	n, err := p.Config[0].GetMasterNode()
-	if err != nil {
-		return false
-	}
-	if n.KubernetesVersion == "" {
-		return false
+	for _, n := range p.Config.Nodes {
+		if n.KubernetesVersion == "" {
+			return false
+		}
 	}
 	return true
 }
 
+// GetMasterNode gets the node specific config for the control plane
 func GetMasterNode(cc MachineConfig) (Node, error) {
 	for _, n := range cc.Nodes {
 		if n.Type == Master {
diff --git a/pkg/minikube/config/types.go b/pkg/minikube/config/types.go
index e688b2cd6c..8c340a3729 100644
--- a/pkg/minikube/config/types.go
+++ b/pkg/minikube/config/types.go
@@ -27,7 +27,7 @@ import (
 type Profile struct {
 	Name   string
 	Status string // running, stopped
-	Config []*MachineConfig
+	Config *MachineConfig
 }
 
 // MachineConfig contains the parameters used to start a cluster.

From 9f2217d8036206bc13b8fcb0505de0a87f7f45a4 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Thu, 12 Dec 2019 17:38:29 -0800
Subject: [PATCH 019/250] sketches of the node commands

---
 cmd/minikube/cmd/config/node/delete.go       |  0
 cmd/minikube/cmd/config/node/node.go         | 25 +++++++++++++
 cmd/minikube/cmd/config/node/start.go        |  0
 cmd/minikube/cmd/config/node/stop.go         |  0
 cmd/minikube/cmd/config/profile_list.go      |  2 +-
 cmd/minikube/cmd/root.go                     |  1 +
 cmd/minikube/cmd/start.go                    | 26 ++++++++-----
 pkg/minikube/bootstrapper/certs.go           | 14 +++----
 pkg/minikube/bootstrapper/certs_test.go      |  2 +-
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 20 +++++-----
 pkg/minikube/cluster/cluster.go              | 27 +++++++-------
 pkg/minikube/cluster/node/node.go            | 39 ++++++++++++++++++++
 pkg/minikube/config/profile.go               |  2 +-
 pkg/minikube/config/profile_test.go          |  4 +-
 pkg/minikube/config/types.go                 | 29 ++++++++-------
 15 files changed, 132 insertions(+), 59 deletions(-)
 create mode 100644 cmd/minikube/cmd/config/node/delete.go
 create mode 100644 cmd/minikube/cmd/config/node/node.go
 create mode 100644 cmd/minikube/cmd/config/node/start.go
 create mode 100644 cmd/minikube/cmd/config/node/stop.go
 create mode 100644 pkg/minikube/cluster/node/node.go

diff --git a/cmd/minikube/cmd/config/node/delete.go b/cmd/minikube/cmd/config/node/delete.go
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/cmd/minikube/cmd/config/node/node.go b/cmd/minikube/cmd/config/node/node.go
new file mode 100644
index 0000000000..9833b4d27f
--- /dev/null
+++ b/cmd/minikube/cmd/config/node/node.go
@@ -0,0 +1,25 @@
+/*
+Copyright 2019 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package config
+
+import "github.com/spf13/cobra"
+
+var NodeCmd = &cobra.Command{
+	Use:   "node",
+	Short: "Node operations",
+	Long:  "Operations on nodes",
+}
diff --git a/cmd/minikube/cmd/config/node/start.go b/cmd/minikube/cmd/config/node/start.go
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/cmd/minikube/cmd/config/node/stop.go b/cmd/minikube/cmd/config/node/stop.go
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/cmd/minikube/cmd/config/profile_list.go b/cmd/minikube/cmd/config/profile_list.go
index 02f8b9ba1d..b9a2c45561 100644
--- a/cmd/minikube/cmd/config/profile_list.go
+++ b/cmd/minikube/cmd/config/profile_list.go
@@ -81,7 +81,7 @@ var printProfilesTable = func() {
 		if err != nil {
 			glog.Infof("error getting host status for %v", err)
 		}
-		validData = append(validData, []string{p.Name, p.Config[0].VMDriver, p.Config[0].KubernetesConfig.NodeIP, strconv.Itoa(p.Config[0].KubernetesConfig.NodePort), p.Config[0].KubernetesConfig.KubernetesVersion, p.Status})
+		validData = append(validData, []string{p.Name, p.Config.VMDriver, p.Config.Nodes[0].IP, strconv.Itoa(p.Config.Nodes[0].Port), p.Config.KubernetesConfig.KubernetesVersion, p.Status})
 	}
 
 	table.AppendBulk(validData)
diff --git a/cmd/minikube/cmd/root.go b/cmd/minikube/cmd/root.go
index 38f84246d9..6a87811245 100644
--- a/cmd/minikube/cmd/root.go
+++ b/cmd/minikube/cmd/root.go
@@ -187,6 +187,7 @@ func init() {
 				configCmd.AddonsCmd,
 				configCmd.ConfigCmd,
 				configCmd.ProfileCmd,
+				configCmd.NodeCmd,
 				updateContextCmd,
 			},
 		},
diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index d1e4fbec98..84d33c68ce 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -339,7 +339,7 @@ func runStart(cmd *cobra.Command, args []string) {
 
 	// exits here in case of --download-only option.
 	handleDownloadOnly(&cacheGroup, k8sVersion)
-	mRunner, preExists, machineAPI, host := startMachine(&config)
+	mRunner, preExists, machineAPI, host := startMachine(&config, &config.Nodes[0])
 	defer machineAPI.Close()
 	// configure the runtime (docker, containerd, crio)
 	cr := configureRuntimes(mRunner, driverName, config.KubernetesConfig)
@@ -347,7 +347,7 @@ func runStart(cmd *cobra.Command, args []string) {
 	waitCacheImages(&cacheGroup)
 
 	// Must be written before bootstrap, otherwise health checks may flake due to stale IP
-	kubeconfig, err := setupKubeconfig(host, &config, config.Name)
+	kubeconfig, err := setupKubeconfig(host, &config.KubernetesConfig, &config.Nodes[0], config.Name)
 	if err != nil {
 		exit.WithError("Failed to setup kubeconfig", err)
 	}
@@ -426,15 +426,15 @@ func displayEnviron(env []string) {
 	}
 }
 
-func setupKubeconfig(h *host.Host, c *cfg.MachineConfig, clusterName string) (*kubeconfig.Settings, error) {
+func setupKubeconfig(h *host.Host, c *cfg.KubernetesConfig, n *cfg.Node, clusterName string) (*kubeconfig.Settings, error) {
 	addr, err := h.Driver.GetURL()
 	if err != nil {
 		exit.WithError("Failed to get driver URL", err)
 	}
 	addr = strings.Replace(addr, "tcp://", "https://", -1)
-	addr = strings.Replace(addr, ":2376", ":"+strconv.Itoa(c.KubernetesConfig.NodePort), -1)
-	if c.KubernetesConfig.APIServerName != constants.APIServerName {
-		addr = strings.Replace(addr, c.KubernetesConfig.NodeIP, c.KubernetesConfig.APIServerName, -1)
+	addr = strings.Replace(addr, ":2376", ":"+strconv.Itoa(n.Port), -1)
+	if c.APIServerName != constants.APIServerName {
+		addr = strings.Replace(addr, n.IP, c.APIServerName, -1)
 	}
 
 	kcs := &kubeconfig.Settings{
@@ -471,7 +471,7 @@ func handleDownloadOnly(cacheGroup *errgroup.Group, k8sVersion string) {
 
 }
 
-func startMachine(config *cfg.MachineConfig) (runner command.Runner, preExists bool, machineAPI libmachine.API, host *host.Host) {
+func startMachine(config *cfg.MachineConfig, node *cfg.Node) (runner command.Runner, preExists bool, machineAPI libmachine.API, host *host.Host) {
 	m, err := machine.NewAPIClient()
 	if err != nil {
 		exit.WithError("Failed to get machine client", err)
@@ -489,7 +489,7 @@ func startMachine(config *cfg.MachineConfig) (runner command.Runner, preExists b
 		out.ErrT(out.FailureType, "Failed to set NO_PROXY Env. Please use `export NO_PROXY=$NO_PROXY,{{.ip}}`.", out.V{"ip": ip})
 	}
 	// Save IP to configuration file for subsequent use
-	config.KubernetesConfig.NodeIP = ip
+	node.IP = ip
 	if err := saveConfig(config); err != nil {
 		exit.WithError("Failed to save config", err)
 	}
@@ -921,8 +921,6 @@ func generateCfgFromFlags(cmd *cobra.Command, k8sVersion string, drvName string)
 		NatNicType:          viper.GetString(natNicType),
 		KubernetesConfig: cfg.KubernetesConfig{
 			KubernetesVersion:      k8sVersion,
-			NodePort:               viper.GetInt(apiServerPort),
-			NodeName:               constants.DefaultNodeName,
 			APIServerName:          viper.GetString(apiServerName),
 			APIServerNames:         apiServerNames,
 			APIServerIPs:           apiServerIPs,
@@ -937,6 +935,14 @@ func generateCfgFromFlags(cmd *cobra.Command, k8sVersion string, drvName string)
 			ShouldLoadCachedImages: viper.GetBool(cacheImages),
 			EnableDefaultCNI:       selectedEnableDefaultCNI,
 		},
+		Nodes: []cfg.Node{
+			cfg.Node{
+				Port:              viper.GetInt(apiServerPort),
+				KubernetesVersion: k8sVersion,
+				Name:              constants.DefaultNodeName,
+				Type:              cfg.Master,
+			},
+		},
 	}
 	return cfg, nil
 }
diff --git a/pkg/minikube/bootstrapper/certs.go b/pkg/minikube/bootstrapper/certs.go
index d0731b24cd..0676b19826 100644
--- a/pkg/minikube/bootstrapper/certs.go
+++ b/pkg/minikube/bootstrapper/certs.go
@@ -59,10 +59,10 @@ var (
 )
 
 // SetupCerts gets the generated credentials required to talk to the APIServer.
-func SetupCerts(cmd command.Runner, k8s config.KubernetesConfig) error {
+func SetupCerts(cmd command.Runner, k8s config.KubernetesConfig, n config.Node) error {
 
 	localPath := localpath.MiniPath()
-	glog.Infof("Setting up %s for IP: %s\n", localPath, k8s.NodeIP)
+	glog.Infof("Setting up %s for IP: %s\n", localPath, n.IP)
 
 	// WARNING: This function was not designed for multiple profiles, so it is VERY racey:
 	//
@@ -78,7 +78,7 @@ func SetupCerts(cmd command.Runner, k8s config.KubernetesConfig) error {
 	}
 	defer releaser.Release()
 
-	if err := generateCerts(k8s); err != nil {
+	if err := generateCerts(k8s, n); err != nil {
 		return errors.Wrap(err, "Error generating certs")
 	}
 	copyableFiles := []assets.CopyableFile{}
@@ -109,8 +109,8 @@ func SetupCerts(cmd command.Runner, k8s config.KubernetesConfig) error {
 	}
 
 	kcs := &kubeconfig.Settings{
-		ClusterName:          k8s.NodeName,
-		ClusterServerAddress: fmt.Sprintf("https://localhost:%d", k8s.NodePort),
+		ClusterName:          n.Name,
+		ClusterServerAddress: fmt.Sprintf("https://localhost:%d", n.Port),
 		ClientCertificate:    path.Join(vmpath.GuestCertsDir, "apiserver.crt"),
 		ClientKey:            path.Join(vmpath.GuestCertsDir, "apiserver.key"),
 		CertificateAuthority: path.Join(vmpath.GuestCertsDir, "ca.crt"),
@@ -143,7 +143,7 @@ func SetupCerts(cmd command.Runner, k8s config.KubernetesConfig) error {
 	return nil
 }
 
-func generateCerts(k8s config.KubernetesConfig) error {
+func generateCerts(k8s config.KubernetesConfig, n config.Node) error {
 	serviceIP, err := util.GetServiceClusterIP(k8s.ServiceCIDR)
 	if err != nil {
 		return errors.Wrap(err, "getting service cluster ip")
@@ -175,7 +175,7 @@ func generateCerts(k8s config.KubernetesConfig) error {
 
 	apiServerIPs := append(
 		k8s.APIServerIPs,
-		[]net.IP{net.ParseIP(k8s.NodeIP), serviceIP, net.ParseIP("10.0.0.1")}...)
+		[]net.IP{net.ParseIP(n.IP), serviceIP, net.ParseIP("10.0.0.1")}...)
 	apiServerNames := append(k8s.APIServerNames, k8s.APIServerName)
 	apiServerAlternateNames := append(
 		apiServerNames,
diff --git a/pkg/minikube/bootstrapper/certs_test.go b/pkg/minikube/bootstrapper/certs_test.go
index 7d65e6c3af..dfbf3e8e59 100644
--- a/pkg/minikube/bootstrapper/certs_test.go
+++ b/pkg/minikube/bootstrapper/certs_test.go
@@ -76,7 +76,7 @@ func TestSetupCerts(t *testing.T) {
 	filesToBeTransferred = append(filesToBeTransferred, filepath.Join(localpath.MiniPath(), "ca.crt"))
 	filesToBeTransferred = append(filesToBeTransferred, filepath.Join(localpath.MiniPath(), "certs", "mycert.pem"))
 
-	if err := SetupCerts(f, k8s); err != nil {
+	if err := SetupCerts(f, k8s, config.Node{}); err != nil {
 		t.Fatalf("Error starting cluster: %v", err)
 	}
 	for _, cert := range filesToBeTransferred {
diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index cfe28ea26e..55a8749227 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -236,10 +236,10 @@ func (k *Bootstrapper) existingConfig() error {
 }
 
 // StartCluster starts the cluster
-func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error {
+func (k *Bootstrapper) StartCluster(k8s config.MachineConfig) error {
 	err := k.existingConfig()
 	if err == nil {
-		return k.restartCluster(k8s)
+		return k.restartCluster(k8s.KubernetesConfig)
 	}
 	glog.Infof("existence check: %v", err)
 
@@ -249,12 +249,12 @@ func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error {
 		glog.Infof("StartCluster complete in %s", time.Since(start))
 	}()
 
-	version, err := parseKubernetesVersion(k8s.KubernetesVersion)
+	version, err := parseKubernetesVersion(k8s.KubernetesConfig.KubernetesVersion)
 	if err != nil {
 		return errors.Wrap(err, "parsing kubernetes version")
 	}
 
-	extraFlags := createFlagsFromExtraArgs(k8s.ExtraOptions)
+	extraFlags := createFlagsFromExtraArgs(k8s.KubernetesConfig.ExtraOptions)
 	r, err := cruntime.New(cruntime.Config{Type: k8s.ContainerRuntime})
 	if err != nil {
 		return err
@@ -279,7 +279,7 @@ func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error {
 		ignore = append(ignore, "SystemVerification")
 	}
 
-	c := exec.Command("/bin/bash", "-c", fmt.Sprintf("%s init --config %s %s --ignore-preflight-errors=%s", invokeKubeadm(k8s.KubernetesVersion), yamlConfigPath, extraFlags, strings.Join(ignore, ",")))
+	c := exec.Command("/bin/bash", "-c", fmt.Sprintf("%s init --config %s %s --ignore-preflight-errors=%s", invokeKubeadm(k8s.KubernetesConfig.KubernetesVersion), yamlConfigPath, extraFlags, strings.Join(ignore, ",")))
 	if rr, err := k.c.RunCmd(c); err != nil {
 		return errors.Wrapf(err, "init failed. cmd: %q", rr.Command())
 	}
@@ -287,7 +287,7 @@ func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error {
 	glog.Infof("Configuring cluster permissions ...")
 
 	elevate := func() error {
-		client, err := k.client(k8s)
+		client, err := k.client(k8s.KubernetesConfig, k8s.Nodes[0])
 		if err != nil {
 			return err
 		}
@@ -357,13 +357,13 @@ func addAddons(files *[]assets.CopyableFile, data interface{}) error {
 }
 
 // client returns a Kubernetes client to use to speak to a kubeadm launched apiserver
-func (k *Bootstrapper) client(k8s config.KubernetesConfig) (*kubernetes.Clientset, error) {
+func (k *Bootstrapper) client(k8s config.KubernetesConfig, n config.Node) (*kubernetes.Clientset, error) {
 	config, err := kapi.ClientConfig(k.contextName)
 	if err != nil {
 		return nil, errors.Wrap(err, "client config")
 	}
 
-	endpoint := fmt.Sprintf("https://%s:%d", k8s.NodeIP, k8s.NodePort)
+	endpoint := fmt.Sprintf("https://%s:%d", n.IP, n.Port)
 	if config.Host != endpoint {
 		glog.Errorf("Overriding stale ClientConfig host %s with %s", config.Host, endpoint)
 		config.Host = endpoint
@@ -392,7 +392,7 @@ func (k *Bootstrapper) waitForAPIServerProcess(start time.Time, timeout time.Dur
 	return nil
 }
 
-func (k *Bootstrapper) waitForAPIServerHealthz(start time.Time, k8s config.KubernetesConfig, timeout time.Duration) error {
+func (k *Bootstrapper) waitForAPIServerHealthz(start time.Time, k8s config.Node, timeout time.Duration) error {
 	glog.Infof("waiting for apiserver healthz status ...")
 	hStart := time.Now()
 	healthz := func() (bool, error) {
@@ -400,7 +400,7 @@ func (k *Bootstrapper) waitForAPIServerHealthz(start time.Time, k8s config.Kuber
 			return false, fmt.Errorf("cluster wait timed out during healthz check")
 		}
 
-		status, err := k.GetAPIServerStatus(net.ParseIP(k8s.NodeIP), k8s.NodePort)
+		status, err := k.GetAPIServerStatus(net.ParseIP(k8s.IP), k8s.Port)
 		if err != nil {
 			glog.Warningf("status: %v", err)
 			return false, nil
diff --git a/pkg/minikube/cluster/cluster.go b/pkg/minikube/cluster/cluster.go
index ca8dc8778c..648553396e 100644
--- a/pkg/minikube/cluster/cluster.go
+++ b/pkg/minikube/cluster/cluster.go
@@ -45,7 +45,6 @@ import (
 	"github.com/spf13/viper"
 
 	"k8s.io/minikube/pkg/minikube/config"
-	cfg "k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/driver"
 	"k8s.io/minikube/pkg/minikube/exit"
@@ -80,33 +79,33 @@ func init() {
 }
 
 // CacheISO downloads and caches ISO.
-func CacheISO(config cfg.MachineConfig) error {
-	if driver.BareMetal(config.VMDriver) {
+func CacheISO(cfg config.MachineConfig) error {
+	if driver.BareMetal(cfg.VMDriver) {
 		return nil
 	}
-	return config.Downloader.CacheMinikubeISOFromURL(config.MinikubeISO)
+	return cfg.Downloader.CacheMinikubeISOFromURL(cfg.MinikubeISO)
 }
 
 // StartHost starts a host VM.
-func StartHost(api libmachine.API, config cfg.MachineConfig) (*host.Host, error) {
-	exists, err := api.Exists(config.Name)
+func StartHost(api libmachine.API, cfg config.MachineConfig) (*host.Host, error) {
+	exists, err := api.Exists(cfg.Name)
 	if err != nil {
-		return nil, errors.Wrapf(err, "exists: %s", config.Name)
+		return nil, errors.Wrapf(err, "exists: %s", cfg.Name)
 	}
 	if !exists {
 		glog.Infoln("Machine does not exist... provisioning new machine")
-		glog.Infof("Provisioning machine with config: %+v", config)
-		return createHost(api, config)
+		glog.Infof("Provisioning machine with config: %+v", cfg)
+		return createHost(api, cfg)
 	}
 
 	glog.Infoln("Skipping create...Using existing machine configuration")
 
-	h, err := api.Load(config.Name)
+	h, err := api.Load(cfg.Name)
 	if err != nil {
 		return nil, errors.Wrap(err, "Error loading existing host. Please try running [minikube delete], then run [minikube start] again.")
 	}
 
-	if exists && config.Name == constants.DefaultMachineName {
+	if exists && cfg.Name == constants.DefaultMachineName {
 		out.T(out.Tip, "Tip: Use 'minikube start -p <name>' to create a new cluster, or 'minikube delete' to delete this one.")
 	}
 
@@ -117,9 +116,9 @@ func StartHost(api libmachine.API, config cfg.MachineConfig) (*host.Host, error)
 	}
 
 	if s == state.Running {
-		out.T(out.Running, `Using the running {{.driver_name}} "{{.profile_name}}" VM ...`, out.V{"driver_name": config.VMDriver, "profile_name": config.Name})
+		out.T(out.Running, `Using the running {{.driver_name}} "{{.profile_name}}" VM ...`, out.V{"driver_name": cfg.VMDriver, "profile_name": cfg.Name})
 	} else {
-		out.T(out.Restarting, `Starting existing {{.driver_name}} VM for "{{.profile_name}}" ...`, out.V{"driver_name": config.VMDriver, "profile_name": config.Name})
+		out.T(out.Restarting, `Starting existing {{.driver_name}} VM for "{{.profile_name}}" ...`, out.V{"driver_name": cfg.VMDriver, "profile_name": cfg.Name})
 		if err := h.Driver.Start(); err != nil {
 			return nil, errors.Wrap(err, "start")
 		}
@@ -128,7 +127,7 @@ func StartHost(api libmachine.API, config cfg.MachineConfig) (*host.Host, error)
 		}
 	}
 
-	e := engineOptions(config)
+	e := engineOptions(cfg)
 	glog.Infof("engine options: %+v", e)
 
 	out.T(out.Waiting, "Waiting for the host to be provisioned ...")
diff --git a/pkg/minikube/cluster/node/node.go b/pkg/minikube/cluster/node/node.go
new file mode 100644
index 0000000000..25176f4410
--- /dev/null
+++ b/pkg/minikube/cluster/node/node.go
@@ -0,0 +1,39 @@
+/*
+Copyright 2019 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cluster
+
+import "k8s.io/minikube/pkg/minikube/config"
+
+// AddNode adds a new node config to a cluster.
+func AddNode(cfg config.MachineConfig, name string) error {
+
+}
+
+// StartNode starts the given node
+func StartNode(node config.Node) error {
+
+}
+
+// StopNode stops the given node
+func StopNode(node config.Node) error {
+
+}
+
+// DeleteNode deletes the given node from the given cluster
+func DeleteNode(cfg config.MachineConfig, node config.Node) error {
+
+}
diff --git a/pkg/minikube/config/profile.go b/pkg/minikube/config/profile.go
index 694cd07a10..85cec366ae 100644
--- a/pkg/minikube/config/profile.go
+++ b/pkg/minikube/config/profile.go
@@ -170,7 +170,7 @@ func LoadProfile(name string, miniHome ...string) (*Profile, error) {
 	cfg, err := DefaultLoader.LoadConfigFromFile(name, miniHome...)
 	p := &Profile{
 		Name:   name,
-		Config: []*MachineConfig{cfg},
+		Config: cfg,
 	}
 	return p, err
 }
diff --git a/pkg/minikube/config/profile_test.go b/pkg/minikube/config/profile_test.go
index a7e436d612..d238111233 100644
--- a/pkg/minikube/config/profile_test.go
+++ b/pkg/minikube/config/profile_test.go
@@ -54,8 +54,8 @@ func TestListProfiles(t *testing.T) {
 		if val[tt.index].Name != tt.expectName {
 			t.Errorf("expected %s got %v", tt.expectName, val[tt.index].Name)
 		}
-		if val[tt.index].Config[0].VMDriver != tt.vmDriver {
-			t.Errorf("expected %s got %v", tt.vmDriver, val[tt.index].Config[0].VMDriver)
+		if val[tt.index].Config.VMDriver != tt.vmDriver {
+			t.Errorf("expected %s got %v", tt.vmDriver, val[tt.index].Config.VMDriver)
 		}
 
 	}
diff --git a/pkg/minikube/config/types.go b/pkg/minikube/config/types.go
index 8c340a3729..efec1ccc47 100644
--- a/pkg/minikube/config/types.go
+++ b/pkg/minikube/config/types.go
@@ -69,19 +69,18 @@ type MachineConfig struct {
 
 // KubernetesConfig contains the parameters used to configure the VM Kubernetes.
 type KubernetesConfig struct {
-	NodePort         int
-	APIServerName    string
-	APIServerNames   []string
-	APIServerIPs     []net.IP
-	DNSDomain        string
-	ContainerRuntime string
-	CRISocket        string
-	NetworkPlugin    string
-	FeatureGates     string
-	ServiceCIDR      string
-	ImageRepository  string
-	ExtraOptions     ExtraOptionSlice
-
+	APIServerName          string
+	APIServerNames         []string
+	APIServerIPs           []net.IP
+	DNSDomain              string
+	ContainerRuntime       string
+	CRISocket              string
+	NetworkPlugin          string
+	FeatureGates           string
+	ServiceCIDR            string
+	ImageRepository        string
+	ExtraOptions           ExtraOptionSlice
+	KubernetesVersion      string
 	ShouldLoadCachedImages bool
 	EnableDefaultCNI       bool
 }
@@ -90,14 +89,18 @@ type KubernetesConfig struct {
 type Node struct {
 	Name              string
 	IP                string
+	Port              int
 	KubernetesVersion string
 	Type              NodeType
 }
 
+// NodeType is an enum to describe the node type.
 type NodeType int
 
 const (
+	// Master is the control plane.
 	Master NodeType = iota
+	// Worker is a... worker.
 	Worker
 )
 

From c295dfd25c7c18229dd811e981f1fc15ef8bcd0b Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Tue, 17 Dec 2019 09:00:48 -0800
Subject: [PATCH 020/250] more node crud stuff

---
 cmd/minikube/cmd/config/node/add.go    | 35 ++++++++++++++++++++++++++
 cmd/minikube/cmd/config/node/delete.go | 33 ++++++++++++++++++++++++
 cmd/minikube/cmd/config/node/node.go   |  8 +++++-
 3 files changed, 75 insertions(+), 1 deletion(-)
 create mode 100644 cmd/minikube/cmd/config/node/add.go

diff --git a/cmd/minikube/cmd/config/node/add.go b/cmd/minikube/cmd/config/node/add.go
new file mode 100644
index 0000000000..0190e3e8be
--- /dev/null
+++ b/cmd/minikube/cmd/config/node/add.go
@@ -0,0 +1,35 @@
+/*
+Copyright 2019 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package config
+
+import "github.com/spf13/cobra"
+
+var name string
+
+var nodeAddCmd = &cobra.Command{
+	Use:   "add",
+	Short: "Adds a node to the given cluster.",
+	Long:  "Adds a node to the given cluster config, without starting it.",
+	Run: func(cmd *cobra.Command, args []string) {
+
+	},
+}
+
+func init() {
+	nodeAddCmd.Flags().StringVar(&name, "name", "", "The name of the node to add. Defaults to a variation of the profile name.")
+	NodeCmd.AddCommand(nodeAddCmd)
+}
diff --git a/cmd/minikube/cmd/config/node/delete.go b/cmd/minikube/cmd/config/node/delete.go
index e69de29bb2..efea52f70c 100644
--- a/cmd/minikube/cmd/config/node/delete.go
+++ b/cmd/minikube/cmd/config/node/delete.go
@@ -0,0 +1,33 @@
+/*
+Copyright 2019 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package config
+
+import "github.com/spf13/cobra"
+
+var nodeDeleteCmd = &cobra.Command{
+	Use:   "delete",
+	Short: "Deletes a node from a cluster.",
+	Long:  "Deletes a node from a cluster.",
+	Run: func(cmd *cobra.Command, args []string) {
+
+	},
+}
+
+func init() {
+	nodeDeleteCmd.Flags().StringVar(&name, "name", "", "The name of the node to delete.")
+	NodeCmd.AddCommand(nodeDeleteCmd)
+}
diff --git a/cmd/minikube/cmd/config/node/node.go b/cmd/minikube/cmd/config/node/node.go
index 9833b4d27f..fa180d67bc 100644
--- a/cmd/minikube/cmd/config/node/node.go
+++ b/cmd/minikube/cmd/config/node/node.go
@@ -16,10 +16,16 @@ limitations under the License.
 
 package config
 
-import "github.com/spf13/cobra"
+import (
+	"github.com/spf13/cobra"
+	"k8s.io/minikube/pkg/minikube/exit"
+)
 
 var NodeCmd = &cobra.Command{
 	Use:   "node",
 	Short: "Node operations",
 	Long:  "Operations on nodes",
+	Run: func(cmd *cobra.Command, args []string) {
+		exit.UsageT("Usage: minikube node [add|start|stop|delete]")
+	},
 }

From d21db02c76341e6ad067e418910eba1faf3fd021 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Tue, 17 Dec 2019 11:58:26 -0800
Subject: [PATCH 021/250] Apply addons to all profiles

---
 go.mod                                        |  2 +-
 go.sum                                        | 25 ++++++
 pkg/minikube/addons/reconcile.go              | 80 +++++++++++++++++--
 pkg/minikube/bootstrapper/bootstrapper.go     |  2 +-
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go  | 23 +++---
 pkg/minikube/bootstrapper/kubeadm/versions.go |  4 +-
 pkg/minikube/constants/constants.go           | 13 +--
 7 files changed, 113 insertions(+), 36 deletions(-)

diff --git a/go.mod b/go.mod
index 8976fb1bec..945cd20bf8 100644
--- a/go.mod
+++ b/go.mod
@@ -10,6 +10,7 @@ require (
 	github.com/Parallels/docker-machine-parallels v1.3.0
 	github.com/Sirupsen/logrus v0.0.0-20170822132746-89742aefa4b2 // indirect
 	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
+	github.com/alecthomas/chroma v0.7.0
 	github.com/blang/semver v3.5.0+incompatible
 	github.com/c4milo/gotoolkit v0.0.0-20170318115440-bcc06269efa9 // indirect
 	github.com/cenkalti/backoff v2.2.1+incompatible
@@ -25,7 +26,6 @@ require (
 	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
 	github.com/google/go-cmp v0.3.0
-	github.com/gorilla/mux v1.7.1 // indirect
 	github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce // indirect
 	github.com/hashicorp/go-getter v1.4.0
 	github.com/hashicorp/go-multierror v0.0.0-20160811015721-8c5f0ad93604 // indirect
diff --git a/go.sum b/go.sum
index 1971629b40..1cfcd62bc9 100644
--- a/go.sum
+++ b/go.sum
@@ -21,6 +21,8 @@ github.com/BurntSushi/toml v0.3.0/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0=
+github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0=
 github.com/GoogleCloudPlatform/k8s-cloud-provider v0.0.0-20181220005116-f8e995905100/go.mod h1:iroGtC8B3tQiqtds1l+mgk/BBOrxbqjH+eUfFQYRc14=
 github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab/go.mod h1:3VYc5hodBMJ5+l/7J4xAyMeuM2PNuepvHlGs8yilUCA=
 github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd h1:sjQovDkwrZp8u+gxLtPgKGjk5hCxuy2hrRejBTA9xFU=
@@ -42,6 +44,15 @@ github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdc
 github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
 github.com/afbjorklund/go-getter v1.4.1-0.20190910175809-eb9f6c26742c h1:18gEt7qzn7CW7qMkfPTFyyotlPbvPQo9o4IDV8jZqP4=
 github.com/afbjorklund/go-getter v1.4.1-0.20190910175809-eb9f6c26742c/go.mod h1:7qxyCd8rBfcShwsvxgIguu4KbS3l8bUCwg2Umn7RjeY=
+github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c=
+github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38/go.mod h1:r7bzyVFMNntcxPZXK3/+KdruV1H5KSlyVY0gc+NgInI=
+github.com/alecthomas/chroma v0.7.0 h1:z+0HgTUmkpRDRz0SRSdMaqOLfJV4F+N1FPDZUZIDUzw=
+github.com/alecthomas/chroma v0.7.0/go.mod h1:1U/PfCsTALWWYHDnsIQkxEBM0+6LLe0v8+RSVMOwxeY=
+github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721/go.mod h1:QO9JBoKquHd+jz9nshCh40fOfO+JzsoXy8qTHF68zU0=
+github.com/alecthomas/kong v0.1.17-0.20190424132513-439c674f7ae0/go.mod h1:+inYUSluD+p4L8KdviBSgzcqEjUQOfC5fQDRFuc36lI=
+github.com/alecthomas/kong v0.2.1-0.20190708041108-0548c6b1afae/go.mod h1:+inYUSluD+p4L8KdviBSgzcqEjUQOfC5fQDRFuc36lI=
+github.com/alecthomas/kong-hcl v0.1.8-0.20190615233001-b21fea9723c8/go.mod h1:MRgZdU3vrFd05IQ89AxUZ0aYdF39BYoNFa324SodPCA=
+github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
 github.com/apache/thrift v0.0.0-20180902110319-2566ecd5d999/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
@@ -104,11 +115,16 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc
 github.com/cyphar/filepath-securejoin v0.0.0-20170720062807-ae69057f2299/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
 github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ=
 github.com/d2g/dhcp4client v0.0.0-20170829104524-6e570ed0a266/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s=
+github.com/daaku/go.zipexe v1.0.0/go.mod h1:z8IiR6TsVLEYKwXAoE/I+8ys/sDkgTzSL0CLnGVd57E=
+github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ=
+github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE=
 github.com/dgrijalva/jwt-go v0.0.0-20160705203006-01aeca54ebda/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dlclark/regexp2 v1.1.6 h1:CqB4MjHw0MFCDj+PHHjiESmHX+N7t0tJzKvC6M97BRg=
+github.com/dlclark/regexp2 v1.1.6/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
 github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
 github.com/docker/distribution v0.0.0-20170726174610-edc3ab29cdff h1:FKH02LHYqSmeWd3GBh0KIkM8JBpw3RrShgtcWShdWJg=
 github.com/docker/distribution v0.0.0-20170726174610-edc3ab29cdff/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
@@ -230,9 +246,13 @@ github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsC
 github.com/gophercloud/gophercloud v0.0.0-20190126172459-c818fa66e4c8/go.mod h1:3WdhXV3rUYy9p6AUW8d94kr+HS62Y4VL9mBnFxsD8q4=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
+github.com/gorilla/csrf v1.6.0/go.mod h1:7tSf8kmjNYr7IWDCYhd3U8Ck34iQ/Yw5CJu7bAkHEGI=
+github.com/gorilla/handlers v1.4.1/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ=
 github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/mux v1.7.1 h1:Dw4jY2nghMMRsh1ol8dv1axHkDwMQK2DHerMNJsIpJU=
 github.com/gorilla/mux v1.7.1/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
@@ -276,6 +296,7 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt
 github.com/intel-go/cpuid v0.0.0-20181003105527-1a4a6f06a1c6 h1:XboatR7lasl05yel5hNXF7kQBw2oFUGdMztcgisfhNU=
 github.com/intel-go/cpuid v0.0.0-20181003105527-1a4a6f06a1c6/go.mod h1:RmeVYf9XrPRbRc3XIx0gLYA8qOFvNoPOfaEZduRlEp4=
 github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
+github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jimmidyson/go-download v0.0.0-20161028105827-7f9a90c8c95b h1:3TknJxYSK1eDe21QorC3C2Yz8jylk6vlJG9YABnFzkU=
 github.com/jimmidyson/go-download v0.0.0-20161028105827-7f9a90c8c95b/go.mod h1:I3WsAhNNoG7a/d8HMlYUywJJlfOs/+/83NEUjuDp4lc=
 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
@@ -382,6 +403,7 @@ github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+
 github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk=
 github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
 github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
+github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E=
 github.com/olekukonko/tablewriter v0.0.0-20160923125401-bdcc175572fd h1:nEatQ6JnwCT9iYD5uqYUiFqq8tJGX25to8KVKXqya7k=
 github.com/olekukonko/tablewriter v0.0.0-20160923125401-bdcc175572fd/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@@ -516,6 +538,8 @@ github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljT
 github.com/ulikunitz/xz v0.5.5 h1:pFrO0lVpTBXLpYw+pnLj6TbvHuyjXMfjGeCwSqCVwok=
 github.com/ulikunitz/xz v0.5.5/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
 github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
 github.com/vishvananda/netlink v0.0.0-20171020171820-b2de5d10e38e/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
 github.com/vishvananda/netns v0.0.0-20171111001504-be1fbeda1936/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI=
 github.com/vmware/govmomi v0.20.1/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU=
@@ -602,6 +626,7 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181004145325-8469e314837c/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
diff --git a/pkg/minikube/addons/reconcile.go b/pkg/minikube/addons/reconcile.go
index cacaca2d8c..5bc6ddd354 100644
--- a/pkg/minikube/addons/reconcile.go
+++ b/pkg/minikube/addons/reconcile.go
@@ -17,12 +17,17 @@ limitations under the License.
 package addons
 
 import (
+	"os"
 	"os/exec"
 	"path"
 
+	"github.com/blang/semver"
 	"github.com/golang/glog"
+	"k8s.io/minikube/pkg/minikube/bootstrapper/kubeadm"
 	"k8s.io/minikube/pkg/minikube/command"
+	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
+	"k8s.io/minikube/pkg/minikube/exit"
 )
 
 // taken from https://github.com/kubernetes/kubernetes/blob/master/cluster/addons/addon-manager/kube-addons.sh
@@ -46,18 +51,37 @@ var kubectlPruneWhitelist = []string{
 }
 
 // Reconcile runs kubectl apply -f on the addons directory
-// to reconcile addons state
+// to reconcile addons state in all running profiles
 func Reconcile(cmd command.Runner) error {
-	if _, err := cmd.RunCmd(kubectlCommand()); err != nil {
-		glog.Warningf("reconciling addons failed: %v", err)
-		return err
+	validProfiles, _, err := config.ListProfiles()
+
+	if len(validProfiles) == 0 || err != nil {
+		exit.UsageT("No minikube profile was found. You can create one using `minikube start`.")
+	}
+
+	for _, p := range validProfiles {
+		v, err := kubernetesVersion(p.Name)
+		if err != nil {
+			glog.Warningf("error getting kubernetes version, skipping profile %s: %v", p.Name, err)
+			continue
+		}
+		c, err := kubectlCommand(v)
+		if err != nil {
+			return err
+		}
+		if _, err := cmd.RunCmd(c); err != nil {
+			glog.Warningf("reconciling addons failed: %v", err)
+			return err
+		}
 	}
 	return nil
 }
 
-func kubectlCommand() *exec.Cmd {
-	kubectlBinary := path.Join("/var/lib/minikube/binaries", constants.KubectlBinaryVersion, constants.KubectlBinary)
-
+func kubectlCommand(version string) (*exec.Cmd, error) {
+	kubectlBinary, err := kubectlBinaryPath(version)
+	if err != nil {
+		return nil, err
+	}
 	// prune will delete any existing objects with the label specified by "-l" which don't appear in /etc/kubernetes/addons
 	// this is how we delete disabled addons
 	args := []string{"KUBECONFIG=/var/lib/minikube/kubeconfig", kubectlBinary, "apply", "-f", "/etc/kubernetes/addons", "-l", "kubernetes.io/cluster-service!=true,addonmanager.kubernetes.io/mode=Reconcile", "--prune=true"}
@@ -66,6 +90,46 @@ func kubectlCommand() *exec.Cmd {
 	}
 	args = append(args, "--recursive")
 
+	if ok, err := appendNamespaceFlag(version); err == nil && ok {
+		args = append(args, "--namespace=kube-system")
+	}
+
 	cmd := exec.Command("sudo", args...)
-	return cmd
+	return cmd, nil
+}
+
+func kubernetesVersion(profile string) (string, error) {
+	cc, err := config.Load(profile)
+	if err != nil && !os.IsNotExist(err) {
+		return "", err
+	}
+	version := constants.DefaultKubernetesVersion
+	if cc != nil {
+		version = cc.KubernetesConfig.KubernetesVersion
+	}
+	return version, nil
+}
+
+// We need to append --namespace=kube-system for Kubernetes versions >=1.17
+// so that prune works as expected. See https://github.com/kubernetes/kubernetes/pull/83084/
+func appendNamespaceFlag(version string) (bool, error) {
+	v, err := kubeadm.ParseKubernetesVersion(version)
+	if err != nil {
+		return false, err
+	}
+	return v.GT(semver.MustParse("1.17.0")), nil
+}
+
+func kubectlBinaryPath(profile string) (string, error) {
+	// TODO: get this for all profiles and run in each one
+	cc, err := config.Load(profile)
+	if err != nil && !os.IsNotExist(err) {
+		return "", err
+	}
+	version := constants.DefaultKubernetesVersion
+	if cc != nil {
+		version = cc.KubernetesConfig.KubernetesVersion
+	}
+	p := path.Join("/var/lib/minikube/binaries", version, "kubectl")
+	return p, nil
 }
diff --git a/pkg/minikube/bootstrapper/bootstrapper.go b/pkg/minikube/bootstrapper/bootstrapper.go
index 9cc29f34ff..aaa714def0 100644
--- a/pkg/minikube/bootstrapper/bootstrapper.go
+++ b/pkg/minikube/bootstrapper/bootstrapper.go
@@ -57,7 +57,7 @@ const (
 func GetCachedBinaryList(bootstrapper string) []string {
 	switch bootstrapper {
 	case BootstrapperTypeKubeadm:
-		return constants.KubeadmBinaries
+		return constants.KubernetesReleaseBinaries
 	default:
 		return []string{}
 	}
diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index 3397411410..9e6f127d7c 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -249,7 +249,7 @@ func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error {
 		glog.Infof("StartCluster complete in %s", time.Since(start))
 	}()
 
-	version, err := parseKubernetesVersion(k8s.KubernetesVersion)
+	version, err := ParseKubernetesVersion(k8s.KubernetesVersion)
 	if err != nil {
 		return errors.Wrap(err, "parsing kubernetes version")
 	}
@@ -481,7 +481,7 @@ func (k *Bootstrapper) restartCluster(k8s config.KubernetesConfig) error {
 		glog.Infof("restartCluster took %s", time.Since(start))
 	}()
 
-	version, err := parseKubernetesVersion(k8s.KubernetesVersion)
+	version, err := ParseKubernetesVersion(k8s.KubernetesVersion)
 	if err != nil {
 		return errors.Wrap(err, "parsing kubernetes version")
 	}
@@ -534,7 +534,7 @@ func (k *Bootstrapper) restartCluster(k8s config.KubernetesConfig) error {
 
 // DeleteCluster removes the components that were started earlier
 func (k *Bootstrapper) DeleteCluster(k8s config.KubernetesConfig) error {
-	version, err := parseKubernetesVersion(k8s.KubernetesVersion)
+	version, err := ParseKubernetesVersion(k8s.KubernetesVersion)
 	if err != nil {
 		return errors.Wrap(err, "parsing kubernetes version")
 	}
@@ -553,7 +553,7 @@ func (k *Bootstrapper) DeleteCluster(k8s config.KubernetesConfig) error {
 
 // PullImages downloads images that will be used by Kubernetes
 func (k *Bootstrapper) PullImages(k8s config.KubernetesConfig) error {
-	version, err := parseKubernetesVersion(k8s.KubernetesVersion)
+	version, err := ParseKubernetesVersion(k8s.KubernetesVersion)
 	if err != nil {
 		return errors.Wrap(err, "parsing kubernetes version")
 	}
@@ -576,7 +576,7 @@ func (k *Bootstrapper) SetupCerts(k8s config.KubernetesConfig) error {
 // NewKubeletConfig generates a new systemd unit containing a configured kubelet
 // based on the options present in the KubernetesConfig.
 func NewKubeletConfig(k8s config.KubernetesConfig, r cruntime.Manager) ([]byte, error) {
-	version, err := parseKubernetesVersion(k8s.KubernetesVersion)
+	version, err := ParseKubernetesVersion(k8s.KubernetesVersion)
 	if err != nil {
 		return nil, errors.Wrap(err, "parsing kubernetes version")
 	}
@@ -707,7 +707,7 @@ func createExtraComponentConfig(extraOptions config.ExtraOptionSlice, version se
 
 // generateConfig generates the kubeadm.yaml file
 func generateConfig(k8s config.KubernetesConfig, r cruntime.Manager) ([]byte, error) {
-	version, err := parseKubernetesVersion(k8s.KubernetesVersion)
+	version, err := ParseKubernetesVersion(k8s.KubernetesVersion)
 	if err != nil {
 		return nil, errors.Wrap(err, "parsing kubernetes version")
 	}
@@ -821,20 +821,15 @@ func invokeKubeadm(version string) string {
 // transferBinaries transfers all required Kubernetes binaries
 func transferBinaries(cfg config.KubernetesConfig, c command.Runner) error {
 	var g errgroup.Group
-	kubernetesBinaries := append(constants.KubeadmBinaries, constants.KubectlBinary)
-	for _, name := range kubernetesBinaries {
+	for _, name := range constants.KubernetesReleaseBinaries {
 		name := name
 		g.Go(func() error {
-			version := cfg.KubernetesVersion
-			if name == constants.KubectlBinary {
-				version = constants.KubectlBinaryVersion
-			}
-			src, err := machine.CacheBinary(name, version, "linux", runtime.GOARCH)
+			src, err := machine.CacheBinary(name, cfg.KubernetesVersion, "linux", runtime.GOARCH)
 			if err != nil {
 				return errors.Wrapf(err, "downloading %s", name)
 			}
 
-			dst := path.Join(binRoot(version), name)
+			dst := path.Join(binRoot(cfg.KubernetesVersion), name)
 			if err := machine.CopyBinary(c, src, dst); err != nil {
 				return errors.Wrapf(err, "copybinary %s -> %s", src, dst)
 			}
diff --git a/pkg/minikube/bootstrapper/kubeadm/versions.go b/pkg/minikube/bootstrapper/kubeadm/versions.go
index 82e157fd79..13c6561c68 100644
--- a/pkg/minikube/bootstrapper/kubeadm/versions.go
+++ b/pkg/minikube/bootstrapper/kubeadm/versions.go
@@ -160,8 +160,8 @@ func Supports(featureName string) bool {
 	return false
 }
 
-// parseKubernetesVersion parses the kubernetes version
-func parseKubernetesVersion(version string) (semver.Version, error) {
+// ParseKubernetesVersion parses the kubernetes version
+func ParseKubernetesVersion(version string) (semver.Version, error) {
 	// Strip leading 'v' prefix from version for semver parsing
 	v, err := semver.Make(version[1:])
 	if err != nil {
diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go
index 2c6719cd1e..a89490c0d5 100644
--- a/pkg/minikube/constants/constants.go
+++ b/pkg/minikube/constants/constants.go
@@ -84,16 +84,9 @@ var ImageRepositories = map[string][]string{
 	"cn":     {"registry.cn-hangzhou.aliyuncs.com/google_containers"},
 }
 
-// KubeadmBinaries are Kubernetes release binaries required for kubeadm
-var KubeadmBinaries = []string{"kubelet", "kubeadm"}
-
-const (
-	// KubectlBinary is the name of the kubectl binary installed in minikube
-	KubectlBinary = "kubectl"
-	// KubectlBinaryVersion is the version of kubectl installed in minikube
-	// it's set to v1.13.2 to match the kubectl version in the addon manager image
-	KubectlBinaryVersion = "v1.13.2"
-)
+// KubernetesReleaseBinaries are Kubernetes release binaries required for
+// kubeadm (kubelet, kubeadm) and the addon manager (kubectl)
+var KubernetesReleaseBinaries = []string{"kubelet", "kubeadm", "kubectl"}
 
 // ImageCacheDir is the path to the image cache directory
 var ImageCacheDir = localpath.MakeMiniPath("cache", "images")

From 84ce2dbd62f8011e981b05c4811ec9e79a72563e Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Tue, 17 Dec 2019 12:09:42 -0800
Subject: [PATCH 022/250] Fixed bug, using version when should use profile

---
 pkg/minikube/addons/reconcile.go | 29 +++++++++++++----------------
 1 file changed, 13 insertions(+), 16 deletions(-)

diff --git a/pkg/minikube/addons/reconcile.go b/pkg/minikube/addons/reconcile.go
index 5bc6ddd354..fdda4f0dec 100644
--- a/pkg/minikube/addons/reconcile.go
+++ b/pkg/minikube/addons/reconcile.go
@@ -17,6 +17,7 @@ limitations under the License.
 package addons
 
 import (
+	"fmt"
 	"os"
 	"os/exec"
 	"path"
@@ -60,12 +61,7 @@ func Reconcile(cmd command.Runner) error {
 	}
 
 	for _, p := range validProfiles {
-		v, err := kubernetesVersion(p.Name)
-		if err != nil {
-			glog.Warningf("error getting kubernetes version, skipping profile %s: %v", p.Name, err)
-			continue
-		}
-		c, err := kubectlCommand(v)
+		c, err := kubectlCommand(p.Name)
 		if err != nil {
 			return err
 		}
@@ -77,8 +73,12 @@ func Reconcile(cmd command.Runner) error {
 	return nil
 }
 
-func kubectlCommand(version string) (*exec.Cmd, error) {
-	kubectlBinary, err := kubectlBinaryPath(version)
+func kubectlCommand(profile string) (*exec.Cmd, error) {
+	kubectlBinary, err := kubectlBinaryPath(profile)
+	if err != nil {
+		return nil, err
+	}
+	v, err := kubernetesVersion(profile)
 	if err != nil {
 		return nil, err
 	}
@@ -90,7 +90,7 @@ func kubectlCommand(version string) (*exec.Cmd, error) {
 	}
 	args = append(args, "--recursive")
 
-	if ok, err := appendNamespaceFlag(version); err == nil && ok {
+	if ok, err := appendNamespaceFlag(v); err == nil && ok {
 		args = append(args, "--namespace=kube-system")
 	}
 
@@ -122,14 +122,11 @@ func appendNamespaceFlag(version string) (bool, error) {
 
 func kubectlBinaryPath(profile string) (string, error) {
 	// TODO: get this for all profiles and run in each one
-	cc, err := config.Load(profile)
-	if err != nil && !os.IsNotExist(err) {
+	v, err := kubernetesVersion(profile)
+	if err != nil {
 		return "", err
 	}
-	version := constants.DefaultKubernetesVersion
-	if cc != nil {
-		version = cc.KubernetesConfig.KubernetesVersion
-	}
-	p := path.Join("/var/lib/minikube/binaries", version, "kubectl")
+	fmt.Printf("got version %s for profile %s\n", v, profile)
+	p := path.Join("/var/lib/minikube/binaries", v, "kubectl")
 	return p, nil
 }

From e46c586764134981f7938f7fb8a17b48e49b27b9 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Tue, 17 Dec 2019 15:53:37 -0800
Subject: [PATCH 023/250] more refactor fun

---
 cmd/minikube/cmd/config/node/add.go          | 12 ++++++-
 cmd/minikube/cmd/start.go                    |  8 ++---
 pkg/minikube/assets/vm_assets.go             | 30 +++++++++++++---
 pkg/minikube/bootstrapper/bootstrapper.go    |  3 +-
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go |  4 +--
 pkg/minikube/config/node.go                  | 36 ++++++++++++++++++++
 pkg/minikube/config/profile.go               |  6 ++--
 pkg/minikube/config/profile_test.go          |  2 +-
 8 files changed, 85 insertions(+), 16 deletions(-)
 create mode 100644 pkg/minikube/config/node.go

diff --git a/cmd/minikube/cmd/config/node/add.go b/cmd/minikube/cmd/config/node/add.go
index 0190e3e8be..8bc0d04cd8 100644
--- a/cmd/minikube/cmd/config/node/add.go
+++ b/cmd/minikube/cmd/config/node/add.go
@@ -16,7 +16,12 @@ limitations under the License.
 
 package config
 
-import "github.com/spf13/cobra"
+import (
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/exit"
+)
 
 var name string
 
@@ -26,6 +31,11 @@ var nodeAddCmd = &cobra.Command{
 	Long:  "Adds a node to the given cluster config, without starting it.",
 	Run: func(cmd *cobra.Command, args []string) {
 
+		profile := viper.GetString(config.MachineProfile)
+		cc, err := config.Load(profile)
+		if err != nil {
+			exit.WithError(err)
+		}
 	},
 }
 
diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 84d33c68ce..e52f434179 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -353,7 +353,7 @@ func runStart(cmd *cobra.Command, args []string) {
 	}
 
 	// setup kubeadm (must come after setupKubeconfig)
-	bs := setupKubeAdm(machineAPI, config)
+	bs := setupKubeAdm(machineAPI, mRunner, config)
 
 	// pull images or restart cluster
 	bootstrapCluster(bs, cr, mRunner, config.KubernetesConfig, preExists, isUpgrade)
@@ -1195,7 +1195,7 @@ func getKubernetesVersion(old *cfg.MachineConfig) (string, bool) {
 }
 
 // setupKubeAdm adds any requested files into the VM before Kubernetes is started
-func setupKubeAdm(mAPI libmachine.API, config cfg.MachineConfig) bootstrapper.Bootstrapper {
+func setupKubeAdm(mAPI libmachine.API, cr command.Runner, config cfg.MachineConfig, node cfg.Node) bootstrapper.Bootstrapper {
 	bs, err := getClusterBootstrapper(mAPI, viper.GetString(cmdcfg.Bootstrapper))
 	if err != nil {
 		exit.WithError("Failed to get bootstrapper", err)
@@ -1207,7 +1207,7 @@ func setupKubeAdm(mAPI libmachine.API, config cfg.MachineConfig) bootstrapper.Bo
 	if err := bs.UpdateCluster(config); err != nil {
 		exit.WithError("Failed to update cluster", err)
 	}
-	if err := bs.SetupCerts(config.KubernetesConfig); err != nil {
+	if err := bs.SetupCerts(cr, config.KubernetesConfig, node); err != nil {
 		exit.WithError("Failed to setup certs", err)
 	}
 	return bs
@@ -1276,5 +1276,5 @@ func configureMounts() {
 
 // saveConfig saves profile cluster configuration in $MINIKUBE_HOME/profiles/<profilename>/config.json
 func saveConfig(clusterCfg *cfg.MachineConfig) error {
-	return cfg.CreateProfile(viper.GetString(cfg.MachineProfile), clusterCfg)
+	return cfg.SaveProfile(viper.GetString(cfg.MachineProfile), clusterCfg)
 }
diff --git a/pkg/minikube/assets/vm_assets.go b/pkg/minikube/assets/vm_assets.go
index 88c92089fc..67c7fcf1a3 100644
--- a/pkg/minikube/assets/vm_assets.go
+++ b/pkg/minikube/assets/vm_assets.go
@@ -38,6 +38,7 @@ type CopyableFile interface {
 	GetTargetName() string
 	GetPermissions() string
 	GetModTime() (time.Time, error)
+	Seek(int64, int) (int64, error)
 }
 
 // BaseAsset is the base asset class
@@ -76,7 +77,7 @@ func (b *BaseAsset) GetModTime() (time.Time, error) {
 // FileAsset is an asset using a file
 type FileAsset struct {
 	BaseAsset
-	reader io.Reader
+	reader *io.SectionReader
 }
 
 // NewMemoryAssetTarget creates a new MemoryAsset, with target
@@ -91,6 +92,11 @@ func NewFileAsset(src, targetDir, targetName, permissions string) (*FileAsset, e
 	if err != nil {
 		return nil, errors.Wrapf(err, "Error opening file asset: %s", src)
 	}
+	info, err := os.Stat(src)
+	if err != nil {
+		return nil, errors.Wrapf(err, "Error getting info for %s", src)
+	}
+	r := io.NewSectionReader(f, 0, info.Size())
 	return &FileAsset{
 		BaseAsset: BaseAsset{
 			AssetName:   src,
@@ -98,7 +104,7 @@ func NewFileAsset(src, targetDir, targetName, permissions string) (*FileAsset, e
 			TargetName:  targetName,
 			Permissions: permissions,
 		},
-		reader: f,
+		reader: r,
 	}, nil
 }
 
@@ -117,6 +123,7 @@ func (f *FileAsset) GetModTime() (time.Time, error) {
 	return fi.ModTime(), err
 }
 
+// Read reads the asset
 func (f *FileAsset) Read(p []byte) (int, error) {
 	if f.reader == nil {
 		return 0, errors.New("Error attempting FileAsset.Read, FileAsset.reader uninitialized")
@@ -124,10 +131,15 @@ func (f *FileAsset) Read(p []byte) (int, error) {
 	return f.reader.Read(p)
 }
 
+// Seek resets the reader to offset
+func (f *FileAsset) Seek(offset int64, whence int) (int64, error) {
+	return f.reader.Seek(offset, whence)
+}
+
 // MemoryAsset is a memory-based asset
 type MemoryAsset struct {
 	BaseAsset
-	reader io.Reader
+	reader *bytes.Reader
 	length int
 }
 
@@ -141,6 +153,11 @@ func (m *MemoryAsset) Read(p []byte) (int, error) {
 	return m.reader.Read(p)
 }
 
+// Seek resets the reader to offset
+func (m *MemoryAsset) Seek(offset int64, whence int) (int64, error) {
+	return m.reader.Seek(offset, whence)
+}
+
 // NewMemoryAsset creates a new MemoryAsset
 func NewMemoryAsset(d []byte, targetDir, targetName, permissions string) *MemoryAsset {
 	return &MemoryAsset{
@@ -157,7 +174,7 @@ func NewMemoryAsset(d []byte, targetDir, targetName, permissions string) *Memory
 // BinAsset is a bindata (binary data) asset
 type BinAsset struct {
 	BaseAsset
-	reader   io.Reader
+	reader   *bytes.Reader
 	template *template.Template
 	length   int
 }
@@ -253,3 +270,8 @@ func (m *BinAsset) Read(p []byte) (int, error) {
 	}
 	return m.reader.Read(p)
 }
+
+// Seek resets the reader to offset
+func (m *BinAsset) Seek(offset int64, whence int) (int64, error) {
+	return m.reader.Seek(offset, whence)
+}
diff --git a/pkg/minikube/bootstrapper/bootstrapper.go b/pkg/minikube/bootstrapper/bootstrapper.go
index 9cc29f34ff..20c9f04d86 100644
--- a/pkg/minikube/bootstrapper/bootstrapper.go
+++ b/pkg/minikube/bootstrapper/bootstrapper.go
@@ -21,6 +21,7 @@ import (
 	"time"
 
 	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
+	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
 )
@@ -43,7 +44,7 @@ type Bootstrapper interface {
 	WaitForCluster(config.KubernetesConfig, time.Duration) error
 	// LogCommands returns a map of log type to a command which will display that log.
 	LogCommands(LogOptions) map[string]string
-	SetupCerts(cfg config.KubernetesConfig) error
+	SetupCerts(command.Runner, config.KubernetesConfig, config.Node) error
 	GetKubeletStatus() (string, error)
 	GetAPIServerStatus(net.IP, int) (string, error)
 }
diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index 55a8749227..2bc9e8d6c9 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -569,8 +569,8 @@ func (k *Bootstrapper) PullImages(k8s config.KubernetesConfig) error {
 }
 
 // SetupCerts sets up certificates within the cluster.
-func (k *Bootstrapper) SetupCerts(k8s config.KubernetesConfig) error {
-	return bootstrapper.SetupCerts(k.c, k8s)
+func (k *Bootstrapper) SetupCerts(k8s config.KubernetesConfig, n config.Node) error {
+	return bootstrapper.SetupCerts(k.c, k8s, n)
 }
 
 // NewKubeletConfig generates a new systemd unit containing a configured kubelet
diff --git a/pkg/minikube/config/node.go b/pkg/minikube/config/node.go
new file mode 100644
index 0000000000..38a920f2d2
--- /dev/null
+++ b/pkg/minikube/config/node.go
@@ -0,0 +1,36 @@
+/*
+Copyright 2019 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package config
+
+// AddNode adds a new node config to an existing cluster.
+func AddNode(cc *MachineConfig, name string, controlPlane bool, k8sVersion string, profileName string) {
+	t := Worker
+	if controlPlane {
+		t = Master
+	}
+	node := Node{
+		Name: name,
+		Type: t,
+	}
+
+	if k8sVersion != "" {
+		node.KubernetesVersion = k8sVersion
+	}
+
+	cc.Nodes = append(cc.Nodes, node)
+	SaveProfile(profileName, cc)
+}
diff --git a/pkg/minikube/config/profile.go b/pkg/minikube/config/profile.go
index 85cec366ae..da3e737fc3 100644
--- a/pkg/minikube/config/profile.go
+++ b/pkg/minikube/config/profile.go
@@ -86,11 +86,11 @@ func ProfileExists(name string, miniHome ...string) bool {
 // CreateEmptyProfile creates an empty profile stores in $MINIKUBE_HOME/profiles/<profilename>/config.json
 func CreateEmptyProfile(name string, miniHome ...string) error {
 	cfg := &MachineConfig{}
-	return CreateProfile(name, cfg, miniHome...)
+	return SaveProfile(name, cfg, miniHome...)
 }
 
-// CreateProfile creates an profile out of the cfg and stores in $MINIKUBE_HOME/profiles/<profilename>/config.json
-func CreateProfile(name string, cfg *MachineConfig, miniHome ...string) error {
+// SaveProfile creates an profile out of the cfg and stores in $MINIKUBE_HOME/profiles/<profilename>/config.json
+func SaveProfile(name string, cfg *MachineConfig, miniHome ...string) error {
 	data, err := json.MarshalIndent(cfg, "", "    ")
 	if err != nil {
 		return err
diff --git a/pkg/minikube/config/profile_test.go b/pkg/minikube/config/profile_test.go
index d238111233..52c3818e38 100644
--- a/pkg/minikube/config/profile_test.go
+++ b/pkg/minikube/config/profile_test.go
@@ -176,7 +176,7 @@ func TestCreateProfile(t *testing.T) {
 	}
 	for _, tc := range testCases {
 		n := tc.name // capturing  loop variable
-		gotErr := CreateProfile(n, tc.cfg, miniDir)
+		gotErr := SaveProfile(n, tc.cfg, miniDir)
 		if gotErr != nil && tc.expectErr == false {
 			t.Errorf("expected CreateEmptyProfile not to error but got err=%v", gotErr)
 		}

From 3f20aa4e8242fd001a7cb6c746b8b92e1672d565 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Thu, 19 Dec 2019 09:45:00 -0800
Subject: [PATCH 024/250] fix config import nonsense in start.go

---
 cmd/minikube/cmd/start.go | 109 +++++++++++++++++++-------------------
 1 file changed, 54 insertions(+), 55 deletions(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index e52f434179..6bb93819d2 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -45,14 +45,13 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"golang.org/x/sync/errgroup"
-	cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config"
+	cmdconfig "k8s.io/minikube/cmd/minikube/cmd/config"
 	"k8s.io/minikube/pkg/minikube/bootstrapper"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/kubeadm"
 	"k8s.io/minikube/pkg/minikube/cluster"
 	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/minikube/config"
-	cfg "k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/cruntime"
 	"k8s.io/minikube/pkg/minikube/driver"
@@ -134,7 +133,7 @@ var (
 	apiServerNames   []string
 	addonList        []string
 	apiServerIPs     []net.IP
-	extraOptions     cfg.ExtraOptionSlice
+	extraOptions     config.ExtraOptionSlice
 )
 
 func init() {
@@ -288,7 +287,7 @@ func runStart(cmd *cobra.Command, args []string) {
 		registryMirror = viper.GetStringSlice("registry_mirror")
 	}
 
-	existing, err := cfg.Load(viper.GetString(config.MachineProfile))
+	existing, err := config.Load(viper.GetString(config.MachineProfile))
 	if err != nil && !os.IsNotExist(err) {
 		exit.WithCodeT(exit.Data, "Unable to load config: {{.error}}", out.V{"error": err})
 	}
@@ -310,13 +309,13 @@ func runStart(cmd *cobra.Command, args []string) {
 	}
 
 	k8sVersion, isUpgrade := getKubernetesVersion(existing)
-	config, err := generateCfgFromFlags(cmd, k8sVersion, driverName)
+	mc, err := generateCfgFromFlags(cmd, k8sVersion, driverName)
 	if err != nil {
 		exit.WithError("Failed to generate config", err)
 	}
 
 	if !driver.BareMetal(driverName) {
-		if err := cluster.CacheISO(config); err != nil {
+		if err := cluster.CacheISO(mc); err != nil {
 			exit.WithError("Failed to cache ISO", err)
 		}
 	}
@@ -329,34 +328,34 @@ func runStart(cmd *cobra.Command, args []string) {
 
 	// Now that the ISO is downloaded, pull images in the background while the VM boots.
 	var cacheGroup errgroup.Group
-	beginCacheImages(&cacheGroup, config.KubernetesConfig.ImageRepository, k8sVersion)
+	beginCacheImages(&cacheGroup, mc.KubernetesConfig.ImageRepository, k8sVersion)
 
 	// Abstraction leakage alert: startHost requires the config to be saved, to satistfy pkg/provision/buildroot.
 	// Hence, saveConfig must be called before startHost, and again afterwards when we know the IP.
-	if err := saveConfig(&config); err != nil {
+	if err := saveConfig(&mc); err != nil {
 		exit.WithError("Failed to save config", err)
 	}
 
 	// exits here in case of --download-only option.
 	handleDownloadOnly(&cacheGroup, k8sVersion)
-	mRunner, preExists, machineAPI, host := startMachine(&config, &config.Nodes[0])
+	mRunner, preExists, machineAPI, host := startMachine(&mc, &mc.Nodes[0])
 	defer machineAPI.Close()
 	// configure the runtime (docker, containerd, crio)
-	cr := configureRuntimes(mRunner, driverName, config.KubernetesConfig)
+	cr := configureRuntimes(mRunner, driverName, mc.KubernetesConfig)
 	showVersionInfo(k8sVersion, cr)
 	waitCacheImages(&cacheGroup)
 
 	// Must be written before bootstrap, otherwise health checks may flake due to stale IP
-	kubeconfig, err := setupKubeconfig(host, &config.KubernetesConfig, &config.Nodes[0], config.Name)
+	kubeconfig, err := setupKubeconfig(host, &mc.KubernetesConfig, &mc.Nodes[0], mc.Name)
 	if err != nil {
 		exit.WithError("Failed to setup kubeconfig", err)
 	}
 
 	// setup kubeadm (must come after setupKubeconfig)
-	bs := setupKubeAdm(machineAPI, mRunner, config)
+	bs := setupKubeAdm(machineAPI, mRunner, mc)
 
 	// pull images or restart cluster
-	bootstrapCluster(bs, cr, mRunner, config.KubernetesConfig, preExists, isUpgrade)
+	bootstrapCluster(bs, cr, mRunner, mc.KubernetesConfig, preExists, isUpgrade)
 	configureMounts()
 
 	// enable addons with start command
@@ -373,11 +372,11 @@ func runStart(cmd *cobra.Command, args []string) {
 
 	// Skip pre-existing, because we already waited for health
 	if viper.GetBool(waitUntilHealthy) && !preExists {
-		if err := bs.WaitForCluster(config.KubernetesConfig, viper.GetDuration(waitTimeout)); err != nil {
+		if err := bs.WaitForCluster(mc.KubernetesConfig, viper.GetDuration(waitTimeout)); err != nil {
 			exit.WithError("Wait failed", err)
 		}
 	}
-	if err := showKubectlInfo(kubeconfig, k8sVersion, config.Name); err != nil {
+	if err := showKubectlInfo(kubeconfig, k8sVersion, mc.Name); err != nil {
 		glog.Errorf("kubectl info: %v", err)
 	}
 }
@@ -393,7 +392,7 @@ func updateDriver(driverName string) {
 
 func enableAddons() {
 	for _, a := range addonList {
-		err := cmdcfg.Set(a, "true")
+		err := cmdconfig.Set(a, "true")
 		if err != nil {
 			exit.WithError("addon enable failed", err)
 		}
@@ -402,8 +401,8 @@ func enableAddons() {
 
 func displayVersion(version string) {
 	prefix := ""
-	if viper.GetString(cfg.MachineProfile) != constants.DefaultMachineName {
-		prefix = fmt.Sprintf("[%s] ", viper.GetString(cfg.MachineProfile))
+	if viper.GetString(config.MachineProfile) != constants.DefaultMachineName {
+		prefix = fmt.Sprintf("[%s] ", viper.GetString(config.MachineProfile))
 	}
 
 	versionState := out.Happy
@@ -426,7 +425,7 @@ func displayEnviron(env []string) {
 	}
 }
 
-func setupKubeconfig(h *host.Host, c *cfg.KubernetesConfig, n *cfg.Node, clusterName string) (*kubeconfig.Settings, error) {
+func setupKubeconfig(h *host.Host, c *config.KubernetesConfig, n *config.Node, clusterName string) (*kubeconfig.Settings, error) {
 	addr, err := h.Driver.GetURL()
 	if err != nil {
 		exit.WithError("Failed to get driver URL", err)
@@ -471,7 +470,7 @@ func handleDownloadOnly(cacheGroup *errgroup.Group, k8sVersion string) {
 
 }
 
-func startMachine(config *cfg.MachineConfig, node *cfg.Node) (runner command.Runner, preExists bool, machineAPI libmachine.API, host *host.Host) {
+func startMachine(config *config.MachineConfig, node *config.Node) (runner command.Runner, preExists bool, machineAPI libmachine.API, host *host.Host) {
 	m, err := machine.NewAPIClient()
 	if err != nil {
 		exit.WithError("Failed to get machine client", err)
@@ -552,7 +551,7 @@ func showKubectlInfo(kcs *kubeconfig.Settings, k8sVersion string, machineName st
 	return nil
 }
 
-func selectDriver(existing *cfg.MachineConfig) string {
+func selectDriver(existing *config.MachineConfig) string {
 	name := viper.GetString("vm-driver")
 	glog.Infof("selectDriver: flag=%q, old=%v", name, existing)
 	options := driver.Choices()
@@ -583,7 +582,7 @@ func selectDriver(existing *cfg.MachineConfig) string {
 }
 
 // validateDriver validates that the selected driver appears sane, exits if not
-func validateDriver(name string, existing *cfg.MachineConfig) {
+func validateDriver(name string, existing *config.MachineConfig) {
 	glog.Infof("validating driver %q against %+v", name, existing)
 	if !driver.Supported(name) {
 		exit.WithCodeT(exit.Unavailable, "The driver '{{.driver}}' is not supported on {{.os}}", out.V{"driver": name, "os": runtime.GOOS})
@@ -620,7 +619,7 @@ func validateDriver(name string, existing *cfg.MachineConfig) {
 		return
 	}
 
-	machineName := viper.GetString(cfg.MachineProfile)
+	machineName := viper.GetString(config.MachineProfile)
 	h, err := api.Load(machineName)
 	if err != nil {
 		glog.Warningf("selectDriver api.Load: %v", err)
@@ -699,8 +698,8 @@ func selectImageRepository(mirrorCountry string) (bool, string, error) {
 
 // Return a minikube command containing the current profile name
 func minikubeCmd() string {
-	if viper.GetString(cfg.MachineProfile) != constants.DefaultMachineName {
-		return fmt.Sprintf("minikube -p %s", cfg.MachineProfile)
+	if viper.GetString(config.MachineProfile) != constants.DefaultMachineName {
+		return fmt.Sprintf("minikube -p %s", config.MachineProfile)
 	}
 	return "minikube"
 }
@@ -730,7 +729,7 @@ func validateUser(drvName string) {
 	if !useForce {
 		os.Exit(exit.Permissions)
 	}
-	_, err = cfg.Load(viper.GetString(config.MachineProfile))
+	_, err = config.Load(viper.GetString(config.MachineProfile))
 	if err == nil || !os.IsNotExist(err) {
 		out.T(out.Tip, "Tip: To remove this root owned cluster, run: sudo {{.cmd}} delete", out.V{"cmd": minikubeCmd()})
 	}
@@ -757,7 +756,7 @@ func validateFlags(cmd *cobra.Command, drvName string) {
 
 	var cpuCount int
 	if driver.BareMetal(drvName) {
-		if viper.GetString(cfg.MachineProfile) != constants.DefaultMachineName {
+		if viper.GetString(config.MachineProfile) != constants.DefaultMachineName {
 			exit.WithCodeT(exit.Config, "The 'none' driver does not support multiple profiles: https://minikube.sigs.k8s.io/docs/reference/drivers/none/")
 		}
 
@@ -789,8 +788,8 @@ func validateFlags(cmd *cobra.Command, drvName string) {
 
 	// check that kubeadm extra args contain only whitelisted parameters
 	for param := range extraOptions.AsMap().Get(kubeadm.Kubeadm) {
-		if !cfg.ContainsParam(kubeadm.KubeadmExtraArgsWhitelist[kubeadm.KubeadmCmdParam], param) &&
-			!cfg.ContainsParam(kubeadm.KubeadmExtraArgsWhitelist[kubeadm.KubeadmConfigParam], param) {
+		if !config.ContainsParam(kubeadm.KubeadmExtraArgsWhitelist[kubeadm.KubeadmCmdParam], param) &&
+			!config.ContainsParam(kubeadm.KubeadmExtraArgsWhitelist[kubeadm.KubeadmConfigParam], param) {
 			exit.UsageT("Sorry, the kubeadm.{{.parameter_name}} parameter is currently not supported by --extra-config", out.V{"parameter_name": param})
 		}
 	}
@@ -818,7 +817,7 @@ func validateRegistryMirror() {
 
 // doCacheBinaries caches Kubernetes binaries in the foreground
 func doCacheBinaries(k8sVersion string) error {
-	return machine.CacheBinariesForBootstrapper(k8sVersion, viper.GetString(cmdcfg.Bootstrapper))
+	return machine.CacheBinariesForBootstrapper(k8sVersion, viper.GetString(cmdconfig.Bootstrapper))
 }
 
 // beginCacheImages caches Docker images in the background
@@ -828,7 +827,7 @@ func beginCacheImages(g *errgroup.Group, imageRepository string, k8sVersion stri
 	}
 
 	g.Go(func() error {
-		return machine.CacheImagesForBootstrapper(imageRepository, k8sVersion, viper.GetString(cmdcfg.Bootstrapper))
+		return machine.CacheImagesForBootstrapper(imageRepository, k8sVersion, viper.GetString(cmdconfig.Bootstrapper))
 	})
 }
 
@@ -842,11 +841,11 @@ func waitCacheImages(g *errgroup.Group) {
 	}
 }
 
-// generateCfgFromFlags generates cfg.Config based on flags and supplied arguments
-func generateCfgFromFlags(cmd *cobra.Command, k8sVersion string, drvName string) (cfg.MachineConfig, error) {
+// generateCfgFromFlags generates config.Config based on flags and supplied arguments
+func generateCfgFromFlags(cmd *cobra.Command, k8sVersion string, drvName string) (config.MachineConfig, config.Node, error) {
 	r, err := cruntime.New(cruntime.Config{Type: viper.GetString(containerRuntime)})
 	if err != nil {
-		return cfg.MachineConfig{}, err
+		return config.MachineConfig{}, config.Node{}, err
 	}
 
 	// Pick good default values for --network-plugin and --enable-default-cni based on runtime.
@@ -887,8 +886,15 @@ func generateCfgFromFlags(cmd *cobra.Command, k8sVersion string, drvName string)
 		out.T(out.SuccessType, "Using image repository {{.name}}", out.V{"name": repository})
 	}
 
-	cfg := cfg.MachineConfig{
-		Name:                viper.GetString(cfg.MachineProfile),
+	masterNode := config.Node{
+		Port:              viper.GetInt(apiServerPort),
+		KubernetesVersion: k8sVersion,
+		Name:              constants.DefaultNodeName,
+		Type:              config.Master,
+	}
+
+	cfg := config.MachineConfig{
+		Name:                viper.GetString(config.MachineProfile),
 		KeepContext:         viper.GetBool(keepContext),
 		EmbedCerts:          viper.GetBool(embedCerts),
 		MinikubeISO:         viper.GetString(isoURL),
@@ -919,7 +925,7 @@ func generateCfgFromFlags(cmd *cobra.Command, k8sVersion string, drvName string)
 		HostDNSResolver:     viper.GetBool(hostDNSResolver),
 		HostOnlyNicType:     viper.GetString(hostOnlyNicType),
 		NatNicType:          viper.GetString(natNicType),
-		KubernetesConfig: cfg.KubernetesConfig{
+		KubernetesConfig: config.KubernetesConfig{
 			KubernetesVersion:      k8sVersion,
 			APIServerName:          viper.GetString(apiServerName),
 			APIServerNames:         apiServerNames,
@@ -935,16 +941,9 @@ func generateCfgFromFlags(cmd *cobra.Command, k8sVersion string, drvName string)
 			ShouldLoadCachedImages: viper.GetBool(cacheImages),
 			EnableDefaultCNI:       selectedEnableDefaultCNI,
 		},
-		Nodes: []cfg.Node{
-			cfg.Node{
-				Port:              viper.GetInt(apiServerPort),
-				KubernetesVersion: k8sVersion,
-				Name:              constants.DefaultNodeName,
-				Type:              cfg.Master,
-			},
-		},
+		Nodes: []config.Node{masterNode},
 	}
-	return cfg, nil
+	return cfg, masterNode, nil
 }
 
 // setDockerProxy sets the proxy environment variables in the docker environment.
@@ -981,7 +980,7 @@ func autoSetDriverOptions(cmd *cobra.Command, drvName string) error {
 // prepareNone prepares the user and host for the joy of the "none" driver
 func prepareNone() {
 	out.T(out.StartingNone, "Configuring local host environment ...")
-	if viper.GetBool(cfg.WantNoneDriverWarning) {
+	if viper.GetBool(config.WantNoneDriverWarning) {
 		out.T(out.Empty, "")
 		out.WarningT("The 'none' driver provides limited isolation and may reduce system security and reliability.")
 		out.WarningT("For more information, see:")
@@ -1008,7 +1007,7 @@ func prepareNone() {
 }
 
 // startHost starts a new minikube host using a VM or None
-func startHost(api libmachine.API, mc cfg.MachineConfig) (*host.Host, bool) {
+func startHost(api libmachine.API, mc config.MachineConfig) (*host.Host, bool) {
 	exists, err := api.Exists(mc.Name)
 	if err != nil {
 		exit.WithError("Failed to check if machine exists", err)
@@ -1126,7 +1125,7 @@ func tryRegistry(r command.Runner) {
 }
 
 // getKubernetesVersion ensures that the requested version is reasonable
-func getKubernetesVersion(old *cfg.MachineConfig) (string, bool) {
+func getKubernetesVersion(old *config.MachineConfig) (string, bool) {
 	paramVersion := viper.GetString(kubernetesVersion)
 	isUpgrade := false
 
@@ -1195,8 +1194,8 @@ func getKubernetesVersion(old *cfg.MachineConfig) (string, bool) {
 }
 
 // setupKubeAdm adds any requested files into the VM before Kubernetes is started
-func setupKubeAdm(mAPI libmachine.API, cr command.Runner, config cfg.MachineConfig, node cfg.Node) bootstrapper.Bootstrapper {
-	bs, err := getClusterBootstrapper(mAPI, viper.GetString(cmdcfg.Bootstrapper))
+func setupKubeAdm(mAPI libmachine.API, cr command.Runner, config config.MachineConfig, node config.Node) bootstrapper.Bootstrapper {
+	bs, err := getClusterBootstrapper(mAPI, viper.GetString(cmdconfig.Bootstrapper))
 	if err != nil {
 		exit.WithError("Failed to get bootstrapper", err)
 	}
@@ -1214,7 +1213,7 @@ func setupKubeAdm(mAPI libmachine.API, cr command.Runner, config cfg.MachineConf
 }
 
 // configureRuntimes does what needs to happen to get a runtime going.
-func configureRuntimes(runner cruntime.CommandRunner, drvName string, k8s cfg.KubernetesConfig) cruntime.Manager {
+func configureRuntimes(runner cruntime.CommandRunner, drvName string, k8s config.KubernetesConfig) cruntime.Manager {
 	config := cruntime.Config{Type: viper.GetString(containerRuntime), Runner: runner, ImageRepository: k8s.ImageRepository, KubernetesVersion: k8s.KubernetesVersion}
 	cr, err := cruntime.New(config)
 	if err != nil {
@@ -1234,7 +1233,7 @@ func configureRuntimes(runner cruntime.CommandRunner, drvName string, k8s cfg.Ku
 }
 
 // bootstrapCluster starts Kubernetes using the chosen bootstrapper
-func bootstrapCluster(bs bootstrapper.Bootstrapper, r cruntime.Manager, runner command.Runner, kc cfg.KubernetesConfig, preexisting bool, isUpgrade bool) {
+func bootstrapCluster(bs bootstrapper.Bootstrapper, r cruntime.Manager, runner command.Runner, kc config.KubernetesConfig, preexisting bool, isUpgrade bool) {
 	if isUpgrade || !preexisting {
 		out.T(out.Pulling, "Pulling images ...")
 		if err := bs.PullImages(kc); err != nil {
@@ -1275,6 +1274,6 @@ func configureMounts() {
 }
 
 // saveConfig saves profile cluster configuration in $MINIKUBE_HOME/profiles/<profilename>/config.json
-func saveConfig(clusterCfg *cfg.MachineConfig) error {
-	return cfg.SaveProfile(viper.GetString(cfg.MachineProfile), clusterCfg)
+func saveConfig(clusterCfg *config.MachineConfig) error {
+	return config.SaveProfile(viper.GetString(config.MachineProfile), clusterCfg)
 }

From 1e5aeb6aa9bf0d7a58e473ac161227628d9df95e Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Thu, 19 Dec 2019 11:27:53 -0800
Subject: [PATCH 025/250] Addressed review comments

---
 cmd/performance/cmd/mkcmp.go                  |  8 +--
 go.mod                                        |  3 -
 pkg/minikube/{performance => perf}/start.go   | 30 +++++-----
 .../{performance => perf}/start_test.go       |  2 +-
 pkg/minikube/performance/binary.go            | 58 -------------------
 5 files changed, 20 insertions(+), 81 deletions(-)
 rename pkg/minikube/{performance => perf}/start.go (79%)
 rename pkg/minikube/{performance => perf}/start_test.go (99%)
 delete mode 100644 pkg/minikube/performance/binary.go

diff --git a/cmd/performance/cmd/mkcmp.go b/cmd/performance/cmd/mkcmp.go
index d146bb9bed..57c5715f4a 100644
--- a/cmd/performance/cmd/mkcmp.go
+++ b/cmd/performance/cmd/mkcmp.go
@@ -23,7 +23,7 @@ import (
 	"os"
 
 	"github.com/spf13/cobra"
-	"k8s.io/minikube/pkg/minikube/performance"
+	"k8s.io/minikube/pkg/minikube/perf"
 )
 
 var rootCmd = &cobra.Command{
@@ -35,11 +35,7 @@ var rootCmd = &cobra.Command{
 		return validateArgs(args)
 	},
 	RunE: func(cmd *cobra.Command, args []string) error {
-		binaries, err := performance.Binaries(args)
-		if err != nil {
-			return err
-		}
-		return performance.CompareMinikubeStart(context.Background(), os.Stdout, binaries)
+		return perf.CompareMinikubeStart(context.Background(), os.Stdout, args)
 	},
 }
 
diff --git a/go.mod b/go.mod
index 46cf856a68..8976fb1bec 100644
--- a/go.mod
+++ b/go.mod
@@ -21,14 +21,11 @@ require (
 	github.com/docker/machine v0.7.1-0.20190718054102-a555e4f7a8f5 // version is 0.7.1 to pin to a555e4f7a8f5
 	github.com/elazarl/goproxy v0.0.0-20190421051319-9d40249d3c2f
 	github.com/elazarl/goproxy/ext v0.0.0-20190421051319-9d40249d3c2f // indirect
-	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/go-ole/go-ole v1.2.4 // indirect
 	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
-	github.com/google/btree v1.0.0 // indirect
 	github.com/google/go-cmp v0.3.0
 	github.com/gorilla/mux v1.7.1 // indirect
-	github.com/grpc-ecosystem/grpc-gateway v1.5.0 // indirect
 	github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce // indirect
 	github.com/hashicorp/go-getter v1.4.0
 	github.com/hashicorp/go-multierror v0.0.0-20160811015721-8c5f0ad93604 // indirect
diff --git a/pkg/minikube/performance/start.go b/pkg/minikube/perf/start.go
similarity index 79%
rename from pkg/minikube/performance/start.go
rename to pkg/minikube/perf/start.go
index c50828ea10..ed4ec1d737 100644
--- a/pkg/minikube/performance/start.go
+++ b/pkg/minikube/perf/start.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package performance
+package perf
 
 import (
 	"context"
@@ -28,14 +28,18 @@ import (
 	"github.com/pkg/errors"
 )
 
-var (
+const (
+	// runs is the number of times each binary will be timed for 'minikube start'
 	runs = 1
+)
+
+var (
 	// For testing
 	collectTimeMinikubeStart = timeMinikubeStart
 )
 
 // CompareMinikubeStart compares the time to run `minikube start` between two minikube binaries
-func CompareMinikubeStart(ctx context.Context, out io.Writer, binaries []*Binary) error {
+func CompareMinikubeStart(ctx context.Context, out io.Writer, binaries []string) error {
 	durations, err := collectTimes(ctx, binaries)
 	if err != nil {
 		return err
@@ -45,7 +49,7 @@ func CompareMinikubeStart(ctx context.Context, out io.Writer, binaries []*Binary
 	return nil
 }
 
-func collectTimes(ctx context.Context, binaries []*Binary) ([][]float64, error) {
+func collectTimes(ctx context.Context, binaries []string) ([][]float64, error) {
 	durations := make([][]float64, len(binaries))
 	for i := range durations {
 		durations[i] = make([]float64, runs)
@@ -56,7 +60,7 @@ func collectTimes(ctx context.Context, binaries []*Binary) ([][]float64, error)
 		for index, binary := range binaries {
 			duration, err := collectTimeMinikubeStart(ctx, binary)
 			if err != nil {
-				return nil, errors.Wrapf(err, "timing run %d with %s", r, binary.path)
+				return nil, errors.Wrapf(err, "timing run %d with %s", r, binary)
 			}
 			durations[index][r] = duration
 		}
@@ -65,22 +69,22 @@ func collectTimes(ctx context.Context, binaries []*Binary) ([][]float64, error)
 	return durations, nil
 }
 
-func average(array []float64) float64 {
+func average(nums []float64) float64 {
 	total := float64(0)
-	for _, a := range array {
+	for _, a := range nums {
 		total += a
 	}
-	return total / float64(len(array))
+	return total / float64(len(nums))
 }
 
 // timeMinikubeStart returns the time it takes to execute `minikube start`
 // It deletes the VM after `minikube start`.
-func timeMinikubeStart(ctx context.Context, binary *Binary) (float64, error) {
-	startCmd := exec.CommandContext(ctx, binary.path, "start")
+func timeMinikubeStart(ctx context.Context, binary string) (float64, error) {
+	startCmd := exec.CommandContext(ctx, binary, "start")
 	startCmd.Stdout = os.Stdout
 	startCmd.Stderr = os.Stderr
 
-	deleteCmd := exec.CommandContext(ctx, binary.path, "delete")
+	deleteCmd := exec.CommandContext(ctx, binary, "delete")
 	defer func() {
 		if err := deleteCmd.Run(); err != nil {
 			log.Printf("error deleting minikube: %v", err)
@@ -93,6 +97,6 @@ func timeMinikubeStart(ctx context.Context, binary *Binary) (float64, error) {
 		return 0, errors.Wrap(err, "starting minikube")
 	}
 
-	startDuration := time.Since(start).Seconds()
-	return startDuration, nil
+	s := time.Since(start).Seconds()
+	return s, nil
 }
diff --git a/pkg/minikube/performance/start_test.go b/pkg/minikube/perf/start_test.go
similarity index 99%
rename from pkg/minikube/performance/start_test.go
rename to pkg/minikube/perf/start_test.go
index ecabbd9617..1c73ba9af5 100644
--- a/pkg/minikube/performance/start_test.go
+++ b/pkg/minikube/perf/start_test.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package performance
+package perf
 
 import (
 	"bytes"
diff --git a/pkg/minikube/performance/binary.go b/pkg/minikube/performance/binary.go
deleted file mode 100644
index 91e2a82d90..0000000000
--- a/pkg/minikube/performance/binary.go
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
-Copyright 2019 The Kubernetes Authors All rights reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package performance
-
-import (
-	"os"
-	"path/filepath"
-)
-
-// Binary holds all necessary information
-// to call a minikube binary
-type Binary struct {
-	path string
-}
-
-// Binaries returns the type *Binary for each provided path
-func Binaries(paths []string) ([]*Binary, error) {
-	var binaries []*Binary
-	for _, path := range paths {
-		b, err := newBinary(path)
-		if err != nil {
-			return nil, err
-		}
-		binaries = append(binaries, b)
-	}
-	return binaries, nil
-}
-
-func newBinary(path string) (*Binary, error) {
-	var err error
-	if !filepath.IsAbs(path) {
-		path, err = filepath.Abs(path)
-		if err != nil {
-			return nil, err
-		}
-	}
-	// make sure binary exists
-	if _, err := os.Stat(path); err != nil {
-		return nil, err
-	}
-	return &Binary{
-		path: path,
-	}, nil
-}

From 624d47b7812b2d0257a5b103aab084c3386c86fa Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Thu, 19 Dec 2019 12:47:20 -0800
Subject: [PATCH 026/250] Update unit tests

---
 pkg/minikube/perf/start_test.go | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)

diff --git a/pkg/minikube/perf/start_test.go b/pkg/minikube/perf/start_test.go
index 1c73ba9af5..539d57500d 100644
--- a/pkg/minikube/perf/start_test.go
+++ b/pkg/minikube/perf/start_test.go
@@ -23,9 +23,9 @@ import (
 	"testing"
 )
 
-func mockCollectTimeMinikubeStart(durations []float64) func(ctx context.Context, binary *Binary) (float64, error) {
+func mockCollectTimeMinikubeStart(durations []float64) func(ctx context.Context, binary string) (float64, error) {
 	index := 0
-	return func(context.Context, *Binary) (float64, error) {
+	return func(context.Context, string) (float64, error) {
 		duration := durations[index]
 		index++
 		return duration, nil
@@ -52,7 +52,7 @@ func TestCompareMinikubeStartOutput(t *testing.T) {
 			defer func() { collectTimeMinikubeStart = originalCollectTimes }()
 
 			buf := bytes.NewBuffer([]byte{})
-			err := CompareMinikubeStart(context.Background(), buf, []*Binary{{}, {}})
+			err := CompareMinikubeStart(context.Background(), buf, []string{"", ""})
 			if err != nil {
 				t.Fatalf("error comparing minikube start: %v", err)
 			}
@@ -69,16 +69,14 @@ func TestCollectTimes(t *testing.T) {
 	tests := []struct {
 		description string
 		durations   []float64
-		runs        int
 		expected    [][]float64
 	}{
 		{
-			description: "two runs",
-			durations:   []float64{1, 2, 3, 4},
-			runs:        2,
+			description: "test collect time",
+			durations:   []float64{1, 2},
 			expected: [][]float64{
-				{1, 3},
-				{2, 4},
+				{1},
+				{2},
 			},
 		},
 	}
@@ -89,8 +87,7 @@ func TestCollectTimes(t *testing.T) {
 			collectTimeMinikubeStart = mockCollectTimeMinikubeStart(test.durations)
 			defer func() { collectTimeMinikubeStart = originalCollectTimes }()
 
-			runs = test.runs
-			actual, err := collectTimes(context.Background(), []*Binary{{}, {}})
+			actual, err := collectTimes(context.Background(), []string{"", ""})
 			if err != nil {
 				t.Fatalf("error collecting times: %v", err)
 			}

From 4ce8d7dc0cb26c6b870bd420f3d55305fb82f4d7 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Thu, 19 Dec 2019 13:58:13 -0800
Subject: [PATCH 027/250] fix boilerplate

---
 pkg/minikube/perf/start.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/minikube/perf/start.go b/pkg/minikube/perf/start.go
index ed4ec1d737..a942863f8c 100644
--- a/pkg/minikube/perf/start.go
+++ b/pkg/minikube/perf/start.go
@@ -1,5 +1,5 @@
 /*
-Copyright 2017 The Kubernetes Authors All rights reserved.
+Copyright 2019 The Kubernetes Authors All rights reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

From f9ed4a64600f7312f032c4d0f7098ee99da28ba4 Mon Sep 17 00:00:00 2001
From: Zhongcheng Lao <Zhongcheng.Lao@microsoft.com>
Date: Tue, 17 Sep 2019 23:29:57 +0800
Subject: [PATCH 028/250] Allow user to override node name

Signed-off-by: Zhongcheng Lao <Zhongcheng.Lao@microsoft.com>
---
 cmd/minikube/cmd/start.go                   | 4 +++-
 pkg/minikube/bootstrapper/bsutil/kubelet.go | 3 +++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index a3ff003985..1b9a25f684 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -95,6 +95,7 @@ const (
 	keepContext           = "keep-context"
 	createMount           = "mount"
 	featureGates          = "feature-gates"
+	nodeName              = "node-name"
 	apiServerName         = "apiserver-name"
 	apiServerPort         = "apiserver-port"
 	dnsDomain             = "dns-domain"
@@ -193,6 +194,7 @@ func initKubernetesFlags() {
 	startCmd.Flags().String(apiServerName, constants.APIServerName, "The apiserver name which is used in the generated certificate for kubernetes.  This can be used if you want to make the apiserver available from outside the machine")
 	startCmd.Flags().StringArrayVar(&apiServerNames, "apiserver-names", nil, "A set of apiserver names which are used in the generated certificate for kubernetes.  This can be used if you want to make the apiserver available from outside the machine")
 	startCmd.Flags().IPSliceVar(&apiServerIPs, "apiserver-ips", nil, "A set of apiserver IP Addresses which are used in the generated certificate for kubernetes.  This can be used if you want to make the apiserver available from outside the machine")
+	startCmd.Flags().String(nodeName, constants.DefaultNodeName, "The node name")
 }
 
 // initDriverFlags inits the commandline flags for vm drivers
@@ -921,7 +923,7 @@ func generateCfgFromFlags(cmd *cobra.Command, k8sVersion string, drvName string)
 		KubernetesConfig: cfg.KubernetesConfig{
 			KubernetesVersion:      k8sVersion,
 			NodePort:               viper.GetInt(apiServerPort),
-			NodeName:               constants.DefaultNodeName,
+			NodeName:               viper.GetString(nodeName),
 			APIServerName:          viper.GetString(apiServerName),
 			APIServerNames:         apiServerNames,
 			APIServerIPs:           apiServerIPs,
diff --git a/pkg/minikube/bootstrapper/bsutil/kubelet.go b/pkg/minikube/bootstrapper/bsutil/kubelet.go
index 48864d80fe..aea5bad07f 100644
--- a/pkg/minikube/bootstrapper/bsutil/kubelet.go
+++ b/pkg/minikube/bootstrapper/bsutil/kubelet.go
@@ -50,6 +50,9 @@ func NewKubeletConfig(k8s config.KubernetesConfig, r cruntime.Manager) ([]byte,
 	if _, ok := extraOpts["node-ip"]; !ok {
 		extraOpts["node-ip"] = k8s.NodeIP
 	}
+	if k8s.NodeName != "" {
+		extraOpts["hostname-override"] = k8s.NodeName
+	}
 
 	pauseImage := images.Pause(k8s.ImageRepository)
 	if _, ok := extraOpts["pod-infra-container-image"]; !ok && k8s.ImageRepository != "" && pauseImage != "" && k8s.ContainerRuntime != remoteContainerRuntime {

From 914c34cc2ab09957ad980d38f844fe05c20de2bb Mon Sep 17 00:00:00 2001
From: Zhongcheng Lao <Zhongcheng.Lao@microsoft.com>
Date: Thu, 19 Sep 2019 08:48:07 +0800
Subject: [PATCH 029/250] Set node name in non-default profile

Signed-off-by: Zhongcheng Lao <Zhongcheng.Lao@microsoft.com>
---
 cmd/minikube/cmd/start.go | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 1b9a25f684..f2b9106bec 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -194,7 +194,7 @@ func initKubernetesFlags() {
 	startCmd.Flags().String(apiServerName, constants.APIServerName, "The apiserver name which is used in the generated certificate for kubernetes.  This can be used if you want to make the apiserver available from outside the machine")
 	startCmd.Flags().StringArrayVar(&apiServerNames, "apiserver-names", nil, "A set of apiserver names which are used in the generated certificate for kubernetes.  This can be used if you want to make the apiserver available from outside the machine")
 	startCmd.Flags().IPSliceVar(&apiServerIPs, "apiserver-ips", nil, "A set of apiserver IP Addresses which are used in the generated certificate for kubernetes.  This can be used if you want to make the apiserver available from outside the machine")
-	startCmd.Flags().String(nodeName, constants.DefaultNodeName, "The node name")
+	startCmd.Flags().String(nodeName, "", "The node name")
 }
 
 // initDriverFlags inits the commandline flags for vm drivers
@@ -888,6 +888,16 @@ func generateCfgFromFlags(cmd *cobra.Command, k8sVersion string, drvName string)
 		out.T(out.SuccessType, "Using image repository {{.name}}", out.V{"name": repository})
 	}
 
+	kubeNodeName := viper.GetString(nodeName)
+	if kubeNodeName == "" {
+		machineName := viper.GetString(cfg.MachineProfile)
+		if machineName != constants.DefaultMachineName {
+			kubeNodeName = fmt.Sprintf("%s-%s", constants.DefaultNodeName, machineName)
+		} else {
+			kubeNodeName = constants.DefaultNodeName
+		}
+	}
+
 	cfg := cfg.MachineConfig{
 		Name:                viper.GetString(cfg.MachineProfile),
 		KeepContext:         viper.GetBool(keepContext),
@@ -923,7 +933,7 @@ func generateCfgFromFlags(cmd *cobra.Command, k8sVersion string, drvName string)
 		KubernetesConfig: cfg.KubernetesConfig{
 			KubernetesVersion:      k8sVersion,
 			NodePort:               viper.GetInt(apiServerPort),
-			NodeName:               viper.GetString(nodeName),
+			NodeName:               kubeNodeName,
 			APIServerName:          viper.GetString(apiServerName),
 			APIServerNames:         apiServerNames,
 			APIServerIPs:           apiServerIPs,

From 73608d60e47df2604c1883df8c7af48bc7b9c08c Mon Sep 17 00:00:00 2001
From: Zhongcheng Lao <Zhongcheng.Lao@microsoft.com>
Date: Fri, 3 Jan 2020 00:32:05 +0800
Subject: [PATCH 030/250] Set node name the same as host name by default for
 none driver

Signed-off-by: Zhongcheng Lao <Zhongcheng.Lao@microsoft.com>
---
 cmd/minikube/cmd/start.go | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index f2b9106bec..064f97627d 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -890,11 +890,19 @@ func generateCfgFromFlags(cmd *cobra.Command, k8sVersion string, drvName string)
 
 	kubeNodeName := viper.GetString(nodeName)
 	if kubeNodeName == "" {
-		machineName := viper.GetString(cfg.MachineProfile)
-		if machineName != constants.DefaultMachineName {
-			kubeNodeName = fmt.Sprintf("%s-%s", constants.DefaultNodeName, machineName)
-		} else {
-			kubeNodeName = constants.DefaultNodeName
+		if drvName == driver.None {
+			// set the node name the same as host name for none driver
+			hostname, _ := os.Hostname()
+			kubeNodeName = hostname
+		}
+
+		if kubeNodeName == "" {
+			machineName := viper.GetString(cfg.MachineProfile)
+			if machineName != constants.DefaultMachineName {
+				kubeNodeName = fmt.Sprintf("%s-%s", constants.DefaultNodeName, machineName)
+			} else {
+				kubeNodeName = constants.DefaultNodeName
+			}
 		}
 	}
 

From 024dfe18dca4c70c667401d0e2949efbc3f54770 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= <anders.f.bjorklund@gmail.com>
Date: Mon, 6 Jan 2020 14:55:18 +0100
Subject: [PATCH 031/250] Refactor: move the crio config to the crio runtime

For some reason it was left with the generic CRI
---
 pkg/minikube/cruntime/cri.go  | 323 ---------------------------------
 pkg/minikube/cruntime/crio.go | 326 ++++++++++++++++++++++++++++++++++
 2 files changed, 326 insertions(+), 323 deletions(-)

diff --git a/pkg/minikube/cruntime/cri.go b/pkg/minikube/cruntime/cri.go
index f053c74404..4cb25646d5 100644
--- a/pkg/minikube/cruntime/cri.go
+++ b/pkg/minikube/cruntime/cri.go
@@ -18,7 +18,6 @@ package cruntime
 
 import (
 	"bytes"
-	"encoding/base64"
 	"fmt"
 	"html/template"
 	"os/exec"
@@ -27,310 +26,9 @@ import (
 
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
-	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
 	"k8s.io/minikube/pkg/minikube/command"
 )
 
-const (
-	// CRIOConfFile is the path to the CRI-O configuration
-	crioConfigFile     = "/etc/crio/crio.conf"
-	crioConfigTemplate = `# The CRI-O configuration file specifies all of the available configuration
-# options and command-line flags for the crio(8) OCI Kubernetes Container Runtime
-# daemon, but in a TOML format that can be more easily modified and versioned.
-#
-# Please refer to crio.conf(5) for details of all configuration options.
-
-# CRI-O supports partial configuration reload during runtime, which can be
-# done by sending SIGHUP to the running process. Currently supported options
-# are explicitly mentioned with: 'This option supports live configuration
-# reload'.
-
-# CRI-O reads its storage defaults from the containers-storage.conf(5) file
-# located at /etc/containers/storage.conf. Modify this storage configuration if
-# you want to change the system's defaults. If you want to modify storage just
-# for CRI-O, you can change the storage configuration options here.
-[crio]
-
-# Path to the "root directory". CRI-O stores all of its data, including
-# containers images, in this directory.
-root = "/var/lib/containers/storage"
-
-# Path to the "run directory". CRI-O stores all of its state in this directory.
-runroot = "/var/run/containers/storage"
-
-# Storage driver used to manage the storage of images and containers. Please
-# refer to containers-storage.conf(5) to see all available storage drivers.
-storage_driver = "overlay"
-
-# List to pass options to the storage driver. Please refer to
-# containers-storage.conf(5) to see all available storage options.
-#storage_option = [
-#]
-
-# If set to false, in-memory locking will be used instead of file-based locking.
-# **Deprecated** this option will be removed in the future.
-file_locking = false
-
-# Path to the lock file.
-# **Deprecated** this option will be removed in the future.
-file_locking_path = "/run/crio.lock"
-
-
-# The crio.api table contains settings for the kubelet/gRPC interface.
-[crio.api]
-
-# Path to AF_LOCAL socket on which CRI-O will listen.
-listen = "/var/run/crio/crio.sock"
-
-# IP address on which the stream server will listen.
-stream_address = "127.0.0.1"
-
-# The port on which the stream server will listen.
-stream_port = "0"
-
-# Enable encrypted TLS transport of the stream server.
-stream_enable_tls = false
-
-# Path to the x509 certificate file used to serve the encrypted stream. This
-# file can change, and CRI-O will automatically pick up the changes within 5
-# minutes.
-stream_tls_cert = ""
-
-# Path to the key file used to serve the encrypted stream. This file can
-# change, and CRI-O will automatically pick up the changes within 5 minutes.
-stream_tls_key = ""
-
-# Path to the x509 CA(s) file used to verify and authenticate client
-# communication with the encrypted stream. This file can change, and CRI-O will
-# automatically pick up the changes within 5 minutes.
-stream_tls_ca = ""
-
-# Maximum grpc send message size in bytes. If not set or <=0, then CRI-O will default to 16 * 1024 * 1024.
-grpc_max_send_msg_size = 16777216
-
-# Maximum grpc receive message size. If not set or <= 0, then CRI-O will default to 16 * 1024 * 1024.
-grpc_max_recv_msg_size = 16777216
-
-# The crio.runtime table contains settings pertaining to the OCI runtime used
-# and options for how to set up and manage the OCI runtime.
-[crio.runtime]
-
-# A list of ulimits to be set in containers by default, specified as
-# "<ulimit name>=<soft limit>:<hard limit>", for example:
-# "nofile=1024:2048"
-# If nothing is set here, settings will be inherited from the CRI-O daemon
-#default_ulimits = [
-#]
-
-# default_runtime is the _name_ of the OCI runtime to be used as the default.
-# The name is matched against the runtimes map below.
-default_runtime = "runc"
-
-# If true, the runtime will not use pivot_root, but instead use MS_MOVE.
-no_pivot = true
-
-# Path to the conmon binary, used for monitoring the OCI runtime.
-conmon = "/usr/libexec/crio/conmon"
-
-# Cgroup setting for conmon
-conmon_cgroup = "pod"
-
-# Environment variable list for the conmon process, used for passing necessary
-# environment variables to conmon or the runtime.
-conmon_env = [
-	"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
-]
-
-# If true, SELinux will be used for pod separation on the host.
-selinux = false
-
-# Path to the seccomp.json profile which is used as the default seccomp profile
-# for the runtime. If not specified, then the internal default seccomp profile
-# will be used.
-seccomp_profile = ""
-
-# Used to change the name of the default AppArmor profile of CRI-O. The default
-# profile name is "crio-default-" followed by the version string of CRI-O.
-apparmor_profile = "crio-default"
-
-# Cgroup management implementation used for the runtime.
-cgroup_manager = "cgroupfs"
-
-# List of default capabilities for containers. If it is empty or commented out,
-# only the capabilities defined in the containers json file by the user/kube
-# will be added.
-default_capabilities = [
-	"CHOWN",
-	"DAC_OVERRIDE",
-	"FSETID",
-	"FOWNER",
-	"NET_RAW",
-	"SETGID",
-	"SETUID",
-	"SETPCAP",
-	"NET_BIND_SERVICE",
-	"SYS_CHROOT",
-	"KILL",
-]
-
-# List of default sysctls. If it is empty or commented out, only the sysctls
-# defined in the container json file by the user/kube will be added.
-default_sysctls = [
-]
-
-# List of additional devices. specified as
-# "<device-on-host>:<device-on-container>:<permissions>", for example: "--device=/dev/sdc:/dev/xvdc:rwm".
-#If it is empty or commented out, only the devices
-# defined in the container json file by the user/kube will be added.
-additional_devices = [
-]
-
-# Path to OCI hooks directories for automatically executed hooks.
-hooks_dir = [
-]
-
-# List of default mounts for each container. **Deprecated:** this option will
-# be removed in future versions in favor of default_mounts_file.
-default_mounts = [
-]
-
-# Path to the file specifying the defaults mounts for each container. The
-# format of the config is /SRC:/DST, one mount per line. Notice that CRI-O reads
-# its default mounts from the following two files:
-#
-#   1) /etc/containers/mounts.conf (i.e., default_mounts_file): This is the
-#      override file, where users can either add in their own default mounts, or
-#      override the default mounts shipped with the package.
-#
-#   2) /usr/share/containers/mounts.conf: This is the default file read for
-#      mounts. If you want CRI-O to read from a different, specific mounts file,
-#      you can change the default_mounts_file. Note, if this is done, CRI-O will
-#      only add mounts it finds in this file.
-#
-#default_mounts_file = ""
-
-# Maximum number of processes allowed in a container.
-pids_limit = 1024
-
-# Maximum sized allowed for the container log file. Negative numbers indicate
-# that no size limit is imposed. If it is positive, it must be >= 8192 to
-# match/exceed conmon's read buffer. The file is truncated and re-opened so the
-# limit is never exceeded.
-log_size_max = -1
-
-# Whether container output should be logged to journald in addition to the kuberentes log file
-log_to_journald = false
-
-# Path to directory in which container exit files are written to by conmon.
-container_exits_dir = "/var/run/crio/exits"
-
-# Path to directory for container attach sockets.
-container_attach_socket_dir = "/var/run/crio"
-
-# If set to true, all containers will run in read-only mode.
-read_only = false
-
-# Changes the verbosity of the logs based on the level it is set to. Options
-# are fatal, panic, error, warn, info, and debug. This option supports live
-# configuration reload.
-log_level = "error"
-
-# The default log directory where all logs will go unless directly specified by the kubelet
-log_dir = "/var/log/crio/pods"
-
-# The UID mappings for the user namespace of each container. A range is
-# specified in the form containerUID:HostUID:Size. Multiple ranges must be
-# separated by comma.
-uid_mappings = ""
-
-# The GID mappings for the user namespace of each container. A range is
-# specified in the form containerGID:HostGID:Size. Multiple ranges must be
-# separated by comma.
-gid_mappings = ""
-
-# The minimal amount of time in seconds to wait before issuing a timeout
-# regarding the proper termination of the container.
-ctr_stop_timeout = 0
-
-# ManageNetworkNSLifecycle determines whether we pin and remove network namespace
-# and manage its lifecycle.
-manage_network_ns_lifecycle = false
-
-# The "crio.runtime.runtimes" table defines a list of OCI compatible runtimes.
-# The runtime to use is picked based on the runtime_handler provided by the CRI.
-# If no runtime_handler is provided, the runtime will be picked based on the level
-# of trust of the workload.
-
-[crio.runtime.runtimes.runc]
-runtime_path = "/usr/bin/runc"
-runtime_type = "oci"
-runtime_root = "/run/runc"
-
-
-# The crio.image table contains settings pertaining to the management of OCI images.
-#
-# CRI-O reads its configured registries defaults from the system wide
-# containers-registries.conf(5) located in /etc/containers/registries.conf. If
-# you want to modify just CRI-O, you can change the registries configuration in
-# this file. Otherwise, leave insecure_registries and registries commented out to
-# use the system's defaults from /etc/containers/registries.conf.
-[crio.image]
-
-# Default transport for pulling images from a remote container storage.
-default_transport = "docker://"
-
-# The path to a file containing credentials necessary for pulling images from
-# secure registries. The file is similar to that of /var/lib/kubelet/config.json
-global_auth_file = ""
-
-# The image used to instantiate infra containers.
-# This option supports live configuration reload.
-pause_image = "{{ .PodInfraContainerImage }}"
-
-# The path to a file containing credentials specific for pulling the pause_image from
-# above. The file is similar to that of /var/lib/kubelet/config.json
-# This option supports live configuration reload.
-pause_image_auth_file = ""
-
-# The command to run to have a container stay in the paused state.
-# This option supports live configuration reload.
-pause_command = "/pause"
-
-# Path to the file which decides what sort of policy we use when deciding
-# whether or not to trust an image that we've pulled. It is not recommended that
-# this option be used, as the default behavior of using the system-wide default
-# policy (i.e., /etc/containers/policy.json) is most often preferred. Please
-# refer to containers-policy.json(5) for more details.
-signature_policy = ""
-
-# Controls how image volumes are handled. The valid values are mkdir, bind and
-# ignore; the latter will ignore volumes entirely.
-image_volumes = "mkdir"
-
-# List of registries to be used when pulling an unqualified image (e.g.,
-# "alpine:latest"). By default, registries is set to "docker.io" for
-# compatibility reasons. Depending on your workload and usecase you may add more
-# registries (e.g., "quay.io", "registry.fedoraproject.org",
-# "registry.opensuse.org", etc.).
-registries = [
-	"docker.io"
-]
-
-
-# The crio.network table containers settings pertaining to the management of
-# CNI plugins.
-[crio.network]
-
-# Path to the directory where CNI configuration files are located.
-network_dir = "/etc/cni/net.d/"
-
-# Paths to directories where CNI plugin binaries are located.
-plugin_dirs = [
-	"/opt/cni/bin/",
-]
-`
-)
-
 // listCRIContainers returns a list of containers using crictl
 func listCRIContainers(cr CommandRunner, filter string) ([]string, error) {
 	var err error
@@ -406,27 +104,6 @@ image-endpoint: unix://{{.Socket}}
 	return nil
 }
 
-// generateCRIOConfig sets up /etc/crio/crio.conf
-func generateCRIOConfig(cr CommandRunner, imageRepository string) error {
-	cPath := crioConfigFile
-	t, err := template.New("crio.conf").Parse(crioConfigTemplate)
-	if err != nil {
-		return err
-	}
-	pauseImage := images.Pause(imageRepository)
-	opts := struct{ PodInfraContainerImage string }{PodInfraContainerImage: pauseImage}
-	var b bytes.Buffer
-	if err := t.Execute(&b, opts); err != nil {
-		return err
-	}
-
-	c := exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo mkdir -p %s && printf %%s \"%s\" | base64 -d | sudo tee %s", path.Dir(cPath), base64.StdEncoding.EncodeToString(b.Bytes()), cPath))
-	if _, err := cr.RunCmd(c); err != nil {
-		return errors.Wrap(err, "generateCRIOConfig.")
-	}
-	return nil
-}
-
 // criContainerLogCmd returns the command to retrieve the log for a container based on ID
 func criContainerLogCmd(id string, len int, follow bool) string {
 	var cmd strings.Builder
diff --git a/pkg/minikube/cruntime/crio.go b/pkg/minikube/cruntime/crio.go
index 1b297d2e2b..346d790709 100644
--- a/pkg/minikube/cruntime/crio.go
+++ b/pkg/minikube/cruntime/crio.go
@@ -17,12 +17,17 @@ limitations under the License.
 package cruntime
 
 import (
+	"bytes"
+	"encoding/base64"
 	"fmt"
 	"os/exec"
+	"path"
 	"strings"
+	"text/template"
 
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
 	"k8s.io/minikube/pkg/minikube/out"
 )
 
@@ -34,6 +39,327 @@ type CRIO struct {
 	KubernetesVersion string
 }
 
+const (
+	// CRIOConfFile is the path to the CRI-O configuration
+	crioConfigFile     = "/etc/crio/crio.conf"
+	crioConfigTemplate = `# The CRI-O configuration file specifies all of the available configuration
+# options and command-line flags for the crio(8) OCI Kubernetes Container Runtime
+# daemon, but in a TOML format that can be more easily modified and versioned.
+#
+# Please refer to crio.conf(5) for details of all configuration options.
+
+# CRI-O supports partial configuration reload during runtime, which can be
+# done by sending SIGHUP to the running process. Currently supported options
+# are explicitly mentioned with: 'This option supports live configuration
+# reload'.
+
+# CRI-O reads its storage defaults from the containers-storage.conf(5) file
+# located at /etc/containers/storage.conf. Modify this storage configuration if
+# you want to change the system's defaults. If you want to modify storage just
+# for CRI-O, you can change the storage configuration options here.
+[crio]
+
+# Path to the "root directory". CRI-O stores all of its data, including
+# containers images, in this directory.
+root = "/var/lib/containers/storage"
+
+# Path to the "run directory". CRI-O stores all of its state in this directory.
+runroot = "/var/run/containers/storage"
+
+# Storage driver used to manage the storage of images and containers. Please
+# refer to containers-storage.conf(5) to see all available storage drivers.
+storage_driver = "overlay"
+
+# List to pass options to the storage driver. Please refer to
+# containers-storage.conf(5) to see all available storage options.
+#storage_option = [
+#]
+
+# If set to false, in-memory locking will be used instead of file-based locking.
+# **Deprecated** this option will be removed in the future.
+file_locking = false
+
+# Path to the lock file.
+# **Deprecated** this option will be removed in the future.
+file_locking_path = "/run/crio.lock"
+
+
+# The crio.api table contains settings for the kubelet/gRPC interface.
+[crio.api]
+
+# Path to AF_LOCAL socket on which CRI-O will listen.
+listen = "/var/run/crio/crio.sock"
+
+# IP address on which the stream server will listen.
+stream_address = "127.0.0.1"
+
+# The port on which the stream server will listen.
+stream_port = "0"
+
+# Enable encrypted TLS transport of the stream server.
+stream_enable_tls = false
+
+# Path to the x509 certificate file used to serve the encrypted stream. This
+# file can change, and CRI-O will automatically pick up the changes within 5
+# minutes.
+stream_tls_cert = ""
+
+# Path to the key file used to serve the encrypted stream. This file can
+# change, and CRI-O will automatically pick up the changes within 5 minutes.
+stream_tls_key = ""
+
+# Path to the x509 CA(s) file used to verify and authenticate client
+# communication with the encrypted stream. This file can change, and CRI-O will
+# automatically pick up the changes within 5 minutes.
+stream_tls_ca = ""
+
+# Maximum grpc send message size in bytes. If not set or <=0, then CRI-O will default to 16 * 1024 * 1024.
+grpc_max_send_msg_size = 16777216
+
+# Maximum grpc receive message size. If not set or <= 0, then CRI-O will default to 16 * 1024 * 1024.
+grpc_max_recv_msg_size = 16777216
+
+# The crio.runtime table contains settings pertaining to the OCI runtime used
+# and options for how to set up and manage the OCI runtime.
+[crio.runtime]
+
+# A list of ulimits to be set in containers by default, specified as
+# "<ulimit name>=<soft limit>:<hard limit>", for example:
+# "nofile=1024:2048"
+# If nothing is set here, settings will be inherited from the CRI-O daemon
+#default_ulimits = [
+#]
+
+# default_runtime is the _name_ of the OCI runtime to be used as the default.
+# The name is matched against the runtimes map below.
+default_runtime = "runc"
+
+# If true, the runtime will not use pivot_root, but instead use MS_MOVE.
+no_pivot = true
+
+# Path to the conmon binary, used for monitoring the OCI runtime.
+conmon = "/usr/libexec/crio/conmon"
+
+# Cgroup setting for conmon
+conmon_cgroup = "pod"
+
+# Environment variable list for the conmon process, used for passing necessary
+# environment variables to conmon or the runtime.
+conmon_env = [
+	"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+]
+
+# If true, SELinux will be used for pod separation on the host.
+selinux = false
+
+# Path to the seccomp.json profile which is used as the default seccomp profile
+# for the runtime. If not specified, then the internal default seccomp profile
+# will be used.
+seccomp_profile = ""
+
+# Used to change the name of the default AppArmor profile of CRI-O. The default
+# profile name is "crio-default-" followed by the version string of CRI-O.
+apparmor_profile = "crio-default"
+
+# Cgroup management implementation used for the runtime.
+cgroup_manager = "cgroupfs"
+
+# List of default capabilities for containers. If it is empty or commented out,
+# only the capabilities defined in the containers json file by the user/kube
+# will be added.
+default_capabilities = [
+	"CHOWN",
+	"DAC_OVERRIDE",
+	"FSETID",
+	"FOWNER",
+	"NET_RAW",
+	"SETGID",
+	"SETUID",
+	"SETPCAP",
+	"NET_BIND_SERVICE",
+	"SYS_CHROOT",
+	"KILL",
+]
+
+# List of default sysctls. If it is empty or commented out, only the sysctls
+# defined in the container json file by the user/kube will be added.
+default_sysctls = [
+]
+
+# List of additional devices. specified as
+# "<device-on-host>:<device-on-container>:<permissions>", for example: "--device=/dev/sdc:/dev/xvdc:rwm".
+#If it is empty or commented out, only the devices
+# defined in the container json file by the user/kube will be added.
+additional_devices = [
+]
+
+# Path to OCI hooks directories for automatically executed hooks.
+hooks_dir = [
+]
+
+# List of default mounts for each container. **Deprecated:** this option will
+# be removed in future versions in favor of default_mounts_file.
+default_mounts = [
+]
+
+# Path to the file specifying the defaults mounts for each container. The
+# format of the config is /SRC:/DST, one mount per line. Notice that CRI-O reads
+# its default mounts from the following two files:
+#
+#   1) /etc/containers/mounts.conf (i.e., default_mounts_file): This is the
+#      override file, where users can either add in their own default mounts, or
+#      override the default mounts shipped with the package.
+#
+#   2) /usr/share/containers/mounts.conf: This is the default file read for
+#      mounts. If you want CRI-O to read from a different, specific mounts file,
+#      you can change the default_mounts_file. Note, if this is done, CRI-O will
+#      only add mounts it finds in this file.
+#
+#default_mounts_file = ""
+
+# Maximum number of processes allowed in a container.
+pids_limit = 1024
+
+# Maximum sized allowed for the container log file. Negative numbers indicate
+# that no size limit is imposed. If it is positive, it must be >= 8192 to
+# match/exceed conmon's read buffer. The file is truncated and re-opened so the
+# limit is never exceeded.
+log_size_max = -1
+
+# Whether container output should be logged to journald in addition to the kuberentes log file
+log_to_journald = false
+
+# Path to directory in which container exit files are written to by conmon.
+container_exits_dir = "/var/run/crio/exits"
+
+# Path to directory for container attach sockets.
+container_attach_socket_dir = "/var/run/crio"
+
+# If set to true, all containers will run in read-only mode.
+read_only = false
+
+# Changes the verbosity of the logs based on the level it is set to. Options
+# are fatal, panic, error, warn, info, and debug. This option supports live
+# configuration reload.
+log_level = "error"
+
+# The default log directory where all logs will go unless directly specified by the kubelet
+log_dir = "/var/log/crio/pods"
+
+# The UID mappings for the user namespace of each container. A range is
+# specified in the form containerUID:HostUID:Size. Multiple ranges must be
+# separated by comma.
+uid_mappings = ""
+
+# The GID mappings for the user namespace of each container. A range is
+# specified in the form containerGID:HostGID:Size. Multiple ranges must be
+# separated by comma.
+gid_mappings = ""
+
+# The minimal amount of time in seconds to wait before issuing a timeout
+# regarding the proper termination of the container.
+ctr_stop_timeout = 0
+
+# ManageNetworkNSLifecycle determines whether we pin and remove network namespace
+# and manage its lifecycle.
+manage_network_ns_lifecycle = false
+
+# The "crio.runtime.runtimes" table defines a list of OCI compatible runtimes.
+# The runtime to use is picked based on the runtime_handler provided by the CRI.
+# If no runtime_handler is provided, the runtime will be picked based on the level
+# of trust of the workload.
+
+[crio.runtime.runtimes.runc]
+runtime_path = "/usr/bin/runc"
+runtime_type = "oci"
+runtime_root = "/run/runc"
+
+
+# The crio.image table contains settings pertaining to the management of OCI images.
+#
+# CRI-O reads its configured registries defaults from the system wide
+# containers-registries.conf(5) located in /etc/containers/registries.conf. If
+# you want to modify just CRI-O, you can change the registries configuration in
+# this file. Otherwise, leave insecure_registries and registries commented out to
+# use the system's defaults from /etc/containers/registries.conf.
+[crio.image]
+
+# Default transport for pulling images from a remote container storage.
+default_transport = "docker://"
+
+# The path to a file containing credentials necessary for pulling images from
+# secure registries. The file is similar to that of /var/lib/kubelet/config.json
+global_auth_file = ""
+
+# The image used to instantiate infra containers.
+# This option supports live configuration reload.
+pause_image = "{{ .PodInfraContainerImage }}"
+
+# The path to a file containing credentials specific for pulling the pause_image from
+# above. The file is similar to that of /var/lib/kubelet/config.json
+# This option supports live configuration reload.
+pause_image_auth_file = ""
+
+# The command to run to have a container stay in the paused state.
+# This option supports live configuration reload.
+pause_command = "/pause"
+
+# Path to the file which decides what sort of policy we use when deciding
+# whether or not to trust an image that we've pulled. It is not recommended that
+# this option be used, as the default behavior of using the system-wide default
+# policy (i.e., /etc/containers/policy.json) is most often preferred. Please
+# refer to containers-policy.json(5) for more details.
+signature_policy = ""
+
+# Controls how image volumes are handled. The valid values are mkdir, bind and
+# ignore; the latter will ignore volumes entirely.
+image_volumes = "mkdir"
+
+# List of registries to be used when pulling an unqualified image (e.g.,
+# "alpine:latest"). By default, registries is set to "docker.io" for
+# compatibility reasons. Depending on your workload and usecase you may add more
+# registries (e.g., "quay.io", "registry.fedoraproject.org",
+# "registry.opensuse.org", etc.).
+registries = [
+	"docker.io"
+]
+
+
+# The crio.network table containers settings pertaining to the management of
+# CNI plugins.
+[crio.network]
+
+# Path to the directory where CNI configuration files are located.
+network_dir = "/etc/cni/net.d/"
+
+# Paths to directories where CNI plugin binaries are located.
+plugin_dirs = [
+	"/opt/cni/bin/",
+]
+`
+)
+
+// generateCRIOConfig sets up /etc/crio/crio.conf
+func generateCRIOConfig(cr CommandRunner, imageRepository string) error {
+	cPath := crioConfigFile
+	t, err := template.New("crio.conf").Parse(crioConfigTemplate)
+	if err != nil {
+		return err
+	}
+	pauseImage := images.Pause(imageRepository)
+	opts := struct{ PodInfraContainerImage string }{PodInfraContainerImage: pauseImage}
+	var b bytes.Buffer
+	if err := t.Execute(&b, opts); err != nil {
+		return err
+	}
+
+	c := exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo mkdir -p %s && printf %%s \"%s\" | base64 -d | sudo tee %s", path.Dir(cPath), base64.StdEncoding.EncodeToString(b.Bytes()), cPath))
+	if _, err := cr.RunCmd(c); err != nil {
+		return errors.Wrap(err, "generateCRIOConfig.")
+	}
+	return nil
+}
+
 // Name is a human readable name for CRIO
 func (r *CRIO) Name() string {
 	return "CRI-O"

From 723f1f941bd0e00606f14130e353b0f1f8957b63 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= <anders.f.bjorklund@gmail.com>
Date: Mon, 6 Jan 2020 15:19:57 +0100
Subject: [PATCH 032/250] Update the crio.conf instead of overwriting it

We are only changing one config value, anyway
---
 pkg/minikube/cruntime/crio.go | 312 +---------------------------------
 1 file changed, 2 insertions(+), 310 deletions(-)

diff --git a/pkg/minikube/cruntime/crio.go b/pkg/minikube/cruntime/crio.go
index 346d790709..13ac34cf00 100644
--- a/pkg/minikube/cruntime/crio.go
+++ b/pkg/minikube/cruntime/crio.go
@@ -17,13 +17,9 @@ limitations under the License.
 package cruntime
 
 import (
-	"bytes"
-	"encoding/base64"
 	"fmt"
 	"os/exec"
-	"path"
 	"strings"
-	"text/template"
 
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
@@ -41,319 +37,15 @@ type CRIO struct {
 
 const (
 	// CRIOConfFile is the path to the CRI-O configuration
-	crioConfigFile     = "/etc/crio/crio.conf"
-	crioConfigTemplate = `# The CRI-O configuration file specifies all of the available configuration
-# options and command-line flags for the crio(8) OCI Kubernetes Container Runtime
-# daemon, but in a TOML format that can be more easily modified and versioned.
-#
-# Please refer to crio.conf(5) for details of all configuration options.
-
-# CRI-O supports partial configuration reload during runtime, which can be
-# done by sending SIGHUP to the running process. Currently supported options
-# are explicitly mentioned with: 'This option supports live configuration
-# reload'.
-
-# CRI-O reads its storage defaults from the containers-storage.conf(5) file
-# located at /etc/containers/storage.conf. Modify this storage configuration if
-# you want to change the system's defaults. If you want to modify storage just
-# for CRI-O, you can change the storage configuration options here.
-[crio]
-
-# Path to the "root directory". CRI-O stores all of its data, including
-# containers images, in this directory.
-root = "/var/lib/containers/storage"
-
-# Path to the "run directory". CRI-O stores all of its state in this directory.
-runroot = "/var/run/containers/storage"
-
-# Storage driver used to manage the storage of images and containers. Please
-# refer to containers-storage.conf(5) to see all available storage drivers.
-storage_driver = "overlay"
-
-# List to pass options to the storage driver. Please refer to
-# containers-storage.conf(5) to see all available storage options.
-#storage_option = [
-#]
-
-# If set to false, in-memory locking will be used instead of file-based locking.
-# **Deprecated** this option will be removed in the future.
-file_locking = false
-
-# Path to the lock file.
-# **Deprecated** this option will be removed in the future.
-file_locking_path = "/run/crio.lock"
-
-
-# The crio.api table contains settings for the kubelet/gRPC interface.
-[crio.api]
-
-# Path to AF_LOCAL socket on which CRI-O will listen.
-listen = "/var/run/crio/crio.sock"
-
-# IP address on which the stream server will listen.
-stream_address = "127.0.0.1"
-
-# The port on which the stream server will listen.
-stream_port = "0"
-
-# Enable encrypted TLS transport of the stream server.
-stream_enable_tls = false
-
-# Path to the x509 certificate file used to serve the encrypted stream. This
-# file can change, and CRI-O will automatically pick up the changes within 5
-# minutes.
-stream_tls_cert = ""
-
-# Path to the key file used to serve the encrypted stream. This file can
-# change, and CRI-O will automatically pick up the changes within 5 minutes.
-stream_tls_key = ""
-
-# Path to the x509 CA(s) file used to verify and authenticate client
-# communication with the encrypted stream. This file can change, and CRI-O will
-# automatically pick up the changes within 5 minutes.
-stream_tls_ca = ""
-
-# Maximum grpc send message size in bytes. If not set or <=0, then CRI-O will default to 16 * 1024 * 1024.
-grpc_max_send_msg_size = 16777216
-
-# Maximum grpc receive message size. If not set or <= 0, then CRI-O will default to 16 * 1024 * 1024.
-grpc_max_recv_msg_size = 16777216
-
-# The crio.runtime table contains settings pertaining to the OCI runtime used
-# and options for how to set up and manage the OCI runtime.
-[crio.runtime]
-
-# A list of ulimits to be set in containers by default, specified as
-# "<ulimit name>=<soft limit>:<hard limit>", for example:
-# "nofile=1024:2048"
-# If nothing is set here, settings will be inherited from the CRI-O daemon
-#default_ulimits = [
-#]
-
-# default_runtime is the _name_ of the OCI runtime to be used as the default.
-# The name is matched against the runtimes map below.
-default_runtime = "runc"
-
-# If true, the runtime will not use pivot_root, but instead use MS_MOVE.
-no_pivot = true
-
-# Path to the conmon binary, used for monitoring the OCI runtime.
-conmon = "/usr/libexec/crio/conmon"
-
-# Cgroup setting for conmon
-conmon_cgroup = "pod"
-
-# Environment variable list for the conmon process, used for passing necessary
-# environment variables to conmon or the runtime.
-conmon_env = [
-	"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
-]
-
-# If true, SELinux will be used for pod separation on the host.
-selinux = false
-
-# Path to the seccomp.json profile which is used as the default seccomp profile
-# for the runtime. If not specified, then the internal default seccomp profile
-# will be used.
-seccomp_profile = ""
-
-# Used to change the name of the default AppArmor profile of CRI-O. The default
-# profile name is "crio-default-" followed by the version string of CRI-O.
-apparmor_profile = "crio-default"
-
-# Cgroup management implementation used for the runtime.
-cgroup_manager = "cgroupfs"
-
-# List of default capabilities for containers. If it is empty or commented out,
-# only the capabilities defined in the containers json file by the user/kube
-# will be added.
-default_capabilities = [
-	"CHOWN",
-	"DAC_OVERRIDE",
-	"FSETID",
-	"FOWNER",
-	"NET_RAW",
-	"SETGID",
-	"SETUID",
-	"SETPCAP",
-	"NET_BIND_SERVICE",
-	"SYS_CHROOT",
-	"KILL",
-]
-
-# List of default sysctls. If it is empty or commented out, only the sysctls
-# defined in the container json file by the user/kube will be added.
-default_sysctls = [
-]
-
-# List of additional devices. specified as
-# "<device-on-host>:<device-on-container>:<permissions>", for example: "--device=/dev/sdc:/dev/xvdc:rwm".
-#If it is empty or commented out, only the devices
-# defined in the container json file by the user/kube will be added.
-additional_devices = [
-]
-
-# Path to OCI hooks directories for automatically executed hooks.
-hooks_dir = [
-]
-
-# List of default mounts for each container. **Deprecated:** this option will
-# be removed in future versions in favor of default_mounts_file.
-default_mounts = [
-]
-
-# Path to the file specifying the defaults mounts for each container. The
-# format of the config is /SRC:/DST, one mount per line. Notice that CRI-O reads
-# its default mounts from the following two files:
-#
-#   1) /etc/containers/mounts.conf (i.e., default_mounts_file): This is the
-#      override file, where users can either add in their own default mounts, or
-#      override the default mounts shipped with the package.
-#
-#   2) /usr/share/containers/mounts.conf: This is the default file read for
-#      mounts. If you want CRI-O to read from a different, specific mounts file,
-#      you can change the default_mounts_file. Note, if this is done, CRI-O will
-#      only add mounts it finds in this file.
-#
-#default_mounts_file = ""
-
-# Maximum number of processes allowed in a container.
-pids_limit = 1024
-
-# Maximum sized allowed for the container log file. Negative numbers indicate
-# that no size limit is imposed. If it is positive, it must be >= 8192 to
-# match/exceed conmon's read buffer. The file is truncated and re-opened so the
-# limit is never exceeded.
-log_size_max = -1
-
-# Whether container output should be logged to journald in addition to the kuberentes log file
-log_to_journald = false
-
-# Path to directory in which container exit files are written to by conmon.
-container_exits_dir = "/var/run/crio/exits"
-
-# Path to directory for container attach sockets.
-container_attach_socket_dir = "/var/run/crio"
-
-# If set to true, all containers will run in read-only mode.
-read_only = false
-
-# Changes the verbosity of the logs based on the level it is set to. Options
-# are fatal, panic, error, warn, info, and debug. This option supports live
-# configuration reload.
-log_level = "error"
-
-# The default log directory where all logs will go unless directly specified by the kubelet
-log_dir = "/var/log/crio/pods"
-
-# The UID mappings for the user namespace of each container. A range is
-# specified in the form containerUID:HostUID:Size. Multiple ranges must be
-# separated by comma.
-uid_mappings = ""
-
-# The GID mappings for the user namespace of each container. A range is
-# specified in the form containerGID:HostGID:Size. Multiple ranges must be
-# separated by comma.
-gid_mappings = ""
-
-# The minimal amount of time in seconds to wait before issuing a timeout
-# regarding the proper termination of the container.
-ctr_stop_timeout = 0
-
-# ManageNetworkNSLifecycle determines whether we pin and remove network namespace
-# and manage its lifecycle.
-manage_network_ns_lifecycle = false
-
-# The "crio.runtime.runtimes" table defines a list of OCI compatible runtimes.
-# The runtime to use is picked based on the runtime_handler provided by the CRI.
-# If no runtime_handler is provided, the runtime will be picked based on the level
-# of trust of the workload.
-
-[crio.runtime.runtimes.runc]
-runtime_path = "/usr/bin/runc"
-runtime_type = "oci"
-runtime_root = "/run/runc"
-
-
-# The crio.image table contains settings pertaining to the management of OCI images.
-#
-# CRI-O reads its configured registries defaults from the system wide
-# containers-registries.conf(5) located in /etc/containers/registries.conf. If
-# you want to modify just CRI-O, you can change the registries configuration in
-# this file. Otherwise, leave insecure_registries and registries commented out to
-# use the system's defaults from /etc/containers/registries.conf.
-[crio.image]
-
-# Default transport for pulling images from a remote container storage.
-default_transport = "docker://"
-
-# The path to a file containing credentials necessary for pulling images from
-# secure registries. The file is similar to that of /var/lib/kubelet/config.json
-global_auth_file = ""
-
-# The image used to instantiate infra containers.
-# This option supports live configuration reload.
-pause_image = "{{ .PodInfraContainerImage }}"
-
-# The path to a file containing credentials specific for pulling the pause_image from
-# above. The file is similar to that of /var/lib/kubelet/config.json
-# This option supports live configuration reload.
-pause_image_auth_file = ""
-
-# The command to run to have a container stay in the paused state.
-# This option supports live configuration reload.
-pause_command = "/pause"
-
-# Path to the file which decides what sort of policy we use when deciding
-# whether or not to trust an image that we've pulled. It is not recommended that
-# this option be used, as the default behavior of using the system-wide default
-# policy (i.e., /etc/containers/policy.json) is most often preferred. Please
-# refer to containers-policy.json(5) for more details.
-signature_policy = ""
-
-# Controls how image volumes are handled. The valid values are mkdir, bind and
-# ignore; the latter will ignore volumes entirely.
-image_volumes = "mkdir"
-
-# List of registries to be used when pulling an unqualified image (e.g.,
-# "alpine:latest"). By default, registries is set to "docker.io" for
-# compatibility reasons. Depending on your workload and usecase you may add more
-# registries (e.g., "quay.io", "registry.fedoraproject.org",
-# "registry.opensuse.org", etc.).
-registries = [
-	"docker.io"
-]
-
-
-# The crio.network table containers settings pertaining to the management of
-# CNI plugins.
-[crio.network]
-
-# Path to the directory where CNI configuration files are located.
-network_dir = "/etc/cni/net.d/"
-
-# Paths to directories where CNI plugin binaries are located.
-plugin_dirs = [
-	"/opt/cni/bin/",
-]
-`
+	crioConfigFile = "/etc/crio/crio.conf"
 )
 
 // generateCRIOConfig sets up /etc/crio/crio.conf
 func generateCRIOConfig(cr CommandRunner, imageRepository string) error {
 	cPath := crioConfigFile
-	t, err := template.New("crio.conf").Parse(crioConfigTemplate)
-	if err != nil {
-		return err
-	}
 	pauseImage := images.Pause(imageRepository)
-	opts := struct{ PodInfraContainerImage string }{PodInfraContainerImage: pauseImage}
-	var b bytes.Buffer
-	if err := t.Execute(&b, opts); err != nil {
-		return err
-	}
 
-	c := exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo mkdir -p %s && printf %%s \"%s\" | base64 -d | sudo tee %s", path.Dir(cPath), base64.StdEncoding.EncodeToString(b.Bytes()), cPath))
+	c := exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo sed -e 's|^pause_image = .*$|pause_image = \"%s\"|' -i %s", pauseImage, cPath))
 	if _, err := cr.RunCmd(c); err != nil {
 		return errors.Wrap(err, "generateCRIOConfig.")
 	}

From d3ff888743aa2a56a6a1fa0ab9299e94d77172a7 Mon Sep 17 00:00:00 2001
From: Zhongcheng Lao <Zhongcheng.Lao@microsoft.com>
Date: Wed, 8 Jan 2020 14:30:40 +0800
Subject: [PATCH 033/250] Use profile name as the node name

Signed-off-by: Zhongcheng Lao <Zhongcheng.Lao@microsoft.com>
---
 cmd/minikube/cmd/start.go | 24 ++++++++----------------
 1 file changed, 8 insertions(+), 16 deletions(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 064f97627d..79aa60c34d 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -194,7 +194,6 @@ func initKubernetesFlags() {
 	startCmd.Flags().String(apiServerName, constants.APIServerName, "The apiserver name which is used in the generated certificate for kubernetes.  This can be used if you want to make the apiserver available from outside the machine")
 	startCmd.Flags().StringArrayVar(&apiServerNames, "apiserver-names", nil, "A set of apiserver names which are used in the generated certificate for kubernetes.  This can be used if you want to make the apiserver available from outside the machine")
 	startCmd.Flags().IPSliceVar(&apiServerIPs, "apiserver-ips", nil, "A set of apiserver IP Addresses which are used in the generated certificate for kubernetes.  This can be used if you want to make the apiserver available from outside the machine")
-	startCmd.Flags().String(nodeName, "", "The node name")
 }
 
 // initDriverFlags inits the commandline flags for vm drivers
@@ -888,22 +887,15 @@ func generateCfgFromFlags(cmd *cobra.Command, k8sVersion string, drvName string)
 		out.T(out.SuccessType, "Using image repository {{.name}}", out.V{"name": repository})
 	}
 
-	kubeNodeName := viper.GetString(nodeName)
-	if kubeNodeName == "" {
-		if drvName == driver.None {
-			// set the node name the same as host name for none driver
-			hostname, _ := os.Hostname()
-			kubeNodeName = hostname
-		}
+	var kubeNodeName string
+	if drvName == driver.None {
+		// set the node name the same as host name for none driver
+		hostname, _ := os.Hostname()
+		kubeNodeName = hostname
+	}
 
-		if kubeNodeName == "" {
-			machineName := viper.GetString(cfg.MachineProfile)
-			if machineName != constants.DefaultMachineName {
-				kubeNodeName = fmt.Sprintf("%s-%s", constants.DefaultNodeName, machineName)
-			} else {
-				kubeNodeName = constants.DefaultNodeName
-			}
-		}
+	if kubeNodeName == "" {
+		kubeNodeName = viper.GetString(cfg.MachineProfile)
 	}
 
 	cfg := cfg.MachineConfig{

From be7b52791ab4dfb32a4450e74f8c75087b09f3f4 Mon Sep 17 00:00:00 2001
From: Zhongcheng Lao <Zhongcheng.Lao@microsoft.com>
Date: Wed, 8 Jan 2020 15:40:48 +0800
Subject: [PATCH 034/250] Remove unused var

Signed-off-by: Zhongcheng Lao <Zhongcheng.Lao@microsoft.com>
---
 cmd/minikube/cmd/start.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index f528acd3f2..b72650dfa2 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -96,7 +96,6 @@ const (
 	keepContext           = "keep-context"
 	createMount           = "mount"
 	featureGates          = "feature-gates"
-	nodeName              = "node-name"
 	apiServerName         = "apiserver-name"
 	apiServerPort         = "apiserver-port"
 	dnsDomain             = "dns-domain"

From 4a06fde1cfeabdb3a492d0d303a04814dea1c52d Mon Sep 17 00:00:00 2001
From: tstromberg <tstromberg@google.com>
Date: Wed, 8 Jan 2020 10:02:11 -0800
Subject: [PATCH 035/250] Check for nil ref and img before passing them into
 go-containerregistry

---
 pkg/minikube/image/cache.go | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/pkg/minikube/image/cache.go b/pkg/minikube/image/cache.go
index 93ac59e59e..2c5aef26fe 100644
--- a/pkg/minikube/image/cache.go
+++ b/pkg/minikube/image/cache.go
@@ -72,10 +72,10 @@ func SaveToDir(images []string, cacheDir string) error {
 }
 
 // saveToTarFile caches an image
-func saveToTarFile(image, dst string) error {
+func saveToTarFile(iname, dst string) error {
 	start := time.Now()
 	defer func() {
-		glog.Infof("cache image %q -> %s to local destination -> %q", image, dst, time.Since(start))
+		glog.Infof("cache image %q -> %s to local destination -> %q", iname, dst, time.Since(start))
 	}()
 
 	if _, err := os.Stat(dst); err == nil {
@@ -92,15 +92,22 @@ func saveToTarFile(image, dst string) error {
 		return errors.Wrapf(err, "making cache image directory: %s", dst)
 	}
 
-	ref, err := name.ParseReference(image, name.WeakValidation)
+	ref, err := name.ParseReference(iname, name.WeakValidation)
 	if err != nil {
-		return errors.Wrapf(err, "parsing image ref name for %s", image)
+		return errors.Wrapf(err, "parsing image ref name for %s", iname)
+	}
+	if ref == nil {
+		return errors.Wrapf(err, "nil reference for %s", iname)
 	}
 
 	img, err := retrieveImage(ref)
 	if err != nil {
 		glog.Warningf("unable to retrieve image: %v", err)
 	}
+	if img == nil {
+		return errors.Wrapf(err, "nil image for %s", iname)
+	}
+
 	glog.Infoln("opening: ", dstPath)
 	f, err := ioutil.TempFile(filepath.Dir(dstPath), filepath.Base(dstPath)+".*.tmp")
 	if err != nil {
@@ -116,7 +123,7 @@ func saveToTarFile(image, dst string) error {
 			}
 		}
 	}()
-	tag, err := name.NewTag(image, name.WeakValidation)
+	tag, err := name.NewTag(iname, name.WeakValidation)
 	if err != nil {
 		return errors.Wrap(err, "newtag")
 	}

From 5d8e9dbfc5a88002f678e00b9a3810dbac8d1caf Mon Sep 17 00:00:00 2001
From: tstromberg <tstromberg@google.com>
Date: Wed, 8 Jan 2020 11:23:15 -0800
Subject: [PATCH 036/250] Support --force for overriding the ssh check

---
 cmd/minikube/cmd/start.go | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index d8b8b59496..7143674c67 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -1097,19 +1097,23 @@ func trySSH(h *host.Host, ip string) {
 	}
 
 	if err := retry.Expo(dial, time.Second, 13*time.Second); err != nil {
-		exit.WithCodeT(exit.IO, `minikube is unable to connect to the VM: {{.error}}
+		if !viper.GetBool(force) {
+			exit.WithCodeT(exit.IO, `minikube is unable to connect to the VM: {{.error}}
 
-This is likely due to one of two reasons:
+	This is likely due to one of two reasons:
 
-- VPN or firewall interference
-- {{.hypervisor}} network configuration issue
+	- VPN or firewall interference
+	- {{.hypervisor}} network configuration issue
 
-Suggested workarounds:
+	Suggested workarounds:
 
-- Disable your local VPN or firewall software
-- Configure your local VPN or firewall to allow access to {{.ip}}
-- Restart or reinstall {{.hypervisor}}
-- Use an alternative --vm-driver`, out.V{"error": err, "hypervisor": h.Driver.DriverName(), "ip": ip})
+	- Disable your local VPN or firewall software
+	- Configure your local VPN or firewall to allow access to {{.ip}}
+	- Restart or reinstall {{.hypervisor}}
+	- Use an alternative --vm-driver
+	- Use --force to override this connectivity check
+	`, out.V{"error": err, "hypervisor": h.Driver.DriverName(), "ip": ip})
+		}
 	}
 }
 

From 05128825209f020306a78d117c51c75fa1178242 Mon Sep 17 00:00:00 2001
From: Zhongcheng Lao <Zhongcheng.Lao@microsoft.com>
Date: Thu, 9 Jan 2020 19:51:29 +0800
Subject: [PATCH 037/250] Do not set host-override for kubelet by default

Signed-off-by: Zhongcheng Lao <Zhongcheng.Lao@microsoft.com>
---
 cmd/minikube/cmd/start.go                    | 8 +-------
 pkg/minikube/bootstrapper/bsutil/versions.go | 2 --
 2 files changed, 1 insertion(+), 9 deletions(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index b72650dfa2..0a2080b2db 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -908,13 +908,7 @@ func generateCfgFromFlags(cmd *cobra.Command, k8sVersion string, drvName string)
 	}
 
 	var kubeNodeName string
-	if drvName == driver.None {
-		// set the node name the same as host name for none driver
-		hostname, _ := os.Hostname()
-		kubeNodeName = hostname
-	}
-
-	if kubeNodeName == "" {
+	if drvName != driver.None {
 		kubeNodeName = viper.GetString(cfg.MachineProfile)
 	}
 
diff --git a/pkg/minikube/bootstrapper/bsutil/versions.go b/pkg/minikube/bootstrapper/bsutil/versions.go
index 39644abced..bb6445ae44 100644
--- a/pkg/minikube/bootstrapper/bsutil/versions.go
+++ b/pkg/minikube/bootstrapper/bsutil/versions.go
@@ -23,7 +23,6 @@ import (
 	"github.com/blang/semver"
 	"github.com/pkg/errors"
 	"k8s.io/minikube/pkg/minikube/config"
-	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/vmpath"
 	"k8s.io/minikube/pkg/util"
 )
@@ -71,7 +70,6 @@ var versionSpecificOpts = []config.VersionedExtraOption{
 		},
 		LessThanOrEqual: semver.MustParse("1.9.10"),
 	},
-	config.NewUnversionedOption(Kubelet, "hostname-override", constants.DefaultNodeName),
 
 	// System pods args
 	config.NewUnversionedOption(Kubelet, "pod-manifest-path", vmpath.GuestManifestsDir),

From 4a1f33e2823f4231e4bc9be29f7c4a1d29bcdcd1 Mon Sep 17 00:00:00 2001
From: Zhongcheng Lao <Zhongcheng.Lao@microsoft.com>
Date: Thu, 9 Jan 2020 16:46:07 +0800
Subject: [PATCH 038/250] Adds cluster name to kubenetes config

Signed-off-by: Zhongcheng Lao <Zhongcheng.Lao@microsoft.com>
---
 cmd/minikube/cmd/start.go                             | 1 +
 pkg/minikube/bootstrapper/bsutil/kubeadm.go           | 2 ++
 pkg/minikube/bootstrapper/bsutil/kubeadm_test.go      | 2 ++
 pkg/minikube/bootstrapper/bsutil/template/template.go | 4 ++--
 pkg/minikube/config/types.go                          | 1 +
 5 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 0a2080b2db..71db6a9208 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -947,6 +947,7 @@ func generateCfgFromFlags(cmd *cobra.Command, k8sVersion string, drvName string)
 		KubernetesConfig: cfg.KubernetesConfig{
 			KubernetesVersion:      k8sVersion,
 			NodePort:               viper.GetInt(apiServerPort),
+			ClusterName:            viper.GetString(cfg.MachineProfile),
 			NodeName:               kubeNodeName,
 			APIServerName:          viper.GetString(apiServerName),
 			APIServerNames:         apiServerNames,
diff --git a/pkg/minikube/bootstrapper/bsutil/kubeadm.go b/pkg/minikube/bootstrapper/bsutil/kubeadm.go
index db580f3622..c5aae34eeb 100644
--- a/pkg/minikube/bootstrapper/bsutil/kubeadm.go
+++ b/pkg/minikube/bootstrapper/bsutil/kubeadm.go
@@ -67,6 +67,7 @@ func GenerateKubeadmYAML(k8s config.KubernetesConfig, r cruntime.Manager) ([]byt
 		APIServerPort     int
 		KubernetesVersion string
 		EtcdDataDir       string
+		ClusterName       string
 		NodeName          string
 		DNSDomain         string
 		CRISocket         string
@@ -82,6 +83,7 @@ func GenerateKubeadmYAML(k8s config.KubernetesConfig, r cruntime.Manager) ([]byt
 		APIServerPort:     nodePort,
 		KubernetesVersion: k8s.KubernetesVersion,
 		EtcdDataDir:       EtcdDataDir(),
+		ClusterName:       k8s.ClusterName,
 		NodeName:          k8s.NodeName,
 		CRISocket:         r.SocketPath(),
 		ImageRepository:   k8s.ImageRepository,
diff --git a/pkg/minikube/bootstrapper/bsutil/kubeadm_test.go b/pkg/minikube/bootstrapper/bsutil/kubeadm_test.go
index c0b90a2b4d..8894e9ba52 100644
--- a/pkg/minikube/bootstrapper/bsutil/kubeadm_test.go
+++ b/pkg/minikube/bootstrapper/bsutil/kubeadm_test.go
@@ -121,6 +121,7 @@ func TestGenerateKubeadmYAMLDNS(t *testing.T) {
 				cfg := tc.cfg
 				cfg.NodeIP = "1.1.1.1"
 				cfg.NodeName = "mk"
+				cfg.ClusterName = "kubernetes"
 				cfg.KubernetesVersion = version + ".0"
 
 				got, err := GenerateKubeadmYAML(cfg, runtime)
@@ -189,6 +190,7 @@ func TestGenerateKubeadmYAML(t *testing.T) {
 				cfg := tc.cfg
 				cfg.NodeIP = "1.1.1.1"
 				cfg.NodeName = "mk"
+				cfg.ClusterName = "kubernetes"
 				cfg.KubernetesVersion = version + ".0"
 
 				got, err := GenerateKubeadmYAML(cfg, runtime)
diff --git a/pkg/minikube/bootstrapper/bsutil/template/template.go b/pkg/minikube/bootstrapper/bsutil/template/template.go
index 92ac7975ae..5cb617f6d8 100644
--- a/pkg/minikube/bootstrapper/bsutil/template/template.go
+++ b/pkg/minikube/bootstrapper/bsutil/template/template.go
@@ -77,7 +77,7 @@ kind: ClusterConfiguration
   {{$i}}: {{$val}}{{end}}
 {{end -}}
 certificatesDir: {{.CertDir}}
-clusterName: kubernetes
+clusterName: {{.ClusterName}}
 controlPlaneEndpoint: localhost:{{.APIServerPort}}
 etcd:
   local:
@@ -129,7 +129,7 @@ kind: ClusterConfiguration
 {{range $i, $val := .FeatureArgs}}{{$i}}: {{$val}}
 {{end -}}{{end -}}
 certificatesDir: {{.CertDir}}
-clusterName: kubernetes
+clusterName: {{.ClusterName}}
 controlPlaneEndpoint: localhost:{{.APIServerPort}}
 dns:
   type: CoreDNS
diff --git a/pkg/minikube/config/types.go b/pkg/minikube/config/types.go
index 79e7f18691..b6863022e4 100644
--- a/pkg/minikube/config/types.go
+++ b/pkg/minikube/config/types.go
@@ -73,6 +73,7 @@ type KubernetesConfig struct {
 	KubernetesVersion string
 	NodeIP            string
 	NodePort          int
+	ClusterName       string
 	NodeName          string
 	APIServerName     string
 	APIServerNames    []string

From 3207b27c11ca8ad5c27d63773ee39ff57d86d3d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= <anders.f.bjorklund@gmail.com>
Date: Thu, 9 Jan 2020 19:47:26 +0100
Subject: [PATCH 039/250] Fix syntax of minikube kubectl example command

Need to terminate the minikube flags with --,
before starting with the kubectl command flags.
---
 cmd/minikube/cmd/kubectl.go                        | 2 +-
 site/content/en/docs/Reference/Commands/kubectl.md | 2 +-
 translations/de.json                               | 2 +-
 translations/es.json                               | 2 +-
 translations/fr.json                               | 2 +-
 translations/ja.json                               | 2 +-
 translations/pl.json                               | 2 +-
 translations/zh-CN.json                            | 2 +-
 8 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/cmd/minikube/cmd/kubectl.go b/cmd/minikube/cmd/kubectl.go
index 715e414cfe..e243e955de 100644
--- a/cmd/minikube/cmd/kubectl.go
+++ b/cmd/minikube/cmd/kubectl.go
@@ -41,7 +41,7 @@ var kubectlCmd = &cobra.Command{
 	Long: `Run the kubernetes client, download it if necessary.
 Examples:
 minikube kubectl -- --help
-kubectl get pods --namespace kube-system`,
+minikube kubectl -- get pods --namespace kube-system`,
 	Run: func(cmd *cobra.Command, args []string) {
 		api, err := machine.NewAPIClient()
 		if err != nil {
diff --git a/site/content/en/docs/Reference/Commands/kubectl.md b/site/content/en/docs/Reference/Commands/kubectl.md
index 54b8747362..58fdf71d47 100644
--- a/site/content/en/docs/Reference/Commands/kubectl.md
+++ b/site/content/en/docs/Reference/Commands/kubectl.md
@@ -22,7 +22,7 @@ minikube kubectl [flags]
 
 ```
 minikube kubectl -- --help
-kubectl get pods --namespace kube-system
+minikube kubectl -- get pods --namespace kube-system
 ```
 
 
diff --git a/translations/de.json b/translations/de.json
index 93f34176cf..fb60ca8e70 100644
--- a/translations/de.json
+++ b/translations/de.json
@@ -290,7 +290,7 @@
 	"Run 'minikube delete' to delete the stale VM": "",
 	"Run kubectl": "",
 	"Run minikube from the C: drive.": "",
-	"Run the kubernetes client, download it if necessary.\nExamples:\nminikube kubectl -- --help\nkubectl get pods --namespace kube-system": "",
+	"Run the kubernetes client, download it if necessary.\nExamples:\nminikube kubectl -- --help\nminikube kubectl -- get pods --namespace kube-system": "",
 	"Run the minikube command as an Administrator": "",
 	"Running on localhost (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...": "",
 	"Set failed": "",
diff --git a/translations/es.json b/translations/es.json
index e1c17cc7cb..b292bf46f7 100644
--- a/translations/es.json
+++ b/translations/es.json
@@ -290,7 +290,7 @@
 	"Run 'minikube delete' to delete the stale VM": "",
 	"Run kubectl": "",
 	"Run minikube from the C: drive.": "",
-	"Run the kubernetes client, download it if necessary.\nExamples:\nminikube kubectl -- --help\nkubectl get pods --namespace kube-system": "",
+	"Run the kubernetes client, download it if necessary.\nExamples:\nminikube kubectl -- --help\nminikube kubectl -- get pods --namespace kube-system": "",
 	"Run the minikube command as an Administrator": "",
 	"Running on localhost (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...": "",
 	"Set failed": "",
diff --git a/translations/fr.json b/translations/fr.json
index 83c5a9ec0c..442bd7bba8 100644
--- a/translations/fr.json
+++ b/translations/fr.json
@@ -291,7 +291,7 @@
 	"Run 'minikube delete' to delete the stale VM": "",
 	"Run kubectl": "",
 	"Run minikube from the C: drive.": "",
-	"Run the kubernetes client, download it if necessary.\nExamples:\nminikube kubectl -- --help\nkubectl get pods --namespace kube-system": "",
+	"Run the kubernetes client, download it if necessary.\nExamples:\nminikube kubectl -- --help\nminikube kubectl -- get pods --namespace kube-system": "",
 	"Run the minikube command as an Administrator": "",
 	"Running on localhost (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...": "",
 	"Set failed": "",
diff --git a/translations/ja.json b/translations/ja.json
index a63609e260..a32aea8462 100644
--- a/translations/ja.json
+++ b/translations/ja.json
@@ -290,7 +290,7 @@
 	"Run 'minikube delete' to delete the stale VM": "",
 	"Run kubectl": "",
 	"Run minikube from the C: drive.": "",
-	"Run the kubernetes client, download it if necessary.\nExamples:\nminikube kubectl -- --help\nkubectl get pods --namespace kube-system": "",
+	"Run the kubernetes client, download it if necessary.\nExamples:\nminikube kubectl -- --help\nminikube kubectl -- get pods --namespace kube-system": "",
 	"Run the minikube command as an Administrator": "",
 	"Running on localhost (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...": "",
 	"Set failed": "",
diff --git a/translations/pl.json b/translations/pl.json
index d5a35d30ca..b9e15b85d8 100644
--- a/translations/pl.json
+++ b/translations/pl.json
@@ -290,7 +290,7 @@
 	"Run 'minikube delete' to delete the stale VM": "",
 	"Run kubectl": "Uruchamia kubectl",
 	"Run minikube from the C: drive.": "",
-	"Run the kubernetes client, download it if necessary.\nExamples:\nminikube kubectl -- --help\nkubectl get pods --namespace kube-system": "",
+	"Run the kubernetes client, download it if necessary.\nExamples:\nminikube kubectl -- --help\nminikube kubectl -- get pods --namespace kube-system": "",
 	"Run the minikube command as an Administrator": "",
 	"Running on localhost (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...": "",
 	"Set failed": "",
diff --git a/translations/zh-CN.json b/translations/zh-CN.json
index df885715ab..d5dd541f8a 100644
--- a/translations/zh-CN.json
+++ b/translations/zh-CN.json
@@ -327,7 +327,7 @@
 	"Run 'kubectl describe pod coredns -n kube-system' and check for a firewall or DNS conflict": "",
 	"Run 'minikube delete' to delete the stale VM, or and ensure that minikube is running as the same user you are issuing this command with": "执行 'minikube delete' 以删除过时的虚拟机，或者确保 minikube 以与您发出此命令的用户相同的用户身份运行",
 	"Run minikube from the C: drive.": "",
-	"Run the kubernetes client, download it if necessary.\nExamples:\nminikube kubectl -- --help\nkubectl get pods --namespace kube-system": "",
+	"Run the kubernetes client, download it if necessary.\nExamples:\nminikube kubectl -- --help\nminikube kubectl -- get pods --namespace kube-system": "",
 	"Run the minikube command as an Administrator": "",
 	"Run: 'chmod 600 $HOME/.kube/config'": "执行 'chmod 600 $HOME/.kube/config'",
 	"Running on localhost (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...": "",

From f98b5b85c78cad893f65acfd06330c05c852de1b Mon Sep 17 00:00:00 2001
From: sshukun <song.shukun@fujitsu.com>
Date: Fri, 10 Jan 2020 17:35:07 +0900
Subject: [PATCH 040/250] Fix some typos and improve documents

---
 pkg/minikube/config/profile.go      | 6 +++---
 pkg/minikube/config/profile_test.go | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/pkg/minikube/config/profile.go b/pkg/minikube/config/profile.go
index 22ad511d51..2abb1ed33d 100644
--- a/pkg/minikube/config/profile.go
+++ b/pkg/minikube/config/profile.go
@@ -73,13 +73,13 @@ func ProfileExists(name string, miniHome ...string) bool {
 	return err == nil
 }
 
-// CreateEmptyProfile creates an empty profile stores in $MINIKUBE_HOME/profiles/<profilename>/config.json
+// CreateEmptyProfile creates an empty profile and stores in $MINIKUBE_HOME/profiles/<profilename>/config.json
 func CreateEmptyProfile(name string, miniHome ...string) error {
 	cfg := &MachineConfig{}
 	return CreateProfile(name, cfg, miniHome...)
 }
 
-// CreateProfile creates an profile out of the cfg and stores in $MINIKUBE_HOME/profiles/<profilename>/config.json
+// CreateProfile creates a profile out of the cfg and stores in $MINIKUBE_HOME/profiles/<profilename>/config.json
 func CreateProfile(name string, cfg *MachineConfig, miniHome ...string) error {
 	data, err := json.MarshalIndent(cfg, "", "    ")
 	if err != nil {
@@ -181,7 +181,7 @@ func profileDirs(miniHome ...string) (dirs []string, err error) {
 	return dirs, err
 }
 
-// profileFilePath returns the Minikube profile config file
+// profileFilePath returns path of profile config file
 func profileFilePath(profile string, miniHome ...string) string {
 	miniPath := localpath.MiniPath()
 	if len(miniHome) > 0 {
diff --git a/pkg/minikube/config/profile_test.go b/pkg/minikube/config/profile_test.go
index a7e436d612..db76be68c2 100644
--- a/pkg/minikube/config/profile_test.go
+++ b/pkg/minikube/config/profile_test.go
@@ -21,7 +21,7 @@ import (
 	"testing"
 )
 
-// TestListProfiles uses a different uses different MINIKUBE_HOME with rest of tests since it relies on file list index
+// TestListProfiles uses a different MINIKUBE_HOME with rest of tests since it relies on file list index
 func TestListProfiles(t *testing.T) {
 	miniDir, err := filepath.Abs("./testdata/profile/.minikube")
 	if err != nil {

From bdc47d0f31a2ba101628c895571342599dcd3442 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B8rn=20Harald=20Fotland?=
 <bfotland@users.noreply.github.com>
Date: Fri, 10 Jan 2020 10:10:15 +0100
Subject: [PATCH 041/250] Update kvm2.md

Suggested missing verb.
---
 site/content/en/docs/Reference/Drivers/kvm2.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/site/content/en/docs/Reference/Drivers/kvm2.md b/site/content/en/docs/Reference/Drivers/kvm2.md
index 066eabb4a9..df13a3f95d 100644
--- a/site/content/en/docs/Reference/Drivers/kvm2.md
+++ b/site/content/en/docs/Reference/Drivers/kvm2.md
@@ -24,7 +24,7 @@ The `minikube start` command supports 3 additional kvm specific flags:
 
 ## Issues
 
-* `minikube` will repeatedly for the root password if user is not in the correct `libvirt` group [#3467](https://github.com/kubernetes/minikube/issues/3467)
+* `minikube` will repeatedly ask for the root password if user is not in the correct `libvirt` group [#3467](https://github.com/kubernetes/minikube/issues/3467)
 * `Machine didn't return an IP after 120 seconds` when firewall prevents VM network access [#3566](https://github.com/kubernetes/minikube/issues/3566)
 * `unable to set user and group to '65534:992` when `dynamic ownership = 1` in `qemu.conf` [#4467](https://github.com/kubernetes/minikube/issues/4467)
 * KVM VM's cannot be used simultaneously with VirtualBox  [#4913](https://github.com/kubernetes/minikube/issues/4913)

From 5ba779f74d135b672edc6275c78763f3c8ff3614 Mon Sep 17 00:00:00 2001
From: tstromberg <tstromberg@google.com>
Date: Fri, 10 Jan 2020 15:10:03 -0800
Subject: [PATCH 042/250] Only grab post-mortem logs if VM is running

---
 test/integration/helpers.go | 55 +++++++++++++++++++++++--------------
 1 file changed, 34 insertions(+), 21 deletions(-)

diff --git a/test/integration/helpers.go b/test/integration/helpers.go
index 1539170f77..2649b723c3 100644
--- a/test/integration/helpers.go
+++ b/test/integration/helpers.go
@@ -34,6 +34,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/docker/machine/libmachine/state"
 	"github.com/shirou/gopsutil/process"
 	core "k8s.io/api/core/v1"
 	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -185,31 +186,43 @@ func CleanupWithLogs(t *testing.T, profile string, cancel context.CancelFunc) {
 	t.Logf("*** %s FAILED at %s", t.Name(), time.Now())
 
 	if *postMortemLogs {
-		t.Logf(">>> %s FAILED: start of post-mortem logs >>>", t.Name())
-
-		rr, rerr := Run(t, exec.Command("kubectl", "--context", profile, "get", "po", "-A", "--show-labels"))
-		if rerr != nil {
-			t.Logf("%s: %v", rr.Command(), rerr)
-		}
-		t.Logf("(dbg) %s:\n%s", rr.Command(), rr.Stdout)
-
-		rr, err := Run(t, exec.Command("kubectl", "--context", profile, "describe", "node"))
-		if err != nil {
-			t.Logf("%s: %v", rr.Command(), err)
-		} else {
-			t.Logf("(dbg) %s:\n%s", rr.Command(), rr.Stdout)
-		}
-
-		rr, err = Run(t, exec.Command(Target(), "-p", profile, "logs", "--problems"))
-		if err != nil {
-			t.Logf("failed logs error: %v", err)
-		}
-		t.Logf("%s logs: %s", t.Name(), rr.Stdout)
-		t.Logf("<<< %s FAILED: end of post-mortem logs <<<", t.Name())
+		clusterLogs(t, profile)
 	}
 	Cleanup(t, profile, cancel)
 }
 
+// clusterLogs shows logs for debugging a failed cluster
+func clusterLogs(t *testing.T, profile string) {
+	st := Status(context.Background(), t, Target(), profile)
+	if st != state.Running.String() {
+		t.Logf("%q is not running, skipping log retrieval (state=%q)", profile, st)
+		return
+	}
+
+	t.Logf("<<< %s FAILED: start of post-mortem logs <<<", t.Name())
+	rr, err := Run(t, exec.Command(Target(), "-p", profile, "logs", "--problems"))
+	if err != nil {
+		t.Logf("failed logs error: %v", err)
+		return
+	}
+	t.Logf("%s logs: %s", t.Name(), rr.Stdout)
+
+	rr, rerr := Run(t, exec.Command("kubectl", "--context", profile, "get", "po", "-A", "--show-labels"))
+	if rerr != nil {
+		t.Logf("%s: %v", rr.Command(), rerr)
+		return
+	}
+	t.Logf("(dbg) %s:\n%s", rr.Command(), rr.Stdout)
+
+	rr, err = Run(t, exec.Command("kubectl", "--context", profile, "describe", "node"))
+	if err != nil {
+		t.Logf("%s: %v", rr.Command(), err)
+	} else {
+		t.Logf("(dbg) %s:\n%s", rr.Command(), rr.Stdout)
+	}
+	t.Logf("<<< %s FAILED: end of post-mortem logs <<<", t.Name())
+}
+
 // podStatusMsg returns a human-readable pod status, for generating debug status
 func podStatusMsg(pod core.Pod) string {
 	var sb strings.Builder

From e6c684e19ebc0a42a47b65aa08df0c95aabf63a2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= <anders.f.bjorklund@gmail.com>
Date: Sun, 12 Jan 2020 15:02:15 +0100
Subject: [PATCH 043/250] Switch from MBR to GPT for the ISO partitioning

---
 deploy/iso/minikube-iso/package/automount/minikube-automount | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/deploy/iso/minikube-iso/package/automount/minikube-automount b/deploy/iso/minikube-iso/package/automount/minikube-automount
index 0327276b4e..17fc8b0279 100755
--- a/deploy/iso/minikube-iso/package/automount/minikube-automount
+++ b/deploy/iso/minikube-iso/package/automount/minikube-automount
@@ -29,7 +29,7 @@ if [ ! -n "$BOOT2DOCKER_DATA" ]; then
             # Let kernel re-read partition table
             partprobe
             # Add the data partition
-            (echo n; echo p; echo 1; echo ; echo ; echo w) | fdisk $UNPARTITIONED_HD
+            (echo g; echo n; echo 1; echo ; echo ; echo w) | fdisk $UNPARTITIONED_HD
             # Let kernel re-read partition table
             partprobe
             # wait for the partition to actually exist, timeout after about 5 seconds
@@ -57,7 +57,7 @@ if [ ! -n "$BOOT2DOCKER_DATA" ]; then
                 if [ $NON_NUL == 0 ]; then
                     # Create the partition, format it and then mount it
                     echo "NEW VMware boot2docker managed disk image ($UNPARTITIONED_HD): formatting it for use"
-                    (echo n; echo p; echo 1; echo ; echo ; echo w) | fdisk $UNPARTITIONED_HD
+                    (echo g; echo n; echo 1; echo ; echo ; echo w) | fdisk $UNPARTITIONED_HD
                     BOOT2DOCKER_DATA=`echo "${UNPARTITIONED_HD}1"`
                     mkfs.ext4 -i 2048 -L $LABEL $BOOT2DOCKER_DATA
                 else

From 9e317ac2bd1f0993769d6e48127f6e70139f729a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= <anders.f.bjorklund@gmail.com>
Date: Wed, 8 Jan 2020 23:08:29 +0100
Subject: [PATCH 044/250] Do not use an arch suffix for the coredns name

They are using multi-arch images instead...
So don't append something like -arm64 to it.

Just use e.g. "k8s.gcr.io/coredns:1.6.5",
and Docker will pull the right platform image.
---
 pkg/minikube/bootstrapper/images/images.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/minikube/bootstrapper/images/images.go b/pkg/minikube/bootstrapper/images/images.go
index 80a06f385e..508faf10f5 100644
--- a/pkg/minikube/bootstrapper/images/images.go
+++ b/pkg/minikube/bootstrapper/images/images.go
@@ -74,7 +74,7 @@ func coreDNS(v semver.Version, mirror string) string {
 	case 11:
 		cv = "1.1.3"
 	}
-	return path.Join(kubernetesRepo(mirror), "coredns"+archTag(false)+cv)
+	return path.Join(kubernetesRepo(mirror), "coredns"+":"+cv)
 }
 
 // etcd returns the image used for etcd

From 71b561e3447f1938a0abdb01255256b43fbb24dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= <anders.f.bjorklund@gmail.com>
Date: Tue, 14 Jan 2020 18:19:38 +0100
Subject: [PATCH 045/250] Add note to remember to end the minikube flags

And add some whitespace before the next section
---
 cmd/minikube/cmd/kubectl.go                        | 3 ++-
 site/content/en/docs/Reference/Commands/kubectl.md | 1 +
 translations/de.json                               | 2 +-
 translations/es.json                               | 2 +-
 translations/fr.json                               | 2 +-
 translations/ja.json                               | 2 +-
 translations/pl.json                               | 2 +-
 translations/zh-CN.json                            | 2 +-
 8 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/cmd/minikube/cmd/kubectl.go b/cmd/minikube/cmd/kubectl.go
index e243e955de..706a9703e2 100644
--- a/cmd/minikube/cmd/kubectl.go
+++ b/cmd/minikube/cmd/kubectl.go
@@ -38,7 +38,8 @@ import (
 var kubectlCmd = &cobra.Command{
 	Use:   "kubectl",
 	Short: "Run kubectl",
-	Long: `Run the kubernetes client, download it if necessary.
+	Long: `Run the kubernetes client, download it if necessary. Remember -- after kubectl!
+
 Examples:
 minikube kubectl -- --help
 minikube kubectl -- get pods --namespace kube-system`,
diff --git a/site/content/en/docs/Reference/Commands/kubectl.md b/site/content/en/docs/Reference/Commands/kubectl.md
index 58fdf71d47..cfca7ec2a4 100644
--- a/site/content/en/docs/Reference/Commands/kubectl.md
+++ b/site/content/en/docs/Reference/Commands/kubectl.md
@@ -11,6 +11,7 @@ description: >
 ### Overview
 
 Run the Kubernetes client, download it if necessary.
+Remember `--` after kubectl!
 
 ### Usage
 
diff --git a/translations/de.json b/translations/de.json
index fb60ca8e70..f56676d146 100644
--- a/translations/de.json
+++ b/translations/de.json
@@ -290,7 +290,7 @@
 	"Run 'minikube delete' to delete the stale VM": "",
 	"Run kubectl": "",
 	"Run minikube from the C: drive.": "",
-	"Run the kubernetes client, download it if necessary.\nExamples:\nminikube kubectl -- --help\nminikube kubectl -- get pods --namespace kube-system": "",
+	"Run the kubernetes client, download it if necessary. Remember -- after kubectl!\n\nExamples:\nminikube kubectl -- --help\nminikube kubectl -- get pods --namespace kube-system": "",
 	"Run the minikube command as an Administrator": "",
 	"Running on localhost (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...": "",
 	"Set failed": "",
diff --git a/translations/es.json b/translations/es.json
index b292bf46f7..bb175f5c33 100644
--- a/translations/es.json
+++ b/translations/es.json
@@ -290,7 +290,7 @@
 	"Run 'minikube delete' to delete the stale VM": "",
 	"Run kubectl": "",
 	"Run minikube from the C: drive.": "",
-	"Run the kubernetes client, download it if necessary.\nExamples:\nminikube kubectl -- --help\nminikube kubectl -- get pods --namespace kube-system": "",
+	"Run the kubernetes client, download it if necessary. Remember -- after kubectl!\n\nExamples:\nminikube kubectl -- --help\nminikube kubectl -- get pods --namespace kube-system": "",
 	"Run the minikube command as an Administrator": "",
 	"Running on localhost (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...": "",
 	"Set failed": "",
diff --git a/translations/fr.json b/translations/fr.json
index 442bd7bba8..1d760fd5a2 100644
--- a/translations/fr.json
+++ b/translations/fr.json
@@ -291,7 +291,7 @@
 	"Run 'minikube delete' to delete the stale VM": "",
 	"Run kubectl": "",
 	"Run minikube from the C: drive.": "",
-	"Run the kubernetes client, download it if necessary.\nExamples:\nminikube kubectl -- --help\nminikube kubectl -- get pods --namespace kube-system": "",
+	"Run the kubernetes client, download it if necessary. Remember -- after kubectl!\n\nExamples:\nminikube kubectl -- --help\nminikube kubectl -- get pods --namespace kube-system": "",
 	"Run the minikube command as an Administrator": "",
 	"Running on localhost (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...": "",
 	"Set failed": "",
diff --git a/translations/ja.json b/translations/ja.json
index a32aea8462..257f69a38a 100644
--- a/translations/ja.json
+++ b/translations/ja.json
@@ -290,7 +290,7 @@
 	"Run 'minikube delete' to delete the stale VM": "",
 	"Run kubectl": "",
 	"Run minikube from the C: drive.": "",
-	"Run the kubernetes client, download it if necessary.\nExamples:\nminikube kubectl -- --help\nminikube kubectl -- get pods --namespace kube-system": "",
+	"Run the kubernetes client, download it if necessary. Remember -- after kubectl!\n\nExamples:\nminikube kubectl -- --help\nminikube kubectl -- get pods --namespace kube-system": "",
 	"Run the minikube command as an Administrator": "",
 	"Running on localhost (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...": "",
 	"Set failed": "",
diff --git a/translations/pl.json b/translations/pl.json
index b9e15b85d8..41491e9b93 100644
--- a/translations/pl.json
+++ b/translations/pl.json
@@ -290,7 +290,7 @@
 	"Run 'minikube delete' to delete the stale VM": "",
 	"Run kubectl": "Uruchamia kubectl",
 	"Run minikube from the C: drive.": "",
-	"Run the kubernetes client, download it if necessary.\nExamples:\nminikube kubectl -- --help\nminikube kubectl -- get pods --namespace kube-system": "",
+	"Run the kubernetes client, download it if necessary. Remember -- after kubectl!\n\nExamples:\nminikube kubectl -- --help\nminikube kubectl -- get pods --namespace kube-system": "",
 	"Run the minikube command as an Administrator": "",
 	"Running on localhost (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...": "",
 	"Set failed": "",
diff --git a/translations/zh-CN.json b/translations/zh-CN.json
index d5dd541f8a..02b0ab7ad2 100644
--- a/translations/zh-CN.json
+++ b/translations/zh-CN.json
@@ -327,7 +327,7 @@
 	"Run 'kubectl describe pod coredns -n kube-system' and check for a firewall or DNS conflict": "",
 	"Run 'minikube delete' to delete the stale VM, or and ensure that minikube is running as the same user you are issuing this command with": "执行 'minikube delete' 以删除过时的虚拟机，或者确保 minikube 以与您发出此命令的用户相同的用户身份运行",
 	"Run minikube from the C: drive.": "",
-	"Run the kubernetes client, download it if necessary.\nExamples:\nminikube kubectl -- --help\nminikube kubectl -- get pods --namespace kube-system": "",
+	"Run the kubernetes client, download it if necessary. Remember -- after kubectl!\n\nExamples:\nminikube kubectl -- --help\nminikube kubectl -- get pods --namespace kube-system": "",
 	"Run the minikube command as an Administrator": "",
 	"Run: 'chmod 600 $HOME/.kube/config'": "执行 'chmod 600 $HOME/.kube/config'",
 	"Running on localhost (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...": "",

From 9a2de2ae7f55c0148540b802a860fb47fa2420ef Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Tue, 14 Jan 2020 10:52:27 -0800
Subject: [PATCH 046/250] Hook up new addon manager logic to existing addons
 code

---
 pkg/addons/addons.go                        |  6 +--
 pkg/{minikube => }/addons/reconcile.go      | 42 +++++++++------------
 pkg/{minikube => }/addons/reconcile_test.go |  0
 3 files changed, 21 insertions(+), 27 deletions(-)
 rename pkg/{minikube => }/addons/reconcile.go (77%)
 rename pkg/{minikube => }/addons/reconcile_test.go (100%)

diff --git a/pkg/addons/addons.go b/pkg/addons/addons.go
index 51ab37907a..184d7952ff 100644
--- a/pkg/addons/addons.go
+++ b/pkg/addons/addons.go
@@ -154,7 +154,7 @@ func enableOrDisableAddon(name, val, profile string) error {
 	}
 
 	data := assets.GenerateTemplateData(cfg.KubernetesConfig)
-	return enableOrDisableAddonInternal(addon, cmd, data, enable)
+	return enableOrDisableAddonInternal(addon, cmd, data, enable, profile)
 }
 
 func isAddonAlreadySet(addon *assets.Addon, enable bool) (bool, error) {
@@ -173,7 +173,7 @@ func isAddonAlreadySet(addon *assets.Addon, enable bool) (bool, error) {
 	return false, nil
 }
 
-func enableOrDisableAddonInternal(addon *assets.Addon, cmd command.Runner, data interface{}, enable bool) error {
+func enableOrDisableAddonInternal(addon *assets.Addon, cmd command.Runner, data interface{}, enable bool, profile string) error {
 	var err error
 
 	updateFile := cmd.Copy
@@ -196,7 +196,7 @@ func enableOrDisableAddonInternal(addon *assets.Addon, cmd command.Runner, data
 			return errors.Wrapf(err, "updating addon %s", addon.AssetName)
 		}
 	}
-	return nil
+	return reconcile(cmd, profile)
 }
 
 // enableOrDisableStorageClasses enables or disables storage classes
diff --git a/pkg/minikube/addons/reconcile.go b/pkg/addons/reconcile.go
similarity index 77%
rename from pkg/minikube/addons/reconcile.go
rename to pkg/addons/reconcile.go
index fdda4f0dec..6854e18070 100644
--- a/pkg/minikube/addons/reconcile.go
+++ b/pkg/addons/reconcile.go
@@ -17,18 +17,17 @@ limitations under the License.
 package addons
 
 import (
-	"fmt"
 	"os"
 	"os/exec"
 	"path"
 
 	"github.com/blang/semver"
 	"github.com/golang/glog"
-	"k8s.io/minikube/pkg/minikube/bootstrapper/kubeadm"
+	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil"
 	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
-	"k8s.io/minikube/pkg/minikube/exit"
 )
 
 // taken from https://github.com/kubernetes/kubernetes/blob/master/cluster/addons/addon-manager/kube-addons.sh
@@ -51,24 +50,16 @@ var kubectlPruneWhitelist = []string{
 	"extensions/v1beta1/Ingress",
 }
 
-// Reconcile runs kubectl apply -f on the addons directory
+// reconcile runs kubectl apply -f on the addons directory
 // to reconcile addons state in all running profiles
-func Reconcile(cmd command.Runner) error {
-	validProfiles, _, err := config.ListProfiles()
-
-	if len(validProfiles) == 0 || err != nil {
-		exit.UsageT("No minikube profile was found. You can create one using `minikube start`.")
+func reconcile(cmd command.Runner, profile string) error {
+	c, err := kubectlCommand(profile)
+	if err != nil {
+		return err
 	}
-
-	for _, p := range validProfiles {
-		c, err := kubectlCommand(p.Name)
-		if err != nil {
-			return err
-		}
-		if _, err := cmd.RunCmd(c); err != nil {
-			glog.Warningf("reconciling addons failed: %v", err)
-			return err
-		}
+	if _, err := cmd.RunCmd(c); err != nil {
+		glog.Warningf("reconciling addons failed: %v", err)
+		return err
 	}
 	return nil
 }
@@ -90,7 +81,11 @@ func kubectlCommand(profile string) (*exec.Cmd, error) {
 	}
 	args = append(args, "--recursive")
 
-	if ok, err := appendNamespaceFlag(v); err == nil && ok {
+	ok, err := shouldAppendNamespaceFlag(v)
+	if err != nil {
+		return nil, errors.Wrap(err, "appending namespace flag")
+	}
+	if ok {
 		args = append(args, "--namespace=kube-system")
 	}
 
@@ -112,12 +107,12 @@ func kubernetesVersion(profile string) (string, error) {
 
 // We need to append --namespace=kube-system for Kubernetes versions >=1.17
 // so that prune works as expected. See https://github.com/kubernetes/kubernetes/pull/83084/
-func appendNamespaceFlag(version string) (bool, error) {
-	v, err := kubeadm.ParseKubernetesVersion(version)
+func shouldAppendNamespaceFlag(version string) (bool, error) {
+	v, err := bsutil.ParseKubernetesVersion(version)
 	if err != nil {
 		return false, err
 	}
-	return v.GT(semver.MustParse("1.17.0")), nil
+	return v.GTE(semver.MustParse("1.17.0")), nil
 }
 
 func kubectlBinaryPath(profile string) (string, error) {
@@ -126,7 +121,6 @@ func kubectlBinaryPath(profile string) (string, error) {
 	if err != nil {
 		return "", err
 	}
-	fmt.Printf("got version %s for profile %s\n", v, profile)
 	p := path.Join("/var/lib/minikube/binaries", v, "kubectl")
 	return p, nil
 }
diff --git a/pkg/minikube/addons/reconcile_test.go b/pkg/addons/reconcile_test.go
similarity index 100%
rename from pkg/minikube/addons/reconcile_test.go
rename to pkg/addons/reconcile_test.go

From f5be457399a6a215698ccb61410891a95b32bdb2 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Tue, 14 Jan 2020 12:12:47 -0800
Subject: [PATCH 047/250] fix unit tests

---
 pkg/addons/reconcile.go      | 23 ++++++++----------
 pkg/addons/reconcile_test.go | 47 ++++++++++++++++++++++++++++++++----
 2 files changed, 52 insertions(+), 18 deletions(-)

diff --git a/pkg/addons/reconcile.go b/pkg/addons/reconcile.go
index 6854e18070..f774cb9393 100644
--- a/pkg/addons/reconcile.go
+++ b/pkg/addons/reconcile.go
@@ -30,6 +30,11 @@ import (
 	"k8s.io/minikube/pkg/minikube/constants"
 )
 
+var (
+	// For testing
+	k8sVersion = kubernetesVersion
+)
+
 // taken from https://github.com/kubernetes/kubernetes/blob/master/cluster/addons/addon-manager/kube-addons.sh
 var kubectlPruneWhitelist = []string{
 	"core/v1/ConfigMap",
@@ -65,14 +70,12 @@ func reconcile(cmd command.Runner, profile string) error {
 }
 
 func kubectlCommand(profile string) (*exec.Cmd, error) {
-	kubectlBinary, err := kubectlBinaryPath(profile)
-	if err != nil {
-		return nil, err
-	}
-	v, err := kubernetesVersion(profile)
+	v, err := k8sVersion(profile)
 	if err != nil {
 		return nil, err
 	}
+	kubectlBinary := kubectlBinaryPath(v)
+
 	// prune will delete any existing objects with the label specified by "-l" which don't appear in /etc/kubernetes/addons
 	// this is how we delete disabled addons
 	args := []string{"KUBECONFIG=/var/lib/minikube/kubeconfig", kubectlBinary, "apply", "-f", "/etc/kubernetes/addons", "-l", "kubernetes.io/cluster-service!=true,addonmanager.kubernetes.io/mode=Reconcile", "--prune=true"}
@@ -115,12 +118,6 @@ func shouldAppendNamespaceFlag(version string) (bool, error) {
 	return v.GTE(semver.MustParse("1.17.0")), nil
 }
 
-func kubectlBinaryPath(profile string) (string, error) {
-	// TODO: get this for all profiles and run in each one
-	v, err := kubernetesVersion(profile)
-	if err != nil {
-		return "", err
-	}
-	p := path.Join("/var/lib/minikube/binaries", v, "kubectl")
-	return p, nil
+func kubectlBinaryPath(version string) string {
+	return path.Join("/var/lib/minikube/binaries", version, "kubectl")
 }
diff --git a/pkg/addons/reconcile_test.go b/pkg/addons/reconcile_test.go
index ffd61b739d..d53e87e698 100644
--- a/pkg/addons/reconcile_test.go
+++ b/pkg/addons/reconcile_test.go
@@ -17,16 +17,53 @@ limitations under the License.
 package addons
 
 import (
+	"fmt"
 	"strings"
 	"testing"
 )
 
 func TestKubectlCommand(t *testing.T) {
-	expected := "sudo KUBECONFIG=/var/lib/minikube/kubeconfig /var/lib/minikube/binaries/v1.13.2/kubectl apply -f /etc/kubernetes/addons -l kubernetes.io/cluster-service!=true,addonmanager.kubernetes.io/mode=Reconcile --prune=true --prune-whitelist core/v1/ConfigMap --prune-whitelist core/v1/Endpoints --prune-whitelist core/v1/Namespace --prune-whitelist core/v1/PersistentVolumeClaim --prune-whitelist core/v1/PersistentVolume --prune-whitelist core/v1/Pod --prune-whitelist core/v1/ReplicationController --prune-whitelist core/v1/Secret --prune-whitelist core/v1/Service --prune-whitelist batch/v1/Job --prune-whitelist batch/v1beta1/CronJob --prune-whitelist apps/v1/DaemonSet --prune-whitelist apps/v1/Deployment --prune-whitelist apps/v1/ReplicaSet --prune-whitelist apps/v1/StatefulSet --prune-whitelist extensions/v1beta1/Ingress --recursive"
-	kc := kubectlCommand()
-	actual := strings.Join(kc.Args, " ")
+	expectedCommand := "sudo KUBECONFIG=/var/lib/minikube/kubeconfig /var/lib/minikube/binaries/%s/kubectl apply -f /etc/kubernetes/addons -l kubernetes.io/cluster-service!=true,addonmanager.kubernetes.io/mode=Reconcile --prune=true --prune-whitelist core/v1/ConfigMap --prune-whitelist core/v1/Endpoints --prune-whitelist core/v1/Namespace --prune-whitelist core/v1/PersistentVolumeClaim --prune-whitelist core/v1/PersistentVolume --prune-whitelist core/v1/Pod --prune-whitelist core/v1/ReplicationController --prune-whitelist core/v1/Secret --prune-whitelist core/v1/Service --prune-whitelist batch/v1/Job --prune-whitelist batch/v1beta1/CronJob --prune-whitelist apps/v1/DaemonSet --prune-whitelist apps/v1/Deployment --prune-whitelist apps/v1/ReplicaSet --prune-whitelist apps/v1/StatefulSet --prune-whitelist extensions/v1beta1/Ingress --recursive"
 
-	if actual != expected {
-		t.Fatalf("expected does not match actual\nExpected: %s\nActual: %s", expected, actual)
+	tests := []struct {
+		description string
+		k8sVersion  string
+		expected    string
+	}{
+		{
+			description: "k8s version < 1.17.0",
+			k8sVersion:  "v1.16.0",
+			expected:    expectedCommand,
+		}, {
+			description: "k8s version == 1.17.0",
+			k8sVersion:  "v1.17.0",
+			expected:    expectedCommand + " --namespace=kube-system",
+		}, {
+			description: "k8s version > 1.17.0",
+			k8sVersion:  "v1.18.0",
+			expected:    expectedCommand + " --namespace=kube-system",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.description, func(t *testing.T) {
+			originalK8sVersion := k8sVersion
+			defer func() { k8sVersion = originalK8sVersion }()
+			k8sVersion = func(_ string) (string, error) {
+				return test.k8sVersion, nil
+			}
+
+			command, err := kubectlCommand("")
+			if err != nil {
+				t.Fatalf("error getting kubectl command: %v", err)
+			}
+			actual := strings.Join(command.Args, " ")
+
+			expected := fmt.Sprintf(test.expected, test.k8sVersion)
+
+			if actual != expected {
+				t.Fatalf("expected does not match actual\nExpected: %s\nActual: %s", expected, actual)
+			}
+		})
 	}
 }

From 5b219088d2934b634f0139fe6756124a563d1457 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Tue, 14 Jan 2020 13:05:39 -0800
Subject: [PATCH 048/250] updaate download only test to include kubectl release
 binary

---
 test/integration/aaa_download_only_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/integration/aaa_download_only_test.go b/test/integration/aaa_download_only_test.go
index 0efa3e7e57..d3e43ee77f 100644
--- a/test/integration/aaa_download_only_test.go
+++ b/test/integration/aaa_download_only_test.go
@@ -82,7 +82,7 @@ func TestDownloadOnly(t *testing.T) {
 				}
 
 				// checking binaries downloaded (kubelet,kubeadm)
-				for _, bin := range constants.KubeadmBinaries {
+				for _, bin := range constants.KubernetesReleaseBinaries {
 					fp := filepath.Join(localpath.MiniPath(), "cache", v, bin)
 					_, err := os.Stat(fp)
 					if err != nil {

From 53fac089138d99a6f0c49778c616850c90edfda6 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Tue, 14 Jan 2020 13:20:58 -0800
Subject: [PATCH 049/250] fix lint

---
 test/integration/functional_test.go | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go
index acec9d6c83..a9a171569b 100644
--- a/test/integration/functional_test.go
+++ b/test/integration/functional_test.go
@@ -169,14 +169,6 @@ func validateKubectlGetPods(ctx context.Context, t *testing.T, profile string) {
 	}
 }
 
-// validateAddonManager asserts that the kube-addon-manager pod is deployed properly
-func validateAddonManager(ctx context.Context, t *testing.T, profile string) {
-	// If --wait=false, this may take a couple of minutes
-	if _, err := PodWait(ctx, t, profile, "kube-system", "component=kube-addon-manager", 10*time.Minute); err != nil {
-		t.Fatalf("wait: %v", err)
-	}
-}
-
 // validateComponentHealth asserts that all Kubernetes components are healthy
 func validateComponentHealth(ctx context.Context, t *testing.T, profile string) {
 	rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "get", "cs", "-o=json"))

From 2c301bd4efe0747f54f1f5bfc0a6be1de365332b Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Tue, 14 Jan 2020 14:25:20 -0800
Subject: [PATCH 050/250] more node changes yay

---
 cmd/minikube/cmd/root.go                  |  1 -
 cmd/minikube/cmd/start_test.go            |  2 +-
 pkg/minikube/bootstrapper/bootstrapper.go | 10 +++----
 pkg/minikube/bootstrapper/kicbs/kicbs.go  | 35 ++++++++++++-----------
 4 files changed, 25 insertions(+), 23 deletions(-)

diff --git a/cmd/minikube/cmd/root.go b/cmd/minikube/cmd/root.go
index 55edb5d48c..2a1d012b29 100644
--- a/cmd/minikube/cmd/root.go
+++ b/cmd/minikube/cmd/root.go
@@ -188,7 +188,6 @@ func init() {
 				configCmd.AddonsCmd,
 				configCmd.ConfigCmd,
 				configCmd.ProfileCmd,
-				configCmd.NodeCmd,
 				updateContextCmd,
 			},
 		},
diff --git a/cmd/minikube/cmd/start_test.go b/cmd/minikube/cmd/start_test.go
index 33de12cc13..32f130aba3 100644
--- a/cmd/minikube/cmd/start_test.go
+++ b/cmd/minikube/cmd/start_test.go
@@ -121,7 +121,7 @@ func TestGenerateCfgFromFlagsHTTPProxyHandling(t *testing.T) {
 			if err := os.Setenv("HTTP_PROXY", test.proxy); err != nil {
 				t.Fatalf("Unexpected error setting HTTP_PROXY: %v", err)
 			}
-			config, err := generateCfgFromFlags(cmd, k8sVersion, "none")
+			config, _, err := generateCfgFromFlags(cmd, k8sVersion, "none")
 			if err != nil {
 				t.Fatalf("Got unexpected error %v during config generation", err)
 			}
diff --git a/pkg/minikube/bootstrapper/bootstrapper.go b/pkg/minikube/bootstrapper/bootstrapper.go
index e11ac16cda..1c3f25b449 100644
--- a/pkg/minikube/bootstrapper/bootstrapper.go
+++ b/pkg/minikube/bootstrapper/bootstrapper.go
@@ -21,7 +21,6 @@ import (
 	"time"
 
 	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
-	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
 )
@@ -38,13 +37,13 @@ type LogOptions struct {
 type Bootstrapper interface {
 	// PullImages pulls images necessary for a cluster. Success should not be required.
 	PullImages(config.KubernetesConfig) error
-	StartCluster(config.KubernetesConfig) error
+	StartCluster(config.MachineConfig) error
 	UpdateCluster(config.MachineConfig) error
 	DeleteCluster(config.KubernetesConfig) error
-	WaitForCluster(config.KubernetesConfig, time.Duration) error
+	WaitForCluster(config.MachineConfig, time.Duration) error
 	// LogCommands returns a map of log type to a command which will display that log.
 	LogCommands(LogOptions) map[string]string
-	SetupCerts(command.Runner, config.KubernetesConfig, config.Node) error
+	SetupCerts(config.KubernetesConfig, config.Node) error
 	GetKubeletStatus() (string, error)
 	GetAPIServerStatus(net.IP, int) (string, error)
 }
@@ -52,7 +51,8 @@ type Bootstrapper interface {
 const (
 	// Kubeadm is the kubeadm bootstrapper type
 	Kubeadm = "kubeadm"
-	KIC     = "kic"
+	// KIC is the kic bootstrapper type
+	KIC = "kic"
 )
 
 // GetCachedBinaryList returns the list of binaries
diff --git a/pkg/minikube/bootstrapper/kicbs/kicbs.go b/pkg/minikube/bootstrapper/kicbs/kicbs.go
index 3c7d2ddd6a..f9fb484a30 100644
--- a/pkg/minikube/bootstrapper/kicbs/kicbs.go
+++ b/pkg/minikube/bootstrapper/kicbs/kicbs.go
@@ -132,8 +132,8 @@ func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error {
 }
 
 // SetupCerts generates the certs the cluster
-func (k *Bootstrapper) SetupCerts(cfg config.KubernetesConfig) error {
-	return bootstrapper.SetupCerts(k.c, cfg)
+func (k *Bootstrapper) SetupCerts(cfg config.KubernetesConfig, n config.Node) error {
+	return bootstrapper.SetupCerts(k.c, cfg, n)
 }
 
 // PullImages downloads images that will be used by Kubernetes
@@ -154,11 +154,12 @@ func (k *Bootstrapper) PullImages(k8s config.KubernetesConfig) error {
 }
 
 // StartCluster starts the cluster
-func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error {
-	k8s.NodeIP = kic.DefaultBindIPV4
+func (k *Bootstrapper) StartCluster(m config.MachineConfig) error {
+	k8s := m.KubernetesConfig
+	m.Nodes[0].IP = kic.DefaultBindIPV4
 	err := bsutil.ExistingConfig(k.c)
 	if err == nil { // if there is an existing cluster don't reconfigure it
-		return k.restartCluster(k8s)
+		return k.restartCluster(m)
 	}
 	glog.Infof("existence check: %v", err)
 
@@ -210,9 +211,9 @@ func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error {
 }
 
 // restartCluster restarts the Kubernetes cluster configured by kubeadm
-func (k *Bootstrapper) restartCluster(k8s config.KubernetesConfig) error {
+func (k *Bootstrapper) restartCluster(m config.MachineConfig) error {
 	glog.Infof("restartCluster start")
-
+	k8s := m.KubernetesConfig
 	start := time.Now()
 	defer func() {
 		glog.Infof("restartCluster took %s", time.Since(start))
@@ -251,12 +252,12 @@ func (k *Bootstrapper) restartCluster(k8s config.KubernetesConfig) error {
 		return errors.Wrap(err, "apiserver healthz")
 	}
 
-	client, err := k.client(k8s)
+	client, err := k.client(k8s, m.Nodes[0])
 	if err != nil {
 		return errors.Wrap(err, "getting k8s client")
 	}
 
-	if err := kverify.SystemPods(client, time.Now(), k8s.NodeIP, k8s.NodePort, kconst.DefaultControlPlaneTimeout); err != nil {
+	if err := kverify.SystemPods(client, time.Now(), m.Nodes[0].IP, m.Nodes[0].Port, kconst.DefaultControlPlaneTimeout); err != nil {
 		return errors.Wrap(err, "system pods")
 	}
 
@@ -272,24 +273,26 @@ func (k *Bootstrapper) restartCluster(k8s config.KubernetesConfig) error {
 }
 
 // WaitForCluster blocks until the cluster appears to be healthy
-func (k *Bootstrapper) WaitForCluster(k8s config.KubernetesConfig, timeout time.Duration) error {
+func (k *Bootstrapper) WaitForCluster(k8s config.MachineConfig, timeout time.Duration) error {
 	start := time.Now()
 	out.T(out.Waiting, "Waiting for cluster to come online ...")
 	if err := kverify.APIServerProcess(k.c, start, timeout); err != nil {
 		return errors.Wrap(err, "wait for api proc")
 	}
 
-	if err := kverify.APIServerIsRunning(start, "127.0.0.1", k8s.NodePort, timeout); err != nil {
+	if err := kverify.APIServerIsRunning(start, "127.0.0.1", k8s.Nodes[0].Port, timeout); err != nil {
 		return err
 	}
 
-	c, err := k.client(k8s) // getting kubernetes client before polling.
+	c, err := k.client(k8s.KubernetesConfig, k8s.Nodes[0]) // getting kubernetes client before polling.
 	if err != nil {
 		return errors.Wrap(err, "get k8s client")
 	}
 
-	if err := kverify.SystemPods(c, start, "127.0.0.1", k8s.NodePort, timeout); err != nil {
-		return errors.Wrap(err, "wait for system pods")
+	for _, n := range k8s.Nodes {
+		if err := kverify.SystemPods(c, start, "127.0.0.1", n.Port, timeout); err != nil {
+			return errors.Wrap(err, "wait for system pods")
+		}
 	}
 
 	return nil
@@ -311,7 +314,7 @@ func (k *Bootstrapper) GetAPIServerStatus(net.IP, int) (string, error) {
 }
 
 // client sets and returns a Kubernetes client to use to speak to a kubeadm launched apiserver
-func (k *Bootstrapper) client(k8s config.KubernetesConfig) (*kubernetes.Clientset, error) {
+func (k *Bootstrapper) client(k8s config.KubernetesConfig, n config.Node) (*kubernetes.Clientset, error) {
 	if k.k8sClient != nil {
 		return k.k8sClient, nil
 	}
@@ -321,7 +324,7 @@ func (k *Bootstrapper) client(k8s config.KubernetesConfig) (*kubernetes.Clientse
 		return nil, errors.Wrap(err, "client config")
 	}
 
-	endpoint := fmt.Sprintf("https://%s", net.JoinHostPort("127.0.0.1", fmt.Sprint(k8s.NodePort)))
+	endpoint := fmt.Sprintf("https://%s", net.JoinHostPort("127.0.0.1", fmt.Sprint(n.Port)))
 	if config.Host != endpoint {
 		glog.Errorf("Overriding stale ClientConfig host %s with %s", config.Host, endpoint)
 		config.Host = endpoint

From dc6c9d7d75dfd022a3e616cf0aced2263a78052e Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Tue, 14 Jan 2020 16:49:13 -0800
Subject: [PATCH 051/250] cfg config fun

---
 cmd/minikube/cmd/start.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index d3cdae2d7b..0bdce61220 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -493,12 +493,12 @@ func handleDownloadOnly(cacheGroup *errgroup.Group, k8sVersion string) {
 
 }
 
-func startMachine(config *config.MachineConfig, node *config.Node) (runner command.Runner, preExists bool, machineAPI libmachine.API, host *host.Host) {
+func startMachine(cfg *config.MachineConfig, node *config.Node) (runner command.Runner, preExists bool, machineAPI libmachine.API, host *host.Host) {
 	m, err := machine.NewAPIClient()
 	if err != nil {
 		exit.WithError("Failed to get machine client", err)
 	}
-	host, preExists = startHost(m, *config)
+	host, preExists = startHost(m, *cfg)
 	runner, err = machine.CommandRunner(host)
 	if err != nil {
 		exit.WithError("Failed to get command runner", err)
@@ -513,7 +513,7 @@ func startMachine(config *config.MachineConfig, node *config.Node) (runner comma
 	}
 	// Save IP to configuration file for subsequent use
 	node.IP = ip
-	if err := saveConfig(config); err != nil {
+	if err := saveConfig(cfg); err != nil {
 		exit.WithError("Failed to save config", err)
 	}
 

From f0372edff6b5677d795c89ff197b48f664e1e6d7 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Wed, 15 Jan 2020 10:59:30 -0800
Subject: [PATCH 052/250] ok at least things build now

---
 pkg/minikube/bootstrapper/bsutil/kubeadm.go  | 9 +++++----
 pkg/minikube/bootstrapper/bsutil/kubelet.go  | 5 +++--
 pkg/minikube/bootstrapper/kicbs/kicbs.go     | 4 ++--
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 4 ++--
 pkg/minikube/cluster/cluster.go              | 6 +++---
 pkg/minikube/registry/drvs/docker/docker.go  | 2 +-
 6 files changed, 16 insertions(+), 14 deletions(-)

diff --git a/pkg/minikube/bootstrapper/bsutil/kubeadm.go b/pkg/minikube/bootstrapper/bsutil/kubeadm.go
index 3887909c83..33b88e0834 100644
--- a/pkg/minikube/bootstrapper/bsutil/kubeadm.go
+++ b/pkg/minikube/bootstrapper/bsutil/kubeadm.go
@@ -35,7 +35,8 @@ import (
 const remoteContainerRuntime = "remote"
 
 // GenerateKubeadmYAML generates the kubeadm.yaml file
-func GenerateKubeadmYAML(k8s config.KubernetesConfig, r cruntime.Manager) ([]byte, error) {
+func GenerateKubeadmYAML(mc config.MachineConfig, r cruntime.Manager) ([]byte, error) {
+	k8s := mc.KubernetesConfig
 	version, err := ParseKubernetesVersion(k8s.KubernetesVersion)
 	if err != nil {
 		return nil, errors.Wrap(err, "parsing kubernetes version")
@@ -53,7 +54,7 @@ func GenerateKubeadmYAML(k8s config.KubernetesConfig, r cruntime.Manager) ([]byt
 	}
 
 	// In case of no port assigned, use default
-	nodePort := k8s.NodePort
+	nodePort := mc.Nodes[0].Port
 	if nodePort <= 0 {
 		nodePort = constants.APIServerPort
 	}
@@ -77,11 +78,11 @@ func GenerateKubeadmYAML(k8s config.KubernetesConfig, r cruntime.Manager) ([]byt
 		CertDir:           vmpath.GuestCertsDir,
 		ServiceCIDR:       constants.DefaultServiceCIDR,
 		PodSubnet:         k8s.ExtraOptions.Get("pod-network-cidr", Kubeadm),
-		AdvertiseAddress:  k8s.NodeIP,
+		AdvertiseAddress:  mc.Nodes[0].IP,
 		APIServerPort:     nodePort,
 		KubernetesVersion: k8s.KubernetesVersion,
 		EtcdDataDir:       EtcdDataDir(),
-		NodeName:          k8s.NodeName,
+		NodeName:          mc.Nodes[0].Name,
 		CRISocket:         r.SocketPath(),
 		ImageRepository:   k8s.ImageRepository,
 		ExtraArgs:         extraComponentConfig,
diff --git a/pkg/minikube/bootstrapper/bsutil/kubelet.go b/pkg/minikube/bootstrapper/bsutil/kubelet.go
index 5c21698deb..f711146a84 100644
--- a/pkg/minikube/bootstrapper/bsutil/kubelet.go
+++ b/pkg/minikube/bootstrapper/bsutil/kubelet.go
@@ -30,7 +30,8 @@ import (
 
 // NewKubeletConfig generates a new systemd unit containing a configured kubelet
 // based on the options present in the KubernetesConfig.
-func NewKubeletConfig(k8s config.KubernetesConfig, r cruntime.Manager) ([]byte, error) {
+func NewKubeletConfig(mc config.MachineConfig, r cruntime.Manager) ([]byte, error) {
+	k8s := mc.KubernetesConfig
 	version, err := ParseKubernetesVersion(k8s.KubernetesVersion)
 	if err != nil {
 		return nil, errors.Wrap(err, "parsing kubernetes version")
@@ -48,7 +49,7 @@ func NewKubeletConfig(k8s config.KubernetesConfig, r cruntime.Manager) ([]byte,
 		extraOpts["network-plugin"] = k8s.NetworkPlugin
 	}
 	if _, ok := extraOpts["node-ip"]; !ok {
-		extraOpts["node-ip"] = k8s.NodeIP
+		extraOpts["node-ip"] = mc.Nodes[0].IP
 	}
 
 	pauseImage := images.Pause(k8s.ImageRepository)
diff --git a/pkg/minikube/bootstrapper/kicbs/kicbs.go b/pkg/minikube/bootstrapper/kicbs/kicbs.go
index f9fb484a30..7225bb9436 100644
--- a/pkg/minikube/bootstrapper/kicbs/kicbs.go
+++ b/pkg/minikube/bootstrapper/kicbs/kicbs.go
@@ -84,12 +84,12 @@ func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error {
 	if err != nil {
 		return errors.Wrap(err, "runtime")
 	}
-	kubeadmCfg, err := bsutil.GenerateKubeadmYAML(cfg.KubernetesConfig, r)
+	kubeadmCfg, err := bsutil.GenerateKubeadmYAML(cfg, r)
 	if err != nil {
 		return errors.Wrap(err, "generating kubeadm cfg")
 	}
 
-	kubeletCfg, err := bsutil.NewKubeletConfig(cfg.KubernetesConfig, r)
+	kubeletCfg, err := bsutil.NewKubeletConfig(cfg, r)
 	if err != nil {
 		return errors.Wrap(err, "generating kubelet config")
 	}
diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index 5a1d6c326a..fc2ba85702 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -385,12 +385,12 @@ func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error {
 	if err != nil {
 		return errors.Wrap(err, "runtime")
 	}
-	kubeadmCfg, err := bsutil.GenerateKubeadmYAML(cfg.KubernetesConfig, r)
+	kubeadmCfg, err := bsutil.GenerateKubeadmYAML(cfg, r)
 	if err != nil {
 		return errors.Wrap(err, "generating kubeadm cfg")
 	}
 
-	kubeletCfg, err := bsutil.NewKubeletConfig(cfg.KubernetesConfig, r)
+	kubeletCfg, err := bsutil.NewKubeletConfig(cfg, r)
 	if err != nil {
 		return errors.Wrap(err, "generating kubelet config")
 	}
diff --git a/pkg/minikube/cluster/cluster.go b/pkg/minikube/cluster/cluster.go
index 4303cc0997..b8db6d045e 100644
--- a/pkg/minikube/cluster/cluster.go
+++ b/pkg/minikube/cluster/cluster.go
@@ -370,7 +370,7 @@ func GetHostDriverIP(api libmachine.API, machineName string) (net.IP, error) {
 func engineOptions(cfg config.MachineConfig) *engine.Options {
 	o := engine.Options{
 		Env:              cfg.DockerEnv,
-		InsecureRegistry: append([]string{constants.DefaultServiceCIDR}, config.InsecureRegistry...),
+		InsecureRegistry: append([]string{constants.DefaultServiceCIDR}, cfg.InsecureRegistry...),
 		RegistryMirror:   cfg.RegistryMirror,
 		ArbitraryFlags:   cfg.DockerOpt,
 		InstallURL:       drivers.DefaultEngineInstallURL,
@@ -456,10 +456,10 @@ func showHostInfo(cfg config.MachineConfig) {
 	} else if driver.IsKIC(cfg.VMDriver) {
 		info, err := getHostInfo() // TODO medyagh: get docker-machine info for non linux
 		if err == nil {
-			out.T(out.StartingVM, "Creating Kubernetes in {{.driver_name}} container with (CPUs={{.number_of_cpus}}), Memory={{.memory_size}}MB ({{.host_memory_size}}MB available) ...", out.V{"driver_name": config.VMDriver, "number_of_cpus": config.CPUs, "number_of_host_cpus": info.CPUs, "memory_size": config.Memory, "host_memory_size": info.Memory})
+			out.T(out.StartingVM, "Creating Kubernetes in {{.driver_name}} container with (CPUs={{.number_of_cpus}}), Memory={{.memory_size}}MB ({{.host_memory_size}}MB available) ...", out.V{"driver_name": cfg.VMDriver, "number_of_cpus": cfg.CPUs, "number_of_host_cpus": info.CPUs, "memory_size": cfg.Memory, "host_memory_size": info.Memory})
 		}
 	} else {
-		out.T(out.StartingVM, "Creating {{.driver_name}} VM (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...", out.V{"driver_name": config.VMDriver, "number_of_cpus": config.CPUs, "memory_size": config.Memory, "disk_size": config.DiskSize})
+		out.T(out.StartingVM, "Creating {{.driver_name}} VM (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...", out.V{"driver_name": cfg.VMDriver, "number_of_cpus": cfg.CPUs, "memory_size": cfg.Memory, "disk_size": cfg.DiskSize})
 	}
 }
 
diff --git a/pkg/minikube/registry/drvs/docker/docker.go b/pkg/minikube/registry/drvs/docker/docker.go
index 42fc292b87..a5b04a82b1 100644
--- a/pkg/minikube/registry/drvs/docker/docker.go
+++ b/pkg/minikube/registry/drvs/docker/docker.go
@@ -48,7 +48,7 @@ func configure(mc config.MachineConfig) interface{} {
 		ImageDigest:  kic.BaseImage,
 		CPU:          mc.CPUs,
 		Memory:       mc.Memory,
-		HostBindPort: mc.KubernetesConfig.NodePort,
+		HostBindPort: mc.Nodes[0].Port,
 		OCIBinary:    oci.Docker,
 	})
 

From f2c385fcde2dbede9bd86aeb63633c0c180edcfd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= <anders.f.bjorklund@gmail.com>
Date: Thu, 9 Jan 2020 20:25:01 +0100
Subject: [PATCH 053/250] Allow building storage-provisioner for other arch

Also allow building the static binary in docker
---
 Makefile                                    | 31 ++++++++++++---------
 deploy/storage-provisioner/Dockerfile       |  3 +-
 deploy/storage-provisioner/Dockerfile-s390x | 17 -----------
 3 files changed, 20 insertions(+), 31 deletions(-)
 delete mode 100644 deploy/storage-provisioner/Dockerfile-s390x

diff --git a/Makefile b/Makefile
index b3523ed011..ded696ccf9 100755
--- a/Makefile
+++ b/Makefile
@@ -81,6 +81,12 @@ BUILD_OS := $(shell uname -s)
 SHA512SUM=$(shell command -v sha512sum || echo "shasum -a 512")
 
 STORAGE_PROVISIONER_TAG := v1.8.1
+# TODO: multi-arch manifest
+ifeq ($(GOARCH),amd64)
+STORAGE_PROVISIONER_IMAGE ?= $(REGISTRY)/storage-provisioner:$(STORAGE_PROVISIONER_TAG)
+else
+STORAGE_PROVISIONER_IMAGE ?= $(REGISTRY)/storage-provisioner-$(GOARCH):$(STORAGE_PROVISIONER_TAG)
+endif
 
 # Set the version information for the Kubernetes servers
 MINIKUBE_LDFLAGS := -X k8s.io/minikube/pkg/version.version=$(VERSION) -X k8s.io/minikube/pkg/version.isoVersion=$(ISO_VERSION) -X k8s.io/minikube/pkg/version.isoPath=$(ISO_BUCKET) -X k8s.io/minikube/pkg/version.gitCommitID=$(COMMIT)
@@ -472,16 +478,19 @@ $(ISO_BUILD_IMAGE): deploy/iso/minikube-iso/Dockerfile
 	@echo ""
 	@echo "$(@) successfully built"
 
-out/storage-provisioner:
-	CGO_ENABLED=0 GOOS=linux go build -o $@ -ldflags=$(PROVISIONER_LDFLAGS) cmd/storage-provisioner/main.go
+out/storage-provisioner: out/storage-provisioner-$(GOARCH)
+	cp $< $@
+
+out/storage-provisioner-%: cmd/storage-provisioner/main.go pkg/storage/storage_provisioner.go
+ifeq ($(MINIKUBE_BUILD_IN_DOCKER),y)
+	$(call DOCKER,$(BUILD_IMAGE),/usr/bin/make $@)
+else
+	CGO_ENABLED=0 GOOS=linux GOARCH=$* go build -o $@ -ldflags=$(PROVISIONER_LDFLAGS) cmd/storage-provisioner/main.go
+endif
 
 .PHONY: storage-provisioner-image
-storage-provisioner-image: out/storage-provisioner ## Build storage-provisioner docker image
-ifeq ($(GOARCH),amd64)
-	docker build -t $(REGISTRY)/storage-provisioner:$(STORAGE_PROVISIONER_TAG) -f deploy/storage-provisioner/Dockerfile  .
-else
-	docker build -t $(REGISTRY)/storage-provisioner-$(GOARCH):$(STORAGE_PROVISIONER_TAG) -f deploy/storage-provisioner/Dockerfile-$(GOARCH) .
-endif
+storage-provisioner-image: out/storage-provisioner-$(GOARCH) ## Build storage-provisioner docker image
+	docker build -t $(STORAGE_PROVISIONER_IMAGE) -f deploy/storage-provisioner/Dockerfile  --build-arg arch=$(GOARCH) .
 
 .PHONY: kic-base-image
 kic-base-image: ## builds the base image used for kic.
@@ -492,11 +501,7 @@ kic-base-image: ## builds the base image used for kic.
 
 .PHONY: push-storage-provisioner-image
 push-storage-provisioner-image: storage-provisioner-image ## Push storage-provisioner docker image using gcloud
-ifeq ($(GOARCH),amd64)
-	gcloud docker -- push $(REGISTRY)/storage-provisioner:$(STORAGE_PROVISIONER_TAG)
-else
-	gcloud docker -- push $(REGISTRY)/storage-provisioner-$(GOARCH):$(STORAGE_PROVISIONER_TAG)
-endif
+	gcloud docker -- push $(STORAGE_PROVISIONER_IMAGE)
 
 .PHONY: out/gvisor-addon
 out/gvisor-addon: pkg/minikube/assets/assets.go pkg/minikube/translate/translations.go ## Build gvisor addon
diff --git a/deploy/storage-provisioner/Dockerfile b/deploy/storage-provisioner/Dockerfile
index bef335d514..cff5132912 100644
--- a/deploy/storage-provisioner/Dockerfile
+++ b/deploy/storage-provisioner/Dockerfile
@@ -13,5 +13,6 @@
 # limitations under the License.
 
 FROM scratch
-COPY out/storage-provisioner storage-provisioner
+ARG arch
+COPY out/storage-provisioner-${arch} storage-provisioner
 CMD ["/storage-provisioner"]
diff --git a/deploy/storage-provisioner/Dockerfile-s390x b/deploy/storage-provisioner/Dockerfile-s390x
deleted file mode 100644
index 5e22f15613..0000000000
--- a/deploy/storage-provisioner/Dockerfile-s390x
+++ /dev/null
@@ -1,17 +0,0 @@
-# Copyright 2016 The Kubernetes Authors All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-FROM s390x/ubuntu:16.04
-COPY out/storage-provisioner storage-provisioner
-CMD ["/storage-provisioner"]

From bfa7c13e34c5b358c8f174a26cdb6485f69be7fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= <anders.f.bjorklund@gmail.com>
Date: Wed, 15 Jan 2020 20:07:25 +0100
Subject: [PATCH 054/250] Stop sending minikube version to storage-provisioner

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index ded696ccf9..6da1bddc31 100755
--- a/Makefile
+++ b/Makefile
@@ -90,7 +90,7 @@ endif
 
 # Set the version information for the Kubernetes servers
 MINIKUBE_LDFLAGS := -X k8s.io/minikube/pkg/version.version=$(VERSION) -X k8s.io/minikube/pkg/version.isoVersion=$(ISO_VERSION) -X k8s.io/minikube/pkg/version.isoPath=$(ISO_BUCKET) -X k8s.io/minikube/pkg/version.gitCommitID=$(COMMIT)
-PROVISIONER_LDFLAGS := "$(MINIKUBE_LDFLAGS) -s -w"
+PROVISIONER_LDFLAGS := "-X k8s.io/minikube/pkg/storage.version=$(STORAGE_PROVISIONER_TAG) -s -w"
 
 MINIKUBEFILES := ./cmd/minikube/
 HYPERKIT_FILES := ./cmd/drivers/hyperkit

From 804ee14a958ed765d8f450a8b424cc4564360e8e Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Wed, 15 Jan 2020 11:36:36 -0800
Subject: [PATCH 055/250] removing node surface stuff for a different PR

---
 cmd/minikube/cmd/config/node/add.go    | 45 --------------------------
 cmd/minikube/cmd/config/node/delete.go | 33 -------------------
 cmd/minikube/cmd/config/node/node.go   | 31 ------------------
 cmd/minikube/cmd/config/node/start.go  |  0
 cmd/minikube/cmd/config/node/stop.go   |  0
 pkg/minikube/cluster/node/node.go      | 39 ----------------------
 6 files changed, 148 deletions(-)
 delete mode 100644 cmd/minikube/cmd/config/node/add.go
 delete mode 100644 cmd/minikube/cmd/config/node/delete.go
 delete mode 100644 cmd/minikube/cmd/config/node/node.go
 delete mode 100644 cmd/minikube/cmd/config/node/start.go
 delete mode 100644 cmd/minikube/cmd/config/node/stop.go
 delete mode 100644 pkg/minikube/cluster/node/node.go

diff --git a/cmd/minikube/cmd/config/node/add.go b/cmd/minikube/cmd/config/node/add.go
deleted file mode 100644
index 8bc0d04cd8..0000000000
--- a/cmd/minikube/cmd/config/node/add.go
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
-Copyright 2019 The Kubernetes Authors All rights reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package config
-
-import (
-	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
-	"k8s.io/minikube/pkg/minikube/config"
-	"k8s.io/minikube/pkg/minikube/exit"
-)
-
-var name string
-
-var nodeAddCmd = &cobra.Command{
-	Use:   "add",
-	Short: "Adds a node to the given cluster.",
-	Long:  "Adds a node to the given cluster config, without starting it.",
-	Run: func(cmd *cobra.Command, args []string) {
-
-		profile := viper.GetString(config.MachineProfile)
-		cc, err := config.Load(profile)
-		if err != nil {
-			exit.WithError(err)
-		}
-	},
-}
-
-func init() {
-	nodeAddCmd.Flags().StringVar(&name, "name", "", "The name of the node to add. Defaults to a variation of the profile name.")
-	NodeCmd.AddCommand(nodeAddCmd)
-}
diff --git a/cmd/minikube/cmd/config/node/delete.go b/cmd/minikube/cmd/config/node/delete.go
deleted file mode 100644
index efea52f70c..0000000000
--- a/cmd/minikube/cmd/config/node/delete.go
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
-Copyright 2019 The Kubernetes Authors All rights reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package config
-
-import "github.com/spf13/cobra"
-
-var nodeDeleteCmd = &cobra.Command{
-	Use:   "delete",
-	Short: "Deletes a node from a cluster.",
-	Long:  "Deletes a node from a cluster.",
-	Run: func(cmd *cobra.Command, args []string) {
-
-	},
-}
-
-func init() {
-	nodeDeleteCmd.Flags().StringVar(&name, "name", "", "The name of the node to delete.")
-	NodeCmd.AddCommand(nodeDeleteCmd)
-}
diff --git a/cmd/minikube/cmd/config/node/node.go b/cmd/minikube/cmd/config/node/node.go
deleted file mode 100644
index fa180d67bc..0000000000
--- a/cmd/minikube/cmd/config/node/node.go
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
-Copyright 2019 The Kubernetes Authors All rights reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package config
-
-import (
-	"github.com/spf13/cobra"
-	"k8s.io/minikube/pkg/minikube/exit"
-)
-
-var NodeCmd = &cobra.Command{
-	Use:   "node",
-	Short: "Node operations",
-	Long:  "Operations on nodes",
-	Run: func(cmd *cobra.Command, args []string) {
-		exit.UsageT("Usage: minikube node [add|start|stop|delete]")
-	},
-}
diff --git a/cmd/minikube/cmd/config/node/start.go b/cmd/minikube/cmd/config/node/start.go
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/cmd/minikube/cmd/config/node/stop.go b/cmd/minikube/cmd/config/node/stop.go
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/pkg/minikube/cluster/node/node.go b/pkg/minikube/cluster/node/node.go
deleted file mode 100644
index 25176f4410..0000000000
--- a/pkg/minikube/cluster/node/node.go
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
-Copyright 2019 The Kubernetes Authors All rights reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package cluster
-
-import "k8s.io/minikube/pkg/minikube/config"
-
-// AddNode adds a new node config to a cluster.
-func AddNode(cfg config.MachineConfig, name string) error {
-
-}
-
-// StartNode starts the given node
-func StartNode(node config.Node) error {
-
-}
-
-// StopNode stops the given node
-func StopNode(node config.Node) error {
-
-}
-
-// DeleteNode deletes the given node from the given cluster
-func DeleteNode(cfg config.MachineConfig, node config.Node) error {
-
-}

From 44c7c48d3e7ac8e425848d95a379e7d0bac8d49c Mon Sep 17 00:00:00 2001
From: Chance Zibolski <czibolski@rigetti.com>
Date: Wed, 15 Jan 2020 11:52:43 -0800
Subject: [PATCH 056/250] pkg/minikube: Configure etcd and kube-proxy metrics
 to listen on minikube node IP

---
 pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta1.go | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta1.go b/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta1.go
index 6b6c59a81b..353bd64357 100644
--- a/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta1.go
+++ b/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta1.go
@@ -60,6 +60,8 @@ dns:
 etcd:
   local:
     dataDir: {{.EtcdDataDir}}
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://{{.AdvertiseAddress}}:2381
 kubernetesVersion: {{.KubernetesVersion}}
 networking:
   dnsDomain: {{if .DNSDomain}}{{.DNSDomain}}{{else}}cluster.local{{end}}
@@ -73,4 +75,8 @@ evictionHard:
   nodefs.available: "0%"
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: {{.AdvertiseAddress}}:10249
 `))

From b7f3ff7ead107dcb3da21b16a3caef4cb2f46e62 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= <anders.f.bjorklund@gmail.com>
Date: Wed, 15 Jan 2020 21:32:40 +0100
Subject: [PATCH 057/250] Stop minikube dashboard from crashing at start

If the profile does not exist, it cannot load
---
 cmd/minikube/cmd/dashboard.go | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/cmd/minikube/cmd/dashboard.go b/cmd/minikube/cmd/dashboard.go
index bdcc204209..7a94293c45 100644
--- a/cmd/minikube/cmd/dashboard.go
+++ b/cmd/minikube/cmd/dashboard.go
@@ -59,11 +59,17 @@ var dashboardCmd = &cobra.Command{
 	Short: "Access the kubernetes dashboard running within the minikube cluster",
 	Long:  `Access the kubernetes dashboard running within the minikube cluster`,
 	Run: func(cmd *cobra.Command, args []string) {
-		cc, err := pkg_config.Load(viper.GetString(config.MachineProfile))
+		profileName := viper.GetString(pkg_config.MachineProfile)
+		cc, err := pkg_config.Load(profileName)
 		if err != nil && !os.IsNotExist(err) {
 			exit.WithError("Error loading profile config", err)
 		}
 
+		if err != nil {
+			out.ErrT(out.Meh, `"{{.name}}" profile does not exist`, out.V{"name": profileName})
+			os.Exit(1)
+		}
+
 		api, err := machine.NewAPIClient()
 		defer func() {
 			err := api.Close()

From ca007f414b82d57b89bec0eafb2104dc3401a84e Mon Sep 17 00:00:00 2001
From: Chance Zibolski <czibolski@rigetti.com>
Date: Wed, 15 Jan 2020 14:02:28 -0800
Subject: [PATCH 058/250] pkg/minikube: Update test fixtures to contain updated
 metrics listen addresses

---
 .../bsutil/testdata/v1.14/containerd-api-port.yaml          | 6 ++++++
 .../bsutil/testdata/v1.14/containerd-pod-network-cidr.yaml  | 6 ++++++
 .../bootstrapper/bsutil/testdata/v1.14/containerd.yaml      | 6 ++++++
 .../bsutil/testdata/v1.14/crio-options-gates.yaml           | 6 ++++++
 pkg/minikube/bootstrapper/bsutil/testdata/v1.14/crio.yaml   | 6 ++++++
 .../bootstrapper/bsutil/testdata/v1.14/default.yaml         | 6 ++++++
 pkg/minikube/bootstrapper/bsutil/testdata/v1.14/dns.yaml    | 6 ++++++
 .../bsutil/testdata/v1.14/image-repository.yaml             | 6 ++++++
 .../bootstrapper/bsutil/testdata/v1.14/options.yaml         | 6 ++++++
 .../bsutil/testdata/v1.15/containerd-api-port.yaml          | 6 ++++++
 .../bsutil/testdata/v1.15/containerd-pod-network-cidr.yaml  | 6 ++++++
 .../bootstrapper/bsutil/testdata/v1.15/containerd.yaml      | 6 ++++++
 .../bsutil/testdata/v1.15/crio-options-gates.yaml           | 6 ++++++
 pkg/minikube/bootstrapper/bsutil/testdata/v1.15/crio.yaml   | 6 ++++++
 .../bootstrapper/bsutil/testdata/v1.15/default.yaml         | 6 ++++++
 pkg/minikube/bootstrapper/bsutil/testdata/v1.15/dns.yaml    | 6 ++++++
 .../bsutil/testdata/v1.15/image-repository.yaml             | 6 ++++++
 .../bootstrapper/bsutil/testdata/v1.15/options.yaml         | 6 ++++++
 .../bsutil/testdata/v1.16/containerd-api-port.yaml          | 6 ++++++
 .../bsutil/testdata/v1.16/containerd-pod-network-cidr.yaml  | 6 ++++++
 .../bootstrapper/bsutil/testdata/v1.16/containerd.yaml      | 6 ++++++
 .../bsutil/testdata/v1.16/crio-options-gates.yaml           | 6 ++++++
 pkg/minikube/bootstrapper/bsutil/testdata/v1.16/crio.yaml   | 6 ++++++
 .../bootstrapper/bsutil/testdata/v1.16/default.yaml         | 6 ++++++
 pkg/minikube/bootstrapper/bsutil/testdata/v1.16/dns.yaml    | 6 ++++++
 .../bsutil/testdata/v1.16/image-repository.yaml             | 6 ++++++
 .../bootstrapper/bsutil/testdata/v1.16/options.yaml         | 6 ++++++
 .../bsutil/testdata/v1.17/containerd-api-port.yaml          | 6 ++++++
 .../bsutil/testdata/v1.17/containerd-pod-network-cidr.yaml  | 6 ++++++
 .../bootstrapper/bsutil/testdata/v1.17/containerd.yaml      | 6 ++++++
 .../bsutil/testdata/v1.17/crio-options-gates.yaml           | 6 ++++++
 pkg/minikube/bootstrapper/bsutil/testdata/v1.17/crio.yaml   | 6 ++++++
 .../bootstrapper/bsutil/testdata/v1.17/default.yaml         | 6 ++++++
 pkg/minikube/bootstrapper/bsutil/testdata/v1.17/dns.yaml    | 6 ++++++
 .../bsutil/testdata/v1.17/image-repository.yaml             | 6 ++++++
 .../bootstrapper/bsutil/testdata/v1.17/options.yaml         | 6 ++++++
 36 files changed, 216 insertions(+)

diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd-api-port.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd-api-port.yaml
index 6d4eeeeba3..c8cab84ec5 100644
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd-api-port.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd-api-port.yaml
@@ -30,6 +30,8 @@ dns:
 etcd:
   local:
     dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.14.0
 networking:
   dnsDomain: cluster.local
@@ -43,3 +45,7 @@ evictionHard:
   nodefs.available: "0%"
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd-pod-network-cidr.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd-pod-network-cidr.yaml
index 3c6997ddec..6d316486d1 100644
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd-pod-network-cidr.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd-pod-network-cidr.yaml
@@ -30,6 +30,8 @@ dns:
 etcd:
   local:
     dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.14.0
 networking:
   dnsDomain: cluster.local
@@ -43,3 +45,7 @@ evictionHard:
   nodefs.available: "0%"
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd.yaml
index d96e27170c..4d8c50928d 100644
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd.yaml
@@ -30,6 +30,8 @@ dns:
 etcd:
   local:
     dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.14.0
 networking:
   dnsDomain: cluster.local
@@ -43,3 +45,7 @@ evictionHard:
   nodefs.available: "0%"
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/crio-options-gates.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/crio-options-gates.yaml
index 1ad22d184b..3e27f96cb0 100644
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/crio-options-gates.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/crio-options-gates.yaml
@@ -40,6 +40,8 @@ dns:
 etcd:
   local:
     dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.14.0
 networking:
   dnsDomain: cluster.local
@@ -53,3 +55,7 @@ evictionHard:
   nodefs.available: "0%"
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/crio.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/crio.yaml
index 062233c224..56baf17506 100644
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/crio.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/crio.yaml
@@ -30,6 +30,8 @@ dns:
 etcd:
   local:
     dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.14.0
 networking:
   dnsDomain: cluster.local
@@ -43,3 +45,7 @@ evictionHard:
   nodefs.available: "0%"
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/default.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/default.yaml
index bedb175306..f6f4595ed1 100644
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/default.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/default.yaml
@@ -30,6 +30,8 @@ dns:
 etcd:
   local:
     dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.14.0
 networking:
   dnsDomain: cluster.local
@@ -43,3 +45,7 @@ evictionHard:
   nodefs.available: "0%"
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/dns.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/dns.yaml
index dc8df12a44..d347333d8d 100644
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/dns.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/dns.yaml
@@ -30,6 +30,8 @@ dns:
 etcd:
   local:
     dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.14.0
 networking:
   dnsDomain: 1.1.1.1
@@ -43,3 +45,7 @@ evictionHard:
   nodefs.available: "0%"
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/image-repository.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/image-repository.yaml
index c9c6846a38..8b419a73a8 100644
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/image-repository.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/image-repository.yaml
@@ -31,6 +31,8 @@ dns:
 etcd:
   local:
     dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.14.0
 networking:
   dnsDomain: cluster.local
@@ -44,3 +46,7 @@ evictionHard:
   nodefs.available: "0%"
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/options.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/options.yaml
index 2d0c7e807a..a0065ab854 100644
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/options.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/options.yaml
@@ -37,6 +37,8 @@ dns:
 etcd:
   local:
     dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.14.0
 networking:
   dnsDomain: cluster.local
@@ -50,3 +52,7 @@ evictionHard:
   nodefs.available: "0%"
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd-api-port.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd-api-port.yaml
index 30bb67574a..790d0f172d 100644
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd-api-port.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd-api-port.yaml
@@ -30,6 +30,8 @@ dns:
 etcd:
   local:
     dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.15.0
 networking:
   dnsDomain: cluster.local
@@ -43,3 +45,7 @@ evictionHard:
   nodefs.available: "0%"
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd-pod-network-cidr.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd-pod-network-cidr.yaml
index 4fc81aa1c5..466f5cf1e0 100644
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd-pod-network-cidr.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd-pod-network-cidr.yaml
@@ -30,6 +30,8 @@ dns:
 etcd:
   local:
     dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.15.0
 networking:
   dnsDomain: cluster.local
@@ -43,3 +45,7 @@ evictionHard:
   nodefs.available: "0%"
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd.yaml
index 5fcd575dee..ca4e5ac535 100644
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd.yaml
@@ -30,6 +30,8 @@ dns:
 etcd:
   local:
     dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.15.0
 networking:
   dnsDomain: cluster.local
@@ -43,3 +45,7 @@ evictionHard:
   nodefs.available: "0%"
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/crio-options-gates.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/crio-options-gates.yaml
index 1d82fb284d..0f4e499a2c 100644
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/crio-options-gates.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/crio-options-gates.yaml
@@ -40,6 +40,8 @@ dns:
 etcd:
   local:
     dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.15.0
 networking:
   dnsDomain: cluster.local
@@ -53,3 +55,7 @@ evictionHard:
   nodefs.available: "0%"
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/crio.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/crio.yaml
index eadb7a3fcd..01f0bef651 100644
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/crio.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/crio.yaml
@@ -30,6 +30,8 @@ dns:
 etcd:
   local:
     dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.15.0
 networking:
   dnsDomain: cluster.local
@@ -43,3 +45,7 @@ evictionHard:
   nodefs.available: "0%"
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/default.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/default.yaml
index 5e9b1e4bfe..d2745820b4 100644
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/default.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/default.yaml
@@ -30,6 +30,8 @@ dns:
 etcd:
   local:
     dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.15.0
 networking:
   dnsDomain: cluster.local
@@ -43,3 +45,7 @@ evictionHard:
   nodefs.available: "0%"
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/dns.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/dns.yaml
index e8e9158ad2..c89c44845f 100644
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/dns.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/dns.yaml
@@ -30,6 +30,8 @@ dns:
 etcd:
   local:
     dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.15.0
 networking:
   dnsDomain: 1.1.1.1
@@ -43,3 +45,7 @@ evictionHard:
   nodefs.available: "0%"
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/image-repository.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/image-repository.yaml
index 1362db1f37..d4a4663a08 100644
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/image-repository.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/image-repository.yaml
@@ -31,6 +31,8 @@ dns:
 etcd:
   local:
     dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.15.0
 networking:
   dnsDomain: cluster.local
@@ -44,3 +46,7 @@ evictionHard:
   nodefs.available: "0%"
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/options.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/options.yaml
index 31462f323f..a690ef381a 100644
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/options.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/options.yaml
@@ -37,6 +37,8 @@ dns:
 etcd:
   local:
     dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.15.0
 networking:
   dnsDomain: cluster.local
@@ -50,3 +52,7 @@ evictionHard:
   nodefs.available: "0%"
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd-api-port.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd-api-port.yaml
index 1befae1db7..f02a7815aa 100644
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd-api-port.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd-api-port.yaml
@@ -30,6 +30,8 @@ dns:
 etcd:
   local:
     dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.16.0
 networking:
   dnsDomain: cluster.local
@@ -43,3 +45,7 @@ evictionHard:
   nodefs.available: "0%"
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd-pod-network-cidr.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd-pod-network-cidr.yaml
index 65ead23137..0171cf876d 100644
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd-pod-network-cidr.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd-pod-network-cidr.yaml
@@ -30,6 +30,8 @@ dns:
 etcd:
   local:
     dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.16.0
 networking:
   dnsDomain: cluster.local
@@ -43,3 +45,7 @@ evictionHard:
   nodefs.available: "0%"
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd.yaml
index 3b269defb3..0f6ac8a836 100644
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd.yaml
@@ -30,6 +30,8 @@ dns:
 etcd:
   local:
     dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.16.0
 networking:
   dnsDomain: cluster.local
@@ -43,3 +45,7 @@ evictionHard:
   nodefs.available: "0%"
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/crio-options-gates.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/crio-options-gates.yaml
index b04f318a08..6e175a4982 100644
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/crio-options-gates.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/crio-options-gates.yaml
@@ -40,6 +40,8 @@ dns:
 etcd:
   local:
     dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.16.0
 networking:
   dnsDomain: cluster.local
@@ -53,3 +55,7 @@ evictionHard:
   nodefs.available: "0%"
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/crio.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/crio.yaml
index c64f2a1cba..d663fcffad 100644
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/crio.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/crio.yaml
@@ -30,6 +30,8 @@ dns:
 etcd:
   local:
     dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.16.0
 networking:
   dnsDomain: cluster.local
@@ -43,3 +45,7 @@ evictionHard:
   nodefs.available: "0%"
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/default.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/default.yaml
index 60d7ae8eea..00649d728a 100644
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/default.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/default.yaml
@@ -30,6 +30,8 @@ dns:
 etcd:
   local:
     dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.16.0
 networking:
   dnsDomain: cluster.local
@@ -43,3 +45,7 @@ evictionHard:
   nodefs.available: "0%"
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/dns.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/dns.yaml
index 08c7528ce3..9800569fe4 100644
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/dns.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/dns.yaml
@@ -30,6 +30,8 @@ dns:
 etcd:
   local:
     dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.16.0
 networking:
   dnsDomain: 1.1.1.1
@@ -43,3 +45,7 @@ evictionHard:
   nodefs.available: "0%"
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/image-repository.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/image-repository.yaml
index 62bf5372ac..1d04d64cce 100644
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/image-repository.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/image-repository.yaml
@@ -31,6 +31,8 @@ dns:
 etcd:
   local:
     dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.16.0
 networking:
   dnsDomain: cluster.local
@@ -44,3 +46,7 @@ evictionHard:
   nodefs.available: "0%"
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/options.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/options.yaml
index 1c0d76eba4..72303f9860 100644
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/options.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/options.yaml
@@ -37,6 +37,8 @@ dns:
 etcd:
   local:
     dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.16.0
 networking:
   dnsDomain: cluster.local
@@ -50,3 +52,7 @@ evictionHard:
   nodefs.available: "0%"
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd-api-port.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd-api-port.yaml
index 872fa683e0..99ea286ed4 100644
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd-api-port.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd-api-port.yaml
@@ -30,6 +30,8 @@ dns:
 etcd:
   local:
     dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.17.0
 networking:
   dnsDomain: cluster.local
@@ -43,3 +45,7 @@ evictionHard:
   nodefs.available: "0%"
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd-pod-network-cidr.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd-pod-network-cidr.yaml
index 7336030107..a7886fc023 100644
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd-pod-network-cidr.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd-pod-network-cidr.yaml
@@ -30,6 +30,8 @@ dns:
 etcd:
   local:
     dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.17.0
 networking:
   dnsDomain: cluster.local
@@ -43,3 +45,7 @@ evictionHard:
   nodefs.available: "0%"
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd.yaml
index 1f5fb46836..a31d40e7f6 100644
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd.yaml
@@ -30,6 +30,8 @@ dns:
 etcd:
   local:
     dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.17.0
 networking:
   dnsDomain: cluster.local
@@ -43,3 +45,7 @@ evictionHard:
   nodefs.available: "0%"
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/crio-options-gates.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/crio-options-gates.yaml
index fe876ff888..ea09a8c0f4 100644
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/crio-options-gates.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/crio-options-gates.yaml
@@ -40,6 +40,8 @@ dns:
 etcd:
   local:
     dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.17.0
 networking:
   dnsDomain: cluster.local
@@ -53,3 +55,7 @@ evictionHard:
   nodefs.available: "0%"
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/crio.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/crio.yaml
index 6704dc431b..0c8d543bb8 100644
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/crio.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/crio.yaml
@@ -30,6 +30,8 @@ dns:
 etcd:
   local:
     dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.17.0
 networking:
   dnsDomain: cluster.local
@@ -43,3 +45,7 @@ evictionHard:
   nodefs.available: "0%"
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/default.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/default.yaml
index 263492d5be..5a933b0b94 100644
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/default.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/default.yaml
@@ -30,6 +30,8 @@ dns:
 etcd:
   local:
     dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.17.0
 networking:
   dnsDomain: cluster.local
@@ -43,3 +45,7 @@ evictionHard:
   nodefs.available: "0%"
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/dns.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/dns.yaml
index 7441511cb0..03809febbf 100644
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/dns.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/dns.yaml
@@ -30,6 +30,8 @@ dns:
 etcd:
   local:
     dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.17.0
 networking:
   dnsDomain: 1.1.1.1
@@ -43,3 +45,7 @@ evictionHard:
   nodefs.available: "0%"
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/image-repository.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/image-repository.yaml
index c746585e4d..474da70a45 100644
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/image-repository.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/image-repository.yaml
@@ -31,6 +31,8 @@ dns:
 etcd:
   local:
     dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.17.0
 networking:
   dnsDomain: cluster.local
@@ -44,3 +46,7 @@ evictionHard:
   nodefs.available: "0%"
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/options.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/options.yaml
index d47790e95e..4c631b6b14 100644
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/options.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/options.yaml
@@ -37,6 +37,8 @@ dns:
 etcd:
   local:
     dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.17.0
 networking:
   dnsDomain: cluster.local
@@ -50,3 +52,7 @@ evictionHard:
   nodefs.available: "0%"
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249

From a13ea9bea0b6c29bf260006509f2c8fcdcca1d2a Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Wed, 15 Jan 2020 15:21:47 -0800
Subject: [PATCH 059/250] add the ability to mark drivers as experimental and
 mark the docker driver as such

---
 cmd/minikube/cmd/start.go                   |  2 +-
 pkg/minikube/driver/driver.go               | 10 +++++++++-
 pkg/minikube/registry/drvs/docker/docker.go |  2 +-
 pkg/minikube/registry/registry.go           |  2 ++
 4 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 55325fee80..fb728dd190 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -200,7 +200,7 @@ func initKubernetesFlags() {
 
 // initDriverFlags inits the commandline flags for vm drivers
 func initDriverFlags() {
-	startCmd.Flags().String("vm-driver", "", fmt.Sprintf("Driver is one of: %v (defaults to auto-detect)", driver.SupportedDrivers()))
+	startCmd.Flags().String("vm-driver", "", fmt.Sprintf("Driver is one of: %v (defaults to auto-detect)", strings.Join(driver.SupportedDrivers(), ", ")))
 	startCmd.Flags().Bool(disableDriverMounts, false, "Disables the filesystem mounts provided by the hypervisors")
 
 	// kvm2
diff --git a/pkg/minikube/driver/driver.go b/pkg/minikube/driver/driver.go
index c084b49846..5b8d82ce0c 100644
--- a/pkg/minikube/driver/driver.go
+++ b/pkg/minikube/driver/driver.go
@@ -57,7 +57,15 @@ var (
 
 // SupportedDrivers returns a list of supported drivers
 func SupportedDrivers() []string {
-	return supportedDrivers
+	var sd []string
+	for _, d := range supportedDrivers {
+		if registry.Driver(d).Priority == registry.Experimental {
+			sd = append(sd, d+" (experimental)")
+			continue
+		}
+		sd = append(sd, d)
+	}
+	return sd
 }
 
 // Supported returns if the driver is supported on this host.
diff --git a/pkg/minikube/registry/drvs/docker/docker.go b/pkg/minikube/registry/drvs/docker/docker.go
index 42fc292b87..bafcf1b093 100644
--- a/pkg/minikube/registry/drvs/docker/docker.go
+++ b/pkg/minikube/registry/drvs/docker/docker.go
@@ -35,7 +35,7 @@ func init() {
 		Config:   configure,
 		Init:     func() drivers.Driver { return kic.NewDriver(kic.Config{OCIBinary: oci.Docker}) },
 		Status:   status,
-		Priority: registry.Discouraged, // experimental
+		Priority: registry.Experimental,
 	}); err != nil {
 		panic(fmt.Sprintf("register failed: %v", err))
 	}
diff --git a/pkg/minikube/registry/registry.go b/pkg/minikube/registry/registry.go
index 0bc86eb0f5..a506a28aeb 100644
--- a/pkg/minikube/registry/registry.go
+++ b/pkg/minikube/registry/registry.go
@@ -33,6 +33,8 @@ const (
 	Unknown Priority = iota
 	// Unhealthy is when a driver does not pass health checks
 	Unhealthy
+	// Experimental is when a driver is not officially supported because it's still experimental
+	Experimental
 	// Discouraged is when a driver has caveats that preclude it's recommendation
 	Discouraged
 	// Deprecated is when a driver has been formally deprecated

From d0fb749842d9a48f50302b1ac072bd24fea25484 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Wed, 15 Jan 2020 16:03:37 -0800
Subject: [PATCH 060/250] add displaysupporteddrivers func

---
 cmd/minikube/cmd/start.go     | 2 +-
 pkg/minikube/driver/driver.go | 8 +++++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index fb728dd190..45285476e2 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -200,7 +200,7 @@ func initKubernetesFlags() {
 
 // initDriverFlags inits the commandline flags for vm drivers
 func initDriverFlags() {
-	startCmd.Flags().String("vm-driver", "", fmt.Sprintf("Driver is one of: %v (defaults to auto-detect)", strings.Join(driver.SupportedDrivers(), ", ")))
+	startCmd.Flags().String("vm-driver", "", fmt.Sprintf("Driver is one of: %v (defaults to auto-detect)", driver.DisplaySupportedDrivers()))
 	startCmd.Flags().Bool(disableDriverMounts, false, "Disables the filesystem mounts provided by the hypervisors")
 
 	// kvm2
diff --git a/pkg/minikube/driver/driver.go b/pkg/minikube/driver/driver.go
index 5b8d82ce0c..006e346d64 100644
--- a/pkg/minikube/driver/driver.go
+++ b/pkg/minikube/driver/driver.go
@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"os"
 	"sort"
+	"strings"
 
 	"github.com/golang/glog"
 	"k8s.io/minikube/pkg/drivers/kic"
@@ -57,6 +58,11 @@ var (
 
 // SupportedDrivers returns a list of supported drivers
 func SupportedDrivers() []string {
+	return supportedDrivers
+}
+
+// DisplaySupportedDrivers returns a string with a list of supported drivers
+func DisplaySupportedDrivers() string {
 	var sd []string
 	for _, d := range supportedDrivers {
 		if registry.Driver(d).Priority == registry.Experimental {
@@ -65,7 +71,7 @@ func SupportedDrivers() []string {
 		}
 		sd = append(sd, d)
 	}
-	return sd
+	return strings.Join(sd, ", ")
 }
 
 // Supported returns if the driver is supported on this host.

From 010d0d4091d777ab88d99ec7d9340750ed3f70b8 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Wed, 15 Jan 2020 16:22:48 -0800
Subject: [PATCH 061/250] adding experimental string to driver detection string

---
 cmd/minikube/cmd/start.go | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 45285476e2..ebb5158124 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -65,6 +65,7 @@ import (
 	"k8s.io/minikube/pkg/minikube/notify"
 	"k8s.io/minikube/pkg/minikube/out"
 	"k8s.io/minikube/pkg/minikube/proxy"
+	"k8s.io/minikube/pkg/minikube/registry"
 	"k8s.io/minikube/pkg/minikube/translate"
 	pkgutil "k8s.io/minikube/pkg/util"
 	"k8s.io/minikube/pkg/util/lock"
@@ -583,23 +584,30 @@ func selectDriver(existing *cfg.MachineConfig) string {
 	driver.SetLibvirtURI(viper.GetString(kvmQemuURI))
 	options := driver.Choices()
 	pick, alts := driver.Choose(name, options)
+	exp := ""
+	if pick.Priority == registry.Experimental {
+		exp = "experimental"
+	}
 
 	if name != "" {
-		out.T(out.Sparkle, `Selecting '{{.driver}}' driver from user configuration (alternates: {{.alternates}})`, out.V{"driver": name, "alternates": alts})
+		out.T(out.Sparkle, `Selecting {{.experimental}} '{{.driver}}' driver from user configuration (alternates: {{.alternates}})`, out.V{"experimental": exp, "driver": name, "alternates": alts})
 		return name
 	}
 
 	// By default, the driver is whatever we used last time
 	if existing != nil {
 		pick, alts := driver.Choose(existing.VMDriver, options)
-		out.T(out.Sparkle, `Selecting '{{.driver}}' driver from existing profile (alternates: {{.alternates}})`, out.V{"driver": existing.VMDriver, "alternates": alts})
+		if pick.Priority == registry.Experimental {
+			exp = "experimental"
+		}
+		out.T(out.Sparkle, `Selecting {{.experimental}} '{{.driver}}' driver from existing profile (alternates: {{.alternates}})`, out.V{"experimental": exp, "driver": existing.VMDriver, "alternates": alts})
 		return pick.Name
 	}
 
 	if len(options) > 1 {
-		out.T(out.Sparkle, `Automatically selected the '{{.driver}}' driver (alternates: {{.alternates}})`, out.V{"driver": pick.Name, "alternates": alts})
+		out.T(out.Sparkle, `Automatically selected the {{.experimental}} '{{.driver}}' driver (alternates: {{.alternates}})`, out.V{"experimental": exp, "driver": pick.Name, "alternates": alts})
 	} else {
-		out.T(out.Sparkle, `Automatically selected the '{{.driver}}' driver`, out.V{"driver": pick.Name})
+		out.T(out.Sparkle, `Automatically selected the {{.experimental}} '{{.driver}}' driver`, out.V{"experimental": exp, "driver": pick.Name})
 	}
 
 	if pick.Name == "" {
@@ -612,7 +620,7 @@ func selectDriver(existing *cfg.MachineConfig) string {
 func validateDriver(name string, existing *cfg.MachineConfig) {
 	glog.Infof("validating driver %q against %+v", name, existing)
 	if !driver.Supported(name) {
-		exit.WithCodeT(exit.Unavailable, "The driver '{{.driver}}' is not supported on {{.os}}", out.V{"driver": name, "os": runtime.GOOS})
+		exit.WithCodeT(exit.Unavailable, "The driver {{.experimental}} '{{.driver}}' is not supported on {{.os}}", out.V{"driver": name, "os": runtime.GOOS})
 	}
 
 	st := driver.Status(name)

From e3ac51ab0c6e7017ecc159dbba6236bcfc870806 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Wed, 15 Jan 2020 17:37:55 -0800
Subject: [PATCH 062/250] add release notes for 1.17.0-beta.0

---
 CHANGELOG.md | 42 +++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 41 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6bb848b274..06955e8881 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,45 @@
 # Release Notes
 
+## Version 1.7.0-beta.0 - 2020-01-15
+
+* add the ability to mark drivers as experimental  [#6326](https://github.com/kubernetes/minikube/pull/6326)
+* Use CGroupDriver function from cruntime for kubelet [#6287](https://github.com/kubernetes/minikube/pull/6287)
+* Experimental Docker support (kic) using the Kind image [#6151](https://github.com/kubernetes/minikube/pull/6151)
+* disable istio provisioner by default [#6315](https://github.com/kubernetes/minikube/pull/6315)
+* Add --dry-run option to start [#6256](https://github.com/kubernetes/minikube/pull/6256)
+* add addons for kic [#6277](https://github.com/kubernetes/minikube/pull/6277)
+* Improve "addon list" by viewing as a table  [#6274](https://github.com/kubernetes/minikube/pull/6274)
+* Handle empty k8s version in existing profile [#6292](https://github.com/kubernetes/minikube/pull/6292)
+* Disable IPv6 in the minikube VM until it can be properly supported [#6241](https://github.com/kubernetes/minikube/pull/6241)
+* Rebuild docker images for each release [#6235](https://github.com/kubernetes/minikube/pull/6235)
+* Fixes IPv6 address handling in kubeadm [#6214](https://github.com/kubernetes/minikube/pull/6214)
+* Upgrade crio to 1.16.1 [#6210](https://github.com/kubernetes/minikube/pull/6210)
+* Upgrade podman to 1.6.4 [#6208](https://github.com/kubernetes/minikube/pull/6208)
+* Enable or disable addons per profile [#6124](https://github.com/kubernetes/minikube/pull/6124)
+* Upgrade buildroot minor version [#6199](https://github.com/kubernetes/minikube/pull/6199)
+* Add systemd patch for booting on AMD Ryzen [#6183](https://github.com/kubernetes/minikube/pull/6183)
+* update zh translation [#6176](https://github.com/kubernetes/minikube/pull/6176)
+* Add istio addon for minikube [#6154](https://github.com/kubernetes/minikube/pull/6154)
+* implemented options command, fixes #5036 [#6144](https://github.com/kubernetes/minikube/pull/6144)
+
+Huge thank you for this release towards our contributors:
+- Anders Björklund
+- andylibrian
+- Dao Cong Tien
+- Dominic Yin
+- fenglixa
+- GennadySpb
+- Kenta Iso
+- Kim Bao Long
+- Medya Ghazizadeh
+- Nguyen Hai Truong
+- Priya Wadhwa
+- Sharif Elgamal
+- Thomas Strömberg
+- ttonline6
+- Zhongcheng Lao
+- Zhou Hao
+
 
 ## Version 1.6.2  - 2019-12-19
 
@@ -11,7 +51,7 @@
 * start: Remove create/delete retry loop [#6129](https://github.com/kubernetes/minikube/pull/6129)
 * Change error text to encourage better issue reports [#6121](https://github.com/kubernetes/minikube/pull/6121)
 
-Huge thank you for this release towards our contributors: 
+Huge thank you for this release towards our contributors:
 - Anukul Sangwan
 - Aresforchina
 - Curtis Carter

From 28148c5f3df567bba49f44a0d7575dcf28f5398a Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Thu, 16 Jan 2020 11:47:01 -0800
Subject: [PATCH 063/250] Remove addon manager

---
 deploy/addons/addon-manager.yaml.tmpl         | 56 -------------------
 go.mod                                        |  2 +-
 go.sum                                        | 27 +--------
 pkg/addons/config.go                          |  5 --
 pkg/minikube/assets/addons.go                 |  8 ---
 pkg/minikube/bootstrapper/images/images.go    |  6 --
 .../bootstrapper/images/images_test.go        |  2 -
 .../bootstrapper/images/kubeadm_test.go       |  7 ---
 pkg/minikube/logs/logs.go                     |  1 -
 9 files changed, 2 insertions(+), 112 deletions(-)
 delete mode 100644 deploy/addons/addon-manager.yaml.tmpl

diff --git a/deploy/addons/addon-manager.yaml.tmpl b/deploy/addons/addon-manager.yaml.tmpl
deleted file mode 100644
index 7dc07dcdaa..0000000000
--- a/deploy/addons/addon-manager.yaml.tmpl
+++ /dev/null
@@ -1,56 +0,0 @@
-# Copyright 2016 The Kubernetes Authors All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-apiVersion: v1
-kind: Pod
-metadata:
-  name: kube-addon-manager
-  namespace: kube-system
-  labels:
-    component: kube-addon-manager
-    version: v9.0.2
-    kubernetes.io/minikube-addons: addon-manager
-spec:
-  hostNetwork: true
-  containers:
-  - name: kube-addon-manager
-    image: {{default "k8s.gcr.io" .ImageRepository}}/kube-addon-manager{{.ExoticArch}}:v9.0.2
-    env:
-    - name: KUBECONFIG
-      value: /var/lib/minikube/kubeconfig
-    - name: TEST_ADDON_CHECK_INTERVAL_SEC
-      value: "5"
-    - name: ADDON_MANAGER_LEADER_ELECTION
-      value: "false"
-    - name: KUBECTL_EXTRA_PRUNE_WHITELIST
-      value: install.istio.io/v1alpha2/IstioControlPlane
-    imagePullPolicy: IfNotPresent
-    resources:
-      requests:
-        cpu: 5m
-        memory: 50Mi
-    volumeMounts:
-    - mountPath: /etc/kubernetes/
-      name: addons
-      readOnly: true
-    - mountPath: /var/lib/minikube/
-      name: kubeconfig
-      readOnly: true
-  volumes:
-  - hostPath:
-      path: /etc/kubernetes/
-    name: addons
-  - hostPath:
-      path: /var/lib/minikube/
-    name: kubeconfig
diff --git a/go.mod b/go.mod
index 9d36285480..0cd9aca2bf 100644
--- a/go.mod
+++ b/go.mod
@@ -10,7 +10,6 @@ require (
 	github.com/Parallels/docker-machine-parallels v1.3.0
 	github.com/Sirupsen/logrus v0.0.0-20170822132746-89742aefa4b2 // indirect
 	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
-	github.com/alecthomas/chroma v0.7.0
 	github.com/blang/semver v3.5.0+incompatible
 	github.com/c4milo/gotoolkit v0.0.0-20170318115440-bcc06269efa9 // indirect
 	github.com/cenkalti/backoff v2.2.1+incompatible
@@ -28,6 +27,7 @@ require (
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
 	github.com/google/go-cmp v0.3.0
 	github.com/googleapis/gnostic v0.3.0 // indirect
+	github.com/gorilla/mux v1.7.3 // indirect
 	github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce // indirect
 	github.com/hashicorp/go-getter v1.4.0
 	github.com/hashicorp/go-multierror v0.0.0-20160811015721-8c5f0ad93604 // indirect
diff --git a/go.sum b/go.sum
index 1fdc1bf42c..2c4f7ca288 100644
--- a/go.sum
+++ b/go.sum
@@ -21,8 +21,6 @@ github.com/BurntSushi/toml v0.3.0/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0=
-github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0=
 github.com/GoogleCloudPlatform/k8s-cloud-provider v0.0.0-20181220005116-f8e995905100/go.mod h1:iroGtC8B3tQiqtds1l+mgk/BBOrxbqjH+eUfFQYRc14=
 github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab/go.mod h1:3VYc5hodBMJ5+l/7J4xAyMeuM2PNuepvHlGs8yilUCA=
 github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd h1:sjQovDkwrZp8u+gxLtPgKGjk5hCxuy2hrRejBTA9xFU=
@@ -44,15 +42,6 @@ github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdc
 github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
 github.com/afbjorklund/go-getter v1.4.1-0.20190910175809-eb9f6c26742c h1:18gEt7qzn7CW7qMkfPTFyyotlPbvPQo9o4IDV8jZqP4=
 github.com/afbjorklund/go-getter v1.4.1-0.20190910175809-eb9f6c26742c/go.mod h1:7qxyCd8rBfcShwsvxgIguu4KbS3l8bUCwg2Umn7RjeY=
-github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c=
-github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38/go.mod h1:r7bzyVFMNntcxPZXK3/+KdruV1H5KSlyVY0gc+NgInI=
-github.com/alecthomas/chroma v0.7.0 h1:z+0HgTUmkpRDRz0SRSdMaqOLfJV4F+N1FPDZUZIDUzw=
-github.com/alecthomas/chroma v0.7.0/go.mod h1:1U/PfCsTALWWYHDnsIQkxEBM0+6LLe0v8+RSVMOwxeY=
-github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721/go.mod h1:QO9JBoKquHd+jz9nshCh40fOfO+JzsoXy8qTHF68zU0=
-github.com/alecthomas/kong v0.1.17-0.20190424132513-439c674f7ae0/go.mod h1:+inYUSluD+p4L8KdviBSgzcqEjUQOfC5fQDRFuc36lI=
-github.com/alecthomas/kong v0.2.1-0.20190708041108-0548c6b1afae/go.mod h1:+inYUSluD+p4L8KdviBSgzcqEjUQOfC5fQDRFuc36lI=
-github.com/alecthomas/kong-hcl v0.1.8-0.20190615233001-b21fea9723c8/go.mod h1:MRgZdU3vrFd05IQ89AxUZ0aYdF39BYoNFa324SodPCA=
-github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
 github.com/apache/thrift v0.0.0-20180902110319-2566ecd5d999/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
@@ -115,16 +104,11 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc
 github.com/cyphar/filepath-securejoin v0.0.0-20170720062807-ae69057f2299/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
 github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ=
 github.com/d2g/dhcp4client v0.0.0-20170829104524-6e570ed0a266/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s=
-github.com/daaku/go.zipexe v1.0.0/go.mod h1:z8IiR6TsVLEYKwXAoE/I+8ys/sDkgTzSL0CLnGVd57E=
-github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ=
-github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE=
 github.com/dgrijalva/jwt-go v0.0.0-20160705203006-01aeca54ebda/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
-github.com/dlclark/regexp2 v1.1.6 h1:CqB4MjHw0MFCDj+PHHjiESmHX+N7t0tJzKvC6M97BRg=
-github.com/dlclark/regexp2 v1.1.6/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
 github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
 github.com/docker/distribution v0.0.0-20170726174610-edc3ab29cdff h1:FKH02LHYqSmeWd3GBh0KIkM8JBpw3RrShgtcWShdWJg=
 github.com/docker/distribution v0.0.0-20170726174610-edc3ab29cdff/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
@@ -250,13 +234,9 @@ github.com/googleapis/gnostic v0.3.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTV
 github.com/gophercloud/gophercloud v0.0.0-20190126172459-c818fa66e4c8/go.mod h1:3WdhXV3rUYy9p6AUW8d94kr+HS62Y4VL9mBnFxsD8q4=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
-github.com/gorilla/csrf v1.6.0/go.mod h1:7tSf8kmjNYr7IWDCYhd3U8Ck34iQ/Yw5CJu7bAkHEGI=
-github.com/gorilla/handlers v1.4.1/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ=
 github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
-github.com/gorilla/mux v1.7.1 h1:Dw4jY2nghMMRsh1ol8dv1axHkDwMQK2DHerMNJsIpJU=
-github.com/gorilla/mux v1.7.1/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
 github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
-github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
@@ -300,7 +280,6 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt
 github.com/intel-go/cpuid v0.0.0-20181003105527-1a4a6f06a1c6 h1:XboatR7lasl05yel5hNXF7kQBw2oFUGdMztcgisfhNU=
 github.com/intel-go/cpuid v0.0.0-20181003105527-1a4a6f06a1c6/go.mod h1:RmeVYf9XrPRbRc3XIx0gLYA8qOFvNoPOfaEZduRlEp4=
 github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
-github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jimmidyson/go-download v0.0.0-20161028105827-7f9a90c8c95b h1:3TknJxYSK1eDe21QorC3C2Yz8jylk6vlJG9YABnFzkU=
 github.com/jimmidyson/go-download v0.0.0-20161028105827-7f9a90c8c95b/go.mod h1:I3WsAhNNoG7a/d8HMlYUywJJlfOs/+/83NEUjuDp4lc=
 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
@@ -409,7 +388,6 @@ github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+
 github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk=
 github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
 github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
-github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E=
 github.com/olekukonko/tablewriter v0.0.0-20160923125401-bdcc175572fd h1:nEatQ6JnwCT9iYD5uqYUiFqq8tJGX25to8KVKXqya7k=
 github.com/olekukonko/tablewriter v0.0.0-20160923125401-bdcc175572fd/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@@ -544,8 +522,6 @@ github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljT
 github.com/ulikunitz/xz v0.5.5 h1:pFrO0lVpTBXLpYw+pnLj6TbvHuyjXMfjGeCwSqCVwok=
 github.com/ulikunitz/xz v0.5.5/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
 github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4=
-github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
 github.com/vishvananda/netlink v0.0.0-20171020171820-b2de5d10e38e/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
 github.com/vishvananda/netns v0.0.0-20171111001504-be1fbeda1936/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI=
 github.com/vmware/govmomi v0.20.1/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU=
@@ -632,7 +608,6 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181004145325-8469e314837c/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
diff --git a/pkg/addons/config.go b/pkg/addons/config.go
index 45d4422332..059ddf7929 100644
--- a/pkg/addons/config.go
+++ b/pkg/addons/config.go
@@ -30,11 +30,6 @@ type Addon struct {
 
 // Addons is a list of all addons
 var Addons = []*Addon{
-	{
-		name:      "addon-manager",
-		set:       SetBool,
-		callbacks: []setFn{enableOrDisableAddon},
-	},
 	{
 		name:      "dashboard",
 		set:       SetBool,
diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go
index af5a01f13a..1c8a0e0773 100644
--- a/pkg/minikube/assets/addons.go
+++ b/pkg/minikube/assets/addons.go
@@ -68,14 +68,6 @@ func (a *Addon) IsEnabled() (bool, error) {
 // Addons is the list of addons
 // TODO: Make dynamically loadable: move this data to a .yaml file within each addon directory
 var Addons = map[string]*Addon{
-	"addon-manager": NewAddon([]*BinAsset{
-		MustBinAsset(
-			"deploy/addons/addon-manager.yaml.tmpl",
-			vmpath.GuestManifestsDir,
-			"addon-manager.yaml.tmpl",
-			"0640",
-			true),
-	}, false, "addon-manager"),
 	"dashboard": NewAddon([]*BinAsset{
 		MustBinAsset("deploy/addons/dashboard/dashboard-clusterrole.yaml", vmpath.GuestAddonsDir, "dashboard-clusterrole.yaml", "0640", false),
 		MustBinAsset("deploy/addons/dashboard/dashboard-clusterrolebinding.yaml", vmpath.GuestAddonsDir, "dashboard-clusterrolebinding.yaml", "0640", false),
diff --git a/pkg/minikube/bootstrapper/images/images.go b/pkg/minikube/bootstrapper/images/images.go
index 80a06f385e..edb89b1b0a 100644
--- a/pkg/minikube/bootstrapper/images/images.go
+++ b/pkg/minikube/bootstrapper/images/images.go
@@ -112,7 +112,6 @@ func archTag(hasTag bool) string {
 // auxiliary returns images that are helpful for running minikube
 func auxiliary(mirror string) []string {
 	return []string{
-		addonManager(mirror),
 		storageProvisioner(mirror),
 		dashboardFrontend(mirror),
 		dashboardMetrics(mirror),
@@ -124,11 +123,6 @@ func storageProvisioner(mirror string) string {
 	return path.Join(minikubeRepo(mirror), "storage-provisioner"+archTag(false)+"v1.8.1")
 }
 
-// addonManager returns the Kubernetes addon manager image
-func addonManager(mirror string) string {
-	return path.Join(kubernetesRepo(mirror), "kube-addon-manager"+archTag(false)+"v9.0.2")
-}
-
 // dashboardFrontend returns the image used for the dashboard frontend
 func dashboardFrontend(repo string) string {
 	if repo == "" {
diff --git a/pkg/minikube/bootstrapper/images/images_test.go b/pkg/minikube/bootstrapper/images/images_test.go
index 3d12adc591..6a2edc5da2 100644
--- a/pkg/minikube/bootstrapper/images/images_test.go
+++ b/pkg/minikube/bootstrapper/images/images_test.go
@@ -24,7 +24,6 @@ import (
 
 func TestAuxiliary(t *testing.T) {
 	want := []string{
-		"k8s.gcr.io/kube-addon-manager:v9.0.2",
 		"gcr.io/k8s-minikube/storage-provisioner:v1.8.1",
 		"kubernetesui/dashboard:v2.0.0-beta8",
 		"kubernetesui/metrics-scraper:v1.0.2",
@@ -37,7 +36,6 @@ func TestAuxiliary(t *testing.T) {
 
 func TestAuxiliaryMirror(t *testing.T) {
 	want := []string{
-		"test.mirror/kube-addon-manager:v9.0.2",
 		"test.mirror/storage-provisioner:v1.8.1",
 		"test.mirror/dashboard:v2.0.0-beta8",
 		"test.mirror/metrics-scraper:v1.0.2",
diff --git a/pkg/minikube/bootstrapper/images/kubeadm_test.go b/pkg/minikube/bootstrapper/images/kubeadm_test.go
index d20a05bc55..1819254f23 100644
--- a/pkg/minikube/bootstrapper/images/kubeadm_test.go
+++ b/pkg/minikube/bootstrapper/images/kubeadm_test.go
@@ -37,7 +37,6 @@ func TestKubeadmImages(t *testing.T) {
 			"k8s.gcr.io/coredns:1.6.5",
 			"k8s.gcr.io/etcd:3.4.3-0",
 			"k8s.gcr.io/pause:3.1",
-			"k8s.gcr.io/kube-addon-manager:v9.0.2",
 			"gcr.io/k8s-minikube/storage-provisioner:v1.8.1",
 			"kubernetesui/dashboard:v2.0.0-beta8",
 			"kubernetesui/metrics-scraper:v1.0.2",
@@ -50,7 +49,6 @@ func TestKubeadmImages(t *testing.T) {
 			"mirror.k8s.io/coredns:1.6.2",
 			"mirror.k8s.io/etcd:3.3.15-0",
 			"mirror.k8s.io/pause:3.1",
-			"mirror.k8s.io/kube-addon-manager:v9.0.2",
 			"mirror.k8s.io/storage-provisioner:v1.8.1",
 			"mirror.k8s.io/dashboard:v2.0.0-beta8",
 			"mirror.k8s.io/metrics-scraper:v1.0.2",
@@ -63,7 +61,6 @@ func TestKubeadmImages(t *testing.T) {
 			"k8s.gcr.io/coredns:1.3.1",
 			"k8s.gcr.io/etcd:3.3.10",
 			"k8s.gcr.io/pause:3.1",
-			"k8s.gcr.io/kube-addon-manager:v9.0.2",
 			"gcr.io/k8s-minikube/storage-provisioner:v1.8.1",
 			"kubernetesui/dashboard:v2.0.0-beta8",
 			"kubernetesui/metrics-scraper:v1.0.2",
@@ -76,7 +73,6 @@ func TestKubeadmImages(t *testing.T) {
 			"k8s.gcr.io/coredns:1.3.1",
 			"k8s.gcr.io/etcd:3.3.10",
 			"k8s.gcr.io/pause:3.1",
-			"k8s.gcr.io/kube-addon-manager:v9.0.2",
 			"gcr.io/k8s-minikube/storage-provisioner:v1.8.1",
 			"kubernetesui/dashboard:v2.0.0-beta8",
 			"kubernetesui/metrics-scraper:v1.0.2",
@@ -89,7 +85,6 @@ func TestKubeadmImages(t *testing.T) {
 			"k8s.gcr.io/coredns:1.2.6",
 			"k8s.gcr.io/etcd:3.2.24",
 			"k8s.gcr.io/pause:3.1",
-			"k8s.gcr.io/kube-addon-manager:v9.0.2",
 			"gcr.io/k8s-minikube/storage-provisioner:v1.8.1",
 			"kubernetesui/dashboard:v2.0.0-beta8",
 			"kubernetesui/metrics-scraper:v1.0.2",
@@ -102,7 +97,6 @@ func TestKubeadmImages(t *testing.T) {
 			"k8s.gcr.io/coredns:1.2.2",
 			"k8s.gcr.io/etcd:3.2.24",
 			"k8s.gcr.io/pause:3.1",
-			"k8s.gcr.io/kube-addon-manager:v9.0.2",
 			"gcr.io/k8s-minikube/storage-provisioner:v1.8.1",
 			"kubernetesui/dashboard:v2.0.0-beta8",
 			"kubernetesui/metrics-scraper:v1.0.2",
@@ -115,7 +109,6 @@ func TestKubeadmImages(t *testing.T) {
 			"k8s.gcr.io/coredns:1.1.3",
 			"k8s.gcr.io/etcd-amd64:3.2.18",
 			"k8s.gcr.io/pause:3.1",
-			"k8s.gcr.io/kube-addon-manager:v9.0.2",
 			"gcr.io/k8s-minikube/storage-provisioner:v1.8.1",
 			"kubernetesui/dashboard:v2.0.0-beta8",
 			"kubernetesui/metrics-scraper:v1.0.2",
diff --git a/pkg/minikube/logs/logs.go b/pkg/minikube/logs/logs.go
index f44455df2b..79328d947e 100644
--- a/pkg/minikube/logs/logs.go
+++ b/pkg/minikube/logs/logs.go
@@ -47,7 +47,6 @@ var importantPods = []string{
 	"coredns",
 	"kube-scheduler",
 	"kube-proxy",
-	"kube-addon-manager",
 	"kubernetes-dashboard",
 	"storage-provisioner",
 	"kube-controller-manager",

From b63212eacf34f9b44d9836abb0ecfddf1cd7b320 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Fri, 17 Jan 2020 14:04:48 -0800
Subject: [PATCH 064/250] patch cgroup pr

---
 pkg/minikube/bootstrapper/kicbs/kicbs.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/minikube/bootstrapper/kicbs/kicbs.go b/pkg/minikube/bootstrapper/kicbs/kicbs.go
index 3c7d2ddd6a..ea7728a042 100644
--- a/pkg/minikube/bootstrapper/kicbs/kicbs.go
+++ b/pkg/minikube/bootstrapper/kicbs/kicbs.go
@@ -80,7 +80,7 @@ func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error {
 			out.FailureT("Unable to load cached images: {{.error}}", out.V{"error": err})
 		}
 	}
-	r, err := cruntime.New(cruntime.Config{Type: cfg.ContainerRuntime, Socket: cfg.KubernetesConfig.CRISocket})
+	r, err := cruntime.New(cruntime.Config{Type: cfg.ContainerRuntime, Socket: cfg.KubernetesConfig.CRISocket, Runner: k.c})
 	if err != nil {
 		return errors.Wrap(err, "runtime")
 	}

From b0968a37db4d20b9a9ab4bcbbf0ed727f5c90ad7 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Fri, 17 Jan 2020 14:20:16 -0800
Subject: [PATCH 065/250] change github-release calls to verbose

---
 hack/jenkins/release_github_page.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/hack/jenkins/release_github_page.sh b/hack/jenkins/release_github_page.sh
index 24a40143a5..02784c881c 100755
--- a/hack/jenkins/release_github_page.sh
+++ b/hack/jenkins/release_github_page.sh
@@ -61,14 +61,14 @@ See [Getting Started](https://minikube.sigs.k8s.io/docs/start/)
 
 # ================================================================================
 # Deleting release from github before creating new one
-github-release delete \
+github-release -v delete \
   --user "${GITHUB_ORGANIZATION}" \
   --repo "${GITHUB_REPO}" \
   --tag "${TAGNAME}" \
   || true
 
 # Creating a new release in github
-github-release release ${RELEASE_FLAGS} \
+github-release -v release ${RELEASE_FLAGS} \
     --user "${GITHUB_ORGANIZATION}" \
     --repo "${GITHUB_REPO}" \
     --tag "${TAGNAME}" \
@@ -106,7 +106,7 @@ do
     n=0
     until [ $n -ge 5 ]
     do
-        github-release upload \
+        github-release -v upload \
           --user "${GITHUB_ORGANIZATION}" \
           --repo "${GITHUB_REPO}" \
           --tag "${TAGNAME}" \

From 0b2d8051bb45c73def4295672b62fd726bc3b2df Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Fri, 17 Jan 2020 13:45:19 -0800
Subject: [PATCH 066/250] remove kicbs

---
 cmd/minikube/cmd/root.go                      |   8 -
 cmd/minikube/cmd/start.go                     |  14 +-
 pkg/minikube/bootstrapper/bootstrapper.go     |   5 +-
 .../bootstrapper/bsutil/kverify/kverify.go    |   4 +-
 pkg/minikube/bootstrapper/kicbs/cni.go        | 134 -------
 pkg/minikube/bootstrapper/kicbs/kicbs.go      | 346 ------------------
 .../bootstrapper/kubeadm/default_cni.go       | 118 ++++++
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go  | 117 ++++--
 pkg/minikube/driver/driver.go                 |   4 +-
 9 files changed, 214 insertions(+), 536 deletions(-)
 delete mode 100644 pkg/minikube/bootstrapper/kicbs/cni.go
 delete mode 100644 pkg/minikube/bootstrapper/kicbs/kicbs.go

diff --git a/cmd/minikube/cmd/root.go b/cmd/minikube/cmd/root.go
index 2a1d012b29..344aee7eff 100644
--- a/cmd/minikube/cmd/root.go
+++ b/cmd/minikube/cmd/root.go
@@ -32,7 +32,6 @@ import (
 	"k8s.io/kubectl/pkg/util/templates"
 	configCmd "k8s.io/minikube/cmd/minikube/cmd/config"
 	"k8s.io/minikube/pkg/minikube/bootstrapper"
-	"k8s.io/minikube/pkg/minikube/bootstrapper/kicbs"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/kubeadm"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
@@ -274,16 +273,9 @@ func getClusterBootstrapper(api libmachine.API, bootstrapperName string) (bootst
 		if err != nil {
 			return nil, errors.Wrap(err, "getting a new kubeadm bootstrapper")
 		}
-	case bootstrapper.KIC:
-		b, err = kicbs.NewBootstrapper(api)
-		if err != nil {
-			return nil, errors.Wrap(err, "getting a new kic bootstrapper")
-		}
-
 	default:
 		return nil, fmt.Errorf("unknown bootstrapper: %s", bootstrapperName)
 	}
-
 	return b, nil
 }
 
diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index ebb5158124..ba3df0215c 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -366,7 +366,7 @@ func runStart(cmd *cobra.Command, args []string) {
 	bs := setupKubeAdm(machineAPI, config)
 
 	// pull images or restart cluster
-	bootstrapCluster(bs, cr, mRunner, config.KubernetesConfig, preExists, isUpgrade)
+	bootstrapCluster(bs, cr, mRunner, config, preExists, isUpgrade)
 	configureMounts()
 
 	// enable addons with start command
@@ -383,7 +383,7 @@ func runStart(cmd *cobra.Command, args []string) {
 
 	// Skip pre-existing, because we already waited for health
 	if viper.GetBool(waitUntilHealthy) && !preExists {
-		if err := bs.WaitForCluster(config.KubernetesConfig, viper.GetDuration(waitTimeout)); err != nil {
+		if err := bs.WaitForCluster(config, viper.GetDuration(waitTimeout)); err != nil {
 			exit.WithError("Wait failed", err)
 		}
 	}
@@ -1027,6 +1027,10 @@ func autoSetDriverOptions(cmd *cobra.Command, drvName string) (err error) {
 		viper.Set(cacheImages, hints.CacheImages)
 	}
 
+	if !cmd.Flags().Changed(enableDefaultCNI) {
+		viper.Set(enableDefaultCNI, hints.EnableDefaultCNI)
+	}
+
 	if !cmd.Flags().Changed(containerRuntime) && hints.ContainerRuntime != "" {
 		viper.Set(containerRuntime, hints.ContainerRuntime)
 		glog.Infof("auto set %s to %q.", containerRuntime, hints.ContainerRuntime)
@@ -1287,16 +1291,16 @@ func configureRuntimes(runner cruntime.CommandRunner, drvName string, k8s cfg.Ku
 }
 
 // bootstrapCluster starts Kubernetes using the chosen bootstrapper
-func bootstrapCluster(bs bootstrapper.Bootstrapper, r cruntime.Manager, runner command.Runner, kc cfg.KubernetesConfig, preexisting bool, isUpgrade bool) {
+func bootstrapCluster(bs bootstrapper.Bootstrapper, r cruntime.Manager, runner command.Runner, c cfg.MachineConfig, preexisting bool, isUpgrade bool) {
 	if isUpgrade || !preexisting {
 		out.T(out.Pulling, "Pulling images ...")
-		if err := bs.PullImages(kc); err != nil {
+		if err := bs.PullImages(c.KubernetesConfig); err != nil {
 			out.T(out.FailureType, "Unable to pull images, which may be OK: {{.error}}", out.V{"error": err})
 		}
 	}
 
 	out.T(out.Launch, "Launching Kubernetes ... ")
-	if err := bs.StartCluster(kc); err != nil {
+	if err := bs.StartCluster(c); err != nil {
 		exit.WithLogEntries("Error starting cluster", err, logs.FindProblems(r, bs, runner))
 	}
 }
diff --git a/pkg/minikube/bootstrapper/bootstrapper.go b/pkg/minikube/bootstrapper/bootstrapper.go
index 9b45006176..f247c84684 100644
--- a/pkg/minikube/bootstrapper/bootstrapper.go
+++ b/pkg/minikube/bootstrapper/bootstrapper.go
@@ -37,10 +37,10 @@ type LogOptions struct {
 type Bootstrapper interface {
 	// PullImages pulls images necessary for a cluster. Success should not be required.
 	PullImages(config.KubernetesConfig) error
-	StartCluster(config.KubernetesConfig) error
+	StartCluster(config.MachineConfig) error
 	UpdateCluster(config.MachineConfig) error
 	DeleteCluster(config.KubernetesConfig) error
-	WaitForCluster(config.KubernetesConfig, time.Duration) error
+	WaitForCluster(config.MachineConfig, time.Duration) error
 	// LogCommands returns a map of log type to a command which will display that log.
 	LogCommands(LogOptions) map[string]string
 	SetupCerts(cfg config.KubernetesConfig) error
@@ -51,7 +51,6 @@ type Bootstrapper interface {
 const (
 	// Kubeadm is the kubeadm bootstrapper type
 	Kubeadm = "kubeadm"
-	KIC     = "kic"
 )
 
 // GetCachedBinaryList returns the list of binaries
diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
index 40cb5c4678..00374eb23b 100644
--- a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
+++ b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
@@ -56,8 +56,8 @@ func APIServerProcess(runner command.Runner, start time.Time, timeout time.Durat
 }
 
 // SystemPods verifies essential pods for running kurnetes is running
-func SystemPods(client *kubernetes.Clientset, start time.Time, ip string, port int, timeout time.Duration) error {
-	glog.Infof("waiting for kube-system pods to appear %s...", net.JoinHostPort(ip, fmt.Sprint(port)))
+func SystemPods(client *kubernetes.Clientset, start time.Time, timeout time.Duration) error {
+	glog.Info("waiting for kube-system pods to appear ...")
 	pStart := time.Now()
 	podStart := time.Time{}
 	podList := func() (bool, error) {
diff --git a/pkg/minikube/bootstrapper/kicbs/cni.go b/pkg/minikube/bootstrapper/kicbs/cni.go
deleted file mode 100644
index caa4534b47..0000000000
--- a/pkg/minikube/bootstrapper/kicbs/cni.go
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
-Copyright 2019 The Kubernetes Authors All rights reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Package kicbs bootstrapper for kic
-package kicbs
-
-const defaultCNIManifest = `
----
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: kindnet
-rules:
-  - apiGroups:
-    - policy
-    resources:
-    - podsecuritypolicies
-    verbs:
-    - use
-    resourceNames: 
-    - kindnet
-  - apiGroups:
-      - ""
-    resources:
-      - nodes
-    verbs:
-      - list
-      - watch
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: kindnet
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: kindnet
-subjects:
-- kind: ServiceAccount
-  name: kindnet
-  namespace: kube-system
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: kindnet
-  namespace: kube-system
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: kindnet
-  namespace: kube-system
-  labels:
-    tier: node
-    app: kindnet
-    k8s-app: kindnet
-spec:
-  selector:
-    matchLabels:
-      app: kindnet
-  template:
-    metadata:
-      labels:
-        tier: node
-        app: kindnet
-        k8s-app: kindnet
-    spec:
-      hostNetwork: true
-      tolerations:
-      - operator: Exists
-        effect: NoSchedule
-      serviceAccountName: kindnet
-      containers:
-      - name: kindnet-cni
-        image: kindest/kindnetd:0.5.3
-        env:
-        - name: HOST_IP
-          valueFrom:
-            fieldRef:
-              fieldPath: status.hostIP
-        - name: POD_IP
-          valueFrom:
-            fieldRef:
-              fieldPath: status.podIP
-        - name: POD_SUBNET
-          value: 10.244.0.0/16
-        volumeMounts:
-        - name: cni-cfg
-          mountPath: /etc/cni/net.d
-        - name: xtables-lock
-          mountPath: /run/xtables.lock
-          readOnly: false
-        - name: lib-modules
-          mountPath: /lib/modules
-          readOnly: true
-        resources:
-          requests:
-            cpu: "100m"
-            memory: "50Mi"
-          limits:
-            cpu: "100m"
-            memory: "50Mi"
-        securityContext:
-          privileged: false
-          capabilities:
-            add: ["NET_RAW", "NET_ADMIN"]
-      volumes:
-      - name: cni-cfg
-        hostPath:
-          path: /etc/cni/net.d
-      - name: xtables-lock
-        hostPath:
-          path: /run/xtables.lock
-          type: FileOrCreate
-      - name: lib-modules
-        hostPath:
-          path: /lib/modules
-
----
-`
diff --git a/pkg/minikube/bootstrapper/kicbs/kicbs.go b/pkg/minikube/bootstrapper/kicbs/kicbs.go
deleted file mode 100644
index ea7728a042..0000000000
--- a/pkg/minikube/bootstrapper/kicbs/kicbs.go
+++ /dev/null
@@ -1,346 +0,0 @@
-/*
-Copyright 2019 The Kubernetes Authors All rights reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Package kicbs bootstrapper for kic
-package kicbs
-
-import (
-	"fmt"
-	"net"
-	"os/exec"
-	"strings"
-	"time"
-
-	"github.com/blang/semver"
-	"github.com/docker/machine/libmachine"
-	"github.com/golang/glog"
-	"github.com/pkg/errors"
-	"github.com/spf13/viper"
-	"k8s.io/client-go/kubernetes"
-	kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants"
-	"k8s.io/minikube/pkg/drivers/kic"
-	"k8s.io/minikube/pkg/kapi"
-	"k8s.io/minikube/pkg/minikube/assets"
-	"k8s.io/minikube/pkg/minikube/bootstrapper"
-	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil"
-	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/kverify"
-	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
-	"k8s.io/minikube/pkg/minikube/command"
-	"k8s.io/minikube/pkg/minikube/config"
-	"k8s.io/minikube/pkg/minikube/constants"
-	"k8s.io/minikube/pkg/minikube/cruntime"
-	"k8s.io/minikube/pkg/minikube/machine"
-	"k8s.io/minikube/pkg/minikube/out"
-	"k8s.io/minikube/pkg/minikube/vmpath"
-)
-
-// Bootstrapper is a bootstrapper using kicbs
-type Bootstrapper struct {
-	c           command.Runner
-	k8sClient   *kubernetes.Clientset // kubernetes client used to verify pods inside cluster
-	contextName string
-}
-
-// NewBootstrapper creates a new kicbs.Bootstrapper
-func NewBootstrapper(api libmachine.API) (*Bootstrapper, error) {
-	name := viper.GetString(config.MachineProfile)
-	h, err := api.Load(name)
-	if err != nil {
-		return nil, errors.Wrap(err, "getting api client")
-	}
-	runner, err := machine.CommandRunner(h)
-	if err != nil {
-		return nil, errors.Wrap(err, "command runner")
-	}
-	return &Bootstrapper{c: runner, contextName: name}, nil
-}
-
-// UpdateCluster updates the cluster
-func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error {
-	images, err := images.Kubeadm(cfg.KubernetesConfig.ImageRepository, cfg.KubernetesConfig.KubernetesVersion)
-	if err != nil {
-		return errors.Wrap(err, "kic images")
-	}
-
-	if cfg.KubernetesConfig.ShouldLoadCachedImages {
-		if err := machine.LoadImages(&cfg, k.c, images, constants.ImageCacheDir); err != nil {
-			out.FailureT("Unable to load cached images: {{.error}}", out.V{"error": err})
-		}
-	}
-	r, err := cruntime.New(cruntime.Config{Type: cfg.ContainerRuntime, Socket: cfg.KubernetesConfig.CRISocket, Runner: k.c})
-	if err != nil {
-		return errors.Wrap(err, "runtime")
-	}
-	kubeadmCfg, err := bsutil.GenerateKubeadmYAML(cfg.KubernetesConfig, r)
-	if err != nil {
-		return errors.Wrap(err, "generating kubeadm cfg")
-	}
-
-	kubeletCfg, err := bsutil.NewKubeletConfig(cfg.KubernetesConfig, r)
-	if err != nil {
-		return errors.Wrap(err, "generating kubelet config")
-	}
-
-	kubeletService, err := bsutil.NewKubeletService(cfg.KubernetesConfig)
-	if err != nil {
-		return errors.Wrap(err, "generating kubelet service")
-	}
-
-	glog.Infof("kubelet %s config:\n%+v", kubeletCfg, cfg.KubernetesConfig)
-
-	stopCmd := exec.Command("/bin/bash", "-c", "pgrep kubelet && sudo systemctl stop kubelet")
-	// stop kubelet to avoid "Text File Busy" error
-	if rr, err := k.c.RunCmd(stopCmd); err != nil {
-		glog.Warningf("unable to stop kubelet: %s command: %q output: %q", err, rr.Command(), rr.Output())
-	}
-
-	if err := bsutil.TransferBinaries(cfg.KubernetesConfig, k.c); err != nil {
-		return errors.Wrap(err, "downloading binaries")
-	}
-
-	cniFile := []byte(defaultCNIManifest)
-
-	files := bsutil.ConfigFileAssets(cfg.KubernetesConfig, kubeadmCfg, kubeletCfg, kubeletService, cniFile)
-
-	if err := bsutil.AddAddons(&files, assets.GenerateTemplateData(cfg.KubernetesConfig)); err != nil {
-		return errors.Wrap(err, "adding addons")
-	}
-
-	for _, f := range files {
-		if err := k.c.Copy(f); err != nil {
-			return errors.Wrapf(err, "copy")
-		}
-	}
-
-	if _, err := k.c.RunCmd(exec.Command("/bin/bash", "-c", "sudo systemctl daemon-reload && sudo systemctl start kubelet")); err != nil {
-		return errors.Wrap(err, "starting kubelet")
-	}
-	return nil
-}
-
-// SetupCerts generates the certs the cluster
-func (k *Bootstrapper) SetupCerts(cfg config.KubernetesConfig) error {
-	return bootstrapper.SetupCerts(k.c, cfg)
-}
-
-// PullImages downloads images that will be used by Kubernetes
-func (k *Bootstrapper) PullImages(k8s config.KubernetesConfig) error {
-	version, err := bsutil.ParseKubernetesVersion(k8s.KubernetesVersion)
-	if err != nil {
-		return errors.Wrap(err, "parsing kubernetes version")
-	}
-	if version.LT(semver.MustParse("1.11.0")) {
-		return fmt.Errorf("pull command is not supported by kubeadm v%s", version)
-	}
-
-	rr, err := k.c.RunCmd(exec.Command("/bin/bash", "-c", fmt.Sprintf("%s config images pull --config %s", bsutil.InvokeKubeadm(k8s.KubernetesVersion), bsutil.KubeadmYamlPath)))
-	if err != nil {
-		return errors.Wrapf(err, "running cmd: %q", rr.Command())
-	}
-	return nil
-}
-
-// StartCluster starts the cluster
-func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error {
-	k8s.NodeIP = kic.DefaultBindIPV4
-	err := bsutil.ExistingConfig(k.c)
-	if err == nil { // if there is an existing cluster don't reconfigure it
-		return k.restartCluster(k8s)
-	}
-	glog.Infof("existence check: %v", err)
-
-	start := time.Now()
-	glog.Infof("StartCluster: %+v", k8s)
-	defer func() {
-		glog.Infof("StartCluster complete in %s", time.Since(start))
-	}()
-
-	extraFlags := bsutil.CreateFlagsFromExtraArgs(k8s.ExtraOptions)
-	r, err := cruntime.New(cruntime.Config{Type: k8s.ContainerRuntime})
-	if err != nil {
-		return err
-	}
-
-	ignore := []string{
-		fmt.Sprintf("DirAvailable-%s", strings.Replace(vmpath.GuestManifestsDir, "/", "-", -1)),
-		fmt.Sprintf("DirAvailable-%s", strings.Replace(vmpath.GuestPersistentDir, "/", "-", -1)),
-		fmt.Sprintf("DirAvailable-%s", strings.Replace(bsutil.EtcdDataDir(), "/", "-", -1)),
-		"FileAvailable--etc-kubernetes-manifests-kube-scheduler.yaml",
-		"FileAvailable--etc-kubernetes-manifests-kube-apiserver.yaml",
-		"FileAvailable--etc-kubernetes-manifests-kube-controller-manager.yaml",
-		"FileAvailable--etc-kubernetes-manifests-etcd.yaml",
-		"FileContent--proc-sys-net-bridge-bridge-nf-call-iptables", // for kic only
-		"Port-10250",         // For "none" users who already have a kubelet online
-		"Swap",               // For "none" users who have swap configured
-		"SystemVerification", // For kic on linux example error: "modprobe: FATAL: Module configs not found in directory /lib/modules/5.2.17-1rodete3-amd64"
-	}
-	ignore = append(ignore, bsutil.SkipAdditionalPreflights[r.Name()]...)
-
-	c := exec.Command("/bin/bash", "-c", fmt.Sprintf("%s init --config %s %s --ignore-preflight-errors=%s", bsutil.InvokeKubeadm(k8s.KubernetesVersion), bsutil.KubeadmYamlPath, extraFlags, strings.Join(ignore, ",")))
-	glog.Infof("starting kubeadm init")
-	if rr, err := k.c.RunCmd(c); err != nil {
-		return errors.Wrapf(err, "init failed. cmd: %q output: %q", rr.Command(), rr.Output())
-	}
-
-	glog.Infof("applying kic overlay network")
-	if err := k.applyOverlayNetwork(); err != nil {
-		return errors.Wrap(err, "applying kic overlay network")
-	}
-
-	glog.Infof("Skipping Configuring cluster permissions for kic...")
-
-	if err := bsutil.AdjustResourceLimits(k.c); err != nil {
-		glog.Warningf("unable to adjust resource limits: %v", err)
-	}
-
-	return nil
-}
-
-// restartCluster restarts the Kubernetes cluster configured by kubeadm
-func (k *Bootstrapper) restartCluster(k8s config.KubernetesConfig) error {
-	glog.Infof("restartCluster start")
-
-	start := time.Now()
-	defer func() {
-		glog.Infof("restartCluster took %s", time.Since(start))
-	}()
-
-	version, err := bsutil.ParseKubernetesVersion(k8s.KubernetesVersion)
-	if err != nil {
-		return errors.Wrap(err, "parsing kubernetes version")
-	}
-
-	phase := "alpha"
-	controlPlane := "controlplane"
-	if version.GTE(semver.MustParse("1.13.0")) {
-		phase = "init"
-		controlPlane = "control-plane"
-	}
-
-	baseCmd := fmt.Sprintf("%s %s", bsutil.InvokeKubeadm(k8s.KubernetesVersion), phase)
-	cmds := []string{
-		fmt.Sprintf("%s phase certs all --config %s", baseCmd, bsutil.KubeadmYamlPath),
-		fmt.Sprintf("%s phase kubeconfig all --config %s", baseCmd, bsutil.KubeadmYamlPath),
-		fmt.Sprintf("%s phase %s all --config %s", baseCmd, controlPlane, bsutil.KubeadmYamlPath),
-		fmt.Sprintf("%s phase etcd local --config %s", baseCmd, bsutil.KubeadmYamlPath),
-	}
-
-	// Run commands one at a time so that it is easier to root cause failures.
-	for _, c := range cmds {
-		rr, err := k.c.RunCmd(exec.Command("/bin/bash", "-c", c))
-		if err != nil {
-			return errors.Wrapf(err, "running cmd: %s", rr.Command())
-		}
-	}
-
-	// We must ensure that the apiserver is healthy before proceeding
-	if err := kverify.APIServerProcess(k.c, time.Now(), kconst.DefaultControlPlaneTimeout); err != nil {
-		return errors.Wrap(err, "apiserver healthz")
-	}
-
-	client, err := k.client(k8s)
-	if err != nil {
-		return errors.Wrap(err, "getting k8s client")
-	}
-
-	if err := kverify.SystemPods(client, time.Now(), k8s.NodeIP, k8s.NodePort, kconst.DefaultControlPlaneTimeout); err != nil {
-		return errors.Wrap(err, "system pods")
-	}
-
-	// Explicitly re-enable kubeadm addons (proxy, coredns) so that they will check for IP or configuration changes.
-	if rr, err := k.c.RunCmd(exec.Command("/bin/bash", "-c", fmt.Sprintf("%s phase addon all --config %s", baseCmd, bsutil.KubeadmYamlPath))); err != nil {
-		return errors.Wrapf(err, fmt.Sprintf("addon phase cmd:%q", rr.Command()))
-	}
-
-	if err := bsutil.AdjustResourceLimits(k.c); err != nil {
-		glog.Warningf("unable to adjust resource limits: %v", err)
-	}
-	return nil
-}
-
-// WaitForCluster blocks until the cluster appears to be healthy
-func (k *Bootstrapper) WaitForCluster(k8s config.KubernetesConfig, timeout time.Duration) error {
-	start := time.Now()
-	out.T(out.Waiting, "Waiting for cluster to come online ...")
-	if err := kverify.APIServerProcess(k.c, start, timeout); err != nil {
-		return errors.Wrap(err, "wait for api proc")
-	}
-
-	if err := kverify.APIServerIsRunning(start, "127.0.0.1", k8s.NodePort, timeout); err != nil {
-		return err
-	}
-
-	c, err := k.client(k8s) // getting kubernetes client before polling.
-	if err != nil {
-		return errors.Wrap(err, "get k8s client")
-	}
-
-	if err := kverify.SystemPods(c, start, "127.0.0.1", k8s.NodePort, timeout); err != nil {
-		return errors.Wrap(err, "wait for system pods")
-	}
-
-	return nil
-}
-
-func (k *Bootstrapper) DeleteCluster(config.KubernetesConfig) error {
-	return fmt.Errorf("the DeleteCluster is not implemented in kicbs yet")
-}
-
-func (k *Bootstrapper) LogCommands(bootstrapper.LogOptions) map[string]string {
-	return map[string]string{}
-}
-
-func (k *Bootstrapper) GetKubeletStatus() (string, error) {
-	return "", fmt.Errorf("the GetKubeletStatus is not implemented in kicbs yet")
-}
-func (k *Bootstrapper) GetAPIServerStatus(net.IP, int) (string, error) {
-	return "", fmt.Errorf("the GetAPIServerStatus is not implemented in kicbs yet")
-}
-
-// client sets and returns a Kubernetes client to use to speak to a kubeadm launched apiserver
-func (k *Bootstrapper) client(k8s config.KubernetesConfig) (*kubernetes.Clientset, error) {
-	if k.k8sClient != nil {
-		return k.k8sClient, nil
-	}
-
-	config, err := kapi.ClientConfig(k.contextName)
-	if err != nil {
-		return nil, errors.Wrap(err, "client config")
-	}
-
-	endpoint := fmt.Sprintf("https://%s", net.JoinHostPort("127.0.0.1", fmt.Sprint(k8s.NodePort)))
-	if config.Host != endpoint {
-		glog.Errorf("Overriding stale ClientConfig host %s with %s", config.Host, endpoint)
-		config.Host = endpoint
-	}
-	c, err := kubernetes.NewForConfig(config)
-	if err == nil {
-		k.k8sClient = c
-	}
-	return c, err
-}
-
-// applyOverlayNetwork applies the CNI plugin needed to make kic work
-func (k *Bootstrapper) applyOverlayNetwork() error {
-	cmd := exec.Command(
-		"kubectl", "create", "--kubeconfig=/etc/kubernetes/admin.conf",
-		"-f", bsutil.DefaultCNIConfigPath,
-	)
-	if rr, err := k.c.RunCmd(cmd); err != nil {
-		return errors.Wrapf(err, "cmd: %s output: %s", rr.Command(), rr.Output())
-	}
-	return nil
-}
diff --git a/pkg/minikube/bootstrapper/kubeadm/default_cni.go b/pkg/minikube/bootstrapper/kubeadm/default_cni.go
index da896eee8d..8eae93c0df 100644
--- a/pkg/minikube/bootstrapper/kubeadm/default_cni.go
+++ b/pkg/minikube/bootstrapper/kubeadm/default_cni.go
@@ -42,3 +42,121 @@ const defaultCNIConfig = `
   }
 }
 `
+
+// kicCNIConfig is the cni plugin needed for kic
+// uses cni plugin created by kind https://github.com/kubernetes-sigs/kind/blob/03a4b519067dc308308cce735065c47a6fda1583/pkg/build/node/cni.go
+const kicCNIConfig = `
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: kindnet
+rules:
+  - apiGroups:
+    - policy
+    resources:
+    - podsecuritypolicies
+    verbs:
+    - use
+    resourceNames: 
+    - kindnet
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+    verbs:
+      - list
+      - watch
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: kindnet
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kindnet
+subjects:
+- kind: ServiceAccount
+  name: kindnet
+  namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kindnet
+  namespace: kube-system
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kindnet
+  namespace: kube-system
+  labels:
+    tier: node
+    app: kindnet
+    k8s-app: kindnet
+spec:
+  selector:
+    matchLabels:
+      app: kindnet
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: kindnet
+        k8s-app: kindnet
+    spec:
+      hostNetwork: true
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      serviceAccountName: kindnet
+      containers:
+      - name: kindnet-cni
+        image: kindest/kindnetd:0.5.3
+        env:
+        - name: HOST_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.hostIP
+        - name: POD_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.podIP
+        - name: POD_SUBNET
+          value: 10.244.0.0/16
+        volumeMounts:
+        - name: cni-cfg
+          mountPath: /etc/cni/net.d
+        - name: xtables-lock
+          mountPath: /run/xtables.lock
+          readOnly: false
+        - name: lib-modules
+          mountPath: /lib/modules
+          readOnly: true
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "50Mi"
+          limits:
+            cpu: "100m"
+            memory: "50Mi"
+        securityContext:
+          privileged: false
+          capabilities:
+            add: ["NET_RAW", "NET_ADMIN"]
+      volumes:
+      - name: cni-cfg
+        hostPath:
+          path: /etc/cni/net.d
+      - name: xtables-lock
+        hostPath:
+          path: /run/xtables.lock
+          type: FileOrCreate
+      - name: lib-modules
+        hostPath:
+          path: /lib/modules
+
+---
+`
diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index 323e5e7609..d5e455174f 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -36,6 +36,7 @@ import (
 	"github.com/spf13/viper"
 	"k8s.io/client-go/kubernetes"
 	kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	"k8s.io/minikube/pkg/drivers/kic"
 	"k8s.io/minikube/pkg/kapi"
 	"k8s.io/minikube/pkg/minikube/assets"
 	"k8s.io/minikube/pkg/minikube/bootstrapper"
@@ -46,6 +47,7 @@ import (
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/cruntime"
+	"k8s.io/minikube/pkg/minikube/driver"
 	"k8s.io/minikube/pkg/minikube/machine"
 	"k8s.io/minikube/pkg/minikube/out"
 	"k8s.io/minikube/pkg/minikube/vmpath"
@@ -139,26 +141,29 @@ func (k *Bootstrapper) createCompatSymlinks() error {
 }
 
 // StartCluster starts the cluster
-func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error {
+func (k *Bootstrapper) StartCluster(cfg config.MachineConfig) error {
+	if driver.IsKIC(cfg.VMDriver) {
+		cfg.KubernetesConfig.NodeIP = kic.DefaultBindIPV4
+	}
 	err := bsutil.ExistingConfig(k.c)
 	if err == nil { // if there is an existing cluster don't reconfigure it
-		return k.restartCluster(k8s)
+		return k.restartCluster(cfg)
 	}
 	glog.Infof("existence check: %v", err)
 
 	start := time.Now()
-	glog.Infof("StartCluster: %+v", k8s)
+	glog.Infof("StartCluster: %+v", cfg)
 	defer func() {
 		glog.Infof("StartCluster complete in %s", time.Since(start))
 	}()
 
-	version, err := bsutil.ParseKubernetesVersion(k8s.KubernetesVersion)
-	if err != nil {
-		return errors.Wrap(err, "parsing kubernetes version")
-	}
+	// version, err := bsutil.ParseKubernetesVersion(cfg.KubernetesConfig.KubernetesVersion)
+	// if err != nil {
+	// 	return errors.Wrap(err, "parsing kubernetes version")
+	// }
 
-	extraFlags := bsutil.CreateFlagsFromExtraArgs(k8s.ExtraOptions)
-	r, err := cruntime.New(cruntime.Config{Type: k8s.ContainerRuntime})
+	extraFlags := bsutil.CreateFlagsFromExtraArgs(cfg.KubernetesConfig.ExtraOptions)
+	r, err := cruntime.New(cruntime.Config{Type: cfg.KubernetesConfig.ContainerRuntime})
 	if err != nil {
 		return err
 	}
@@ -173,31 +178,44 @@ func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error {
 		"FileAvailable--etc-kubernetes-manifests-etcd.yaml",
 		"Port-10250", // For "none" users who already have a kubelet online
 		"Swap",       // For "none" users who have swap configured
+		"SystemVerification",
 	}
 	ignore = append(ignore, bsutil.SkipAdditionalPreflights[r.Name()]...)
+	// if cfg.KubernetesConfig.EnableDefaultCNI {
+	// 	// if err := k.applyOverlayNetwork(); err != nil {
+	// 	// 	return errors.Wrap(err, "applying overlay network")
+	// 	// }
+	// }
 
 	// Allow older kubeadm versions to function with newer Docker releases.
-	if version.LT(semver.MustParse("1.13.0")) {
-		glog.Infof("Older Kubernetes release detected (%s), disabling SystemVerification check.", version)
-		ignore = append(ignore, "SystemVerification")
-	}
+	// For kic on linux example error: "modprobe: FATAL: Module configs not found in directory /lib/modules/5.2.17-1rodete3-amd64"
+	// fmt.Println("medya dbg: ", cfg.VMDriver)
+	// if version.LT(semver.MustParse("1.13.0")) || driver.IsKIC(cfg.VMDriver) {
+	// 	glog.Infof("Older Kubernetes release detected (%s), disabling SystemVerification check.", version)
+	// 	ignore = append(ignore, "SystemVerification")
+	// }
 
-	c := exec.Command("/bin/bash", "-c", fmt.Sprintf("%s init --config %s %s --ignore-preflight-errors=%s", bsutil.InvokeKubeadm(k8s.KubernetesVersion), bsutil.KubeadmYamlPath, extraFlags, strings.Join(ignore, ",")))
-	if rr, err := k.c.RunCmd(c); err != nil {
-		return errors.Wrapf(err, "init failed. cmd: %q", rr.Command())
+	fmt.Printf("medya dbg:ignore is %+v ", ignore)
+	c := exec.Command("/bin/bash", "-c", fmt.Sprintf("%s init --config %s %s --ignore-preflight-errors=%s", bsutil.InvokeKubeadm(cfg.KubernetesConfig.KubernetesVersion), bsutil.KubeadmYamlPath, extraFlags, strings.Join(ignore, ",")))
+	rr, err := k.c.RunCmd(c)
+	if err != nil {
+		return errors.Wrapf(err, "init failed. output: %q", rr.Output())
 	}
+	fmt.Println(rr.Output())
 
-	glog.Infof("Configuring cluster permissions ...")
-	elevate := func() error {
-		client, err := k.client(k8s)
-		if err != nil {
-			return err
+	if !driver.IsKIC(cfg.VMDriver) { // TODO: skip for both after verifications https://github.com/kubernetes/minikube/issues/6239
+		glog.Infof("Configuring cluster permissions ...")
+		elevate := func() error {
+			client, err := k.client(cfg)
+			if err != nil {
+				return err
+			}
+			return bsutil.ElevateKubeSystemPrivileges(client)
 		}
-		return bsutil.ElevateKubeSystemPrivileges(client)
-	}
 
-	if err := retry.Expo(elevate, time.Millisecond*500, 120*time.Second); err != nil {
-		return errors.Wrap(err, "timed out waiting to elevate kube-system RBAC privileges")
+		if err := retry.Expo(elevate, time.Millisecond*500, 120*time.Second); err != nil {
+			return errors.Wrap(err, "timed out waiting to elevate kube-system RBAC privileges")
+		}
 	}
 
 	if err := bsutil.AdjustResourceLimits(k.c); err != nil {
@@ -208,7 +226,7 @@ func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error {
 }
 
 // client sets and returns a Kubernetes client to use to speak to a kubeadm launched apiserver
-func (k *Bootstrapper) client(k8s config.KubernetesConfig) (*kubernetes.Clientset, error) {
+func (k *Bootstrapper) client(cfg config.MachineConfig) (*kubernetes.Clientset, error) {
 	if k.k8sClient != nil {
 		return k.k8sClient, nil
 	}
@@ -218,7 +236,8 @@ func (k *Bootstrapper) client(k8s config.KubernetesConfig) (*kubernetes.Clientse
 		return nil, errors.Wrap(err, "client config")
 	}
 
-	endpoint := fmt.Sprintf("https://%s", net.JoinHostPort(k8s.NodeIP, strconv.Itoa(k8s.NodePort)))
+	ip, port := k.clientEndpointAddr(cfg)
+	endpoint := fmt.Sprintf("https://%s", net.JoinHostPort(ip, strconv.Itoa(port)))
 	if config.Host != endpoint {
 		glog.Errorf("Overriding stale ClientConfig host %s with %s", config.Host, endpoint)
 		config.Host = endpoint
@@ -231,26 +250,27 @@ func (k *Bootstrapper) client(k8s config.KubernetesConfig) (*kubernetes.Clientse
 }
 
 // WaitForCluster blocks until the cluster appears to be healthy
-func (k *Bootstrapper) WaitForCluster(k8s config.KubernetesConfig, timeout time.Duration) error {
+func (k *Bootstrapper) WaitForCluster(cfg config.MachineConfig, timeout time.Duration) error {
 	start := time.Now()
+	ip, port := k.clientEndpointAddr(cfg)
 	out.T(out.Waiting, "Waiting for cluster to come online ...")
 	if err := kverify.APIServerProcess(k.c, start, timeout); err != nil {
 		return err
 	}
-	if err := kverify.APIServerIsRunning(start, k8s.NodeIP, k8s.NodePort, timeout); err != nil {
+	if err := kverify.APIServerIsRunning(start, ip, port, timeout); err != nil {
 		return err
 	}
 
-	c, err := k.client(k8s)
+	c, err := k.client(cfg)
 	if err != nil {
 		return errors.Wrap(err, "get k8s client")
 	}
 
-	return kverify.SystemPods(c, start, k8s.NodeIP, k8s.NodePort, timeout)
+	return kverify.SystemPods(c, start, timeout)
 }
 
 // restartCluster restarts the Kubernetes cluster configured by kubeadm
-func (k *Bootstrapper) restartCluster(k8s config.KubernetesConfig) error {
+func (k *Bootstrapper) restartCluster(cfg config.MachineConfig) error {
 	glog.Infof("restartCluster start")
 
 	start := time.Now()
@@ -258,7 +278,7 @@ func (k *Bootstrapper) restartCluster(k8s config.KubernetesConfig) error {
 		glog.Infof("restartCluster took %s", time.Since(start))
 	}()
 
-	version, err := bsutil.ParseKubernetesVersion(k8s.KubernetesVersion)
+	version, err := bsutil.ParseKubernetesVersion(cfg.KubernetesConfig.KubernetesVersion)
 	if err != nil {
 		return errors.Wrap(err, "parsing kubernetes version")
 	}
@@ -274,7 +294,7 @@ func (k *Bootstrapper) restartCluster(k8s config.KubernetesConfig) error {
 		glog.Errorf("failed to create compat symlinks: %v", err)
 	}
 
-	baseCmd := fmt.Sprintf("%s %s", bsutil.InvokeKubeadm(k8s.KubernetesVersion), phase)
+	baseCmd := fmt.Sprintf("%s %s", bsutil.InvokeKubeadm(cfg.KubernetesConfig.KubernetesVersion), phase)
 	cmds := []string{
 		fmt.Sprintf("%s phase certs all --config %s", baseCmd, bsutil.KubeadmYamlPath),
 		fmt.Sprintf("%s phase kubeconfig all --config %s", baseCmd, bsutil.KubeadmYamlPath),
@@ -295,12 +315,12 @@ func (k *Bootstrapper) restartCluster(k8s config.KubernetesConfig) error {
 		return errors.Wrap(err, "apiserver healthz")
 	}
 
-	client, err := k.client(k8s)
+	client, err := k.client(cfg)
 	if err != nil {
 		return errors.Wrap(err, "getting k8s client")
 	}
 
-	if err := kverify.SystemPods(client, time.Now(), k8s.NodeIP, k8s.NodePort, kconst.DefaultControlPlaneTimeout); err != nil {
+	if err := kverify.SystemPods(client, time.Now(), kconst.DefaultControlPlaneTimeout); err != nil {
 		return errors.Wrap(err, "system pods")
 	}
 
@@ -403,6 +423,10 @@ func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error {
 	var cniFile []byte = nil
 	if cfg.KubernetesConfig.EnableDefaultCNI {
 		cniFile = []byte(defaultCNIConfig)
+		if driver.IsKIC(cfg.VMDriver) {
+			cniFile = []byte(kicCNIConfig)
+		}
+
 	}
 	files := bsutil.ConfigFileAssets(cfg.KubernetesConfig, kubeadmCfg, kubeletCfg, kubeletService, cniFile)
 
@@ -420,3 +444,24 @@ func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error {
 	}
 	return nil
 }
+
+// // applyOverlayNetwork applies the CNI plugin needed to make kic work
+// func (k *Bootstrapper) applyOverlayNetwork() error {
+// 	cmd := exec.Command(
+// 		"kubectl", "create", "--kubeconfig=/var/lib/minikube/kubeconfig",
+// 		"-f", bsutil.DefaultCNIConfigPath,
+// 	)
+// 	if rr, err := k.c.RunCmd(cmd); err != nil {
+// 		return errors.Wrapf(err, "cmd: %s output: %s", rr.Command(), rr.Output())
+// 	}
+// 	return nil
+// }
+
+// clientEndpointAddr returns ip and port accessible for the kubernetes clients to talk to the cluster
+func (k *Bootstrapper) clientEndpointAddr(cfg config.MachineConfig) (string, int) {
+	if driver.IsKIC(cfg.VMDriver) {
+		// because docker container ip on non-linux is not accesible
+		return kic.DefaultBindIPV4, cfg.KubernetesConfig.NodePort
+	}
+	return cfg.KubernetesConfig.NodeIP, cfg.KubernetesConfig.NodePort
+}
diff --git a/pkg/minikube/driver/driver.go b/pkg/minikube/driver/driver.go
index 006e346d64..9d32c14da3 100644
--- a/pkg/minikube/driver/driver.go
+++ b/pkg/minikube/driver/driver.go
@@ -24,7 +24,6 @@ import (
 
 	"github.com/golang/glog"
 	"k8s.io/minikube/pkg/drivers/kic"
-	"k8s.io/minikube/pkg/minikube/bootstrapper"
 	"k8s.io/minikube/pkg/minikube/registry"
 )
 
@@ -100,6 +99,7 @@ type FlagHints struct {
 	CacheImages      bool
 	ContainerRuntime string
 	Bootstrapper     string
+	EnableDefaultCNI bool
 }
 
 // FlagDefaults returns suggested defaults based on a driver
@@ -110,8 +110,8 @@ func FlagDefaults(name string) FlagHints {
 		// only for kic, till other run-times are available we auto-set containerd.
 		if name == Docker {
 			fh.ContainerRuntime = "containerd"
-			fh.Bootstrapper = bootstrapper.KIC
 			fh.ExtraOptions = append(fh.ExtraOptions, fmt.Sprintf("kubeadm.pod-network-cidr=%s", kic.DefaultPodCIDR))
+			fh.EnableDefaultCNI = true
 		}
 		return fh
 	}

From 7d07d0d61bda08f5d6d17fd186ead2e862d06948 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Fri, 17 Jan 2020 15:38:33 -0800
Subject: [PATCH 067/250] ignore iptables for kic

---
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 57 ++++++++++----------
 1 file changed, 30 insertions(+), 27 deletions(-)

diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index d5e455174f..8ade8fbf00 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -157,10 +157,10 @@ func (k *Bootstrapper) StartCluster(cfg config.MachineConfig) error {
 		glog.Infof("StartCluster complete in %s", time.Since(start))
 	}()
 
-	// version, err := bsutil.ParseKubernetesVersion(cfg.KubernetesConfig.KubernetesVersion)
-	// if err != nil {
-	// 	return errors.Wrap(err, "parsing kubernetes version")
-	// }
+	version, err := bsutil.ParseKubernetesVersion(cfg.KubernetesConfig.KubernetesVersion)
+	if err != nil {
+		return errors.Wrap(err, "parsing kubernetes version")
+	}
 
 	extraFlags := bsutil.CreateFlagsFromExtraArgs(cfg.KubernetesConfig.ExtraOptions)
 	r, err := cruntime.New(cruntime.Config{Type: cfg.KubernetesConfig.ContainerRuntime})
@@ -181,27 +181,30 @@ func (k *Bootstrapper) StartCluster(cfg config.MachineConfig) error {
 		"SystemVerification",
 	}
 	ignore = append(ignore, bsutil.SkipAdditionalPreflights[r.Name()]...)
-	// if cfg.KubernetesConfig.EnableDefaultCNI {
-	// 	// if err := k.applyOverlayNetwork(); err != nil {
-	// 	// 	return errors.Wrap(err, "applying overlay network")
-	// 	// }
-	// }
 
 	// Allow older kubeadm versions to function with newer Docker releases.
 	// For kic on linux example error: "modprobe: FATAL: Module configs not found in directory /lib/modules/5.2.17-1rodete3-amd64"
-	// fmt.Println("medya dbg: ", cfg.VMDriver)
-	// if version.LT(semver.MustParse("1.13.0")) || driver.IsKIC(cfg.VMDriver) {
-	// 	glog.Infof("Older Kubernetes release detected (%s), disabling SystemVerification check.", version)
-	// 	ignore = append(ignore, "SystemVerification")
-	// }
+	if version.LT(semver.MustParse("1.13.0")) || driver.IsKIC(cfg.VMDriver) {
+		glog.Infof("Older Kubernetes release detected (%s), disabling SystemVerification check.", version)
+		ignore = append(ignore, "SystemVerification")
+	}
+
+	if driver.IsKIC(cfg.VMDriver) { // to bypass this error: /proc/sys/net/bridge/bridge-nf-call-iptables does not exist
+		ignore = append(ignore, "FileContent--proc-sys-net-bridge-bridge-nf-call-iptables")
+
+	}
 
-	fmt.Printf("medya dbg:ignore is %+v ", ignore)
 	c := exec.Command("/bin/bash", "-c", fmt.Sprintf("%s init --config %s %s --ignore-preflight-errors=%s", bsutil.InvokeKubeadm(cfg.KubernetesConfig.KubernetesVersion), bsutil.KubeadmYamlPath, extraFlags, strings.Join(ignore, ",")))
 	rr, err := k.c.RunCmd(c)
 	if err != nil {
 		return errors.Wrapf(err, "init failed. output: %q", rr.Output())
 	}
-	fmt.Println(rr.Output())
+
+	if cfg.KubernetesConfig.EnableDefaultCNI {
+		if err := k.applyOverlayNetwork(); err != nil {
+			return errors.Wrap(err, "applying overlay network")
+		}
+	}
 
 	if !driver.IsKIC(cfg.VMDriver) { // TODO: skip for both after verifications https://github.com/kubernetes/minikube/issues/6239
 		glog.Infof("Configuring cluster permissions ...")
@@ -445,17 +448,17 @@ func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error {
 	return nil
 }
 
-// // applyOverlayNetwork applies the CNI plugin needed to make kic work
-// func (k *Bootstrapper) applyOverlayNetwork() error {
-// 	cmd := exec.Command(
-// 		"kubectl", "create", "--kubeconfig=/var/lib/minikube/kubeconfig",
-// 		"-f", bsutil.DefaultCNIConfigPath,
-// 	)
-// 	if rr, err := k.c.RunCmd(cmd); err != nil {
-// 		return errors.Wrapf(err, "cmd: %s output: %s", rr.Command(), rr.Output())
-// 	}
-// 	return nil
-// }
+// applyOverlayNetwork applies the CNI plugin needed to make kic work
+func (k *Bootstrapper) applyOverlayNetwork() error {
+	cmd := exec.Command(
+		"kubectl", "create", "--kubeconfig=/etc/kubernetes/admin.conf",
+		"-f", bsutil.DefaultCNIConfigPath,
+	)
+	if rr, err := k.c.RunCmd(cmd); err != nil {
+		return errors.Wrapf(err, "cmd: %s output: %s", rr.Command(), rr.Output())
+	}
+	return nil
+}
 
 // clientEndpointAddr returns ip and port accessible for the kubernetes clients to talk to the cluster
 func (k *Bootstrapper) clientEndpointAddr(cfg config.MachineConfig) (string, int) {

From df3dc976b0060f7f73dfa3acdf10d7c433dbbbf6 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Fri, 17 Jan 2020 15:44:56 -0800
Subject: [PATCH 068/250] let's actually save the node config

---
 cmd/minikube/cmd/start.go | 13 ++++++++++++-
 go.mod                    |  2 +-
 go.sum                    | 27 +--------------------------
 3 files changed, 14 insertions(+), 28 deletions(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 76588c4f84..42b6113012 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -514,7 +514,8 @@ func startMachine(cfg *config.MachineConfig, node *config.Node) (runner command.
 	}
 	// Save IP to configuration file for subsequent use
 	node.IP = ip
-	if err := saveConfig(cfg); err != nil {
+
+	if err := saveNodeToConfig(cfg, node); err != nil {
 		exit.WithError("Failed to save config", err)
 	}
 
@@ -1336,3 +1337,13 @@ func configureMounts() {
 func saveConfig(clusterCfg *config.MachineConfig) error {
 	return config.SaveProfile(viper.GetString(config.MachineProfile), clusterCfg)
 }
+
+func saveNodeToConfig(cfg *config.MachineConfig, node *config.Node) error {
+	for i, n := range cfg.Nodes {
+		if n.Name == node.Name {
+			cfg.Nodes[i] = *node
+			break
+		}
+	}
+	return saveConfig(cfg)
+}
diff --git a/go.mod b/go.mod
index 9d36285480..0cd9aca2bf 100644
--- a/go.mod
+++ b/go.mod
@@ -10,7 +10,6 @@ require (
 	github.com/Parallels/docker-machine-parallels v1.3.0
 	github.com/Sirupsen/logrus v0.0.0-20170822132746-89742aefa4b2 // indirect
 	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
-	github.com/alecthomas/chroma v0.7.0
 	github.com/blang/semver v3.5.0+incompatible
 	github.com/c4milo/gotoolkit v0.0.0-20170318115440-bcc06269efa9 // indirect
 	github.com/cenkalti/backoff v2.2.1+incompatible
@@ -28,6 +27,7 @@ require (
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
 	github.com/google/go-cmp v0.3.0
 	github.com/googleapis/gnostic v0.3.0 // indirect
+	github.com/gorilla/mux v1.7.3 // indirect
 	github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce // indirect
 	github.com/hashicorp/go-getter v1.4.0
 	github.com/hashicorp/go-multierror v0.0.0-20160811015721-8c5f0ad93604 // indirect
diff --git a/go.sum b/go.sum
index 1fdc1bf42c..2c4f7ca288 100644
--- a/go.sum
+++ b/go.sum
@@ -21,8 +21,6 @@ github.com/BurntSushi/toml v0.3.0/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0=
-github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0=
 github.com/GoogleCloudPlatform/k8s-cloud-provider v0.0.0-20181220005116-f8e995905100/go.mod h1:iroGtC8B3tQiqtds1l+mgk/BBOrxbqjH+eUfFQYRc14=
 github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab/go.mod h1:3VYc5hodBMJ5+l/7J4xAyMeuM2PNuepvHlGs8yilUCA=
 github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd h1:sjQovDkwrZp8u+gxLtPgKGjk5hCxuy2hrRejBTA9xFU=
@@ -44,15 +42,6 @@ github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdc
 github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
 github.com/afbjorklund/go-getter v1.4.1-0.20190910175809-eb9f6c26742c h1:18gEt7qzn7CW7qMkfPTFyyotlPbvPQo9o4IDV8jZqP4=
 github.com/afbjorklund/go-getter v1.4.1-0.20190910175809-eb9f6c26742c/go.mod h1:7qxyCd8rBfcShwsvxgIguu4KbS3l8bUCwg2Umn7RjeY=
-github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c=
-github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38/go.mod h1:r7bzyVFMNntcxPZXK3/+KdruV1H5KSlyVY0gc+NgInI=
-github.com/alecthomas/chroma v0.7.0 h1:z+0HgTUmkpRDRz0SRSdMaqOLfJV4F+N1FPDZUZIDUzw=
-github.com/alecthomas/chroma v0.7.0/go.mod h1:1U/PfCsTALWWYHDnsIQkxEBM0+6LLe0v8+RSVMOwxeY=
-github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721/go.mod h1:QO9JBoKquHd+jz9nshCh40fOfO+JzsoXy8qTHF68zU0=
-github.com/alecthomas/kong v0.1.17-0.20190424132513-439c674f7ae0/go.mod h1:+inYUSluD+p4L8KdviBSgzcqEjUQOfC5fQDRFuc36lI=
-github.com/alecthomas/kong v0.2.1-0.20190708041108-0548c6b1afae/go.mod h1:+inYUSluD+p4L8KdviBSgzcqEjUQOfC5fQDRFuc36lI=
-github.com/alecthomas/kong-hcl v0.1.8-0.20190615233001-b21fea9723c8/go.mod h1:MRgZdU3vrFd05IQ89AxUZ0aYdF39BYoNFa324SodPCA=
-github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
 github.com/apache/thrift v0.0.0-20180902110319-2566ecd5d999/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
@@ -115,16 +104,11 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc
 github.com/cyphar/filepath-securejoin v0.0.0-20170720062807-ae69057f2299/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
 github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ=
 github.com/d2g/dhcp4client v0.0.0-20170829104524-6e570ed0a266/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s=
-github.com/daaku/go.zipexe v1.0.0/go.mod h1:z8IiR6TsVLEYKwXAoE/I+8ys/sDkgTzSL0CLnGVd57E=
-github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ=
-github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE=
 github.com/dgrijalva/jwt-go v0.0.0-20160705203006-01aeca54ebda/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
-github.com/dlclark/regexp2 v1.1.6 h1:CqB4MjHw0MFCDj+PHHjiESmHX+N7t0tJzKvC6M97BRg=
-github.com/dlclark/regexp2 v1.1.6/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
 github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
 github.com/docker/distribution v0.0.0-20170726174610-edc3ab29cdff h1:FKH02LHYqSmeWd3GBh0KIkM8JBpw3RrShgtcWShdWJg=
 github.com/docker/distribution v0.0.0-20170726174610-edc3ab29cdff/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
@@ -250,13 +234,9 @@ github.com/googleapis/gnostic v0.3.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTV
 github.com/gophercloud/gophercloud v0.0.0-20190126172459-c818fa66e4c8/go.mod h1:3WdhXV3rUYy9p6AUW8d94kr+HS62Y4VL9mBnFxsD8q4=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
-github.com/gorilla/csrf v1.6.0/go.mod h1:7tSf8kmjNYr7IWDCYhd3U8Ck34iQ/Yw5CJu7bAkHEGI=
-github.com/gorilla/handlers v1.4.1/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ=
 github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
-github.com/gorilla/mux v1.7.1 h1:Dw4jY2nghMMRsh1ol8dv1axHkDwMQK2DHerMNJsIpJU=
-github.com/gorilla/mux v1.7.1/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
 github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
-github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
@@ -300,7 +280,6 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt
 github.com/intel-go/cpuid v0.0.0-20181003105527-1a4a6f06a1c6 h1:XboatR7lasl05yel5hNXF7kQBw2oFUGdMztcgisfhNU=
 github.com/intel-go/cpuid v0.0.0-20181003105527-1a4a6f06a1c6/go.mod h1:RmeVYf9XrPRbRc3XIx0gLYA8qOFvNoPOfaEZduRlEp4=
 github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
-github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jimmidyson/go-download v0.0.0-20161028105827-7f9a90c8c95b h1:3TknJxYSK1eDe21QorC3C2Yz8jylk6vlJG9YABnFzkU=
 github.com/jimmidyson/go-download v0.0.0-20161028105827-7f9a90c8c95b/go.mod h1:I3WsAhNNoG7a/d8HMlYUywJJlfOs/+/83NEUjuDp4lc=
 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
@@ -409,7 +388,6 @@ github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+
 github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk=
 github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
 github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
-github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E=
 github.com/olekukonko/tablewriter v0.0.0-20160923125401-bdcc175572fd h1:nEatQ6JnwCT9iYD5uqYUiFqq8tJGX25to8KVKXqya7k=
 github.com/olekukonko/tablewriter v0.0.0-20160923125401-bdcc175572fd/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@@ -544,8 +522,6 @@ github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljT
 github.com/ulikunitz/xz v0.5.5 h1:pFrO0lVpTBXLpYw+pnLj6TbvHuyjXMfjGeCwSqCVwok=
 github.com/ulikunitz/xz v0.5.5/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
 github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4=
-github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
 github.com/vishvananda/netlink v0.0.0-20171020171820-b2de5d10e38e/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
 github.com/vishvananda/netns v0.0.0-20171111001504-be1fbeda1936/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI=
 github.com/vmware/govmomi v0.20.1/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU=
@@ -632,7 +608,6 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181004145325-8469e314837c/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

From 47d7a689b870cbe935117b7ed1d549a442077b3f Mon Sep 17 00:00:00 2001
From: inductor <kohei.ota@zozo.com>
Date: Sat, 18 Jan 2020 10:05:44 +0900
Subject: [PATCH 069/250] update minikube translation json for ja

---
 translations/ja.json | 124 +++++++++++++++++++++++++++++++++----------
 1 file changed, 95 insertions(+), 29 deletions(-)

diff --git a/translations/ja.json b/translations/ja.json
index a63609e260..65ee169b94 100644
--- a/translations/ja.json
+++ b/translations/ja.json
@@ -8,8 +8,10 @@
 	"'none' driver does not support 'minikube docker-env' command": "",
 	"'none' driver does not support 'minikube mount' command": "",
 	"'none' driver does not support 'minikube ssh' command": "",
+	"'{{.driver}}' driver reported an issue: {{.error}}": "",
+	"- {{.profile}}": "",
 	"A VPN or firewall is interfering with HTTP access to the minikube VM. Alternatively, try a different VM driver: https://minikube.sigs.k8s.io/docs/start/": "",
-	"A firewall is blocking Docker within the minikube VM from reaching the internet. You may need to configure it to use a proxy.": "",
+	"A firewall is blocking Docker the minikube VM from reaching the internet. You may need to configure it to use a proxy.": "",
 	"A firewall is interfering with minikube's ability to make outgoing HTTPS requests. You may need to change the value of the HTTPS_PROXY environment variable.": "",
 	"A firewall is likely blocking minikube from reaching the internet. You may need to configure minikube to use a proxy.": "",
 	"A set of apiserver IP Addresses which are used in the generated certificate for kubernetes.  This can be used if you want to make the apiserver available from outside the machine": "",
@@ -32,21 +34,33 @@
 	"Amount of RAM allocated to the minikube VM (format: \u003cnumber\u003e[\u003cunit\u003e], where unit = b, k, m or g).": "",
 	"Amount of time to wait for a service in seconds": "",
 	"Amount of time to wait for service in seconds": "",
+	"Another hypervisor, such as VirtualBox, is conflicting with KVM. Please stop the other hypervisor, or use --vm-driver to switch to it.": "",
+	"Automatically selected the {{.experimental}} '{{.driver}}' driver": "",
+	"Automatically selected the {{.experimental}} '{{.driver}}' driver (alternates: {{.alternates}})": "",
 	"Available Commands": "",
 	"Basic Commands:": "",
+	"Block until the apiserver is servicing API requests": "",
 	"Cannot find directory {{.path}} for mount": "",
+	"Cannot use both --output and --format options": "",
+	"Check output of 'journalctl -xeu kubelet', try passing --extra-config=kubelet.cgroup-driver=systemd to minikube start": "",
+	"Check that SELinux is disabled, and that the provided apiserver flags are valid": "",
 	"Check that minikube is running and that you have specified the correct namespace (-n flag) if required.": "",
+	"Check that the provided apiserver flags are valid": "",
 	"Check that your --kubernetes-version has a leading 'v'. For example: 'v1.1.14'": "",
-	"Check that your apiserver flags are valid, or run 'minikube delete'": "",
 	"Check your firewall rules for interference, and run 'virt-host-validate' to check for KVM configuration issues. If you are running minikube within a VM, consider using --vm-driver=none": "",
 	"Configuration and Management Commands:": "",
+	"Configure a default route on this Linux host, or use another --vm-driver that does not require it": "",
 	"Configure an external network switch following the official documentation, then add `--hyperv-virtual-switch=\u003cswitch-name\u003e` to `minikube start`": "",
 	"Configures the addon w/ADDON_NAME within minikube (example: minikube addons configure registry-creds). For a list of available addons use: minikube addons list": "",
 	"Configuring local host environment ...": "",
 	"Confirm that you have a working internet connection and that your VM has not run out of resources by using: 'minikube logs'": "",
 	"Confirm that you have supplied the correct value to --hyperv-virtual-switch using the 'Get-VMSwitch' command": "",
+	"Could not get profile flag": "",
+	"Could not process error from failed deletion": "",
+	"Could not process errors from failed deletion": "",
 	"Country code of the image mirror to be used. Leave empty to use the global one. For Chinese mainland users, set it to cn.": "使用するイメージミラーの国コード。グローバルのものを使用する場合は空のままにします。中国本土のユーザーの場合は、「cn」に設定します。",
 	"Created a new profile : {{.profile_name}}": "",
+	"Creating Kubernetes in {{.driver_name}} container with (CPUs={{.number_of_cpus}}), Memory={{.memory_size}}MB ({{.host_memory_size}}MB available) ...": "",
 	"Creating a new profile failed": "",
 	"Creating mount {{.name}} ...": "マウント {{.name}} を作成しています...",
 	"Creating {{.driver_name}} VM (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...": "",
@@ -82,25 +96,31 @@
 	"ERROR creating `registry-creds-ecr` secret: {{.error}}": "",
 	"ERROR creating `registry-creds-gcr` secret: {{.error}}": "",
 	"Either systemctl is not installed, or Docker is broken. Run 'sudo systemctl start docker' and 'journalctl -u docker'": "",
+	"Enable addons. see `minikube addons list` for a list of valid addon names.": "",
 	"Enable experimental NVIDIA GPU support in minikube": "minikube での試験運用版 NVIDIA GPU の対応を有効にします",
 	"Enable host resolver for NAT DNS requests (virtualbox driver only)": "NAT DNS リクエスト用のホストリゾルバを有効にします（virtualbox ドライバのみ）",
+	"Enable istio needs {{.minMem}} MB of memory and {{.minCpus}} CPUs.": "",
 	"Enable proxy for NAT DNS requests (virtualbox driver only)": "NAT DNS リクエスト用のプロキシを有効にします（virtualbox ドライバのみ）",
 	"Enable the default CNI plugin (/etc/cni/net.d/k8s.conf). Used in conjunction with \\\"--network-plugin=cni\\": "デフォルトの CNI プラグイン（/etc/cni/net.d/k8s.conf）を有効にします。\\\"--network-plugin=cni\\\" と組み合わせて使用されます。",
 	"Enable the default CNI plugin (/etc/cni/net.d/k8s.conf). Used in conjunction with \\\"--network-plugin=cni\\\".": "",
 	"Enables the addon w/ADDON_NAME within minikube (example: minikube addons enable dashboard). For a list of available addons use: minikube addons list": "",
 	"Enabling dashboard ...": "",
+	"Ensure that CRI-O is installed and healthy: Run 'sudo systemctl start crio' and 'journalctl -u crio'. Alternatively, use --container-runtime=docker": "",
+	"Ensure that Docker is installed and healthy: Run 'sudo systemctl start docker' and 'journalctl -u docker'. Alternatively, select another value for --vm-driver": "",
+	"Ensure that the user listed in /etc/libvirt/qemu.conf has access to your home directory": "",
+	"Ensure that your value for HTTPS_PROXY points to an HTTPS proxy rather than an HTTP proxy": "",
 	"Environment variables to pass to the Docker daemon. (format: key=value)": "Docker デーモンに渡す環境変数（形式: Key=Value）",
 	"Error checking driver version: {{.error}}": "ドライバのバージョンの確認中にエラーが発生しました。{{.error}}",
-	"Error creating list template": "",
+	"Error converting status to json": "",
 	"Error creating minikube directory": "",
 	"Error creating status template": "",
 	"Error creating view template": "",
-	"Error executing list template": "",
 	"Error executing status template": "",
 	"Error executing template": "",
 	"Error executing view template": "",
 	"Error finding port for mount": "",
 	"Error getting IP": "",
+	"Error getting addons status": "",
 	"Error getting bootstrapper": "",
 	"Error getting client": "",
 	"Error getting client: {{.error}}": "",
@@ -110,7 +130,7 @@
 	"Error getting host": "",
 	"Error getting host status": "",
 	"Error getting machine logs": "",
-	"Error getting machine status": "",
+	"Error getting profiles to delete": "",
 	"Error getting service status": "",
 	"Error getting service with namespace: {{.namespace}} and labels {{.labelName}}:{{.addonName}}: {{.error}}": "",
 	"Error getting the host IP address to use from within the VM": "",
@@ -124,7 +144,6 @@
 	"Error parsing minikube version: {{.error}}": "minikube バージョンの解析中にエラーが発生しました。{{.error}}",
 	"Error parsing vmDriver version: {{.error}}": "vmDriver バージョンの解析中にエラーが発生しました。{{.error}}",
 	"Error reading {{.path}}: {{.error}}": "",
-	"Error restarting cluster": "",
 	"Error setting shell variables": "",
 	"Error starting cluster": "",
 	"Error starting mount": "",
@@ -136,12 +155,12 @@
 	"Error: [{{.id}}] {{.error}}": "",
 	"Examples": "",
 	"Exiting": "終了しています",
-	"Exiting due to driver incompatibility": "",
+	"Exiting.": "",
 	"Failed runtime": "",
 	"Failed to cache ISO": "",
 	"Failed to cache and load images": "",
 	"Failed to cache binaries": "",
-	"Failed to cache images": "",
+	"Failed to cache images to tar": "",
 	"Failed to change permissions for {{.minikube_dir_path}}: {{.error}}": "{{.minikube_dir_path}} に対する権限を変更できませんでした。{{.error}}",
 	"Failed to check if machine exists": "",
 	"Failed to check main repository and mirrors for images for images": "",
@@ -160,7 +179,7 @@
 	"Failed to get service URL: {{.error}}": "",
 	"Failed to kill mount process: {{.error}}": "マウント プロセスを強制終了できませんでした。{{.error}}",
 	"Failed to list cached images": "",
-	"Failed to remove profile": "",
+	"Failed to reload cached images": "",
 	"Failed to save config": "",
 	"Failed to set NO_PROXY Env. Please use `export NO_PROXY=$NO_PROXY,{{.ip}}": "NO_PROXY 環境変数を設定できませんでした。「export NO_PROXY=$NO_PROXY,{{.ip}}」を使用してください。",
 	"Failed to set NO_PROXY Env. Please use `export NO_PROXY=$NO_PROXY,{{.ip}}`.": "",
@@ -185,14 +204,17 @@
 	"Gets the status of a local kubernetes cluster": "",
 	"Gets the status of a local kubernetes cluster.\n\tExit status contains the status of minikube's VM, cluster and kubernetes encoded on it's bits in this order from right to left.\n\tEg: 7 meaning: 1 (for minikube NOK) + 2 (for cluster NOK) + 4 (for kubernetes NOK)": "",
 	"Gets the value of PROPERTY_NAME from the minikube config file": "",
+	"Getting machine config failed": "",
 	"Global Flags": "",
-	"Go template format string for the addon list output.  The format for Go templates can be found here: https://golang.org/pkg/text/template/\nFor the list of accessible variables for the template, see the struct values here: https://godoc.org/k8s.io/minikube/cmd/minikube/cmd/config#AddonListTemplate": "",
 	"Go template format string for the cache list output.  The format for Go templates can be found here: https://golang.org/pkg/text/template/\nFor the list of accessible variables for the template, see the struct values here: https://godoc.org/k8s.io/minikube/cmd/minikube/cmd#CacheListTemplate": "",
 	"Go template format string for the config view output.  The format for Go templates can be found here: https://golang.org/pkg/text/template/\nFor the list of accessible variables for the template, see the struct values here: https://godoc.org/k8s.io/minikube/cmd/minikube/cmd/config#ConfigViewTemplate": "",
 	"Go template format string for the status output.  The format for Go templates can be found here: https://golang.org/pkg/text/template/\nFor the list accessible variables for the template, see the struct values here: https://godoc.org/k8s.io/minikube/cmd/minikube/cmd#Status": "",
 	"Group ID:     {{.groupID}}": "",
 	"Have you set up libvirt correctly?": "",
 	"Hide the hypervisor signature from the guest in minikube (kvm2 driver only)": "minikube でゲストに対し、ハイパーバイザ署名を非表示にします（kvm2 ドライバのみ）",
+	"Hyperkit is broken. Upgrade to the latest hyperkit version and/or Docker for Desktop. Alternatively, you may choose an alternate --vm-driver": "",
+	"Hyperkit networking is broken. Upgrade to the latest hyperkit version and/or Docker for Desktop. Alternatively, you may choose an alternate --vm-driver": "",
+	"If set, automatically updates drivers to the latest version. Defaults to true.": "",
 	"If the above advice does not help, please let us know:": "",
 	"If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --vm-driver=none.": "true の場合、現在のブートストラッパの Docker イメージをキャッシュに保存して、マシンに読み込みます。--vm-driver=none の場合は常に false です。",
 	"If true, only download and cache files for later use - don't install or start anything.": "true の場合、後で使用できるようにファイルのダウンロードとキャッシュ保存だけが行われます。インストールも起動も行われません。",
@@ -206,6 +228,7 @@
 	"Invalid size passed in argument: {{.error}}": "",
 	"IsEnabled failed": "",
 	"Kill the mount process spawned by minikube start": "",
+	"Kubernetes {{.new}} is now available. If you would like to upgrade, specify: --kubernetes-version={{.new}}": "",
 	"Kubernetes {{.version}} is not supported by this release of minikube": "",
 	"Launching Kubernetes ...": "Kubernetes を起動しています...",
 	"Launching proxy ...": "",
@@ -230,6 +253,9 @@
 	"Mounting host path {{.sourcePath}} into VM as {{.destinationPath}} ...": "",
 	"Mounts the specified directory into minikube": "",
 	"Mounts the specified directory into minikube.": "",
+	"Multiple errors deleting profiles": "",
+	"Multiple minikube profiles were found -": "",
+	"NIC Type used for host only network. One of Am79C970A, Am79C973, 82540EM, 82543GC, 82545EM, or virtio (virtualbox driver only)": "",
 	"NOTE: This process must stay alive for the mount to be accessible ...": "",
 	"Networking and Connectivity Commands:": "",
 	"No minikube profile was found. You can create one using `minikube start`.": "",
@@ -243,6 +269,7 @@
 	"Open the addons URL with https instead of http": "",
 	"Open the service URL with https instead of http": "",
 	"Opening kubernetes service  {{.namespace_name}}/{{.service_name}} in default browser...": "",
+	"Opening service {{.namespace_name}}/{{.service_name}} in default browser...": "",
 	"Opening {{.url}} in your default browser...": "",
 	"Opens the addon w/ADDON_NAME within minikube (example: minikube addons open dashboard). For a list of available addons use: minikube addons list": "",
 	"Options:      {{.options}}": "",
@@ -264,9 +291,10 @@
 	"Problems detected in {{.entry}}:": "",
 	"Problems detected in {{.name}}:": "",
 	"Profile gets or sets the current minikube profile": "",
+	"Profile name \"{{.profilename}}\" is minikube keyword. To delete profile use command minikube delete -p \u003cprofile name\u003e": "",
 	"Provide VM UUID to restore MAC address (hyperkit driver only)": "MAC アドレスを復元するための VM UUID を指定します（hyperkit ドライバのみ）",
 	"Pulling images ...": "",
-	"Reboot to complete VirtualBox installation, and verify that VirtualBox is not blocked by your system": "",
+	"Reboot to complete VirtualBox installation, verify that VirtualBox is not blocked by your system, and/or use another hypervisor": "",
 	"Rebuild libvirt with virt-network support": "",
 	"Received {{.name}} signal": "",
 	"Registry mirrors to pass to the Docker daemon": "Docker デーモンに渡すレジストリ ミラー",
@@ -275,11 +303,10 @@
 	"Related issues:": "",
 	"Relaunching Kubernetes using {{.bootstrapper}} ...": "{{.bootstrapper}} を使用して Kubernetes を再起動しています...",
 	"Removing {{.directory}} ...": "{{.directory}} を削除しています...",
-	"Requested CPU count {{.requested_cpus}} is less than the minimum allowed of {{.minimum_cpus}}": "",
+	"Requested cpu count {{.requested_cpus}} is less than the minimum allowed of {{.minimum_cpus}}": "",
 	"Requested disk size {{.requested_size}} is less than minimum of {{.minimum_size}}": "リクエストされたディスクサイズ {{.requested_size}} が最小値 {{.minimum_size}} 未満です",
 	"Requested memory allocation ({{.memory}}MB) is less than the default memory allocation of {{.default_memorysize}}MB. Beware that minikube might not work correctly or crash unexpectedly.": "リクエストされたメモリ割り当て（{{.memory}} MB）がデフォルトのメモリ割り当て {{.default_memorysize}} MB 未満です。minikube が正常に動作しないか、予期せずクラッシュする可能性があることに注意してください。",
 	"Requested memory allocation {{.requested_size}} is less than the minimum allowed of {{.minimum_size}}": "リクエストされたメモリ割り当て {{.requested_size}} が許可される最小値 {{.minimum_size}} 未満です",
-	"Retriable failure: {{.error}}": "",
 	"Retrieve the ssh identity key path of the specified cluster": "",
 	"Retrieve the ssh identity key path of the specified cluster.": "",
 	"Retrieves the IP address of the running cluster": "",
@@ -287,22 +314,27 @@
 	"Retrieves the IP address of the running cluster, checks it\n\t\t\twith IP in kubeconfig, and corrects kubeconfig if incorrect.": "",
 	"Returns the value of PROPERTY_NAME from the minikube config file.  Can be overwritten at runtime by flags or environmental variables.": "",
 	"Run 'kubectl describe pod coredns -n kube-system' and check for a firewall or DNS conflict": "",
-	"Run 'minikube delete' to delete the stale VM": "",
+	"Run 'minikube delete' to delete the stale VM, or and ensure that minikube is running as the same user you are issuing this command with": "",
 	"Run kubectl": "",
 	"Run minikube from the C: drive.": "",
 	"Run the kubernetes client, download it if necessary.\nExamples:\nminikube kubectl -- --help\nkubectl get pods --namespace kube-system": "",
 	"Run the minikube command as an Administrator": "",
+	"Run: 'chmod 600 $HOME/.kube/config'": "",
 	"Running on localhost (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...": "",
+	"Selecting {{.experimental}} '{{.driver}}' driver from existing profile (alternates: {{.alternates}})": "",
+	"Selecting {{.experimental}} '{{.driver}}' driver from user configuration (alternates: {{.alternates}})": "",
 	"Set failed": "",
+	"Set flag to delete all profiles": "",
+	"Set this flag to delete the '.minikube' folder from your user directory.": "",
 	"Sets an individual value in a minikube config file": "",
 	"Sets the PROPERTY_NAME config value to PROPERTY_VALUE\n\tThese values can be overwritten by flags or environment variables at runtime.": "",
 	"Sets up docker env variables; similar to '$(docker-machine env)'": "",
 	"Sets up docker env variables; similar to '$(docker-machine env)'.": "",
 	"Setting profile failed": "",
+	"Show a list of global command-line options (applies to all commands).": "",
 	"Show only log entries which point to known problems": "",
 	"Show only the most recent journal entries, and continuously print new entries as they are appended to the journal.": "",
 	"Skipped switching kubectl context for {{.profile_name}} because --keep-context was set.": "",
-	"Sorry that minikube crashed. If this was unexpected, we would love to hear from you:": "",
 	"Sorry, Kubernetes {{.version}} is not supported by this release of minikube": "",
 	"Sorry, completion support is not yet implemented for {{.name}}": "",
 	"Sorry, the kubeadm.{{.parameter_name}} parameter is currently not supported by --extra-config": "申し訳ありません。現在、kubeadm.{{.parameter_name}} パラメータは --extra-config でサポートされていません",
@@ -319,14 +351,21 @@
 	"Stopping \"{{.profile_name}}\" in {{.driver_name}} ...": "",
 	"Stops a local kubernetes cluster running in Virtualbox. This command stops the VM\nitself, leaving all files intact. The cluster can be started again with the \"start\" command.": "",
 	"Stops a running local kubernetes cluster": "",
+	"Successfully deleted all profiles": "",
+	"Successfully deleted profile \\\"{{.name}}\\\"": "",
 	"Successfully mounted {{.sourcePath}} to {{.destinationPath}}": "",
+	"Successfully powered off Hyper-V. minikube driver -- {{.driver}}": "",
+	"Successfully purged minikube directory located at - [{{.minikubeDirectory}}]": "",
 	"Suggestion: {{.advice}}": "",
+	"Suggestion: {{.fix}}": "",
 	"Target directory {{.path}} must be an absolute path": "",
 	"The \"{{.driver_name}}\" driver requires root privileges. Please run minikube using 'sudo minikube --vm-driver={{.driver_name}}": "「{{.driver_name}}」ドライバにはルート権限が必要です。「sudo minikube --vm-driver={{.driver_name}}」を使用して minikube を実行してください",
 	"The \"{{.driver_name}}\" driver requires root privileges. Please run minikube using 'sudo minikube --vm-driver={{.driver_name}}'.": "",
 	"The \"{{.driver_name}}\" driver should not be used with root privileges.": "",
 	"The \"{{.name}}\" cluster has been deleted.": "「{{.name}}」クラスタが削除されました。",
 	"The \"{{.name}}\" cluster has been deleted.__1": "「{{.name}}」クラスタが削除されました。",
+	"The 'none' driver does not respect the --cpus flag": "",
+	"The 'none' driver does not respect the --memory flag": "",
 	"The 'none' driver does not support multiple profiles: https://minikube.sigs.k8s.io/docs/reference/drivers/none/": "",
 	"The 'none' driver provides limited isolation and may reduce system security and reliability.": "ドライバに「none」を指定すると、分離が制限され、システムのセキュリティと信頼性が低下する可能性があります。",
 	"The '{{.driver}}' driver requires elevated permissions. The following commands will be executed:\\n\\n{{ .example }}\\n": "",
@@ -351,14 +390,18 @@
 	"The docker host is currently not running": "",
 	"The docker service is currently not active": "",
 	"The driver '{{.driver}}' is not supported on {{.os}}": "ドライバ「{{.driver}}」は、{{.os}} ではサポートされていません",
+	"The driver {{.experimental}} '{{.driver}}' is not supported on {{.os}}": "",
 	"The existing \"{{.profile_name}}\" VM that was created using the \"{{.old_driver}}\" driver, and is incompatible with the \"{{.driver}}\" driver.": "",
 	"The hyperv virtual switch name. Defaults to first found. (hyperv driver only)": "hyperv 仮想スイッチ名。最初に見つかったものにデフォルト設定されます（hyperv ドライバのみ）",
+	"The hypervisor does not appear to be configured properly. Run 'minikube start --alsologtostderr -v=1' and inspect the error code": "",
 	"The initial time interval for each check that wait performs in seconds": "",
 	"The kubernetes version that the minikube VM will use (ex: v1.2.3)": "minikube VM で使用される Kubernetes バージョン（例: v1.2.3）",
+	"The machine-driver specified is failing to start. Try running 'docker-machine-driver-\u003ctype\u003e version'": "",
 	"The minikube VM is offline. Please run 'minikube start' to start it again.": "",
 	"The name of the network plugin": "ネットワーク プラグインの名前",
 	"The name of the network plugin.": "",
 	"The number of bytes to use for 9p packet payload": "",
+	"The output format. One of 'json', 'table'": "",
 	"The path on the file system where the docs in markdown need to be saved": "",
 	"The service namespace": "",
 	"The services namespace": "",
@@ -367,7 +410,6 @@
 	"The value passed to --format is invalid: {{.error}}": "",
 	"The vmwarefusion driver is deprecated and support for it will be removed in a future release.\n\t\t\tPlease consider switching to the new vmware unified driver, which is intended to replace the vmwarefusion driver.\n\t\t\tSee https://minikube.sigs.k8s.io/docs/reference/drivers/vmware/ for more information.\n\t\t\tTo disable this message, run [minikube config set ShowDriverDeprecationNotification false]": "",
 	"The {{.driver_name}} driver should not be used with root privileges.": "{{.driver_name}} ドライバをルート権限で使用しないでください。",
-	"There appears to be another hypervisor conflicting with KVM. Please stop the other hypervisor, or use --vm-driver to switch to it.": "",
 	"There's a new version for '{{.driver_executable}}'. Please consider upgrading. {{.documentation_url}}": "「{{.driver_executable}}」の新しいバージョンがあります。アップグレードを検討してください。{{.documentation_url}}",
 	"These changes will take effect upon a minikube delete and then a minikube start": "",
 	"This addon does not have an endpoint defined for the 'addons open' command.\nYou can add one by annotating a service with the label {{.labelName}}:{{.addonName}}": "",
@@ -381,11 +423,14 @@
 	"To connect to this cluster, use: kubectl --context={{.name}}__1": "このクラスタに接続するには、「kubectl --context={{.name}}」を使用します",
 	"To connect to this cluster, use: kubectl --context={{.profile_name}}": "",
 	"To disable this notice, run: 'minikube config set WantUpdateNotification false'\\n": "",
-	"To proceed, either:\n      1) Delete the existing VM using: '{{.command}} delete'\n      or\n      2) Restart with the existing driver: '{{.command}} start --vm-driver={{.old_driver}}'": "",
+	"To proceed, either:\n\n    1) Delete the existing \"{{.profile_name}}\" cluster using: '{{.command}} delete'\n\n    * or *\n\n    2) Start the existing \"{{.profile_name}}\" cluster using: '{{.command}} start --vm-driver={{.old_driver}}'": "",
+	"To see addons list for other profiles use: `minikube addons -p name list`": "",
 	"To start minikube with HyperV Powershell must be in your PATH`": "",
 	"To use kubectl or minikube commands as your own user, you may need to relocate them. For example, to overwrite your own settings, run:": "kubectl か minikube コマンドを独自のユーザーとして使用するには、そのコマンドの再配置が必要な場合があります。たとえば、独自の設定を上書きするには、以下を実行します。",
 	"Troubleshooting Commands:": "",
+	"Trying to delete invalid profile {{.profile}}": "",
 	"Unable to bind flags": "",
+	"Unable to determine a default driver to use. Try specifying --vm-driver, or see https://minikube.sigs.k8s.io/docs/start/": "",
 	"Unable to enable dashboard": "",
 	"Unable to fetch latest version info": "",
 	"Unable to generate docs": "",
@@ -394,18 +439,20 @@
 	"Unable to get bootstrapper: {{.error}}": "ブートストラッパを取得できません。{{.error}}",
 	"Unable to get current user": "",
 	"Unable to get runtime": "",
-	"Unable to get the status of the cluster.": "",
+	"Unable to get the status of the {{.name}} cluster.": "",
 	"Unable to kill mount process: {{.error}}": "",
 	"Unable to load cached images from config file.": "キャッシュに保存されているイメージを構成ファイルから読み込むことができません。",
 	"Unable to load cached images: {{.error}}": "",
 	"Unable to load config: {{.error}}": "構成を読み込むことができません。{{.error}}",
 	"Unable to parse \"{{.kubernetes_version}}\": {{.error}}": "「{{.kubernetes_version}}」を解析できません。{{.error}}",
+	"Unable to parse default Kubernetes version from constants: {{.error}}": "",
 	"Unable to parse oldest Kubernetes version from constants: {{.error}}": "",
 	"Unable to pull images, which may be OK: {{.error}}": "イメージを pull できませんが、問題ありません。{{.error}}",
 	"Unable to remove machine directory: %v": "",
-	"Unable to start VM": "",
+	"Unable to start VM. Please investigate and run 'minikube delete' if possible": "",
 	"Unable to stop VM": "",
 	"Unable to update {{.driver}} driver: {{.error}}": "",
+	"Unable to verify SSH connectivity: {{.error}}. Will retry...": "",
 	"Uninstalling Kubernetes {{.kubernetes_version}} using {{.bootstrapper_name}} ...": "{{.bootstrapper_name}} を使用して Kubernetes {{.kubernetes_version}} をアンインストールしています...",
 	"Unmounting {{.path}} ...": "",
 	"Unset the KUBECONFIG environment variable, or verify that it does not point to an empty or otherwise invalid path": "",
@@ -416,15 +463,20 @@
 	"Usage": "",
 	"Usage: minikube completion SHELL": "",
 	"Usage: minikube delete": "",
+	"Usage: minikube delete --all --purge": "",
 	"Use \"{{.CommandPath}} [command] --help\" for more information about a command.": "",
+	"Use 'kubect get po -A' to find the correct and namespace name": "",
 	"Use VirtualBox to remove the conflicting VM and/or network interfaces": "",
 	"Use native Golang SSH client (default true). Set to 'false' to use the command line 'ssh' command when accessing the docker machine. Useful for the machine drivers when they will not start with 'Waiting for SSH'.": "",
 	"User ID:      {{.userID}}": "",
 	"Userspace file server is shutdown": "",
 	"Userspace file server:": "",
 	"Using image repository {{.name}}": "イメージ リポジトリ {{.name}} を使用しています",
+	"Using the '{{.runtime}}' runtime with the 'none' driver is an untested configuration!": "",
 	"Using the running {{.driver_name}} \"{{.profile_name}}\" VM ...": "",
 	"VM driver is one of: %v": "VM ドライバは次のいずれかです。%v",
+	"VM is unable to access {{.repository}}, you may need to configure a proxy or set --image-repository": "",
+	"VM may be unable to resolve external DNS records": "",
 	"Verify that your HTTP_PROXY and HTTPS_PROXY environment variables are set correctly.": "",
 	"Verify the IP address of the running cluster in kubeconfig.": "",
 	"Verifying dashboard health ...": "",
@@ -434,12 +486,14 @@
 	"VirtualBox cannot create a network, probably because it conflicts with an existing network that minikube no longer knows about. Try running 'minikube delete'": "",
 	"VirtualBox is broken. Disable real-time anti-virus software, reboot, and reinstall VirtualBox if the problem continues.": "",
 	"VirtualBox is broken. Reinstall VirtualBox, reboot, and run 'minikube delete'.": "",
+	"VirtualBox is unable to find its network interface. Try upgrading to the latest release and rebooting.": "",
+	"Virtualization support is disabled on your computer. If you are running minikube within a VM, try '--vm-driver=none'. Otherwise, consult your systems BIOS manual for how to enable virtualization.": "",
 	"Wait failed": "",
 	"Wait failed: {{.error}}": "",
 	"Wait until Kubernetes core services are healthy before exiting": "Kubernetes コアサービスが正常になるまで待機してから終了してください",
-	"Wait until Kubernetes core services are healthy before exiting.": "",
+	"Waiting for cluster to come online ...": "",
 	"Waiting for the host to be provisioned ...": "",
-	"Waiting for:": "",
+	"Warning: Your kubectl is pointing to stale minikube-vm.\\nTo fix the kubectl context, run `minikube update-context`": "",
 	"Where to root the NFS Shares, defaults to /nfsshares (hyperkit driver only)": "NFS 共有のルートに指定する場所。デフォルトは /nfsshares（hyperkit ドライバのみ）",
 	"You appear to be using a proxy, but your NO_PROXY environment does not include the minikube IP ({{.ip_address}}). Please see {{.documentation_url}} for more details": "プロキシを使用しようとしていますが、現在の NO_PROXY 環境に minikube IP（{{.ip_address}}）は含まれていません。詳細については、{{.documentation_url}} をご覧ください",
 	"You can delete them using the following command(s):": "",
@@ -452,50 +506,57 @@
 	"Your minikube vm is not running, try minikube start.": "",
 	"addon '{{.name}}' is currently not enabled.\nTo enable this addon run:\nminikube addons enable {{.name}}": "",
 	"addon '{{.name}}' is not a valid addon packaged with minikube.\nTo see the list of available addons run:\nminikube addons list": "",
-	"addon list failed": "",
-	"addons modifies minikube addons files using subcommands like \"minikube addons enable heapster\"": "",
+	"addon enable failed": "",
+	"addons modifies minikube addons files using subcommands like \"minikube addons enable dashboard\"": "",
 	"api load": "",
 	"bash completion failed": "",
-	"browser failed to open url: {{.error}}": "",
 	"call with cleanup=true to remove old tunnels": "",
 	"command runner": "",
 	"config modifies minikube config files using subcommands like \"minikube config set vm-driver kvm\"\nConfigurable fields:\\n\\n": "",
 	"config view failed": "",
 	"dashboard service is not running: {{.error}}": "",
 	"disable failed": "",
+	"dry-run mode. Validates configuration, but does does not mutate system state": "",
+	"dry-run validation complete!": "",
 	"enable failed": "",
 	"error creating clientset": "",
 	"error creating machine client": "",
-	"error getting driver": "",
 	"error parsing the input ip address for mount": "",
 	"error starting tunnel": "",
 	"failed to open browser: {{.error}}": "",
 	"if true, will embed the certs in kubeconfig.": "",
+	"kubeadm detected a TCP port conflict with another process: probably another local Kubernetes installation. Run lsof -p\u003cport\u003e to find the process and kill it": "",
 	"kubectl and minikube configuration will be stored in {{.home_folder}}": "kubectl と minikube の構成は {{.home_folder}} に保存されます",
 	"kubectl not found in PATH, but is required for the dashboard. Installation guide: https://kubernetes.io/docs/tasks/tools/install-kubectl/": "",
 	"kubectl proxy": "",
 	"logdir set failed": "",
 	"max time to wait per Kubernetes core services to be healthy.": "",
-	"minikube is not running, so the service cannot be accessed": "",
+	"minikube addons list --output OUTPUT. json, list": "",
+	"minikube is exiting due to an error. If the above message is not useful, open an issue:": "",
 	"minikube is unable to access the Google Container Registry. You may need to configure it to use a HTTP proxy.": "",
+	"minikube is unable to connect to the VM: {{.error}}\n\nThis is likely due to one of two reasons:\n\n- VPN or firewall interference\n- {{.hypervisor}} network configuration issue\n\nSuggested workarounds:\n\n- Disable your local VPN or firewall software\n- Configure your local VPN or firewall to allow access to {{.ip}}\n- Restart or reinstall {{.hypervisor}}\n- Use an alternative --vm-driver": "",
 	"minikube profile was successfully set to {{.profile_name}}": "",
+	"minikube status --output OUTPUT. json, text": "",
 	"minikube {{.version}} is available! Download it: {{.url}}": "",
+	"mkcmp is used to compare performance of two minikube binaries": "",
 	"mount argument \"{{.value}}\" must be in form: \u003csource directory\u003e:\u003ctarget directory\u003e": "",
 	"mount failed": "",
 	"not enough arguments ({{.ArgCount}}).\\nusage: minikube config set PROPERTY_NAME PROPERTY_VALUE": "",
 	"profile sets the current minikube profile, or gets the current profile if no arguments are provided.  This is used to run and manage multiple minikube instance.  You can return to the default minikube profile by running `minikube profile default`": "",
+	"reload cached images.": "",
+	"reloads images previously added using the 'cache add' subcommand": "",
 	"service {{.namespace_name}}/{{.service_name}} has no node port": "",
 	"stat failed": "",
 	"toom any arguments ({{.ArgCount}}).\\nusage: minikube config set PROPERTY_NAME PROPERTY_VALUE": "",
-	"tunnel creates a route to services deployed with type LoadBalancer and sets their Ingress to their ClusterIP": "",
+	"tunnel creates a route to services deployed with type LoadBalancer and sets their Ingress to their ClusterIP. for a detailed example see https://minikube.sigs.k8s.io/docs/tasks/loadbalancer": "",
 	"tunnel makes services of type LoadBalancer accessible on localhost": "",
 	"unable to bind flags": "",
+	"unable to delete minikube config folder": "",
 	"unable to set logtostderr": "",
 	"unset failed": "",
-	"unset minikube profile": "",
 	"unsets PROPERTY_NAME from the minikube config file.  Can be overwritten by flags or environmental variables": "",
 	"unsets an individual value in a minikube config file": "",
-	"unsupported driver: {{.name}}": "",
+	"unsupported or missing driver: {{.name}}": "",
 	"update config": "",
 	"usage: minikube addons configure ADDON_NAME": "",
 	"usage: minikube addons disable ADDON_NAME": "",
@@ -503,15 +564,20 @@
 	"usage: minikube addons list": "",
 	"usage: minikube addons open ADDON_NAME": "",
 	"usage: minikube config unset PROPERTY_NAME": "",
+	"usage: minikube delete": "",
+	"usage: minikube delete --all": "",
 	"usage: minikube profile [MINIKUBE_PROFILE_NAME]": "",
 	"zsh completion failed": "",
 	"{{.addonName}} was successfully enabled": "",
+	"{{.driver}} does not appear to be installed": "",
+	"{{.driver}} does not appear to be installed, but is specified by an existing profile. Please run 'minikube delete' or install {{.driver}}": "",
 	"{{.extra_option_component_name}}.{{.key}}={{.value}}": "",
 	"{{.machine}} IP has been updated to point at {{.ip}}": "",
 	"{{.machine}} IP was already correctly configured for {{.ip}}": "",
 	"{{.name}} cluster does not exist": "",
 	"{{.name}} has no available configuration options": "",
 	"{{.name}} was successfully configured": "",
+	"{{.path}} is version {{.client_version}}, and is incompatible with Kubernetes {{.cluster_version}}. You will need to update {{.path}} or use 'minikube kubectl' to connect with this cluster": "",
 	"{{.prefix}}minikube {{.version}} on {{.platform}}": "{{.platform}} 上の {{.prefix}}minikube {{.version}}",
 	"{{.type}} is not yet a supported filesystem. We will try anyways!": "",
 	"{{.url}} is not accessible: {{.error}}": ""

From 1939f497cf08d75b7dbf6b3178296006642bdd02 Mon Sep 17 00:00:00 2001
From: inductor <kohei.ota@zozo.com>
Date: Sat, 18 Jan 2020 10:14:04 +0900
Subject: [PATCH 070/250] add some translations to jp with updated strings

---
 translations/ja.json | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/translations/ja.json b/translations/ja.json
index 65ee169b94..62a0eb2b4b 100644
--- a/translations/ja.json
+++ b/translations/ja.json
@@ -1,14 +1,14 @@
 {
-	"\"{{.minikube_addon}}\" was successfully disabled": "",
-	"\"{{.name}}\" cluster does not exist. Proceeding ahead with cleanup.": "",
-	"\"{{.name}}\" profile does not exist": "「{{.name}}」プロファイルは存在しません",
-	"\"{{.profile_name}}\" VM does not exist, nothing to stop": "",
-	"\"{{.profile_name}}\" host does not exist, unable to show an IP": "",
-	"\"{{.profile_name}}\" stopped.": "",
-	"'none' driver does not support 'minikube docker-env' command": "",
-	"'none' driver does not support 'minikube mount' command": "",
-	"'none' driver does not support 'minikube ssh' command": "",
-	"'{{.driver}}' driver reported an issue: {{.error}}": "",
+	"\"{{.minikube_addon}}\" was successfully disabled": "「{{.minikube_addon}}」が無効化されました",
+	"\"{{.name}}\" cluster does not exist. Proceeding ahead with cleanup.": "「{{.name}}」というクラスターは存在しません。クリーンアップ処理を続行します。",
+	"\"{{.name}}\" profile does not exist": "「{{.name}}」というプロファイルは存在しません",
+	"\"{{.profile_name}}\" VM does not exist, nothing to stop": "「{{.profile_name}}」というVMは存在しません。停止すべき対象がありません",
+	"\"{{.profile_name}}\" host does not exist, unable to show an IP": "「{{.profile_name}}」というホストは存在しません。IPを表示できません",
+	"\"{{.profile_name}}\" stopped.": "「{{.profile_name}}」が停止しました。",
+	"'none' driver does not support 'minikube docker-env' command": "「none」ドライバーは「minikube docker-env」コマンドをサポートしていません",
+	"'none' driver does not support 'minikube mount' command": "「none」ドライバーは「minikube mount」コマンドをサポートしていません",
+	"'none' driver does not support 'minikube ssh' command": "「none」ドライバーは「minikube ssh」コマンドをサポートしていません",
+	"'{{.driver}}' driver reported an issue: {{.error}}": "「{{.driver}}」ドライバーがエラーを報告しました: {{.error}}",
 	"- {{.profile}}": "",
 	"A VPN or firewall is interfering with HTTP access to the minikube VM. Alternatively, try a different VM driver: https://minikube.sigs.k8s.io/docs/start/": "",
 	"A firewall is blocking Docker the minikube VM from reaching the internet. You may need to configure it to use a proxy.": "",
@@ -19,9 +19,9 @@
 	"A set of apiserver names which are used in the generated certificate for kubernetes.  This can be used if you want to make the apiserver available from outside the machine": "",
 	"A set of apiserver names which are used in the generated certificate for kubernetes. This can be used if you want to make the apiserver available from outside the machine": "Kubernetes 用に生成された証明書で使用される一連の API サーバー名。マシンの外部から API サーバーを利用できるようにする場合に使用します。",
 	"A set of key=value pairs that describe configuration that may be passed to different components.\nThe key should be '.' separated, and the first part before the dot is the component to apply the configuration to.\nValid components are: kubelet, kubeadm, apiserver, controller-manager, etcd, proxy, scheduler\nValid kubeadm parameters:": "さまざまなコンポーネントに渡される可能性のある構成を記述する一連の key=value ペア。\nキーは「.」で区切る必要があり、このドットより前の部分は構成の適用先のコンポーネントを表します。\n有効なコンポーネントは、kubelet、kubeadm、apiserver、controller-manager、etcd、proxy、scheduler です。\n有効な kubeadm パラメータ:",
-	"A set of key=value pairs that describe feature gates for alpha/experimental features.": "アルファ版または試験運用版の機能の機能ゲートを記述する一連の key=value ペア。",
+	"A set of key=value pairs that describe feature gates for alpha/experimental features.": "アルファ版または試験運用版の機能のフィーチャーゲートを記述する一連の key=value ペアです。",
 	"Access the kubernetes dashboard running within the minikube cluster": "",
-	"Add an image to local cache.": "",
+	"Add an image to local cache.": "イメージをローカルキャッシュに追加します",
 	"Add machine IP to NO_PROXY environment variable": "",
 	"Add or delete an image from the local cache.": "",
 	"Additional help topics": "",
@@ -155,7 +155,7 @@
 	"Error: [{{.id}}] {{.error}}": "",
 	"Examples": "",
 	"Exiting": "終了しています",
-	"Exiting.": "",
+	"Exiting.": "終了しています。",
 	"Failed runtime": "",
 	"Failed to cache ISO": "",
 	"Failed to cache and load images": "",

From 3e0724c9380262045152d509d3765183a8f62d11 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= <anders.f.bjorklund@gmail.com>
Date: Sat, 18 Jan 2020 10:05:03 +0100
Subject: [PATCH 071/250] Document the current config better with defaults

Using gzip for kernel and bzip2 for the rootfs
---
 deploy/iso/minikube-iso/configs/minikube_defconfig | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/deploy/iso/minikube-iso/configs/minikube_defconfig b/deploy/iso/minikube-iso/configs/minikube_defconfig
index 7d11f9a705..d91b4ed1ea 100644
--- a/deploy/iso/minikube-iso/configs/minikube_defconfig
+++ b/deploy/iso/minikube-iso/configs/minikube_defconfig
@@ -19,6 +19,8 @@ BR2_ROOTFS_USERS_TABLES="$(BR2_EXTERNAL_MINIKUBE_PATH)/board/coreos/minikube/use
 BR2_ROOTFS_OVERLAY="$(BR2_EXTERNAL_MINIKUBE_PATH)/board/coreos/minikube/rootfs-overlay"
 BR2_GLOBAL_PATCH_DIR="$(BR2_EXTERNAL_MINIKUBE_PATH)/board/coreos/minikube/patches"
 BR2_LINUX_KERNEL=y
+BR2_LINUX_KERNEL_BZIMAGE=y
+BR2_LINUX_KERNEL_GZIP=y
 BR2_LINUX_KERNEL_USE_CUSTOM_CONFIG=y
 BR2_LINUX_KERNEL_CUSTOM_CONFIG_FILE="$(BR2_EXTERNAL_MINIKUBE_PATH)/board/coreos/minikube/linux_defconfig"
 BR2_LINUX_KERNEL_NEEDS_HOST_LIBELF=y
@@ -55,6 +57,7 @@ BR2_PACKAGE_SYSTEMD_MACHINED=y
 BR2_PACKAGE_SYSTEMD_VCONSOLE=y
 BR2_PACKAGE_UTIL_LINUX_NSENTER=y
 BR2_PACKAGE_UTIL_LINUX_SCHEDUTILS=y
+BR2_TARGET_ROOTFS_CPIO=y
 BR2_TARGET_ROOTFS_CPIO_BZIP2=y
 BR2_TARGET_ROOTFS_ISO9660=y
 BR2_TARGET_ROOTFS_ISO9660_BOOT_MENU="$(BR2_EXTERNAL_MINIKUBE_PATH)/board/coreos/minikube/isolinux.cfg"

From 857c8f5eb098676a6649818b35d2327abab87923 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= <anders.f.bjorklund@gmail.com>
Date: Sat, 18 Jan 2020 10:07:19 +0100
Subject: [PATCH 072/250] Change the compression methods used on the iso

Using lz4 for kernel and gzip for the rootfs
---
 deploy/iso/minikube-iso/configs/minikube_defconfig | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/deploy/iso/minikube-iso/configs/minikube_defconfig b/deploy/iso/minikube-iso/configs/minikube_defconfig
index d91b4ed1ea..caaf4d3adb 100644
--- a/deploy/iso/minikube-iso/configs/minikube_defconfig
+++ b/deploy/iso/minikube-iso/configs/minikube_defconfig
@@ -20,7 +20,7 @@ BR2_ROOTFS_OVERLAY="$(BR2_EXTERNAL_MINIKUBE_PATH)/board/coreos/minikube/rootfs-o
 BR2_GLOBAL_PATCH_DIR="$(BR2_EXTERNAL_MINIKUBE_PATH)/board/coreos/minikube/patches"
 BR2_LINUX_KERNEL=y
 BR2_LINUX_KERNEL_BZIMAGE=y
-BR2_LINUX_KERNEL_GZIP=y
+BR2_LINUX_KERNEL_LZ4=y
 BR2_LINUX_KERNEL_USE_CUSTOM_CONFIG=y
 BR2_LINUX_KERNEL_CUSTOM_CONFIG_FILE="$(BR2_EXTERNAL_MINIKUBE_PATH)/board/coreos/minikube/linux_defconfig"
 BR2_LINUX_KERNEL_NEEDS_HOST_LIBELF=y
@@ -58,7 +58,7 @@ BR2_PACKAGE_SYSTEMD_VCONSOLE=y
 BR2_PACKAGE_UTIL_LINUX_NSENTER=y
 BR2_PACKAGE_UTIL_LINUX_SCHEDUTILS=y
 BR2_TARGET_ROOTFS_CPIO=y
-BR2_TARGET_ROOTFS_CPIO_BZIP2=y
+BR2_TARGET_ROOTFS_CPIO_GZIP=y
 BR2_TARGET_ROOTFS_ISO9660=y
 BR2_TARGET_ROOTFS_ISO9660_BOOT_MENU="$(BR2_EXTERNAL_MINIKUBE_PATH)/board/coreos/minikube/isolinux.cfg"
 BR2_TARGET_SYSLINUX=y

From 16ed1756e4cf923ee13f3015112394a1553dd424 Mon Sep 17 00:00:00 2001
From: Naoki Oketani <okepy.naoki@gmail.com>
Date: Sun, 19 Jan 2020 21:54:15 +0900
Subject: [PATCH 073/250] fix grammer and remove extra spaces

---
 site/content/en/docs/Tasks/loadbalancer.md | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/site/content/en/docs/Tasks/loadbalancer.md b/site/content/en/docs/Tasks/loadbalancer.md
index 2c3d280e1f..3807363165 100644
--- a/site/content/en/docs/Tasks/loadbalancer.md
+++ b/site/content/en/docs/Tasks/loadbalancer.md
@@ -9,7 +9,7 @@ description: >
 
 ## Overview
 
-A LoadBalancer service is the standard way to expose a service to the internet. With this method, each service gets it's own IP address.
+A LoadBalancer service is the standard way to expose a service to the internet. With this method, each service gets its own IP address.
 
 
 ## Using `minikube tunnel`
@@ -53,11 +53,11 @@ Status:
 
 #### Create a kubernetes deployment 
 ```
-  kubectl create deployment hello-minikube1 --image=k8s.gcr.io/echoserver:1.4
+kubectl create deployment hello-minikube1 --image=k8s.gcr.io/echoserver:1.4
 ```
 #### Create a kubernetes service type LoadBalancer
 ```
-  kubectl expose deployment hello-minikube1 --type=LoadBalancer --port=8080
+kubectl expose deployment hello-minikube1 --type=LoadBalancer --port=8080
 ```
 
 ### Check external IP 
@@ -65,13 +65,13 @@ Status:
 kubectl get svc
 ```
 <pre>
- $ kc get svc
-NAME             TYPE           CLUSTER-IP      EXTERNAL-IP     PORT(S)          AGE
+$ kc get svc
+NAME              TYPE           CLUSTER-IP      EXTERNAL-IP     PORT(S)          AGE
 hello-minikube1   LoadBalancer   10.96.184.178   10.96.184.178   8080:30791/TCP   40s
 </pre>
 
 
-note that without minikube tunnel, it would kubernetes would be showing external IP as "pending".
+note that without minikube tunnel, kubernetes would be showing external IP as "pending".
 
 ### Try in your browser
 open in your browser (make sure there is no proxy set)
@@ -80,7 +80,7 @@ http://REPLACE_WITH_EXTERNAL_IP:8080
 ```
 
 
-Each service will get it's own external ip.
+Each service will get its own external ip.
 
 ----
 ### DNS resolution (experimental)

From 45272d4f8e2c96cdc0ecf10bab5c204e2c33136d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= <anders.f.bjorklund@gmail.com>
Date: Mon, 13 Jan 2020 22:12:04 +0100
Subject: [PATCH 074/250] Add varlink program for using with podman-remote

Using podman-remote requires "varlink bridge"
---
 deploy/iso/minikube-iso/package/Config.in            | 1 +
 deploy/iso/minikube-iso/package/varlink/Config.in    | 3 +++
 deploy/iso/minikube-iso/package/varlink/varlink.hash | 3 +++
 deploy/iso/minikube-iso/package/varlink/varlink.mk   | 7 +++++++
 4 files changed, 14 insertions(+)
 create mode 100644 deploy/iso/minikube-iso/package/varlink/Config.in
 create mode 100644 deploy/iso/minikube-iso/package/varlink/varlink.hash
 create mode 100644 deploy/iso/minikube-iso/package/varlink/varlink.mk

diff --git a/deploy/iso/minikube-iso/package/Config.in b/deploy/iso/minikube-iso/package/Config.in
index f9a04fdfbe..75aaaf10c6 100644
--- a/deploy/iso/minikube-iso/package/Config.in
+++ b/deploy/iso/minikube-iso/package/Config.in
@@ -1,6 +1,7 @@
 menu "System tools"
     source "$BR2_EXTERNAL_MINIKUBE_PATH/package/runc-master/Config.in"
     source "$BR2_EXTERNAL_MINIKUBE_PATH/package/podman/Config.in"
+    source "$BR2_EXTERNAL_MINIKUBE_PATH/package/varlink/Config.in"
     source "$BR2_EXTERNAL_MINIKUBE_PATH/package/conmon-master/Config.in"
     source "$BR2_EXTERNAL_MINIKUBE_PATH/package/crio-bin/Config.in"
     source "$BR2_EXTERNAL_MINIKUBE_PATH/package/crictl-bin/Config.in"
diff --git a/deploy/iso/minikube-iso/package/varlink/Config.in b/deploy/iso/minikube-iso/package/varlink/Config.in
new file mode 100644
index 0000000000..ef047e1560
--- /dev/null
+++ b/deploy/iso/minikube-iso/package/varlink/Config.in
@@ -0,0 +1,3 @@
+config BR2_PACKAGE_VARLINK
+	bool "varlink"
+	default y
diff --git a/deploy/iso/minikube-iso/package/varlink/varlink.hash b/deploy/iso/minikube-iso/package/varlink/varlink.hash
new file mode 100644
index 0000000000..214978c84e
--- /dev/null
+++ b/deploy/iso/minikube-iso/package/varlink/varlink.hash
@@ -0,0 +1,3 @@
+sha256 3857f109574750403b233b5fdf73f1852d8decc33dac8f73bd49f2003b69ad22 16.tar.gz
+sha256 0dcb451f32033154c56710c216e67f245923fe2b011321271f6670e5a2285ce6 17.tar.gz
+sha256 7a32543643116ad105da4ddb2f8030de7dcad1cdb3feb1a214ae5e7b65a6a198 18.tar.gz
diff --git a/deploy/iso/minikube-iso/package/varlink/varlink.mk b/deploy/iso/minikube-iso/package/varlink/varlink.mk
new file mode 100644
index 0000000000..ade2392f6f
--- /dev/null
+++ b/deploy/iso/minikube-iso/package/varlink/varlink.mk
@@ -0,0 +1,7 @@
+VARLINK_VERSION = 18
+VARLINK_SITE = https://github.com/varlink/libvarlink/archive
+VARLINK_SOURCE = $(VARLINK_VERSION).tar.gz
+VARLINK_LICENSE = Apache-2.0
+VARLINK_LICENSE_FILES = LICENSE
+
+$(eval $(meson-package))

From a681f1f00f3da1ac1957712b59152d05997a53c2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= <anders.f.bjorklund@gmail.com>
Date: Mon, 20 Jan 2020 21:18:34 +0100
Subject: [PATCH 075/250] Remove stray space from driver selection message

---
 cmd/minikube/cmd/start.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index ebb5158124..03663df478 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -586,11 +586,11 @@ func selectDriver(existing *cfg.MachineConfig) string {
 	pick, alts := driver.Choose(name, options)
 	exp := ""
 	if pick.Priority == registry.Experimental {
-		exp = "experimental"
+		exp = "experimental "
 	}
 
 	if name != "" {
-		out.T(out.Sparkle, `Selecting {{.experimental}} '{{.driver}}' driver from user configuration (alternates: {{.alternates}})`, out.V{"experimental": exp, "driver": name, "alternates": alts})
+		out.T(out.Sparkle, `Selecting {{.experimental}}'{{.driver}}' driver from user configuration (alternates: {{.alternates}})`, out.V{"experimental": exp, "driver": name, "alternates": alts})
 		return name
 	}
 

From 5a3c13815ed5bb198a92b2183d9542af4462ddb3 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Tue, 21 Jan 2020 15:14:51 -0800
Subject: [PATCH 076/250] network overlay only for kic

---
 cmd/minikube/cmd/start.go                     |  4 ---
 go.mod                                        |  2 +-
 go.sum                                        | 27 +-----------------
 pkg/drivers/kic/kic.go                        |  5 +++-
 .../bootstrapper/kubeadm/default_cni.go       | 12 ++++----
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go  | 28 ++++++++++---------
 pkg/minikube/driver/driver.go                 |  2 --
 7 files changed, 27 insertions(+), 53 deletions(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index ba3df0215c..7a0d2e90c1 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -1027,10 +1027,6 @@ func autoSetDriverOptions(cmd *cobra.Command, drvName string) (err error) {
 		viper.Set(cacheImages, hints.CacheImages)
 	}
 
-	if !cmd.Flags().Changed(enableDefaultCNI) {
-		viper.Set(enableDefaultCNI, hints.EnableDefaultCNI)
-	}
-
 	if !cmd.Flags().Changed(containerRuntime) && hints.ContainerRuntime != "" {
 		viper.Set(containerRuntime, hints.ContainerRuntime)
 		glog.Infof("auto set %s to %q.", containerRuntime, hints.ContainerRuntime)
diff --git a/go.mod b/go.mod
index 9d36285480..0cd9aca2bf 100644
--- a/go.mod
+++ b/go.mod
@@ -10,7 +10,6 @@ require (
 	github.com/Parallels/docker-machine-parallels v1.3.0
 	github.com/Sirupsen/logrus v0.0.0-20170822132746-89742aefa4b2 // indirect
 	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
-	github.com/alecthomas/chroma v0.7.0
 	github.com/blang/semver v3.5.0+incompatible
 	github.com/c4milo/gotoolkit v0.0.0-20170318115440-bcc06269efa9 // indirect
 	github.com/cenkalti/backoff v2.2.1+incompatible
@@ -28,6 +27,7 @@ require (
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
 	github.com/google/go-cmp v0.3.0
 	github.com/googleapis/gnostic v0.3.0 // indirect
+	github.com/gorilla/mux v1.7.3 // indirect
 	github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce // indirect
 	github.com/hashicorp/go-getter v1.4.0
 	github.com/hashicorp/go-multierror v0.0.0-20160811015721-8c5f0ad93604 // indirect
diff --git a/go.sum b/go.sum
index 1fdc1bf42c..2c4f7ca288 100644
--- a/go.sum
+++ b/go.sum
@@ -21,8 +21,6 @@ github.com/BurntSushi/toml v0.3.0/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0=
-github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0=
 github.com/GoogleCloudPlatform/k8s-cloud-provider v0.0.0-20181220005116-f8e995905100/go.mod h1:iroGtC8B3tQiqtds1l+mgk/BBOrxbqjH+eUfFQYRc14=
 github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab/go.mod h1:3VYc5hodBMJ5+l/7J4xAyMeuM2PNuepvHlGs8yilUCA=
 github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd h1:sjQovDkwrZp8u+gxLtPgKGjk5hCxuy2hrRejBTA9xFU=
@@ -44,15 +42,6 @@ github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdc
 github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
 github.com/afbjorklund/go-getter v1.4.1-0.20190910175809-eb9f6c26742c h1:18gEt7qzn7CW7qMkfPTFyyotlPbvPQo9o4IDV8jZqP4=
 github.com/afbjorklund/go-getter v1.4.1-0.20190910175809-eb9f6c26742c/go.mod h1:7qxyCd8rBfcShwsvxgIguu4KbS3l8bUCwg2Umn7RjeY=
-github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c=
-github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38/go.mod h1:r7bzyVFMNntcxPZXK3/+KdruV1H5KSlyVY0gc+NgInI=
-github.com/alecthomas/chroma v0.7.0 h1:z+0HgTUmkpRDRz0SRSdMaqOLfJV4F+N1FPDZUZIDUzw=
-github.com/alecthomas/chroma v0.7.0/go.mod h1:1U/PfCsTALWWYHDnsIQkxEBM0+6LLe0v8+RSVMOwxeY=
-github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721/go.mod h1:QO9JBoKquHd+jz9nshCh40fOfO+JzsoXy8qTHF68zU0=
-github.com/alecthomas/kong v0.1.17-0.20190424132513-439c674f7ae0/go.mod h1:+inYUSluD+p4L8KdviBSgzcqEjUQOfC5fQDRFuc36lI=
-github.com/alecthomas/kong v0.2.1-0.20190708041108-0548c6b1afae/go.mod h1:+inYUSluD+p4L8KdviBSgzcqEjUQOfC5fQDRFuc36lI=
-github.com/alecthomas/kong-hcl v0.1.8-0.20190615233001-b21fea9723c8/go.mod h1:MRgZdU3vrFd05IQ89AxUZ0aYdF39BYoNFa324SodPCA=
-github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
 github.com/apache/thrift v0.0.0-20180902110319-2566ecd5d999/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
@@ -115,16 +104,11 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc
 github.com/cyphar/filepath-securejoin v0.0.0-20170720062807-ae69057f2299/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
 github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ=
 github.com/d2g/dhcp4client v0.0.0-20170829104524-6e570ed0a266/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s=
-github.com/daaku/go.zipexe v1.0.0/go.mod h1:z8IiR6TsVLEYKwXAoE/I+8ys/sDkgTzSL0CLnGVd57E=
-github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ=
-github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE=
 github.com/dgrijalva/jwt-go v0.0.0-20160705203006-01aeca54ebda/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
-github.com/dlclark/regexp2 v1.1.6 h1:CqB4MjHw0MFCDj+PHHjiESmHX+N7t0tJzKvC6M97BRg=
-github.com/dlclark/regexp2 v1.1.6/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
 github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
 github.com/docker/distribution v0.0.0-20170726174610-edc3ab29cdff h1:FKH02LHYqSmeWd3GBh0KIkM8JBpw3RrShgtcWShdWJg=
 github.com/docker/distribution v0.0.0-20170726174610-edc3ab29cdff/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
@@ -250,13 +234,9 @@ github.com/googleapis/gnostic v0.3.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTV
 github.com/gophercloud/gophercloud v0.0.0-20190126172459-c818fa66e4c8/go.mod h1:3WdhXV3rUYy9p6AUW8d94kr+HS62Y4VL9mBnFxsD8q4=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
-github.com/gorilla/csrf v1.6.0/go.mod h1:7tSf8kmjNYr7IWDCYhd3U8Ck34iQ/Yw5CJu7bAkHEGI=
-github.com/gorilla/handlers v1.4.1/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ=
 github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
-github.com/gorilla/mux v1.7.1 h1:Dw4jY2nghMMRsh1ol8dv1axHkDwMQK2DHerMNJsIpJU=
-github.com/gorilla/mux v1.7.1/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
 github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
-github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
@@ -300,7 +280,6 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt
 github.com/intel-go/cpuid v0.0.0-20181003105527-1a4a6f06a1c6 h1:XboatR7lasl05yel5hNXF7kQBw2oFUGdMztcgisfhNU=
 github.com/intel-go/cpuid v0.0.0-20181003105527-1a4a6f06a1c6/go.mod h1:RmeVYf9XrPRbRc3XIx0gLYA8qOFvNoPOfaEZduRlEp4=
 github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
-github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jimmidyson/go-download v0.0.0-20161028105827-7f9a90c8c95b h1:3TknJxYSK1eDe21QorC3C2Yz8jylk6vlJG9YABnFzkU=
 github.com/jimmidyson/go-download v0.0.0-20161028105827-7f9a90c8c95b/go.mod h1:I3WsAhNNoG7a/d8HMlYUywJJlfOs/+/83NEUjuDp4lc=
 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
@@ -409,7 +388,6 @@ github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+
 github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk=
 github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
 github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
-github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E=
 github.com/olekukonko/tablewriter v0.0.0-20160923125401-bdcc175572fd h1:nEatQ6JnwCT9iYD5uqYUiFqq8tJGX25to8KVKXqya7k=
 github.com/olekukonko/tablewriter v0.0.0-20160923125401-bdcc175572fd/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@@ -544,8 +522,6 @@ github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljT
 github.com/ulikunitz/xz v0.5.5 h1:pFrO0lVpTBXLpYw+pnLj6TbvHuyjXMfjGeCwSqCVwok=
 github.com/ulikunitz/xz v0.5.5/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
 github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4=
-github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
 github.com/vishvananda/netlink v0.0.0-20171020171820-b2de5d10e38e/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
 github.com/vishvananda/netns v0.0.0-20171111001504-be1fbeda1936/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI=
 github.com/vmware/govmomi v0.20.1/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU=
@@ -632,7 +608,6 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181004145325-8469e314837c/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go
index ef05e30323..bcd08b436c 100644
--- a/pkg/drivers/kic/kic.go
+++ b/pkg/drivers/kic/kic.go
@@ -38,9 +38,12 @@ const DefaultPodCIDR = "10.244.0.0/16"
 // DefaultBindIPV4 is The default IP the container will bind to.
 const DefaultBindIPV4 = "127.0.0.1"
 
-// BaseImage is the base image is used to spin up kic containers
+// BaseImage is the base image is used to spin up kic containers created by kind.
 const BaseImage = "gcr.io/k8s-minikube/kicbase:v0.0.1@sha256:c4ad2938877d2ae0d5b7248a5e7182ff58c0603165c3bedfe9d503e2d380a0db"
 
+// OverlayImage is the cni plugin used for overlay image, created by kind.
+const OverlayImage = "kindest/kindnetd:0.5.3"
+
 // Driver represents a kic driver https://minikube.sigs.k8s.io/docs/reference/drivers/kic/
 type Driver struct {
 	*drivers.BaseDriver
diff --git a/pkg/minikube/bootstrapper/kubeadm/default_cni.go b/pkg/minikube/bootstrapper/kubeadm/default_cni.go
index 8eae93c0df..fb5b0c55f5 100644
--- a/pkg/minikube/bootstrapper/kubeadm/default_cni.go
+++ b/pkg/minikube/bootstrapper/kubeadm/default_cni.go
@@ -16,6 +16,8 @@ limitations under the License.
 
 package kubeadm
 
+import "html/template"
+
 // defaultCNIConfig is the CNI config which is provisioned when --enable-default-cni
 // has been passed to `minikube start`.
 //
@@ -43,10 +45,8 @@ const defaultCNIConfig = `
 }
 `
 
-// kicCNIConfig is the cni plugin needed for kic
-// uses cni plugin created by kind https://github.com/kubernetes-sigs/kind/blob/03a4b519067dc308308cce735065c47a6fda1583/pkg/build/node/cni.go
-const kicCNIConfig = `
----
+// kicCNIConfig is the cni plugin needed for kic uses cni plugin created by kind https://github.com/kubernetes-sigs/kind/blob/03a4b519067dc308308cce735065c47a6fda1583/pkg/build/node/cni.go
+var kicCNIConfig = template.Must(template.New("kubeletServiceTemplate").Parse(`---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -114,7 +114,7 @@ spec:
       serviceAccountName: kindnet
       containers:
       - name: kindnet-cni
-        image: kindest/kindnetd:0.5.3
+        image: {{.ImageName}}
         env:
         - name: HOST_IP
           valueFrom:
@@ -159,4 +159,4 @@ spec:
           path: /lib/modules
 
 ---
-`
+`))
diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index 8ade8fbf00..1f27f156cd 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -17,7 +17,9 @@ limitations under the License.
 package kubeadm
 
 import (
+	"bytes"
 	"os/exec"
+	"path"
 
 	"fmt"
 	"net"
@@ -200,9 +202,9 @@ func (k *Bootstrapper) StartCluster(cfg config.MachineConfig) error {
 		return errors.Wrapf(err, "init failed. output: %q", rr.Output())
 	}
 
-	if cfg.KubernetesConfig.EnableDefaultCNI {
-		if err := k.applyOverlayNetwork(); err != nil {
-			return errors.Wrap(err, "applying overlay network")
+	if driver.IsKIC(cfg.VMDriver) {
+		if err := k.applyKicOverlay(cfg); err != nil {
+			return errors.Wrap(err, "applying kic overlay network")
 		}
 	}
 
@@ -426,10 +428,6 @@ func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error {
 	var cniFile []byte = nil
 	if cfg.KubernetesConfig.EnableDefaultCNI {
 		cniFile = []byte(defaultCNIConfig)
-		if driver.IsKIC(cfg.VMDriver) {
-			cniFile = []byte(kicCNIConfig)
-		}
-
 	}
 	files := bsutil.ConfigFileAssets(cfg.KubernetesConfig, kubeadmCfg, kubeletCfg, kubeletService, cniFile)
 
@@ -448,12 +446,16 @@ func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error {
 	return nil
 }
 
-// applyOverlayNetwork applies the CNI plugin needed to make kic work
-func (k *Bootstrapper) applyOverlayNetwork() error {
-	cmd := exec.Command(
-		"kubectl", "create", "--kubeconfig=/etc/kubernetes/admin.conf",
-		"-f", bsutil.DefaultCNIConfigPath,
-	)
+// applyKicOverlay applies the CNI plugin needed to make kic work
+func (k *Bootstrapper) applyKicOverlay(cfg config.MachineConfig) error {
+	cmd := exec.Command("sudo",
+		path.Join("/var/lib/minikube/binaries", cfg.KubernetesConfig.KubernetesVersion, "kubectl"), "create", "--kubeconfig=/var/lib/minikube/kubeconfig",
+		"-f", "-")
+	b := bytes.Buffer{}
+	if err := kicCNIConfig.Execute(&b, struct{ ImageName string }{ImageName: kic.OverlayImage}); err != nil {
+		return err
+	}
+	cmd.Stdin = bytes.NewReader(b.Bytes())
 	if rr, err := k.c.RunCmd(cmd); err != nil {
 		return errors.Wrapf(err, "cmd: %s output: %s", rr.Command(), rr.Output())
 	}
diff --git a/pkg/minikube/driver/driver.go b/pkg/minikube/driver/driver.go
index 9d32c14da3..0dd832a229 100644
--- a/pkg/minikube/driver/driver.go
+++ b/pkg/minikube/driver/driver.go
@@ -99,7 +99,6 @@ type FlagHints struct {
 	CacheImages      bool
 	ContainerRuntime string
 	Bootstrapper     string
-	EnableDefaultCNI bool
 }
 
 // FlagDefaults returns suggested defaults based on a driver
@@ -111,7 +110,6 @@ func FlagDefaults(name string) FlagHints {
 		if name == Docker {
 			fh.ContainerRuntime = "containerd"
 			fh.ExtraOptions = append(fh.ExtraOptions, fmt.Sprintf("kubeadm.pod-network-cidr=%s", kic.DefaultPodCIDR))
-			fh.EnableDefaultCNI = true
 		}
 		return fh
 	}

From 6ca7b74f8a8151e2d41bcc1adf1c3fa562ced984 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Tue, 21 Jan 2020 17:39:06 -0800
Subject: [PATCH 077/250] implement reomve file for kic runner

---
 pkg/minikube/command/kic_runner.go | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/pkg/minikube/command/kic_runner.go b/pkg/minikube/command/kic_runner.go
index 8dbcf11e13..bf357c356e 100644
--- a/pkg/minikube/command/kic_runner.go
+++ b/pkg/minikube/command/kic_runner.go
@@ -169,7 +169,11 @@ func (k *kicRunner) Copy(f assets.CopyableFile) error {
 
 // Remove removes a file
 func (k *kicRunner) Remove(f assets.CopyableFile) error {
-	return fmt.Errorf("not implemented yet for kic runner")
+	fp := path.Join(f.GetTargetDir(), f.GetTargetName())
+	if rr, err := k.RunCmd(exec.Command("sudo", "rm", fp)); err != nil {
+		return errors.Wrapf(err, "removing file %q output: %s", fp, rr.Output())
+	}
+	return nil
 }
 
 // isTerminal returns true if the writer w is a terminal

From 7fe786aca460e6507bfc251119d5d37b1ac3e439 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Tue, 21 Jan 2020 19:46:05 -0800
Subject: [PATCH 078/250] Merge upstream

---
 pkg/minikube/config/config.go       |  22 ++-
 pkg/minikube/config/profile.go.orig | 201 ----------------------------
 pkg/minikube/cruntime/cri.go        |  83 +++++++++---
 pkg/minikube/out/style_enum.go      |   1 +
 4 files changed, 83 insertions(+), 224 deletions(-)
 delete mode 100644 pkg/minikube/config/profile.go.orig

diff --git a/pkg/minikube/config/config.go b/pkg/minikube/config/config.go
index 7b7501ad15..7abd2a8d81 100644
--- a/pkg/minikube/config/config.go
+++ b/pkg/minikube/config/config.go
@@ -24,7 +24,6 @@ import (
 	"io/ioutil"
 	"os"
 
-	"github.com/spf13/viper"
 	"k8s.io/minikube/pkg/minikube/localpath"
 )
 
@@ -124,14 +123,19 @@ func encode(w io.Writer, m MinikubeConfig) error {
 }
 
 // Load loads the kubernetes and machine config for the current machine
-func Load() (*MachineConfig, error) {
-	machine := viper.GetString(MachineProfile)
-	return DefaultLoader.LoadConfigFromFile(machine)
+func Load(profile string) (*MachineConfig, error) {
+	return DefaultLoader.LoadConfigFromFile(profile)
+}
+
+// Write writes the kubernetes and machine config for the current machine
+func Write(profile string, cc *MachineConfig) error {
+	return DefaultLoader.WriteConfigToFile(profile, cc)
 }
 
 // Loader loads the kubernetes and machine config based on the machine profile name
 type Loader interface {
 	LoadConfigFromFile(profile string, miniHome ...string) (*MachineConfig, error)
+	WriteConfigToFile(profileName string, cc *MachineConfig, miniHome ...string) error
 }
 
 type simpleConfigLoader struct{}
@@ -158,3 +162,13 @@ func (c *simpleConfigLoader) LoadConfigFromFile(profileName string, miniHome ...
 	}
 	return &cc, nil
 }
+
+func (c *simpleConfigLoader) WriteConfigToFile(profileName string, cc *MachineConfig, miniHome ...string) error {
+	// Move to profile package
+	path := profileFilePath(profileName, miniHome...)
+	contents, err := json.MarshalIndent(cc, "", "	")
+	if err != nil {
+		return err
+	}
+	return ioutil.WriteFile(path, contents, 0644)
+}
diff --git a/pkg/minikube/config/profile.go.orig b/pkg/minikube/config/profile.go.orig
deleted file mode 100644
index 3546ec2e93..0000000000
--- a/pkg/minikube/config/profile.go.orig
+++ /dev/null
@@ -1,201 +0,0 @@
-/*
-Copyright 2019 The Kubernetes Authors All rights reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package config
-
-import (
-	"encoding/json"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"strings"
-
-	"github.com/golang/glog"
-	"k8s.io/minikube/pkg/minikube/localpath"
-	"k8s.io/minikube/pkg/util/lock"
-)
-
-var keywords = []string{"start", "stop", "status", "delete", "config", "open", "profile", "addons", "cache", "logs"}
-
-// IsValid checks if the profile has the essential info needed for a profile
-func (p *Profile) IsValid() bool {
-	if p.Config == nil {
-		return false
-	}
-	if len(p.Config) == 0 {
-		return false
-	}
-	// This will become a loop for multinode
-	if p.Config[0] == nil {
-		return false
-	}
-	if p.Config[0].VMDriver == "" {
-		return false
-	}
-	if p.Config[0].KubernetesConfig.KubernetesVersion == "" {
-		return false
-	}
-	return true
-}
-
-// ProfileNameInReservedKeywords checks if the profile is an internal keywords
-func ProfileNameInReservedKeywords(name string) bool {
-	for _, v := range keywords {
-		if strings.EqualFold(v, name) {
-			return true
-		}
-	}
-	return false
-}
-
-// ProfileExists returns true if there is a profile config (regardless of being valid)
-func ProfileExists(name string, miniHome ...string) bool {
-	miniPath := localpath.MiniPath()
-	if len(miniHome) > 0 {
-		miniPath = miniHome[0]
-	}
-
-	p := profileFilePath(name, miniPath)
-	_, err := os.Stat(p)
-	return err == nil
-}
-
-// CreateEmptyProfile creates an empty profile stores in $MINIKUBE_HOME/profiles/<profilename>/config.json
-func CreateEmptyProfile(name string, miniHome ...string) error {
-	cfg := &MachineConfig{}
-	return CreateProfile(name, cfg, miniHome...)
-}
-
-// CreateProfile creates an profile out of the cfg and stores in $MINIKUBE_HOME/profiles/<profilename>/config.json
-func CreateProfile(name string, cfg *MachineConfig, miniHome ...string) error {
-	data, err := json.MarshalIndent(cfg, "", "    ")
-	if err != nil {
-		return err
-	}
-	path := profileFilePath(name, miniHome...)
-	glog.Infof("Saving config to %s ...", path)
-	if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
-		return err
-	}
-
-	// If no config file exists, don't worry about swapping paths
-	if _, err := os.Stat(path); os.IsNotExist(err) {
-		if err := lock.WriteFile(path, data, 0600); err != nil {
-			return err
-		}
-		return nil
-	}
-
-	tf, err := ioutil.TempFile(filepath.Dir(path), "config.json.tmp")
-	if err != nil {
-		return err
-	}
-	defer os.Remove(tf.Name())
-
-	if err = lock.WriteFile(tf.Name(), data, 0600); err != nil {
-		return err
-	}
-
-	if err = tf.Close(); err != nil {
-		return err
-	}
-
-	if err = os.Remove(path); err != nil {
-		return err
-	}
-
-	if err = os.Rename(tf.Name(), path); err != nil {
-		return err
-	}
-	return nil
-}
-
-// DeleteProfile deletes a profile and removes the profile dir
-func DeleteProfile(profile string, miniHome ...string) error {
-	miniPath := localpath.MiniPath()
-	if len(miniHome) > 0 {
-		miniPath = miniHome[0]
-	}
-	return os.RemoveAll(ProfileFolderPath(profile, miniPath))
-}
-
-// ListProfiles returns all valid and invalid (if any) minikube profiles
-// invalidPs are the profiles that have a directory or config file but not usable
-// invalidPs would be suggested to be deleted
-func ListProfiles(miniHome ...string) (validPs []*Profile, inValidPs []*Profile, err error) {
-	pDirs, err := profileDirs(miniHome...)
-	if err != nil {
-		return nil, nil, err
-	}
-	for _, n := range pDirs {
-		p, err := LoadProfile(n, miniHome...)
-		if err != nil {
-			inValidPs = append(inValidPs, p)
-			continue
-		}
-		if !p.IsValid() {
-			inValidPs = append(inValidPs, p)
-			continue
-		}
-		validPs = append(validPs, p)
-	}
-	return validPs, inValidPs, nil
-}
-
-// LoadProfile loads type Profile based on its name
-func LoadProfile(name string, miniHome ...string) (*Profile, error) {
-	cfg, err := DefaultLoader.LoadConfigFromFile(name, miniHome...)
-	p := &Profile{
-		Name:   name,
-		Config: []*MachineConfig{cfg},
-	}
-	return p, err
-}
-
-// profileDirs gets all the folders in the user's profiles folder regardless of valid or invalid config
-func profileDirs(miniHome ...string) (dirs []string, err error) {
-	miniPath := localpath.MiniPath()
-	if len(miniHome) > 0 {
-		miniPath = miniHome[0]
-	}
-	pRootDir := filepath.Join(miniPath, "profiles")
-	items, err := ioutil.ReadDir(pRootDir)
-	for _, f := range items {
-		if f.IsDir() {
-			dirs = append(dirs, f.Name())
-		}
-	}
-	return dirs, err
-}
-
-// profileFilePath returns the Minikube profile config file
-func profileFilePath(profile string, miniHome ...string) string {
-	miniPath := localpath.MiniPath()
-	if len(miniHome) > 0 {
-		miniPath = miniHome[0]
-	}
-
-	return filepath.Join(miniPath, "profiles", profile, "config.json")
-}
-
-// ProfileFolderPath returns path of profile folder
-func ProfileFolderPath(profile string, miniHome ...string) string {
-	miniPath := localpath.MiniPath()
-	if len(miniHome) > 0 {
-		miniPath = miniHome[0]
-	}
-	return filepath.Join(miniPath, "profiles", profile)
-}
diff --git a/pkg/minikube/cruntime/cri.go b/pkg/minikube/cruntime/cri.go
index ca77a58dab..b3618272a4 100644
--- a/pkg/minikube/cruntime/cri.go
+++ b/pkg/minikube/cruntime/cri.go
@@ -31,10 +31,8 @@ import (
 
 // container maps to 'runc list -f json'
 type container struct {
-	pid         int
-	id          string
-	status      string
-	annotations map[string]string
+	ID     string
+	Status string
 }
 
 // listCRIContainers returns a list of containers
@@ -55,7 +53,8 @@ func listCRIContainers(cr CommandRunner, root string, o ListOptions) ([]string,
 	// Avoid an id named ""
 	var ids []string
 	seen := map[string]bool{}
-	for _, id := range strings.Split(rr.Stderr.String(), "\n") {
+	for _, id := range strings.Split(rr.Stdout.String(), "\n") {
+		glog.Infof("found id: %q", id)
 		if id != "" && !seen[id] {
 			ids = append(ids, id)
 			seen[id] = true
@@ -71,39 +70,51 @@ func listCRIContainers(cr CommandRunner, root string, o ListOptions) ([]string,
 
 	// crictl does not understand paused pods
 	cs := []container{}
-	args = []string{"runc", "list", "-f", "json"}
+	args = []string{"runc"}
 	if root != "" {
 		args = append(args, "--root", root)
 	}
 
-	if _, err := cr.RunCmd(exec.Command("sudo", args...)); err != nil {
+	args = append(args, "list", "-f", "json")
+	rr, err = cr.RunCmd(exec.Command("sudo", args...))
+	if err != nil {
 		return nil, errors.Wrap(err, "runc")
 	}
-
-	d := json.NewDecoder(bytes.NewReader(rr.Stdout.Bytes()))
+	content := rr.Stdout.Bytes()
+	glog.Infof("JSON = %s", content)
+	d := json.NewDecoder(bytes.NewReader(content))
 	if err := d.Decode(&cs); err != nil {
 		return nil, err
 	}
 
+	if len(cs) == 0 {
+		return nil, fmt.Errorf("list returned 0 containers, but ps returned %d", len(ids))
+	}
+
+	glog.Infof("list returned %d containers", len(cs))
 	var fids []string
 	for _, c := range cs {
-		if !seen[c.id] {
+		glog.Infof("container: %+v", c)
+		if !seen[c.ID] {
+			glog.Infof("skipping %s - not in ps", c.ID)
 			continue
 		}
-		if o.State.String() != c.status {
+		if o.State != All && o.State.String() != c.Status {
+			glog.Infof("skipping %s: state = %q, want %q", c, c.Status, o.State)
 			continue
 		}
-		fids = append(fids, c.id)
+		fids = append(fids, c.ID)
 	}
 	return fids, nil
 }
 
-// pauseCRIContainers pauses a list of containers
+// pauseContainers pauses a list of containers
 func pauseCRIContainers(cr CommandRunner, root string, ids []string) error {
-	args := []string{"runc", "pause"}
+	args := []string{"runc"}
 	if root != "" {
 		args = append(args, "--root", root)
 	}
+	args = append(args, "pause")
 
 	for _, id := range ids {
 		cargs := append(args, id)
@@ -114,12 +125,23 @@ func pauseCRIContainers(cr CommandRunner, root string, ids []string) error {
 	return nil
 }
 
+// getCrictlPath returns the absolute path of crictl
+func getCrictlPath(cr CommandRunner) string {
+	cmd := "crictl"
+	rr, err := cr.RunCmd(exec.Command("which", cmd))
+	if err != nil {
+		return cmd
+	}
+	return strings.Split(rr.Stdout.String(), "\n")[0]
+}
+
 // unpauseCRIContainers pauses a list of containers
 func unpauseCRIContainers(cr CommandRunner, root string, ids []string) error {
-	args := []string{"runc", "unpause"}
+	args := []string{"runc"}
 	if root != "" {
 		args = append(args, "--root", root)
 	}
+	args = append(args, "resume")
 
 	for _, id := range ids {
 		cargs := append(args, id)
@@ -137,7 +159,8 @@ func killCRIContainers(cr CommandRunner, ids []string) error {
 	}
 	glog.Infof("Killing containers: %s", ids)
 
-	args := append([]string{"crictl", "rm"}, ids...)
+	crictl := getCrictlPath(cr)
+	args := append([]string{crictl, "rm"}, ids...)
 	c := exec.Command("sudo", args...)
 	if _, err := cr.RunCmd(c); err != nil {
 		return errors.Wrap(err, "crictl")
@@ -151,7 +174,9 @@ func stopCRIContainers(cr CommandRunner, ids []string) error {
 		return nil
 	}
 	glog.Infof("Stopping containers: %s", ids)
-	args := append([]string{"crictl", "rm"}, ids...)
+
+	crictl := getCrictlPath(cr)
+	args := append([]string{crictl, "rm"}, ids...)
 	c := exec.Command("sudo", args...)
 	if _, err := cr.RunCmd(c); err != nil {
 		return errors.Wrap(err, "crictl")
@@ -181,10 +206,30 @@ image-endpoint: unix://{{.Socket}}
 	return nil
 }
 
+// getCRIInfo returns current information
+func getCRIInfo(cr CommandRunner) (map[string]interface{}, error) {
+	args := []string{"crictl", "info"}
+	c := exec.Command("sudo", args...)
+	rr, err := cr.RunCmd(c)
+	if err != nil {
+		return nil, errors.Wrap(err, "get cri info")
+	}
+	info := rr.Stdout.String()
+	jsonMap := make(map[string]interface{})
+	err = json.Unmarshal([]byte(info), &jsonMap)
+	if err != nil {
+		return nil, err
+	}
+	return jsonMap, nil
+}
+
 // criContainerLogCmd returns the command to retrieve the log for a container based on ID
-func criContainerLogCmd(id string, len int, follow bool) string {
+func criContainerLogCmd(cr CommandRunner, id string, len int, follow bool) string {
+	crictl := getCrictlPath(cr)
 	var cmd strings.Builder
-	cmd.WriteString("sudo crictl logs ")
+	cmd.WriteString("sudo ")
+	cmd.WriteString(crictl)
+	cmd.WriteString(" logs ")
 	if len > 0 {
 		cmd.WriteString(fmt.Sprintf("--tail %d ", len))
 	}
diff --git a/pkg/minikube/out/style_enum.go b/pkg/minikube/out/style_enum.go
index 3787dbe904..0d0c6e9ae1 100644
--- a/pkg/minikube/out/style_enum.go
+++ b/pkg/minikube/out/style_enum.go
@@ -86,4 +86,5 @@ const (
 	Sparkle
 	Pause
 	Unpause
+	DryRun
 )

From be65e4c2ad9168e69761220029dde7681feab3a3 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Tue, 21 Jan 2020 19:46:42 -0800
Subject: [PATCH 079/250] Merge upstream

---
 cmd/minikube/cmd/pause.go         | 23 +++++++++++++++-----
 cmd/minikube/cmd/unpause.go       | 35 ++++++++++++++++++++++---------
 pkg/minikube/cruntime/cruntime.go | 13 +++++++++++-
 pkg/minikube/cruntime/docker.go   | 29 +++++++++++++++++++++++--
 pkg/minikube/kubelet/kubelet.go   |  4 ++--
 5 files changed, 84 insertions(+), 20 deletions(-)

diff --git a/cmd/minikube/cmd/pause.go b/cmd/minikube/cmd/pause.go
index 731c5f87d0..797cffa25d 100644
--- a/cmd/minikube/cmd/pause.go
+++ b/cmd/minikube/cmd/pause.go
@@ -17,7 +17,11 @@ limitations under the License.
 package cmd
 
 import (
+	"os"
+
+	"github.com/golang/glog"
 	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
 
 	"k8s.io/minikube/pkg/minikube/cluster"
 	"k8s.io/minikube/pkg/minikube/config"
@@ -33,16 +37,25 @@ var pauseCmd = &cobra.Command{
 	Use:   "pause",
 	Short: "pause containers",
 	Run: func(cmd *cobra.Command, args []string) {
+		cname := viper.GetString(config.MachineProfile)
 		api, err := machine.NewAPIClient()
 		if err != nil {
 			exit.WithError("Error getting client", err)
 		}
 		defer api.Close()
-		cc, err := config.Load()
-		if err != nil {
-			exit.WithError("Error getting config", err)
+		cc, err := config.Load(cname)
+
+		if err != nil && !os.IsNotExist(err) {
+			exit.WithError("Error loading profile config", err)
 		}
-		host, err := cluster.CheckIfHostExistsAndLoad(api, cc.Name)
+
+		if err != nil {
+			out.ErrT(out.Meh, `"{{.name}}" profile does not exist`, out.V{"name": cname})
+			os.Exit(1)
+		}
+
+		glog.Infof("config: %+v", cc)
+		host, err := cluster.CheckIfHostExistsAndLoad(api, cname)
 		if err != nil {
 			exit.WithError("Error getting host", err)
 		}
@@ -52,7 +65,7 @@ var pauseCmd = &cobra.Command{
 			exit.WithError("Failed to get command runner", err)
 		}
 
-		config := cruntime.Config{Type: cc.ContainerRuntime}
+		config := cruntime.Config{Type: cc.ContainerRuntime, Runner: r}
 		cr, err := cruntime.New(config)
 		if err != nil {
 			exit.WithError("Failed runtime", err)
diff --git a/cmd/minikube/cmd/unpause.go b/cmd/minikube/cmd/unpause.go
index 3e45eaf711..e0df53060e 100644
--- a/cmd/minikube/cmd/unpause.go
+++ b/cmd/minikube/cmd/unpause.go
@@ -17,8 +17,13 @@ limitations under the License.
 package cmd
 
 import (
-	"github.com/spf13/cobra"
+	"os"
 
+	"github.com/golang/glog"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+
+	"k8s.io/minikube/pkg/minikube/cluster"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/cruntime"
 	"k8s.io/minikube/pkg/minikube/exit"
@@ -32,22 +37,27 @@ var unpauseCmd = &cobra.Command{
 	Use:   "unpause",
 	Short: "unpause Kubernetes",
 	Run: func(cmd *cobra.Command, args []string) {
+		cname := viper.GetString(config.MachineProfile)
 		api, err := machine.NewAPIClient()
 		if err != nil {
 			exit.WithError("Error getting client", err)
 		}
 		defer api.Close()
+		cc, err := config.Load(cname)
 
-		cc, err := config.Load()
-		if err != nil {
-			exit.WithError("Error getting config", err)
+		if err != nil && !os.IsNotExist(err) {
+			exit.WithError("Error loading profile config", err)
 		}
-		host, err := api.Load(cc.Name)
 
-		config := cruntime.Config{Type: cc.ContainerRuntime}
-		cr, err := cruntime.New(config)
 		if err != nil {
-			exit.WithError("Failed runtime", err)
+			out.ErrT(out.Meh, `"{{.name}}" profile does not exist`, out.V{"name": cname})
+			os.Exit(1)
+		}
+
+		glog.Infof("config: %+v", cc)
+		host, err := cluster.CheckIfHostExistsAndLoad(api, cname)
+		if err != nil {
+			exit.WithError("Error getting host", err)
 		}
 
 		r, err := machine.CommandRunner(host)
@@ -55,10 +65,15 @@ var unpauseCmd = &cobra.Command{
 			exit.WithError("Failed to get command runner", err)
 		}
 
-		err = pause.Pause(cr, r)
+		config := cruntime.Config{Type: cc.ContainerRuntime, Runner: r}
+		cr, err := cruntime.New(config)
+		if err != nil {
+			exit.WithError("Failed runtime", err)
+		}
+		err = pause.Unpause(cr, r)
 		if err != nil {
 			exit.WithError("Pause", err)
 		}
-		out.T(out.Pause, "The '{{.name}}' cluster is now paused", out.V{"name": cc.Name})
+		out.T(out.Unpause, "The '{{.name}}' cluster is now unpaused", out.V{"name": cc.Name})
 	},
 }
diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go
index 9ca3e078bf..df81613678 100644
--- a/pkg/minikube/cruntime/cruntime.go
+++ b/pkg/minikube/cruntime/cruntime.go
@@ -36,7 +36,7 @@ const (
 )
 
 func (cs ContainerState) String() string {
-	return [...]string{"All", "Running", "Paused"}[cs]
+	return [...]string{"all", "running", "paused"}[cs]
 }
 
 // CommandRunner is the subset of command.Runner this package consumes
@@ -61,6 +61,8 @@ type Manager interface {
 	// Style is an associated StyleEnum for Name()
 	Style() out.StyleEnum
 
+	// CGroupDriver returns cgroup driver ("cgroupfs" or "systemd")
+	CGroupDriver() (string, error)
 	// KubeletOptions returns kubelet options for a runtime.
 	KubeletOptions() map[string]string
 	// SocketPath returns the path to the socket file for a given runtime
@@ -71,6 +73,9 @@ type Manager interface {
 	// Load an image idempotently into the runtime on a host
 	LoadImage(string) error
 
+	// ImageExists takes image name and image sha checks if an it exists
+	ImageExists(string, string) bool
+
 	// ListContainers returns a list of managed by this container runtime
 	ListContainers(ListOptions) ([]string, error)
 	// KillContainers removes containers based on ID
@@ -122,6 +127,12 @@ func New(c Config) (Manager, error) {
 	}
 }
 
+// ContainerStatusCommand works across container runtimes with good formatting
+func ContainerStatusCommand() string {
+	// Fallback to 'docker ps' if it fails (none driver)
+	return "sudo `which crictl || echo crictl` ps -a || sudo docker ps -a"
+}
+
 // disableOthers disables all other runtimes except for me.
 func disableOthers(me Manager, cr CommandRunner) error {
 	// valid values returned by manager.Name()
diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go
index 1bc2ab6920..0b5297f381 100644
--- a/pkg/minikube/cruntime/docker.go
+++ b/pkg/minikube/cruntime/docker.go
@@ -102,6 +102,20 @@ func (r *Docker) Disable() error {
 	return nil
 }
 
+// ImageExists checks if an image exists
+func (r *Docker) ImageExists(name string, sha string) bool {
+	// expected output looks like [SHA_ALGO:SHA]
+	c := exec.Command("docker", "inspect", "--format='{{.Id}}'", name)
+	rr, err := r.Runner.RunCmd(c)
+	if err != nil {
+		return false
+	}
+	if !strings.Contains(rr.Output(), sha) {
+		return false
+	}
+	return true
+}
+
 // LoadImage loads an image into this runtime
 func (r *Docker) LoadImage(path string) error {
 	glog.Infof("Loading image: %s", path)
@@ -113,6 +127,17 @@ func (r *Docker) LoadImage(path string) error {
 
 }
 
+// CGroupDriver returns cgroup driver ("cgroupfs" or "systemd")
+func (r *Docker) CGroupDriver() (string, error) {
+	// Note: the server daemon has to be running, for this call to return successfully
+	c := exec.Command("docker", "info", "--format", "'{{.CgroupDriver}}'")
+	rr, err := r.Runner.RunCmd(c)
+	if err != nil {
+		return "", err
+	}
+	return strings.Split(rr.Stdout.String(), "\n")[0], nil
+}
+
 // KubeletOptions returns kubelet options for a runtime.
 func (r *Docker) KubeletOptions() map[string]string {
 	return map[string]string{
@@ -127,9 +152,9 @@ func (r *Docker) ListContainers(o ListOptions) ([]string, error) {
 	case All:
 		args = append(args, "-a")
 	case Paused:
-		args = append(args, "--filter status=paused")
+		args = append(args, "--filter", "status=paused")
 	}
-	args = append(args, fmt.Sprintf("--filter=name=%s", KubernetesContainerPrefix+o.Name), "--format=\"{{.ID}}\"")
+	args = append(args, fmt.Sprintf("--filter=name=%s", KubernetesContainerPrefix+o.Name), "--format={{.ID}}")
 	rr, err := r.Runner.RunCmd(exec.Command("docker", args...))
 	if err != nil {
 		return nil, errors.Wrapf(err, "docker")
diff --git a/pkg/minikube/kubelet/kubelet.go b/pkg/minikube/kubelet/kubelet.go
index c8e0436a2f..315f492878 100644
--- a/pkg/minikube/kubelet/kubelet.go
+++ b/pkg/minikube/kubelet/kubelet.go
@@ -87,7 +87,7 @@ func Check(cr command.Runner) error {
 // Disable disables the Kubelet
 func Disable(cr command.Runner) error {
 	glog.Infof("disabling kubelet ...")
-	c := exec.Command("systemctl", "disable", "kubelet")
+	c := exec.Command("sudo", "systemctl", "disable", "kubelet")
 	if _, err := cr.RunCmd(c); err != nil {
 		return errors.Wrap(err, "disable")
 	}
@@ -97,7 +97,7 @@ func Disable(cr command.Runner) error {
 // Enable enables the Kubelet
 func Enable(cr command.Runner) error {
 	glog.Infof("enabling kubelet ...")
-	c := exec.Command("systemctl", "enable", "kubelet")
+	c := exec.Command("sudo", "systemctl", "enable", "kubelet")
 	if _, err := cr.RunCmd(c); err != nil {
 		return errors.Wrap(err, "enable")
 	}

From 0f40eef9a7d556aee63e140a8b1db635d24fde77 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Tue, 21 Jan 2020 20:06:03 -0800
Subject: [PATCH 080/250] Merge conflicts

---
 pkg/minikube/cruntime/crio.go | 327 +---------------------------------
 1 file changed, 1 insertion(+), 326 deletions(-)

diff --git a/pkg/minikube/cruntime/crio.go b/pkg/minikube/cruntime/crio.go
index 648e350dee..dc4670cca7 100644
--- a/pkg/minikube/cruntime/crio.go
+++ b/pkg/minikube/cruntime/crio.go
@@ -17,13 +17,9 @@ limitations under the License.
 package cruntime
 
 import (
-	"bytes"
-	"encoding/base64"
 	"fmt"
 	"os/exec"
-	"path"
 	"strings"
-	"text/template"
 
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
@@ -33,302 +29,7 @@ import (
 
 const (
 	// CRIOConfFile is the path to the CRI-O configuration
-	crioConfigFile     = "/etc/crio/crio.conf"
-	crioConfigTemplate = `# The CRI-O configuration file specifies all of the available configuration
-# options and command-line flags for the crio(8) OCI Kubernetes Container Runtime
-# daemon, but in a TOML format that can be more easily modified and versioned.
-#
-# Please refer to crio.conf(5) for details of all configuration options.
-
-# CRI-O supports partial configuration reload during runtime, which can be
-# done by sending SIGHUP to the running process. Currently supported options
-# are explicitly mentioned with: 'This option supports live configuration
-# reload'.
-
-# CRI-O reads its storage defaults from the containers-storage.conf(5) file
-# located at /etc/containers/storage.conf. Modify this storage configuration if
-# you want to change the system's defaults. If you want to modify storage just
-# for CRI-O, you can change the storage configuration options here.
-[crio]
-
-# Path to the "root directory". CRI-O stores all of its data, including
-# containers images, in this directory.
-root = "/var/lib/containers/storage"
-
-# Path to the "run directory". CRI-O stores all of its state in this directory.
-runroot = "/var/run/containers/storage"
-
-# Storage driver used to manage the storage of images and containers. Please
-# refer to containers-storage.conf(5) to see all available storage drivers.
-storage_driver = "overlay"
-
-# List to pass options to the storage driver. Please refer to
-# containers-storage.conf(5) to see all available storage options.
-#storage_option = [
-#]
-
-# If set to false, in-memory locking will be used instead of file-based locking.
-# **Deprecated** this option will be removed in the future.
-file_locking = false
-
-# Path to the lock file.
-# **Deprecated** this option will be removed in the future.
-file_locking_path = "/run/crio.lock"
-
-
-# The crio.api table contains settings for the kubelet/gRPC interface.
-[crio.api]
-
-# Path to AF_LOCAL socket on which CRI-O will listen.
-listen = "/var/run/crio/crio.sock"
-
-# IP address on which the stream server will listen.
-stream_address = "127.0.0.1"
-
-# The port on which the stream server will listen.
-stream_port = "0"
-
-# Enable encrypted TLS transport of the stream server.
-stream_enable_tls = false
-
-# Path to the x509 certificate file used to serve the encrypted stream. This
-# file can change, and CRI-O will automatically pick up the changes within 5
-# minutes.
-stream_tls_cert = ""
-
-# Path to the key file used to serve the encrypted stream. This file can
-# change, and CRI-O will automatically pick up the changes within 5 minutes.
-stream_tls_key = ""
-
-# Path to the x509 CA(s) file used to verify and authenticate client
-# communication with the encrypted stream. This file can change, and CRI-O will
-# automatically pick up the changes within 5 minutes.
-stream_tls_ca = ""
-
-# Maximum grpc send message size in bytes. If not set or <=0, then CRI-O will default to 16 * 1024 * 1024.
-grpc_max_send_msg_size = 16777216
-
-# Maximum grpc receive message size. If not set or <= 0, then CRI-O will default to 16 * 1024 * 1024.
-grpc_max_recv_msg_size = 16777216
-
-# The crio.runtime table contains settings pertaining to the OCI runtime used
-# and options for how to set up and manage the OCI runtime.
-[crio.runtime]
-
-# A list of ulimits to be set in containers by default, specified as
-# "<ulimit name>=<soft limit>:<hard limit>", for example:
-# "nofile=1024:2048"
-# If nothing is set here, settings will be inherited from the CRI-O daemon
-#default_ulimits = [
-#]
-
-# default_runtime is the _name_ of the OCI runtime to be used as the default.
-# The name is matched against the runtimes map below.
-default_runtime = "runc"
-
-# If true, the runtime will not use pivot_root, but instead use MS_MOVE.
-no_pivot = true
-
-# Path to the conmon binary, used for monitoring the OCI runtime.
-conmon = "/usr/libexec/crio/conmon"
-
-# Cgroup setting for conmon
-conmon_cgroup = "pod"
-
-# Environment variable list for the conmon process, used for passing necessary
-# environment variables to conmon or the runtime.
-conmon_env = [
-	"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
-]
-
-# If true, SELinux will be used for pod separation on the host.
-selinux = false
-
-# Path to the seccomp.json profile which is used as the default seccomp profile
-# for the runtime. If not specified, then the internal default seccomp profile
-# will be used.
-seccomp_profile = ""
-
-# Used to change the name of the default AppArmor profile of CRI-O. The default
-# profile name is "crio-default-" followed by the version string of CRI-O.
-apparmor_profile = "crio-default"
-
-# Cgroup management implementation used for the runtime.
-cgroup_manager = "cgroupfs"
-
-# List of default capabilities for containers. If it is empty or commented out,
-# only the capabilities defined in the containers json file by the user/kube
-# will be added.
-default_capabilities = [
-	"CHOWN",
-	"DAC_OVERRIDE",
-	"FSETID",
-	"FOWNER",
-	"NET_RAW",
-	"SETGID",
-	"SETUID",
-	"SETPCAP",
-	"NET_BIND_SERVICE",
-	"SYS_CHROOT",
-	"KILL",
-]
-
-# List of default sysctls. If it is empty or commented out, only the sysctls
-# defined in the container json file by the user/kube will be added.
-default_sysctls = [
-]
-
-# List of additional devices. specified as
-# "<device-on-host>:<device-on-container>:<permissions>", for example: "--device=/dev/sdc:/dev/xvdc:rwm".
-#If it is empty or commented out, only the devices
-# defined in the container json file by the user/kube will be added.
-additional_devices = [
-]
-
-# Path to OCI hooks directories for automatically executed hooks.
-hooks_dir = [
-]
-
-# List of default mounts for each container. **Deprecated:** this option will
-# be removed in future versions in favor of default_mounts_file.
-default_mounts = [
-]
-
-# Path to the file specifying the defaults mounts for each container. The
-# format of the config is /SRC:/DST, one mount per line. Notice that CRI-O reads
-# its default mounts from the following two files:
-#
-#   1) /etc/containers/mounts.conf (i.e., default_mounts_file): This is the
-#      override file, where users can either add in their own default mounts, or
-#      override the default mounts shipped with the package.
-#
-#   2) /usr/share/containers/mounts.conf: This is the default file read for
-#      mounts. If you want CRI-O to read from a different, specific mounts file,
-#      you can change the default_mounts_file. Note, if this is done, CRI-O will
-#      only add mounts it finds in this file.
-#
-#default_mounts_file = ""
-
-# Maximum number of processes allowed in a container.
-pids_limit = 1024
-
-# Maximum sized allowed for the container log file. Negative numbers indicate
-# that no size limit is imposed. If it is positive, it must be >= 8192 to
-# match/exceed conmon's read buffer. The file is truncated and re-opened so the
-# limit is never exceeded.
-log_size_max = -1
-
-# Whether container output should be logged to journald in addition to the kuberentes log file
-log_to_journald = false
-
-# Path to directory in which container exit files are written to by conmon.
-container_exits_dir = "/var/run/crio/exits"
-
-# Path to directory for container attach sockets.
-container_attach_socket_dir = "/var/run/crio"
-
-# If set to true, all containers will run in read-only mode.
-read_only = false
-
-# Changes the verbosity of the logs based on the level it is set to. Options
-# are fatal, panic, error, warn, info, and debug. This option supports live
-# configuration reload.
-log_level = "error"
-
-# The default log directory where all logs will go unless directly specified by the kubelet
-log_dir = "/var/log/crio/pods"
-
-# The UID mappings for the user namespace of each container. A range is
-# specified in the form containerUID:HostUID:Size. Multiple ranges must be
-# separated by comma.
-uid_mappings = ""
-
-# The GID mappings for the user namespace of each container. A range is
-# specified in the form containerGID:HostGID:Size. Multiple ranges must be
-# separated by comma.
-gid_mappings = ""
-
-# The minimal amount of time in seconds to wait before issuing a timeout
-# regarding the proper termination of the container.
-ctr_stop_timeout = 0
-
-# ManageNetworkNSLifecycle determines whether we pin and remove network namespace
-# and manage its lifecycle.
-manage_network_ns_lifecycle = false
-
-# The "crio.runtime.runtimes" table defines a list of OCI compatible runtimes.
-# The runtime to use is picked based on the runtime_handler provided by the CRI.
-# If no runtime_handler is provided, the runtime will be picked based on the level
-# of trust of the workload.
-
-[crio.runtime.runtimes.runc]
-runtime_path = "/usr/bin/runc"
-runtime_type = "oci"
-runtime_root = "/run/runc"
-
-
-# The crio.image table contains settings pertaining to the management of OCI images.
-#
-# CRI-O reads its configured registries defaults from the system wide
-# containers-registries.conf(5) located in /etc/containers/registries.conf. If
-# you want to modify just CRI-O, you can change the registries configuration in
-# this file. Otherwise, leave insecure_registries and registries commented out to
-# use the system's defaults from /etc/containers/registries.conf.
-[crio.image]
-
-# Default transport for pulling images from a remote container storage.
-default_transport = "docker://"
-
-# The path to a file containing credentials necessary for pulling images from
-# secure registries. The file is similar to that of /var/lib/kubelet/config.json
-global_auth_file = ""
-
-# The image used to instantiate infra containers.
-# This option supports live configuration reload.
-pause_image = "{{ .PodInfraContainerImage }}"
-
-# The path to a file containing credentials specific for pulling the pause_image from
-# above. The file is similar to that of /var/lib/kubelet/config.json
-# This option supports live configuration reload.
-pause_image_auth_file = ""
-
-# The command to run to have a container stay in the paused state.
-# This option supports live configuration reload.
-pause_command = "/pause"
-
-# Path to the file which decides what sort of policy we use when deciding
-# whether or not to trust an image that we've pulled. It is not recommended that
-# this option be used, as the default behavior of using the system-wide default
-# policy (i.e., /etc/containers/policy.json) is most often preferred. Please
-# refer to containers-policy.json(5) for more details.
-signature_policy = ""
-
-# Controls how image volumes are handled. The valid values are mkdir, bind and
-# ignore; the latter will ignore volumes entirely.
-image_volumes = "mkdir"
-
-# List of registries to be used when pulling an unqualified image (e.g.,
-# "alpine:latest"). By default, registries is set to "docker.io" for
-# compatibility reasons. Depending on your workload and usecase you may add more
-# registries (e.g., "quay.io", "registry.fedoraproject.org",
-# "registry.opensuse.org", etc.).
-registries = [
-	"docker.io"
-]
-
-
-# The crio.network table containers settings pertaining to the management of
-# CNI plugins.
-[crio.network]
-
-# Path to the directory where CNI configuration files are located.
-network_dir = "/etc/cni/net.d/"
-
-# Paths to directories where CNI plugin binaries are located.
-plugin_dirs = [
-	"/opt/cni/bin/",
-]
-`
+	crioConfigFile = "/etc/crio/crio.conf"
 )
 
 // CRIO contains CRIO runtime state
@@ -339,11 +40,6 @@ type CRIO struct {
 	KubernetesVersion string
 }
 
-const (
-	// CRIOConfFile is the path to the CRI-O configuration
-	crioConfigFile = "/etc/crio/crio.conf"
-)
-
 // generateCRIOConfig sets up /etc/crio/crio.conf
 func generateCRIOConfig(cr CommandRunner, imageRepository string) error {
 	cPath := crioConfigFile
@@ -529,24 +225,3 @@ func (r *CRIO) ContainerLogCmd(id string, len int, follow bool) string {
 func (r *CRIO) SystemLogCmd(len int) string {
 	return fmt.Sprintf("sudo journalctl -u crio -n %d", len)
 }
-
-// generateCRIOConfig sets up /etc/crio/crio.conf
-func generateCRIOConfig(cr CommandRunner, imageRepository string) error {
-	cPath := crioConfigFile
-	t, err := template.New("crio.conf").Parse(crioConfigTemplate)
-	if err != nil {
-		return err
-	}
-	pauseImage := images.Pause(imageRepository)
-	opts := struct{ PodInfraContainerImage string }{PodInfraContainerImage: pauseImage}
-	var b bytes.Buffer
-	if err := t.Execute(&b, opts); err != nil {
-		return err
-	}
-
-	c := exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo mkdir -p %s && printf %%s \"%s\" | base64 -d | sudo tee %s", path.Dir(cPath), base64.StdEncoding.EncodeToString(b.Bytes()), cPath))
-	if _, err := cr.RunCmd(c); err != nil {
-		return errors.Wrap(err, "generateCRIOConfig.")
-	}
-	return nil
-}

From 09b2780fb2e1503f071cbc20d7627cd518781c02 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Tue, 21 Jan 2020 22:06:05 -0800
Subject: [PATCH 081/250] Add support for pausing/unpausing specific namespaces

---
 cmd/minikube/cmd/pause.go                | 112 ++++++++++++++---------
 cmd/minikube/cmd/unpause.go              |  21 ++++-
 pkg/minikube/{pause => cluster}/pause.go |  17 +++-
 pkg/minikube/cruntime/cri.go             |  41 ++++++---
 pkg/minikube/cruntime/cruntime.go        |   2 +
 pkg/minikube/cruntime/docker.go          |   9 +-
 6 files changed, 139 insertions(+), 63 deletions(-)
 rename pkg/minikube/{pause => cluster}/pause.go (80%)

diff --git a/cmd/minikube/cmd/pause.go b/cmd/minikube/cmd/pause.go
index 797cffa25d..dc9e3b61d8 100644
--- a/cmd/minikube/cmd/pause.go
+++ b/cmd/minikube/cmd/pause.go
@@ -1,5 +1,5 @@
 /*
-Copyright 2016 The Kubernetes Authors All rights reserved.
+Copyright 2020 The Kubernetes Authors All rights reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"os"
+	"strings"
 
 	"github.com/golang/glog"
 	"github.com/spf13/cobra"
@@ -29,52 +30,77 @@ import (
 	"k8s.io/minikube/pkg/minikube/exit"
 	"k8s.io/minikube/pkg/minikube/machine"
 	"k8s.io/minikube/pkg/minikube/out"
-	"k8s.io/minikube/pkg/minikube/pause"
+)
+
+var (
+	namespaces    []string
+	allNamespaces bool
 )
 
 // pauseCmd represents the docker-pause command
 var pauseCmd = &cobra.Command{
 	Use:   "pause",
 	Short: "pause containers",
-	Run: func(cmd *cobra.Command, args []string) {
-		cname := viper.GetString(config.MachineProfile)
-		api, err := machine.NewAPIClient()
-		if err != nil {
-			exit.WithError("Error getting client", err)
-		}
-		defer api.Close()
-		cc, err := config.Load(cname)
-
-		if err != nil && !os.IsNotExist(err) {
-			exit.WithError("Error loading profile config", err)
-		}
-
-		if err != nil {
-			out.ErrT(out.Meh, `"{{.name}}" profile does not exist`, out.V{"name": cname})
-			os.Exit(1)
-		}
-
-		glog.Infof("config: %+v", cc)
-		host, err := cluster.CheckIfHostExistsAndLoad(api, cname)
-		if err != nil {
-			exit.WithError("Error getting host", err)
-		}
-
-		r, err := machine.CommandRunner(host)
-		if err != nil {
-			exit.WithError("Failed to get command runner", err)
-		}
-
-		config := cruntime.Config{Type: cc.ContainerRuntime, Runner: r}
-		cr, err := cruntime.New(config)
-		if err != nil {
-			exit.WithError("Failed runtime", err)
-		}
-
-		err = pause.Pause(cr, r)
-		if err != nil {
-			exit.WithError("Pause", err)
-		}
-		out.T(out.Pause, "The '{{.name}}' cluster is now paused", out.V{"name": cc.Name})
-	},
+	Run:   runPause,
+}
+
+func runPause(cmd *cobra.Command, args []string) {
+	cname := viper.GetString(config.MachineProfile)
+	api, err := machine.NewAPIClient()
+	if err != nil {
+		exit.WithError("Error getting client", err)
+	}
+	defer api.Close()
+	cc, err := config.Load(cname)
+
+	if err != nil && !os.IsNotExist(err) {
+		exit.WithError("Error loading profile config", err)
+	}
+
+	if err != nil {
+		out.ErrT(out.Meh, `"{{.name}}" profile does not exist`, out.V{"name": cname})
+		os.Exit(1)
+	}
+
+	glog.Infof("config: %+v", cc)
+	host, err := cluster.CheckIfHostExistsAndLoad(api, cname)
+	if err != nil {
+		exit.WithError("Error getting host", err)
+	}
+
+	r, err := machine.CommandRunner(host)
+	if err != nil {
+		exit.WithError("Failed to get command runner", err)
+	}
+
+	config := cruntime.Config{Type: cc.ContainerRuntime, Runner: r}
+	cr, err := cruntime.New(config)
+	if err != nil {
+		exit.WithError("Failed runtime", err)
+	}
+
+	glog.Infof("namespaces: %v keys: %v", namespaces, viper.AllSettings())
+	if allNamespaces {
+		namespaces = nil //all
+	} else {
+		if len(namespaces) == 0 {
+			exit.WithCodeT(exit.BadUsage, "Use -A to specify all namespaces")
+		}
+	}
+
+	err = cluster.Pause(cr, r, namespaces)
+	if err != nil {
+		exit.WithError("Pause", err)
+	}
+
+	if namespaces == nil {
+		out.T(out.Pause, "Paused all namespaces in the '{{.name}}' cluster", out.V{"name": cc.Name})
+	} else {
+		out.T(out.Pause, "Paused the following namespaces in '{{.name}}': {{.namespaces}}", out.V{"name": cc.Name, "namespaces": strings.Join(namespaces, ", ")})
+	}
+}
+
+func init() {
+	pauseCmd.Flags().StringSliceVarP(&namespaces, "--namespaces", "n", cluster.DefaultNamespaces, "namespaces to pause")
+	pauseCmd.Flags().BoolVarP(&allNamespaces, "all-namespaces", "A", false, "If set, pause all namespaces")
 }
diff --git a/cmd/minikube/cmd/unpause.go b/cmd/minikube/cmd/unpause.go
index dbaf18ffb4..7791c54e0b 100644
--- a/cmd/minikube/cmd/unpause.go
+++ b/cmd/minikube/cmd/unpause.go
@@ -1,5 +1,5 @@
 /*
-Copyright 2016 The Kubernetes Authors All rights reserved.
+Copyright 2020 The Kubernetes Authors All rights reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -29,7 +29,6 @@ import (
 	"k8s.io/minikube/pkg/minikube/exit"
 	"k8s.io/minikube/pkg/minikube/machine"
 	"k8s.io/minikube/pkg/minikube/out"
-	"k8s.io/minikube/pkg/minikube/pause"
 )
 
 // unpauseCmd represents the docker-pause command
@@ -69,10 +68,26 @@ var unpauseCmd = &cobra.Command{
 		if err != nil {
 			exit.WithError("Failed runtime", err)
 		}
-		err = pause.Unpause(cr, r)
+
+		glog.Infof("namespaces: %v keys: %v", namespaces, viper.AllSettings())
+		if allNamespaces {
+			namespaces = nil //all
+		} else {
+			if len(namespaces) == 0 {
+				exit.WithCodeT(exit.BadUsage, "Use -A to specify all namespaces")
+			}
+		}
+
+		err = cluster.Unpause(cr, r, namespaces)
+
 		if err != nil {
 			exit.WithError("Pause", err)
 		}
 		out.T(out.Unpause, "The '{{.name}}' cluster is now unpaused", out.V{"name": cc.Name})
 	},
 }
+
+func init() {
+	unpauseCmd.Flags().StringSliceVarP(&namespaces, "--namespaces", "n", cluster.DefaultNamespaces, "namespaces to unpause")
+	unpauseCmd.Flags().BoolVarP(&allNamespaces, "all-namespaces", "A", false, "If set, unpause all namespaces")
+}
diff --git a/pkg/minikube/pause/pause.go b/pkg/minikube/cluster/pause.go
similarity index 80%
rename from pkg/minikube/pause/pause.go
rename to pkg/minikube/cluster/pause.go
index 8800fabfef..b963b4b002 100644
--- a/pkg/minikube/pause/pause.go
+++ b/pkg/minikube/cluster/pause.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package pause
+package cluster
 
 import (
 	"github.com/golang/glog"
@@ -24,15 +24,22 @@ import (
 	"k8s.io/minikube/pkg/minikube/kubelet"
 )
 
+// DefaultNamespaces are namespaces used by minikube, including addons
+var DefaultNamespaces = []string{
+	"kube-system",
+	"kubernetes-dashboard",
+	"storage-gluster",
+}
+
 // Pause pauses a Kubernetes cluster
-func Pause(cr cruntime.Manager, r command.Runner) error {
+func Pause(cr cruntime.Manager, r command.Runner, namespaces []string) error {
 	if err := kubelet.Disable(r); err != nil {
 		return errors.Wrap(err, "kubelet disable")
 	}
 	if err := kubelet.Stop(r); err != nil {
 		return errors.Wrap(err, "kubelet stop")
 	}
-	ids, err := cr.ListContainers(cruntime.ListOptions{State: cruntime.Running})
+	ids, err := cr.ListContainers(cruntime.ListOptions{State: cruntime.Running, Namespaces: namespaces})
 	if err != nil {
 		return errors.Wrap(err, "list running")
 	}
@@ -45,8 +52,8 @@ func Pause(cr cruntime.Manager, r command.Runner) error {
 }
 
 // Unpause unpauses a Kubernetes cluster
-func Unpause(cr cruntime.Manager, r command.Runner) error {
-	ids, err := cr.ListContainers(cruntime.ListOptions{State: cruntime.Paused})
+func Unpause(cr cruntime.Manager, r command.Runner, namespaces []string) error {
+	ids, err := cr.ListContainers(cruntime.ListOptions{State: cruntime.Paused, Namespaces: namespaces})
 	if err != nil {
 		return errors.Wrap(err, "list paused")
 	}
diff --git a/pkg/minikube/cruntime/cri.go b/pkg/minikube/cruntime/cri.go
index b3618272a4..2cf50c78e2 100644
--- a/pkg/minikube/cruntime/cri.go
+++ b/pkg/minikube/cruntime/cri.go
@@ -27,6 +27,7 @@ import (
 
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/minikube/command"
 )
 
 // container maps to 'runc list -f json'
@@ -35,19 +36,37 @@ type container struct {
 	Status string
 }
 
+// crictlList returns the output of 'crictl ps' in an efficient manner
+func crictlList(cr CommandRunner, root string, o ListOptions) (*command.RunResult, error) {
+	glog.Infof("listing CRI containers in root %s: %+v", root, o)
+
+	// Use -a because otherwise paused containers are missed
+	baseCmd := []string{"crictl", "ps", "-a", "--quiet"}
+
+	if o.Name != "" {
+		baseCmd = append(baseCmd, fmt.Sprintf("--name=%s", o.Name))
+	}
+
+	// shortcut for all namespaces
+	if len(o.Namespaces) == 0 {
+		return cr.RunCmd(exec.Command("sudo", baseCmd...))
+	}
+
+	// Gather containers for all namespaces without causing extraneous shells to be launched
+	cmds := []string{}
+	for _, ns := range o.Namespaces {
+		cmd := fmt.Sprintf("%s --label io.kubernetes.pod.namespace=%s", strings.Join(baseCmd, " "), ns)
+		cmds = append(cmds, cmd)
+	}
+
+	return cr.RunCmd(exec.Command("sudo", "-s", "eval", strings.Join(cmds, "; ")))
+}
+
 // listCRIContainers returns a list of containers
 func listCRIContainers(cr CommandRunner, root string, o ListOptions) ([]string, error) {
-	// First use crictl, because it reliably matches names
-	args := []string{"crictl", "ps", "--quiet"}
-	if o.State == All {
-		args = append(args, "-a")
-	}
-	if o.Name != "" {
-		args = append(args, fmt.Sprintf("--name=%s", o.Name))
-	}
-	rr, err := cr.RunCmd(exec.Command("sudo", args...))
+	rr, err := crictlList(cr, root, o)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "crictl list")
 	}
 
 	// Avoid an id named ""
@@ -70,7 +89,7 @@ func listCRIContainers(cr CommandRunner, root string, o ListOptions) ([]string,
 
 	// crictl does not understand paused pods
 	cs := []container{}
-	args = []string{"runc"}
+	args := []string{"runc"}
 	if root != "" {
 		args = append(args, "--root", root)
 	}
diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go
index df81613678..2705a012ce 100644
--- a/pkg/minikube/cruntime/cruntime.go
+++ b/pkg/minikube/cruntime/cruntime.go
@@ -111,6 +111,8 @@ type ListOptions struct {
 	State ContainerState
 	// Name is a name filter
 	Name string
+	// Namespaces is the namespaces to look into
+	Namespaces []string
 }
 
 // New returns an appropriately configured runtime
diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go
index 0b5297f381..a58cff35b1 100644
--- a/pkg/minikube/cruntime/docker.go
+++ b/pkg/minikube/cruntime/docker.go
@@ -154,7 +154,14 @@ func (r *Docker) ListContainers(o ListOptions) ([]string, error) {
 	case Paused:
 		args = append(args, "--filter", "status=paused")
 	}
-	args = append(args, fmt.Sprintf("--filter=name=%s", KubernetesContainerPrefix+o.Name), "--format={{.ID}}")
+
+	nameFilter := KubernetesContainerPrefix + o.Name
+	if len(o.Namespaces) > 0 {
+		// Example result: k8s.*(kube-system|kubernetes-dashboard)
+		nameFilter = fmt.Sprintf("%s.*_(%s)_", nameFilter, strings.Join(o.Namespaces, "|"))
+	}
+
+	args = append(args, fmt.Sprintf("--filter=name=%s", nameFilter), "--format={{.ID}}")
 	rr, err := r.Runner.RunCmd(exec.Command("docker", args...))
 	if err != nil {
 		return nil, errors.Wrapf(err, "docker")

From e6999caea376744a27ced7c5ea28350cce7d4cbd Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Wed, 22 Jan 2020 09:48:16 -0800
Subject: [PATCH 082/250] Add paused state to apiserver status

---
 cmd/minikube/cmd/status.go                   | 165 ++++++++++---------
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go |  23 ++-
 2 files changed, 108 insertions(+), 80 deletions(-)

diff --git a/cmd/minikube/cmd/status.go b/cmd/minikube/cmd/status.go
index 91376babb7..171cf82e5a 100644
--- a/cmd/minikube/cmd/status.go
+++ b/cmd/minikube/cmd/status.go
@@ -23,8 +23,10 @@ import (
 	"strings"
 	"text/template"
 
+	"github.com/docker/machine/libmachine"
 	"github.com/docker/machine/libmachine/state"
 	"github.com/golang/glog"
+	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config"
@@ -40,16 +42,13 @@ import (
 var statusFormat string
 var output string
 
-// KubeconfigStatus represents the kubeconfig status
-var KubeconfigStatus = struct {
-	Configured    string
-	Misconfigured string
-}{
-	Configured:    `Configured`,
-	Misconfigured: `Misconfigured`,
-}
+const (
+	// Additional states used by kubeconfig
+	Configured    = "Configured"    // analagous to state.Saved
+	Misconfigured = "Misconfigured" // analagous to state.Error
+)
 
-// Status represents the status
+// Status holds string representations of libmachine.state.State
 type Status struct {
 	Host       string
 	Kubelet    string
@@ -81,7 +80,6 @@ var statusCmd = &cobra.Command{
 			exit.UsageT("Cannot use both --output and --format options")
 		}
 
-		var returnCode = 0
 		api, err := machine.NewAPIClient()
 		if err != nil {
 			exit.WithCodeT(exit.Unavailable, "Error getting client: {{.error}}", out.V{"error": err})
@@ -89,81 +87,91 @@ var statusCmd = &cobra.Command{
 		defer api.Close()
 
 		machineName := viper.GetString(config.MachineProfile)
-
-		hostSt, err := cluster.GetHostStatus(api, machineName)
+		st, err := status(api, machineName)
 		if err != nil {
-			exit.WithError("Error getting host status", err)
-		}
-
-		kubeletSt := state.None.String()
-		kubeconfigSt := state.None.String()
-		apiserverSt := state.None.String()
-
-		if hostSt == state.Running.String() {
-			clusterBootstrapper, err := getClusterBootstrapper(api, viper.GetString(cmdcfg.Bootstrapper))
-			if err != nil {
-				exit.WithError("Error getting bootstrapper", err)
-			}
-			kubeletSt, err = clusterBootstrapper.GetKubeletStatus()
-			if err != nil {
-				glog.Warningf("kubelet err: %v", err)
-				returnCode |= clusterNotRunningStatusFlag
-			} else if kubeletSt != state.Running.String() {
-				returnCode |= clusterNotRunningStatusFlag
-			}
-
-			ip, err := cluster.GetHostDriverIP(api, machineName)
-			if err != nil {
-				glog.Errorln("Error host driver ip status:", err)
-			}
-
-			apiserverPort, err := kubeconfig.Port(machineName)
-			if err != nil {
-				// Fallback to presuming default apiserver port
-				apiserverPort = constants.APIServerPort
-			}
-
-			apiserverSt, err = clusterBootstrapper.GetAPIServerStatus(ip, apiserverPort)
-			if err != nil {
-				glog.Errorln("Error apiserver status:", err)
-			} else if apiserverSt != state.Running.String() {
-				returnCode |= clusterNotRunningStatusFlag
-			}
-
-			ks, err := kubeconfig.IsClusterInConfig(ip, machineName)
-			if err != nil {
-				glog.Errorln("Error kubeconfig status:", err)
-			}
-			if ks {
-				kubeconfigSt = KubeconfigStatus.Configured
-			} else {
-				kubeconfigSt = KubeconfigStatus.Misconfigured
-				returnCode |= k8sNotRunningStatusFlag
-			}
-		} else {
-			returnCode |= minikubeNotRunningStatusFlag
-		}
-
-		status := Status{
-			Host:       hostSt,
-			Kubelet:    kubeletSt,
-			APIServer:  apiserverSt,
-			Kubeconfig: kubeconfigSt,
+			glog.Errorf("status error: %v", err)
 		}
 
 		switch strings.ToLower(output) {
 		case "text":
-			printStatusText(status)
+			printStatusText(st)
 		case "json":
-			printStatusJSON(status)
+			printStatusJSON(st)
 		default:
 			exit.WithCodeT(exit.BadUsage, fmt.Sprintf("invalid output format: %s. Valid values: 'text', 'json'", output))
 		}
 
-		os.Exit(returnCode)
+		os.Exit(exitCode(st))
 	},
 }
 
+func exitCode(st *Status) int {
+	c := 0
+	if st.Host != state.Running.String() {
+		c |= minikubeNotRunningStatusFlag
+	}
+	if st.APIServer != state.Running.String() || st.Kubelet != state.Running.String() {
+		c |= clusterNotRunningStatusFlag
+	}
+	if st.Kubeconfig != Configured {
+		c |= k8sNotRunningStatusFlag
+	}
+	return c
+}
+
+func status(api libmachine.API, name string) (*Status, error) {
+	st := &Status{}
+	hs, err := cluster.GetHostStatus(api, name)
+	if err != nil {
+		return st, errors.Wrap(err, "host")
+	}
+	st.Host = hs
+	if st.Host != state.Running.String() {
+		return st, nil
+	}
+
+	bs, err := getClusterBootstrapper(api, viper.GetString(cmdcfg.Bootstrapper))
+	if err != nil {
+		return st, errors.Wrap(err, "bootstrapper")
+	}
+
+	st.Kubelet, err = bs.GetKubeletStatus()
+	if err != nil {
+		glog.Warningf("kubelet err: %v", err)
+		st.Kubelet = state.Error.String()
+	}
+
+	ip, err := cluster.GetHostDriverIP(api, name)
+	if err != nil {
+		glog.Errorln("Error host driver ip status:", err)
+		st.APIServer = state.Error.String()
+		return st, err
+	}
+
+	port, err := kubeconfig.Port(name)
+	if err != nil {
+		glog.Warningf("unable to get port: %v", err)
+		port = constants.APIServerPort
+	}
+
+	st.APIServer, err = bs.GetAPIServerStatus(ip, port)
+	if err != nil {
+		glog.Errorln("Error apiserver status:", err)
+		st.APIServer = state.Error.String()
+	}
+
+	ks, err := kubeconfig.IsClusterInConfig(ip, name)
+	if err != nil {
+		glog.Errorln("Error kubeconfig status:", err)
+	}
+	if ks {
+		st.Kubeconfig = Configured
+	} else {
+		st.Kubeconfig = Misconfigured
+	}
+	return st, nil
+}
+
 func init() {
 	statusCmd.Flags().StringVarP(&statusFormat, "format", "f", defaultStatusFormat,
 		`Go template format string for the status output.  The format for Go templates can be found here: https://golang.org/pkg/text/template/
@@ -172,25 +180,24 @@ For the list accessible variables for the template, see the struct values here:
 		`minikube status --output OUTPUT. json, text`)
 }
 
-var printStatusText = func(status Status) {
+var printStatusText = func(st *Status) {
 	tmpl, err := template.New("status").Parse(statusFormat)
 	if err != nil {
 		exit.WithError("Error creating status template", err)
 	}
-	err = tmpl.Execute(os.Stdout, status)
+	err = tmpl.Execute(os.Stdout, st)
 	if err != nil {
 		exit.WithError("Error executing status template", err)
 	}
-	if status.Kubeconfig == KubeconfigStatus.Misconfigured {
+	if st.Kubeconfig == Misconfigured {
 		out.WarningT("Warning: Your kubectl is pointing to stale minikube-vm.\nTo fix the kubectl context, run `minikube update-context`")
 	}
 }
 
-var printStatusJSON = func(status Status) {
-
-	jsonString, err := json.Marshal(status)
+var printStatusJSON = func(st *Status) {
+	js, err := json.Marshal(st)
 	if err != nil {
 		exit.WithError("Error converting status to json", err)
 	}
-	out.String(string(jsonString))
+	out.String(string(js))
 }
diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index 323e5e7609..61f8b69697 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -18,6 +18,7 @@ package kubeadm
 
 import (
 	"os/exec"
+	"path"
 
 	"fmt"
 	"net"
@@ -77,9 +78,11 @@ func NewBootstrapper(api libmachine.API) (*Bootstrapper, error) {
 func (k *Bootstrapper) GetKubeletStatus() (string, error) {
 	rr, err := k.c.RunCmd(exec.Command("sudo", "systemctl", "is-active", "kubelet"))
 	if err != nil {
-		return "", errors.Wrapf(err, "getting kublet status. command: %q", rr.Command())
+		// Do not return now, as we still have parsing to do!
+		glog.Warningf("%s returned error: %v", rr.Command(), err)
 	}
 	s := strings.TrimSpace(rr.Stdout.String())
+	glog.Infof("kubelet is-active: %s", s)
 	switch s {
 	case "active":
 		return state.Running.String(), nil
@@ -93,6 +96,24 @@ func (k *Bootstrapper) GetKubeletStatus() (string, error) {
 
 // GetAPIServerStatus returns the api-server status
 func (k *Bootstrapper) GetAPIServerStatus(ip net.IP, apiserverPort int) (string, error) {
+	// sudo, in case hidepid is set
+	rr, err := k.c.RunCmd(exec.Command("sudo", "pgrep", "kube-apiserver"))
+	if err != nil {
+		return state.Stopped.String(), nil
+	}
+	pid := strings.TrimSpace(rr.Stdout.String())
+
+	// Check if apiserver is in an uninteruptible sleep
+	rr, err = k.c.RunCmd(exec.Command("sudo", "cut", "-d", " ", "-f3", path.Join("/proc", pid, "stat")))
+	if err != nil {
+		return state.Error.String(), err
+	}
+	st := strings.TrimSpace(rr.Stdout.String())
+	glog.Infof("apiserver process state: %s", st)
+	if st == "S" {
+		return state.Paused.String(), nil
+	}
+
 	return kverify.APIServerStatus(ip, apiserverPort)
 }
 

From 43259cc363bb93d7a4a2959fc175d4f7b550d0b2 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Wed, 22 Jan 2020 11:00:10 -0800
Subject: [PATCH 083/250] Reorganize buildroot.go by container runtime and
 remove unnecessary "sudo systemctl restart crio" to save time on 'minikube
 start'

---
 pkg/provision/buildroot.go | 72 +++++++++++++++++++-------------------
 1 file changed, 36 insertions(+), 36 deletions(-)

diff --git a/pkg/provision/buildroot.go b/pkg/provision/buildroot.go
index ef2f5a2358..b06f60ae92 100644
--- a/pkg/provision/buildroot.go
+++ b/pkg/provision/buildroot.go
@@ -170,10 +170,25 @@ WantedBy=multi-user.target
 		return nil, err
 	}
 
-	return &provision.DockerOptions{
+	dockerCfg := &provision.DockerOptions{
 		EngineOptions:     engineCfg.String(),
 		EngineOptionsPath: "/lib/systemd/system/docker.service",
-	}, nil
+	}
+
+	log.Info("Setting Docker configuration on the remote daemon...")
+
+	if _, err = p.SSHCommand(fmt.Sprintf("sudo mkdir -p %s && printf %%s \"%s\" | sudo tee %s", path.Dir(dockerCfg.EngineOptionsPath), dockerCfg.EngineOptions, dockerCfg.EngineOptionsPath)); err != nil {
+		return nil, err
+	}
+
+	if err := p.Service("docker", serviceaction.Enable); err != nil {
+		return nil, err
+	}
+
+	if err := p.Service("docker", serviceaction.Restart); err != nil {
+		return nil, err
+	}
+	return dockerCfg, nil
 }
 
 func rootFileSystemType(p *BuildrootProvisioner) (string, error) {
@@ -218,7 +233,7 @@ func (p *BuildrootProvisioner) Provision(swarmOptions swarm.Options, authOptions
 	}
 
 	log.Debugf("setting minikube options for container-runtime")
-	if err := setMinikubeOptions(p); err != nil {
+	if err := setContainerRuntimeOptions(p); err != nil {
 		log.Debugf("Error setting container-runtime options during provisioning %v", err)
 		return err
 	}
@@ -239,7 +254,24 @@ func setRemoteAuthOptions(p provision.Provisioner) auth.Options {
 	return authOptions
 }
 
-func setMinikubeOptions(p *BuildrootProvisioner) error {
+func setContainerRuntimeOptions(p *BuildrootProvisioner) error {
+	c, err := config.Load(p.Driver.GetMachineName())
+	if err != nil {
+		return errors.Wrap(err, "getting cluster config")
+	}
+
+	switch c.ContainerRuntime {
+	case "crio", "cri-o":
+		return p.setCrioOptions()
+	case "containerd":
+		return nil
+	default:
+		_, err := p.GenerateDockerOptions(engine.DefaultPort)
+		return err
+	}
+}
+
+func (p *BuildrootProvisioner) setCrioOptions() error {
 	// pass through --insecure-registry
 	var (
 		crioOptsTmpl = `
@@ -260,11 +292,6 @@ CRIO_MINIKUBE_OPTIONS='{{ range .EngineOptions.InsecureRegistry }}--insecure-reg
 		return err
 	}
 
-	// This is unlikely to cause issues unless the user has explicitly requested CRIO, so just log a warning.
-	if err := p.Service("crio", serviceaction.Restart); err != nil {
-		log.Warn("Unable to restart crio service. Error: %v", err)
-	}
-
 	return nil
 }
 
@@ -314,33 +341,6 @@ func configureAuth(p *BuildrootProvisioner) error {
 		return err
 	}
 
-	config, err := config.Load(p.Driver.GetMachineName())
-	if err != nil {
-		return errors.Wrap(err, "getting cluster config")
-	}
-
-	dockerCfg, err := p.GenerateDockerOptions(engine.DefaultPort)
-	if err != nil {
-		return errors.Wrap(err, "generating docker options")
-	}
-
-	log.Info("Setting Docker configuration on the remote daemon...")
-
-	if _, err = p.SSHCommand(fmt.Sprintf("sudo mkdir -p %s && printf %%s \"%s\" | sudo tee %s", path.Dir(dockerCfg.EngineOptionsPath), dockerCfg.EngineOptions, dockerCfg.EngineOptionsPath)); err != nil {
-		return err
-	}
-
-	if config.ContainerRuntime == "" {
-
-		if err := p.Service("docker", serviceaction.Enable); err != nil {
-			return err
-		}
-
-		if err := p.Service("docker", serviceaction.Restart); err != nil {
-			return err
-		}
-	}
-
 	return nil
 }
 

From 4be56d6d31e50a32765ee0d088318e21d588ebca Mon Sep 17 00:00:00 2001
From: tstromberg <tstromberg@google.com>
Date: Wed, 22 Jan 2020 12:14:05 -0800
Subject: [PATCH 084/250] Double dry-run time limit, add logging

---
 test/integration/functional_test.go | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go
index a9a171569b..b47684d40c 100644
--- a/test/integration/functional_test.go
+++ b/test/integration/functional_test.go
@@ -303,12 +303,12 @@ func validateDNS(ctx context.Context, t *testing.T, profile string) {
 
 // validateDryRun asserts that the dry-run mode quickly exits with the right code
 func validateDryRun(ctx context.Context, t *testing.T, profile string) {
-	// dry-run mode should always be able to finish quickly
-	mctx, cancel := context.WithTimeout(ctx, 2*time.Second)
+	// dry-run mode should always be able to finish quickly (<5s)
+	mctx, cancel := context.WithTimeout(ctx, 4*time.Second)
 	defer cancel()
 
 	// Too little memory!
-	startArgs := append([]string{"start", "-p", profile, "--dry-run", "--memory", "250MB"}, StartArgs()...)
+	startArgs := append([]string{"start", "-p", profile, "--dry-run", "--memory", "250MB", "--alsologtostderr", "-v=1"}, StartArgs()...)
 	c := exec.CommandContext(mctx, Target(), startArgs...)
 	rr, err := Run(t, c)
 
@@ -317,9 +317,9 @@ func validateDryRun(ctx context.Context, t *testing.T, profile string) {
 		t.Errorf("dry-run(250MB) exit code = %d, wanted = %d: %v", rr.ExitCode, wantCode, err)
 	}
 
-	dctx, cancel := context.WithTimeout(ctx, 2*time.Second)
+	dctx, cancel := context.WithTimeout(ctx, 4*time.Second)
 	defer cancel()
-	startArgs = append([]string{"start", "-p", profile, "--dry-run"}, StartArgs()...)
+	startArgs = append([]string{"start", "-p", profile, "--dry-run", "--alsologtostderr", "-v=1"}, StartArgs()...)
 	c = exec.CommandContext(dctx, Target(), startArgs...)
 	rr, err = Run(t, c)
 	if rr.ExitCode != 0 || err != nil {

From b115f181c430f379073da042354cf92f17ae0fd6 Mon Sep 17 00:00:00 2001
From: tstromberg <tstromberg@google.com>
Date: Wed, 22 Jan 2020 12:14:54 -0800
Subject: [PATCH 085/250] Increase time limit to 5s

---
 test/integration/functional_test.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go
index b47684d40c..df85356bb5 100644
--- a/test/integration/functional_test.go
+++ b/test/integration/functional_test.go
@@ -304,7 +304,7 @@ func validateDNS(ctx context.Context, t *testing.T, profile string) {
 // validateDryRun asserts that the dry-run mode quickly exits with the right code
 func validateDryRun(ctx context.Context, t *testing.T, profile string) {
 	// dry-run mode should always be able to finish quickly (<5s)
-	mctx, cancel := context.WithTimeout(ctx, 4*time.Second)
+	mctx, cancel := context.WithTimeout(ctx, 5*time.Second)
 	defer cancel()
 
 	// Too little memory!
@@ -317,7 +317,7 @@ func validateDryRun(ctx context.Context, t *testing.T, profile string) {
 		t.Errorf("dry-run(250MB) exit code = %d, wanted = %d: %v", rr.ExitCode, wantCode, err)
 	}
 
-	dctx, cancel := context.WithTimeout(ctx, 4*time.Second)
+	dctx, cancel := context.WithTimeout(ctx, 5*time.Second)
 	defer cancel()
 	startArgs = append([]string{"start", "-p", profile, "--dry-run", "--alsologtostderr", "-v=1"}, StartArgs()...)
 	c = exec.CommandContext(dctx, Target(), startArgs...)

From 31f22f9e623fee006fe6a1e360235a52246b418d Mon Sep 17 00:00:00 2001
From: tstromberg <tstromberg@google.com>
Date: Wed, 22 Jan 2020 12:38:00 -0800
Subject: [PATCH 086/250] Version bump before we update the ISO

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index b3523ed011..e3c9831d01 100755
--- a/Makefile
+++ b/Makefile
@@ -15,7 +15,7 @@
 # Bump these on release - and please check ISO_VERSION for correctness.
 VERSION_MAJOR ?= 1
 VERSION_MINOR ?= 7
-VERSION_BUILD ?= 0-beta.0
+VERSION_BUILD ?= 0-beta.1
 RAW_VERSION=$(VERSION_MAJOR).$(VERSION_MINOR).${VERSION_BUILD}
 VERSION ?= v$(RAW_VERSION)
 

From 180b23f2c8368f7138873d20eee3980f26b4463d Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Tue, 21 Jan 2020 12:46:44 -0800
Subject: [PATCH 087/250] Save addons in subdirectories, separated by namespace

This way we can pass the correct --namespace flag into 'kubectl apply'
such that it works with '--prune' for kubectl >=v.17.0
---
 pkg/addons/reconcile.go       |  32 ++++++--
 pkg/minikube/assets/addons.go | 140 +++++++++++++++++-----------------
 2 files changed, 96 insertions(+), 76 deletions(-)

diff --git a/pkg/addons/reconcile.go b/pkg/addons/reconcile.go
index f774cb9393..e4096d71ff 100644
--- a/pkg/addons/reconcile.go
+++ b/pkg/addons/reconcile.go
@@ -17,6 +17,8 @@ limitations under the License.
 package addons
 
 import (
+	"fmt"
+	"io/ioutil"
 	"os"
 	"os/exec"
 	"path"
@@ -24,10 +26,12 @@ import (
 	"github.com/blang/semver"
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/minikube/assets"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil"
 	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
+	"k8s.io/minikube/pkg/minikube/vmpath"
 )
 
 var (
@@ -58,18 +62,32 @@ var kubectlPruneWhitelist = []string{
 // reconcile runs kubectl apply -f on the addons directory
 // to reconcile addons state in all running profiles
 func reconcile(cmd command.Runner, profile string) error {
-	c, err := kubectlCommand(profile)
+	// Walk through all subdirs in /etc/kubernetes/addons and reconcile addons
+	dirs, err := ioutil.ReadDir(vmpath.GuestAddonsDir)
 	if err != nil {
 		return err
 	}
-	if _, err := cmd.RunCmd(c); err != nil {
-		glog.Warningf("reconciling addons failed: %v", err)
-		return err
+
+	for _, d := range dirs {
+		if !d.IsDir() {
+			continue
+		}
+		namespace := d.Name()
+		glog.Infof("Reconciling addons in namespace %s", namespace)
+		c, err := kubectlCommand(profile, namespace)
+		if err != nil {
+			return err
+		}
+		if _, err := cmd.RunCmd(c); err != nil {
+			glog.Warningf("reconciling addons failed: %v", err)
+			return err
+		}
 	}
+
 	return nil
 }
 
-func kubectlCommand(profile string) (*exec.Cmd, error) {
+func kubectlCommand(profile, namespace string) (*exec.Cmd, error) {
 	v, err := k8sVersion(profile)
 	if err != nil {
 		return nil, err
@@ -78,7 +96,7 @@ func kubectlCommand(profile string) (*exec.Cmd, error) {
 
 	// prune will delete any existing objects with the label specified by "-l" which don't appear in /etc/kubernetes/addons
 	// this is how we delete disabled addons
-	args := []string{"KUBECONFIG=/var/lib/minikube/kubeconfig", kubectlBinary, "apply", "-f", "/etc/kubernetes/addons", "-l", "kubernetes.io/cluster-service!=true,addonmanager.kubernetes.io/mode=Reconcile", "--prune=true"}
+	args := []string{"KUBECONFIG=/var/lib/minikube/kubeconfig", kubectlBinary, "apply", "-f", assets.TargetDirForAddon(namespace), "-l", "kubernetes.io/cluster-service!=true,addonmanager.kubernetes.io/mode=Reconcile", "--prune=true"}
 	for _, k := range kubectlPruneWhitelist {
 		args = append(args, []string{"--prune-whitelist", k}...)
 	}
@@ -89,7 +107,7 @@ func kubectlCommand(profile string) (*exec.Cmd, error) {
 		return nil, errors.Wrap(err, "appending namespace flag")
 	}
 	if ok {
-		args = append(args, "--namespace=kube-system")
+		args = append(args, fmt.Sprintf("--namespace=%s", namespace))
 	}
 
 	cmd := exec.Command("sudo", args...)
diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go
index af5a01f13a..98b5d2ccc2 100644
--- a/pkg/minikube/assets/addons.go
+++ b/pkg/minikube/assets/addons.go
@@ -1,12 +1,9 @@
 /*
 Copyright 2016 The Kubernetes Authors All rights reserved.
-
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
-
     http://www.apache.org/licenses/LICENSE-2.0
-
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -40,7 +37,7 @@ type Addon struct {
 }
 
 // NewAddon creates a new Addon
-func NewAddon(assets []*BinAsset, enabled bool, addonName string) *Addon {
+func NewAddon(assets []*BinAsset, enabled bool, addonName, namespace string) *Addon {
 	a := &Addon{
 		Assets:    assets,
 		enabled:   enabled,
@@ -71,235 +68,235 @@ var Addons = map[string]*Addon{
 	"addon-manager": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/addon-manager.yaml.tmpl",
-			vmpath.GuestManifestsDir,
+			TargetDirForAddon("kube-system"),
 			"addon-manager.yaml.tmpl",
 			"0640",
 			true),
-	}, false, "addon-manager"),
+	}, false, "addon-manager", "kube-system"),
 	"dashboard": NewAddon([]*BinAsset{
-		MustBinAsset("deploy/addons/dashboard/dashboard-clusterrole.yaml", vmpath.GuestAddonsDir, "dashboard-clusterrole.yaml", "0640", false),
-		MustBinAsset("deploy/addons/dashboard/dashboard-clusterrolebinding.yaml", vmpath.GuestAddonsDir, "dashboard-clusterrolebinding.yaml", "0640", false),
-		MustBinAsset("deploy/addons/dashboard/dashboard-configmap.yaml", vmpath.GuestAddonsDir, "dashboard-configmap.yaml", "0640", false),
-		MustBinAsset("deploy/addons/dashboard/dashboard-dp.yaml", vmpath.GuestAddonsDir, "dashboard-dp.yaml", "0640", false),
-		MustBinAsset("deploy/addons/dashboard/dashboard-ns.yaml", vmpath.GuestAddonsDir, "dashboard-ns.yaml", "0640", false),
-		MustBinAsset("deploy/addons/dashboard/dashboard-role.yaml", vmpath.GuestAddonsDir, "dashboard-role.yaml", "0640", false),
-		MustBinAsset("deploy/addons/dashboard/dashboard-rolebinding.yaml", vmpath.GuestAddonsDir, "dashboard-rolebinding.yaml", "0640", false),
-		MustBinAsset("deploy/addons/dashboard/dashboard-sa.yaml", vmpath.GuestAddonsDir, "dashboard-sa.yaml", "0640", false),
-		MustBinAsset("deploy/addons/dashboard/dashboard-secret.yaml", vmpath.GuestAddonsDir, "dashboard-secret.yaml", "0640", false),
-		MustBinAsset("deploy/addons/dashboard/dashboard-svc.yaml", vmpath.GuestAddonsDir, "dashboard-svc.yaml", "0640", false),
-	}, false, "dashboard"),
+		MustBinAsset("deploy/addons/dashboard/dashboard-clusterrole.yaml", TargetDirForAddon("kubernetes-dashboard"), "dashboard-clusterrole.yaml", "0640", false),
+		MustBinAsset("deploy/addons/dashboard/dashboard-clusterrolebinding.yaml", TargetDirForAddon("kubernetes-dashboard"), "dashboard-clusterrolebinding.yaml", "0640", false),
+		MustBinAsset("deploy/addons/dashboard/dashboard-configmap.yaml", TargetDirForAddon("kubernetes-dashboard"), "dashboard-configmap.yaml", "0640", false),
+		MustBinAsset("deploy/addons/dashboard/dashboard-dp.yaml", TargetDirForAddon("kubernetes-dashboard"), "dashboard-dp.yaml", "0640", false),
+		MustBinAsset("deploy/addons/dashboard/dashboard-ns.yaml", TargetDirForAddon("kubernetes-dashboard"), "dashboard-ns.yaml", "0640", false),
+		MustBinAsset("deploy/addons/dashboard/dashboard-role.yaml", TargetDirForAddon("kubernetes-dashboard"), "dashboard-role.yaml", "0640", false),
+		MustBinAsset("deploy/addons/dashboard/dashboard-rolebinding.yaml", TargetDirForAddon("kubernetes-dashboard"), "dashboard-rolebinding.yaml", "0640", false),
+		MustBinAsset("deploy/addons/dashboard/dashboard-sa.yaml", TargetDirForAddon("kubernetes-dashboard"), "dashboard-sa.yaml", "0640", false),
+		MustBinAsset("deploy/addons/dashboard/dashboard-secret.yaml", TargetDirForAddon("kubernetes-dashboard"), "dashboard-secret.yaml", "0640", false),
+		MustBinAsset("deploy/addons/dashboard/dashboard-svc.yaml", TargetDirForAddon("kubernetes-dashboard"), "dashboard-svc.yaml", "0640", false),
+	}, false, "dashboard", "kubernetes-dashboard"),
 	"default-storageclass": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/storageclass/storageclass.yaml.tmpl",
-			vmpath.GuestAddonsDir,
+			TargetDirForAddon("kube-system"),
 			"storageclass.yaml",
 			"0640",
 			false),
-	}, true, "default-storageclass"),
+	}, true, "default-storageclass", "kube-system"),
 	"storage-provisioner": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/storage-provisioner/storage-provisioner.yaml.tmpl",
-			vmpath.GuestAddonsDir,
+			TargetDirForAddon("kube-system"),
 			"storage-provisioner.yaml",
 			"0640",
 			true),
-	}, true, "storage-provisioner"),
+	}, true, "storage-provisioner", "kube-system"),
 	"storage-provisioner-gluster": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/storage-provisioner-gluster/storage-gluster-ns.yaml.tmpl",
-			vmpath.GuestAddonsDir,
+			TargetDirForAddon("kube-system"),
 			"storage-gluster-ns.yaml",
 			"0640",
 			false),
 		MustBinAsset(
 			"deploy/addons/storage-provisioner-gluster/glusterfs-daemonset.yaml.tmpl",
-			vmpath.GuestAddonsDir,
+			TargetDirForAddon("kube-system"),
 			"glusterfs-daemonset.yaml",
 			"0640",
 			false),
 		MustBinAsset(
 			"deploy/addons/storage-provisioner-gluster/heketi-deployment.yaml.tmpl",
-			vmpath.GuestAddonsDir,
+			TargetDirForAddon("kube-system"),
 			"heketi-deployment.yaml",
 			"0640",
 			false),
 		MustBinAsset(
 			"deploy/addons/storage-provisioner-gluster/storage-provisioner-glusterfile.yaml.tmpl",
-			vmpath.GuestAddonsDir,
+			TargetDirForAddon("kube-system"),
 			"storage-privisioner-glusterfile.yaml",
 			"0640",
 			false),
-	}, false, "storage-provisioner-gluster"),
+	}, false, "storage-provisioner-gluster", "kube-system"),
 	"efk": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/efk/elasticsearch-rc.yaml.tmpl",
-			vmpath.GuestAddonsDir,
+			TargetDirForAddon("kube-system"),
 			"elasticsearch-rc.yaml",
 			"0640",
 			true),
 		MustBinAsset(
 			"deploy/addons/efk/elasticsearch-svc.yaml.tmpl",
-			vmpath.GuestAddonsDir,
+			TargetDirForAddon("kube-system"),
 			"elasticsearch-svc.yaml",
 			"0640",
 			false),
 		MustBinAsset(
 			"deploy/addons/efk/fluentd-es-rc.yaml.tmpl",
-			vmpath.GuestAddonsDir,
+			TargetDirForAddon("kube-system"),
 			"fluentd-es-rc.yaml",
 			"0640",
 			true),
 		MustBinAsset(
 			"deploy/addons/efk/fluentd-es-configmap.yaml.tmpl",
-			vmpath.GuestAddonsDir,
+			TargetDirForAddon("kube-system"),
 			"fluentd-es-configmap.yaml",
 			"0640",
 			false),
 		MustBinAsset(
 			"deploy/addons/efk/kibana-rc.yaml.tmpl",
-			vmpath.GuestAddonsDir,
+			TargetDirForAddon("kube-system"),
 			"kibana-rc.yaml",
 			"0640",
 			false),
 		MustBinAsset(
 			"deploy/addons/efk/kibana-svc.yaml.tmpl",
-			vmpath.GuestAddonsDir,
+			TargetDirForAddon("kube-system"),
 			"kibana-svc.yaml",
 			"0640",
 			false),
-	}, false, "efk"),
+	}, false, "efk", "kube-system"),
 	"ingress": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/ingress/ingress-configmap.yaml.tmpl",
-			vmpath.GuestAddonsDir,
+			TargetDirForAddon("kube-system"),
 			"ingress-configmap.yaml",
 			"0640",
 			false),
 		MustBinAsset(
 			"deploy/addons/ingress/ingress-rbac.yaml.tmpl",
-			vmpath.GuestAddonsDir,
+			TargetDirForAddon("kube-system"),
 			"ingress-rbac.yaml",
 			"0640",
 			false),
 		MustBinAsset(
 			"deploy/addons/ingress/ingress-dp.yaml.tmpl",
-			vmpath.GuestAddonsDir,
+			TargetDirForAddon("kube-system"),
 			"ingress-dp.yaml",
 			"0640",
 			true),
-	}, false, "ingress"),
+	}, false, "ingress", "kube-system"),
 	"istio-provisioner": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/istio-provisioner/istio-operator.yaml.tmpl",
-			vmpath.GuestAddonsDir,
+			TargetDirForAddon("istio-operator"),
 			"istio-operator.yaml",
 			"0640",
 			true),
-	}, false, "istio-provisioner"),
+	}, false, "istio-provisioner", "istio-operator"),
 	"istio": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/istio/istio-default-profile.yaml.tmpl",
-			vmpath.GuestAddonsDir,
+			TargetDirForAddon("istio-operator"),
 			"istio-default-profile.yaml",
 			"0640",
 			false),
-	}, false, "istio"),
+	}, false, "istio", "istio-operator"),
 	"metrics-server": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/metrics-server/metrics-apiservice.yaml.tmpl",
-			vmpath.GuestAddonsDir,
+			TargetDirForAddon("kube-system"),
 			"metrics-apiservice.yaml",
 			"0640",
 			false),
 		MustBinAsset(
 			"deploy/addons/metrics-server/metrics-server-deployment.yaml.tmpl",
-			vmpath.GuestAddonsDir,
+			TargetDirForAddon("kube-system"),
 			"metrics-server-deployment.yaml",
 			"0640",
 			true),
 		MustBinAsset(
 			"deploy/addons/metrics-server/metrics-server-service.yaml.tmpl",
-			vmpath.GuestAddonsDir,
+			TargetDirForAddon("kube-system"),
 			"metrics-server-service.yaml",
 			"0640",
 			false),
-	}, false, "metrics-server"),
+	}, false, "metrics-server", "kube-system"),
 	"registry": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/registry/registry-rc.yaml.tmpl",
-			vmpath.GuestAddonsDir,
+			TargetDirForAddon("kube-system"),
 			"registry-rc.yaml",
 			"0640",
 			false),
 		MustBinAsset(
 			"deploy/addons/registry/registry-svc.yaml.tmpl",
-			vmpath.GuestAddonsDir,
+			TargetDirForAddon("kube-system"),
 			"registry-svc.yaml",
 			"0640",
 			false),
 		MustBinAsset(
 			"deploy/addons/registry/registry-proxy.yaml.tmpl",
-			vmpath.GuestAddonsDir,
+			TargetDirForAddon("kube-system"),
 			"registry-proxy.yaml",
 			"0640",
 			false),
-	}, false, "registry"),
+	}, false, "registry", "kube-system"),
 	"registry-creds": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/registry-creds/registry-creds-rc.yaml.tmpl",
-			vmpath.GuestAddonsDir,
+			TargetDirForAddon("kube-system"),
 			"registry-creds-rc.yaml",
 			"0640",
 			false),
-	}, false, "registry-creds"),
+	}, false, "registry-creds", "kube-system"),
 	"freshpod": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/freshpod/freshpod-rc.yaml.tmpl",
-			vmpath.GuestAddonsDir,
+			TargetDirForAddon("kube-system"),
 			"freshpod-rc.yaml",
 			"0640",
 			true),
-	}, false, "freshpod"),
+	}, false, "freshpod", "kube-system"),
 	"nvidia-driver-installer": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/gpu/nvidia-driver-installer.yaml.tmpl",
-			vmpath.GuestAddonsDir,
+			TargetDirForAddon("kube-system"),
 			"nvidia-driver-installer.yaml",
 			"0640",
 			true),
-	}, false, "nvidia-driver-installer"),
+	}, false, "nvidia-driver-installer", "kube-system"),
 	"nvidia-gpu-device-plugin": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/gpu/nvidia-gpu-device-plugin.yaml.tmpl",
-			vmpath.GuestAddonsDir,
+			TargetDirForAddon("kube-system"),
 			"nvidia-gpu-device-plugin.yaml",
 			"0640",
 			true),
-	}, false, "nvidia-gpu-device-plugin"),
+	}, false, "nvidia-gpu-device-plugin", "kube-system"),
 	"logviewer": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/logviewer/logviewer-dp-and-svc.yaml.tmpl",
-			vmpath.GuestAddonsDir,
+			TargetDirForAddon("kube-system"),
 			"logviewer-dp-and-svc.yaml",
 			"0640",
 			false),
 		MustBinAsset(
 			"deploy/addons/logviewer/logviewer-rbac.yaml.tmpl",
-			vmpath.GuestAddonsDir,
+			TargetDirForAddon("kube-system"),
 			"logviewer-rbac.yaml",
 			"0640",
 			false),
-	}, false, "logviewer"),
+	}, false, "logviewer", "kube-system"),
 	"gvisor": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/gvisor/gvisor-pod.yaml.tmpl",
-			vmpath.GuestAddonsDir,
+			TargetDirForAddon("kube-system"),
 			"gvisor-pod.yaml",
 			"0640",
 			true),
 		MustBinAsset(
 			"deploy/addons/gvisor/gvisor-runtimeclass.yaml",
-			vmpath.GuestAddonsDir,
+			TargetDirForAddon("kube-system"),
 			"gvisor-runtimeclass.yaml",
 			"0640",
 			false),
@@ -309,35 +306,40 @@ var Addons = map[string]*Addon{
 			constants.GvisorConfigTomlTargetName,
 			"0640",
 			true),
-	}, false, "gvisor"),
+	}, false, "gvisor", "kube-system"),
 	"helm-tiller": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/helm-tiller/helm-tiller-dp.tmpl",
-			vmpath.GuestAddonsDir,
+			TargetDirForAddon("kube-system"),
 			"helm-tiller-dp.yaml",
 			"0640",
 			true),
 		MustBinAsset(
 			"deploy/addons/helm-tiller/helm-tiller-rbac.tmpl",
-			vmpath.GuestAddonsDir,
+			TargetDirForAddon("kube-system"),
 			"helm-tiller-rbac.yaml",
 			"0640",
 			true),
 		MustBinAsset(
 			"deploy/addons/helm-tiller/helm-tiller-svc.tmpl",
-			vmpath.GuestAddonsDir,
+			TargetDirForAddon("kube-system"),
 			"helm-tiller-svc.yaml",
 			"0640",
 			true),
-	}, false, "helm-tiller"),
+	}, false, "helm-tiller", "kube-system"),
 	"ingress-dns": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/ingress-dns/ingress-dns-pod.yaml",
-			vmpath.GuestAddonsDir,
+			TargetDirForAddon("kube-system"),
 			"ingress-dns-pod.yaml",
 			"0640",
 			false),
-	}, false, "ingress-dns"),
+	}, false, "ingress-dns", "kube-system"),
+}
+
+// TargetDirForAddon appends the namespace as a directory to /etc/kubernetes/addons
+func TargetDirForAddon(namespace string) string {
+	return filepath.Join(vmpath.GuestAddonsDir, namespace)
 }
 
 // AddMinikubeDirAssets adds all addons and files to the list

From 50cd0dbfba630ddcb9f12c07817dd0a7353c60d8 Mon Sep 17 00:00:00 2001
From: tstromberg <tstromberg@google.com>
Date: Wed, 22 Jan 2020 13:55:01 -0800
Subject: [PATCH 088/250] First stab at freezer checks

---
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 25 ++++++++++++++++----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index 3504477388..4582c6dbfa 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -106,14 +106,29 @@ func (k *Bootstrapper) GetAPIServerStatus(ip net.IP, apiserverPort int) (string,
 	}
 	pid := strings.TrimSpace(rr.Stdout.String())
 
-	// Check if apiserver is in an uninteruptible sleep
-	rr, err = k.c.RunCmd(exec.Command("sudo", "cut", "-d", " ", "-f3", path.Join("/proc", pid, "stat")))
+	// Get the freezer cgroup entry for this pid
+	rr, err = k.c.RunCmd(exec.Command("sudo", "egrep", "^[-09]+:freezer:", path.Join("/proc", pid, "cgroup")))
 	if err != nil {
 		return state.Error.String(), err
 	}
-	st := strings.TrimSpace(rr.Stdout.String())
-	glog.Infof("apiserver process state: %s", st)
-	if st == "S" {
+	freezer := strings.TrimSpace(rr.Stdout.String())
+	glog.Infof("apiserver freezer: %q", freezer)
+
+	fparts := strings.Split(freezer, ":")
+	if len(fparts) == 3 {
+		glog.Warningf("unable to parse freezer: %s", freezer)
+		return kverify.APIServerStatus(ip, apiserverPort)
+	}
+
+	rr, err = k.c.RunCmd(exec.Command("sudo", "cat", path.Join("/sys/fs/cgroup/freezer", fparts[2], "freezer.state")))
+	if err != nil {
+		glog.Errorf("unable to get freezer state: %s", rr.Stderr)
+		return kverify.APIServerStatus(ip, apiserverPort)
+	}
+
+	fs := strings.TrimSpace(rr.Stdout.String())
+	glog.Infof("freezer state: %q", fs)
+	if fs == "FREEZING" || fs == "FROZEN" {
 		return state.Paused.String(), nil
 	}
 

From c83ee09417865d1ccf970a39431957551dace70c Mon Sep 17 00:00:00 2001
From: tstromberg <tstromberg@google.com>
Date: Wed, 22 Jan 2020 14:10:53 -0800
Subject: [PATCH 089/250] Update Kubernetes libraries to v1.17.2

---
 go.mod                   |  84 +++--
 go.sum                   | 657 +++++++++++++++++++++++++--------------
 pkg/drivers/none/none.go |   2 +-
 3 files changed, 463 insertions(+), 280 deletions(-)

diff --git a/go.mod b/go.mod
index 0cd9aca2bf..30367be52c 100644
--- a/go.mod
+++ b/go.mod
@@ -2,10 +2,6 @@ module k8s.io/minikube
 
 go 1.13
 
-require github.com/google/go-containerregistry v0.0.0-20180731221751-697ee0b3d46e
-
-require k8s.io/kubernetes v1.15.2
-
 require (
 	github.com/Parallels/docker-machine-parallels v1.3.0
 	github.com/Sirupsen/logrus v0.0.0-20170822132746-89742aefa4b2 // indirect
@@ -17,7 +13,7 @@ require (
 	github.com/cloudfoundry-attic/jibber_jabber v0.0.0-20151120183258-bcc4c8345a21
 	github.com/cloudfoundry/jibber_jabber v0.0.0-20151120183258-bcc4c8345a21 // indirect
 	github.com/docker/docker v1.13.1
-	github.com/docker/go-units v0.3.3
+	github.com/docker/go-units v0.4.0
 	github.com/docker/machine v0.7.1-0.20190718054102-a555e4f7a8f5 // version is 0.7.1 to pin to a555e4f7a8f5
 	github.com/elazarl/goproxy v0.0.0-20190421051319-9d40249d3c2f
 	github.com/elazarl/goproxy/ext v0.0.0-20190421051319-9d40249d3c2f // indirect
@@ -26,6 +22,7 @@ require (
 	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
 	github.com/google/go-cmp v0.3.0
+	github.com/google/go-containerregistry v0.0.0-20180731221751-697ee0b3d46e
 	github.com/googleapis/gnostic v0.3.0 // indirect
 	github.com/gorilla/mux v1.7.3 // indirect
 	github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce // indirect
@@ -34,6 +31,7 @@ require (
 	github.com/hashicorp/go-retryablehttp v0.5.4
 	github.com/hooklift/assert v0.0.0-20170704181755-9d1defd6d214 // indirect
 	github.com/hooklift/iso9660 v0.0.0-20170318115843-1cf07e5970d8
+	github.com/imdario/mergo v0.3.8 // indirect
 	github.com/intel-go/cpuid v0.0.0-20181003105527-1a4a6f06a1c6 // indirect
 	github.com/jimmidyson/go-download v0.0.0-20161028105827-7f9a90c8c95b
 	github.com/johanneswuerbach/nfsexports v0.0.0-20181204082207-1aa528dcb345
@@ -48,10 +46,10 @@ require (
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/libvirt/libvirt-go v3.4.0+incompatible
 	github.com/machine-drivers/docker-machine-driver-vmware v0.1.1
-	github.com/mattn/go-isatty v0.0.8
+	github.com/mattn/go-isatty v0.0.9
 	github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936
 	github.com/moby/hyperkit v0.0.0-20171020124204-a12cd7250bcd
-	github.com/olekukonko/tablewriter v0.0.0-20160923125401-bdcc175572fd
+	github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5
 	github.com/otiai10/copy v1.0.2
 	github.com/pborman/uuid v1.2.0
 	github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2
@@ -61,30 +59,26 @@ require (
 	github.com/pmezard/go-difflib v1.0.0
 	github.com/samalba/dockerclient v0.0.0-20160414174713-91d7393ff859 // indirect
 	github.com/shirou/gopsutil v2.18.12+incompatible
-	github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 // indirect
-	github.com/sirupsen/logrus v1.4.1 // indirect
 	github.com/spf13/cobra v0.0.5
-	github.com/spf13/pflag v1.0.3
+	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.3.2
 	github.com/xeipuuv/gojsonpointer v0.0.0-20151027082146-e0fe6f683076 // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20150808065054-e02fc20de94c // indirect
 	github.com/xeipuuv/gojsonschema v0.0.0-20160623135812-c539bca196be
 	github.com/zchee/go-vmnet v0.0.0-20161021174912-97ebf9174097
-	golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d
-	golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4
-	golang.org/x/sync v0.0.0-20190423024810-112230192c58
-	golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb
+	golang.org/x/build v0.0.0-20190927031335-2835ba2e683f
+	golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586
+	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
+	golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456
 	golang.org/x/text v0.3.2
-	gopkg.in/airbrake/gobrake.v2 v2.0.9 // indirect
-	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
-	gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 // indirect
 	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
 	gotest.tools v2.2.0+incompatible
-	k8s.io/api v0.0.0
-	k8s.io/apimachinery v0.0.0
+	k8s.io/api v0.17.2
+	k8s.io/apimachinery v0.17.2
 	k8s.io/client-go v11.0.0+incompatible
-	k8s.io/klog v0.3.3 // indirect
-	k8s.io/kubectl v0.0.0-00010101000000-000000000000
+	k8s.io/kubectl v0.0.0
+	k8s.io/kubernetes v1.17.2
+	k8s.io/utils v0.0.0-20200122174043-1e243dd1a584 // indirect
 	sigs.k8s.io/sig-storage-lib-external-provisioner v4.0.0+incompatible
 )
 
@@ -92,28 +86,28 @@ replace (
 	git.apache.org/thrift.git => github.com/apache/thrift v0.0.0-20180902110319-2566ecd5d999
 	github.com/docker/machine => github.com/machine-drivers/machine v0.7.1-0.20191109154235-b39d5b50de51
 	github.com/hashicorp/go-getter => github.com/afbjorklund/go-getter v1.4.1-0.20190910175809-eb9f6c26742c
-	k8s.io/api => k8s.io/kubernetes/staging/src/k8s.io/api v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/apiextensions-apiserver => k8s.io/kubernetes/staging/src/k8s.io/apiextensions-apiserver v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/apimachinery => k8s.io/kubernetes/staging/src/k8s.io/apimachinery v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/apiserver => k8s.io/kubernetes/staging/src/k8s.io/apiserver v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/cli-runtime => k8s.io/kubernetes/staging/src/k8s.io/cli-runtime v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/client-go => k8s.io/kubernetes/staging/src/k8s.io/client-go v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/cloud-provider => k8s.io/kubernetes/staging/src/k8s.io/cloud-provider v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/cluster-bootstrap => k8s.io/kubernetes/staging/src/k8s.io/cluster-bootstrap v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/code-generator => k8s.io/kubernetes/staging/src/k8s.io/code-generator v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/component-base => k8s.io/kubernetes/staging/src/k8s.io/component-base v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/cri-api => k8s.io/kubernetes/staging/src/k8s.io/cri-api v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/csi-translation-lib => k8s.io/kubernetes/staging/src/k8s.io/csi-translation-lib v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/kube-aggregator => k8s.io/kubernetes/staging/src/k8s.io/kube-aggregator v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/kube-controller-manager => k8s.io/kubernetes/staging/src/k8s.io/kube-controller-manager v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/kube-proxy => k8s.io/kubernetes/staging/src/k8s.io/kube-proxy v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/kube-scheduler => k8s.io/kubernetes/staging/src/k8s.io/kube-scheduler v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/kubectl => k8s.io/kubernetes/staging/src/k8s.io/kubectl v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/kubelet => k8s.io/kubernetes/staging/src/k8s.io/kubelet v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/legacy-cloud-providers => k8s.io/kubernetes/staging/src/k8s.io/legacy-cloud-providers v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/metrics => k8s.io/kubernetes/staging/src/k8s.io/metrics v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/node-api => k8s.io/kubernetes/staging/src/k8s.io/node-api v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/sample-apiserver => k8s.io/kubernetes/staging/src/k8s.io/sample-apiserver v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/sample-cli-plugin => k8s.io/kubernetes/staging/src/k8s.io/sample-cli-plugin v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/sample-controller => k8s.io/kubernetes/staging/src/k8s.io/sample-controller v0.0.0-20190623232353-8c3b7d7679cc
+	k8s.io/api => k8s.io/api v0.17.2
+	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.17.2
+	k8s.io/apimachinery => k8s.io/apimachinery v0.17.3-beta.0
+	k8s.io/apiserver => k8s.io/apiserver v0.17.2
+	k8s.io/cli-runtime => k8s.io/cli-runtime v0.17.2
+	k8s.io/client-go => k8s.io/client-go v0.17.2
+	k8s.io/cloud-provider => k8s.io/cloud-provider v0.17.2
+	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.17.2
+	k8s.io/code-generator => k8s.io/code-generator v0.17.3-beta.0
+	k8s.io/component-base => k8s.io/component-base v0.17.2
+	k8s.io/cri-api => k8s.io/cri-api v0.17.3-beta.0
+	k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.17.2
+	k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.17.2
+	k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.17.2
+	k8s.io/kube-proxy => k8s.io/kube-proxy v0.17.2
+	k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.17.2
+	k8s.io/kubectl => k8s.io/kubectl v0.17.2
+	k8s.io/kubelet => k8s.io/kubelet v0.17.2
+	k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.17.2
+	k8s.io/metrics => k8s.io/metrics v0.17.2
+	k8s.io/node-api => k8s.io/node-api v0.17.2
+	k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.17.2
+	k8s.io/sample-cli-plugin => k8s.io/sample-cli-plugin v0.17.2
+	k8s.io/sample-controller => k8s.io/sample-controller v0.17.2
 )
diff --git a/go.sum b/go.sum
index 2c4f7ca288..d4cc8b5767 100644
--- a/go.sum
+++ b/go.sum
@@ -1,6 +1,5 @@
 bitbucket.org/bertimus9/systemstat v0.0.0-20180207000608-0eeff89b0690/go.mod h1:Ulb78X89vxKYgdL24HMTiXYHlyHEvruOj1ZPlqeNEZM=
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
 cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
@@ -9,109 +8,130 @@ cloud.google.com/go v0.45.1 h1:lRi0CHyU+ytlvylOlFKKq0af6JncuyoRh1J+QJBqQx0=
 cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
-dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU=
-dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU=
-dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4=
-dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU=
-github.com/Azure/azure-sdk-for-go v21.4.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
+github.com/Azure/azure-sdk-for-go v35.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
-github.com/Azure/go-autorest v11.1.2+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
-github.com/BurntSushi/toml v0.3.0/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
+github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0=
+github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA=
+github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
+github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
+github.com/Azure/go-autorest/autorest/to v0.2.0/go.mod h1:GunWKJp1AEqgMaGLV+iocmRAJWqST1wQYhyyjXJ3SJc=
+github.com/Azure/go-autorest/autorest/validation v0.1.0/go.mod h1:Ha3z/SqBeaalWQvokg3NZAlQTalVMtOIAs1aGK7G6u8=
+github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
+github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/GoogleCloudPlatform/k8s-cloud-provider v0.0.0-20181220005116-f8e995905100/go.mod h1:iroGtC8B3tQiqtds1l+mgk/BBOrxbqjH+eUfFQYRc14=
+github.com/GoogleCloudPlatform/k8s-cloud-provider v0.0.0-20190822182118-27a4ced34534/go.mod h1:iroGtC8B3tQiqtds1l+mgk/BBOrxbqjH+eUfFQYRc14=
 github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab/go.mod h1:3VYc5hodBMJ5+l/7J4xAyMeuM2PNuepvHlGs8yilUCA=
 github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd h1:sjQovDkwrZp8u+gxLtPgKGjk5hCxuy2hrRejBTA9xFU=
 github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E=
 github.com/Microsoft/go-winio v0.4.11 h1:zoIOcVf0xPN1tnMVbTtEdI+P8OofVk3NObnwOQ6nK2Q=
 github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
-github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=
+github.com/Microsoft/hcsshim v0.0.0-20190417211021-672e52e9209d/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
+github.com/OpenPeeDeeP/depguard v1.0.0/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o=
+github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmUx/1V+TNhjQvM=
 github.com/Parallels/docker-machine-parallels v1.3.0 h1:RG1fyf3v1GwXMCeHRiZkB4tL9phFZEv6ixcvRZ1raN8=
 github.com/Parallels/docker-machine-parallels v1.3.0/go.mod h1:HCOMm3Hulq/xuEVQMyZOuQlA+dSZpFY5kdCTZWjMVis=
+github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/Rican7/retry v0.1.0/go.mod h1:FgOROf8P5bebcC1DS0PdOQiqGUridaZvikzUmkFW6gg=
 github.com/Sirupsen/logrus v0.0.0-20170822132746-89742aefa4b2 h1:k1A7eIeUk6rnX2yuagwljW/pDezkK8oSpvPumT9zdZY=
 github.com/Sirupsen/logrus v0.0.0-20170822132746-89742aefa4b2/go.mod h1:rmk17hk6i8ZSAJkSDa7nOxamrG+SP4P0mm+DAvExv4U=
+github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM=
 github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
 github.com/afbjorklund/go-getter v1.4.1-0.20190910175809-eb9f6c26742c h1:18gEt7qzn7CW7qMkfPTFyyotlPbvPQo9o4IDV8jZqP4=
 github.com/afbjorklund/go-getter v1.4.1-0.20190910175809-eb9f6c26742c/go.mod h1:7qxyCd8rBfcShwsvxgIguu4KbS3l8bUCwg2Umn7RjeY=
+github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
-github.com/apache/thrift v0.0.0-20180902110319-2566ecd5d999/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
+github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/auth0/go-jwt-middleware v0.0.0-20170425171159-5493cabe49f7/go.mod h1:LWMyo4iOLWXHGdBki7NIht1kHru/0wM179h+d3g8ATM=
+github.com/aws/aws-sdk-go v1.15.78 h1:LaXy6lWR0YK7LKyuU0QWy2ws/LWTPfYV/UgfiBu4tvY=
 github.com/aws/aws-sdk-go v1.15.78/go.mod h1:E3/ieXAlvM0XWO57iftYVDLLvQ824smPP3ATZkfNZeM=
 github.com/aws/aws-sdk-go v1.16.26 h1:GWkl3rkRO/JGRTWoLLIqwf7AWC4/W/1hMOUZqmX0js4=
 github.com/aws/aws-sdk-go v1.16.26/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
-github.com/bazelbuild/bazel-gazelle v0.0.0-20181012220611-c728ce9f663e/go.mod h1:uHBSeeATKpVazAACZBDPL/Nk/UhQDDsJWDlqYJo8/Us=
-github.com/bazelbuild/buildtools v0.0.0-20180226164855-80c7f0d45d7e/go.mod h1:5JP0TXzWDHXv8qvxRC4InIazwdyDseBDbzESUMKk1yU=
+github.com/bazelbuild/bazel-gazelle v0.18.2/go.mod h1:D0ehMSbS+vesFsLGiD6JXu3mVEzOlfUl8wNnq+x/9p0=
+github.com/bazelbuild/bazel-gazelle v0.19.1-0.20191105222053-70208cbdc798/go.mod h1:rPwzNHUqEzngx1iVBfO/2X2npKaT3tqPqqHW6rVsn/A=
+github.com/bazelbuild/buildtools v0.0.0-20190731111112-f720930ceb60/go.mod h1:5JP0TXzWDHXv8qvxRC4InIazwdyDseBDbzESUMKk1yU=
+github.com/bazelbuild/buildtools v0.0.0-20190917191645-69366ca98f89/go.mod h1:5JP0TXzWDHXv8qvxRC4InIazwdyDseBDbzESUMKk1yU=
+github.com/bazelbuild/rules_go v0.0.0-20190719190356-6dae44dc5cab/go.mod h1:MC23Dc/wkXEyk3Wpq6lCqz0ZAYOZDw2DR5y3N1q2i7M=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas=
 github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4=
+github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/bifurcation/mint v0.0.0-20180715133206-93c51c6ce115/go.mod h1:zVt7zX3K/aDCk9Tj+VM7YymsX66ERvzCJzw8rFCX2JU=
 github.com/blang/semver v3.5.0+incompatible h1:CGxCgetQ64DKk7rdZ++Vfnb1+ogGNnB17OJKJXD2Cfs=
 github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
 github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
 github.com/c4milo/gotoolkit v0.0.0-20170318115440-bcc06269efa9 h1:+ziP/wVJWuAORkjv7386TRidVKY57X0bXBZFMeFlW+U=
 github.com/c4milo/gotoolkit v0.0.0-20170318115440-bcc06269efa9/go.mod h1:txokOny9wavBtq2PWuHmj1P+eFwpCsj+gQeNNANChfU=
+github.com/caddyserver/caddy v1.0.3/go.mod h1:G+ouvOY32gENkJC+jhgl62TyhvqEsFaDiZ4uw0RzP1E=
+github.com/cenkalti/backoff v2.1.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
 github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
 github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
 github.com/cespare/prettybench v0.0.0-20150116022406-03b8cfe5406c/go.mod h1:Xe6ZsFhtM8HrDku0pxJ3/Lr51rwykrzgFwpmTzleatY=
 github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw=
+github.com/checkpoint-restore/go-criu v0.0.0-20190109184317-bdb7599cd87b/go.mod h1:TrMrLQfeENAPYPRsJuq3jsqdlRh3lvi6trTZJG8+tho=
+github.com/cheekybits/genny v0.0.0-20170328200008-9127e812e1e9/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
 github.com/cheggaaa/pb v1.0.27 h1:wIkZHkNfC7R6GI5w7l/PdAdzXzlrbcI3p8OAlnkTsnc=
 github.com/cheggaaa/pb v1.0.27/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s=
 github.com/cheggaaa/pb/v3 v3.0.1 h1:m0BngUk2LuSRYdx4fujDKNRXNDpbNCfptPfVT2m6OJY=
 github.com/cheggaaa/pb/v3 v3.0.1/go.mod h1:SqqeMF/pMOIu3xgGoxtPYhMNQP258xE4x/XRTYua+KU=
-github.com/client9/misspell v0.0.0-20170928000206-9ce5d979ffda/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/cloudflare/cfssl v0.0.0-20180726162950-56268a613adf/go.mod h1:yMWuSON2oQp+43nFtAV/uvKQIFpSPerB57DCt9t8sSA=
 github.com/cloudfoundry-attic/jibber_jabber v0.0.0-20151120183258-bcc4c8345a21 h1:Yg2hDs4b13Evkpj42FU2idX2cVXVFqQSheXYKM86Qsk=
 github.com/cloudfoundry-attic/jibber_jabber v0.0.0-20151120183258-bcc4c8345a21/go.mod h1:MgJyK38wkzZbiZSKeIeFankxxSA8gayko/nr5x5bgBA=
 github.com/cloudfoundry/jibber_jabber v0.0.0-20151120183258-bcc4c8345a21 h1:tuijfIjZyjZaHq9xDUh0tNitwXshJpbLkqMOJv4H3do=
 github.com/cloudfoundry/jibber_jabber v0.0.0-20151120183258-bcc4c8345a21/go.mod h1:po7NpZ/QiTKzBKyrsEAxwnTamCoh8uDk/egRpQ7siIc=
 github.com/clusterhq/flocker-go v0.0.0-20160920122132-2b8b7259d313/go.mod h1:P1wt9Z3DP8O6W3rvwCt0REIlshg1InHImaLW0t3ObY0=
-github.com/codedellemc/goscaleio v0.0.0-20170830184815-20e2ce2cf885/go.mod h1:JIHmDHNZO4tmA3y3RHp6+Gap6kFsNf55W9Pn/3YS9IY=
+github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/codegangsta/negroni v1.0.0/go.mod h1:v0y3T5G7Y1UlFfyxFn/QLRU4a2EuNau2iZY63YTKWo0=
-github.com/container-storage-interface/spec v1.1.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4=
+github.com/container-storage-interface/spec v1.2.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4=
 github.com/containerd/console v0.0.0-20170925154832-84eeaae905fa/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=
 github.com/containerd/containerd v1.0.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
 github.com/containerd/typeurl v0.0.0-20190228175220-2a93cfde8c20/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc=
-github.com/containernetworking/cni v0.6.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
-github.com/coreos/bbolt v1.3.1-coreos.6/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
+github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
+github.com/coredns/corefile-migration v1.0.4/go.mod h1:OFwBp/Wc9dJt5cAZzHWMNhK1r5L0p0jDwIBc6j8NC8E=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
-github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
-github.com/coreos/go-oidc v0.0.0-20180117170138-065b426bd416/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
-github.com/coreos/go-semver v0.0.0-20180108230905-e214231b295a/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
-github.com/coreos/rkt v1.30.0/go.mod h1:O634mlH6U7qk87poQifK6M2rsFNt+FyUTWNMnP1hF1U=
-github.com/cpuguy83/go-md2man v1.0.4/go.mod h1:N6JayAiVKtlHSnuTCeuLSQVs75hb8q+dYQLjr7cDsKY=
 github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk=
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
-github.com/cyphar/filepath-securejoin v0.0.0-20170720062807-ae69057f2299/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
-github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ=
-github.com/d2g/dhcp4client v0.0.0-20170829104524-6e570ed0a266/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s=
+github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
+github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
+github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE=
-github.com/dgrijalva/jwt-go v0.0.0-20160705203006-01aeca54ebda/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
-github.com/docker/distribution v0.0.0-20170726174610-edc3ab29cdff h1:FKH02LHYqSmeWd3GBh0KIkM8JBpw3RrShgtcWShdWJg=
-github.com/docker/distribution v0.0.0-20170726174610-edc3ab29cdff/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
+github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
+github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0 h1:w3NnFcKR5241cfmQU5ZZAsf0xcpId6mWOupTvJlUX2U=
 github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo=
@@ -120,9 +140,12 @@ github.com/docker/go-connections v0.3.0 h1:3lOnM9cSzgGwx8VfK/NGOW5fLQ0GjIlCkaktF
 github.com/docker/go-connections v0.3.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
 github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk=
 github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
-github.com/docker/libnetwork v0.0.0-20180830151422-a9cd636e3789/go.mod h1:93m0aTqz6z+g32wla4l4WxTrdtvBRmVzYRkYvasA5Z8=
+github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
+github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/docker/libnetwork v0.8.0-dev.2.0.20190624125649-f0e46a78ea34/go.mod h1:93m0aTqz6z+g32wla4l4WxTrdtvBRmVzYRkYvasA5Z8=
 github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 h1:cenwrSVm+Z7QLSV/BsnenAOcDXdX4cMv4wP0B/5QbPg=
 github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
+github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/elazarl/goproxy v0.0.0-20190421051319-9d40249d3c2f h1:8GDPb0tCY8LQ+OJ3dbHb5sA6YZWXFORQYZx5sdsTlMs=
@@ -132,77 +155,139 @@ github.com/elazarl/goproxy/ext v0.0.0-20190421051319-9d40249d3c2f/go.mod h1:gNh8
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/euank/go-kmsg-parser v2.0.0+incompatible/go.mod h1:MhmAMZ8V4CYH4ybgdRwPr2TU5ThnS43puaKEMpja1uw=
-github.com/evanphx/json-patch v0.0.0-20190203023257-5858425f7550/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch v4.2.0+incompatible h1:fUDGZCv/7iAN7u0puUVhvKCcsR6vRfwrJatElLBEf0I=
 github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M=
 github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
-github.com/fatih/camelcase v0.0.0-20160318181535-f6a740d52f96/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
+github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
+github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/ghodss/yaml v0.0.0-20180820084758-c7ce16629ff4/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
 github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
+github.com/go-acme/lego v2.5.0+incompatible/go.mod h1:yzMNe9CasVUhkquNvti5nAtPmG94USbYxYrZfTkIn0M=
+github.com/go-bindata/go-bindata v3.1.1+incompatible/go.mod h1:xK8Dsgwmeed+BBsSy2XTopBn/8uK2HWuGSnA11C3Joo=
+github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
+github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
 github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI=
 github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
 github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI=
 github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik=
-github.com/go-openapi/analysis v0.17.2/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik=
+github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik=
+github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk=
+github.com/go-openapi/analysis v0.19.5/go.mod h1:hkEAkxagaIvIP7VTn8ygJNkd4kAYON2rCu0v0ObL0AU=
 github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0=
-github.com/go-openapi/errors v0.17.2/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0=
+github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0=
+github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94=
+github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0=
 github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
-github.com/go-openapi/jsonpointer v0.19.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
+github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
+github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
+github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
+github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg=
 github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
-github.com/go-openapi/jsonreference v0.19.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
+github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
+github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
+github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
 github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
-github.com/go-openapi/loads v0.17.2/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
+github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
+github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
+github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs=
+github.com/go-openapi/loads v0.19.4/go.mod h1:zZVHonKd8DXyxyw4yfnVjPzBjIQcLt0CCsn0N0ZrQsk=
 github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA=
-github.com/go-openapi/runtime v0.17.2/go.mod h1:QO936ZXeisByFmZEO1IS1Dqhtf4QV1sYYFtIq6Ld86Q=
+github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64=
+github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29gLDlFGtJ8NL4=
+github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc=
 github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
-github.com/go-openapi/spec v0.17.2/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
+github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
+github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY=
+github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
 github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU=
+github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU=
+github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY=
+github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU=
+github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
 github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
-github.com/go-openapi/swag v0.17.2/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
-github.com/go-openapi/validate v0.17.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4=
+github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
+github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4=
+github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA=
+github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4=
 github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
-github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
-github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415 h1:WSBJMqJbLxsn+bTCPyPYZfqHdJmc8MK4wrBjMft6BAM=
-github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4=
+github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ=
+github.com/go-toolsmith/astequal v0.0.0-20180903214952-dcb477bfacd6/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
+github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
+github.com/go-toolsmith/astfmt v0.0.0-20180903215011-8f8ee99c3086/go.mod h1:mP93XdblcopXwlyN4X4uodxXQhldPGZbcEJIimQHrkg=
+github.com/go-toolsmith/astfmt v1.0.0/go.mod h1:cnWmsOAuq4jJY6Ct5YWlVLmcmLMn1JUPuQIHCY7CJDw=
+github.com/go-toolsmith/astinfo v0.0.0-20180906194353-9809ff7efb21/go.mod h1:dDStQCHtmZpYOmjRP/8gHHnCCch3Zz3oEgCdZVdtweU=
+github.com/go-toolsmith/astp v0.0.0-20180903215135-0af7e3c24f30/go.mod h1:SV2ur98SGypH1UjcPpCatrV5hPazG6+IfNHbkDXBRrk=
+github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdYrA8meBmLI=
+github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8/go.mod h1:WoMrjiy4zvdS+Bg6z9jZH82QXwkcgCBX6nOfnmdaHks=
+github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc=
+github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8=
+github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
+github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
+github.com/godbus/dbus v0.0.0-20181101234600-2ff6f7ffd60f/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
 github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I=
+github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3 h1:zN2lZNZRflqFyxVaTIU61KNKQ9C0055u9CAfpmqUvo4=
 github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3/go.mod h1:nPpo7qLxd6XL3hWJG/O60sR8ZKfMCiIoNap5GvD12KU=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
-github.com/golang/mock v0.0.0-20160127222235-bd3c8e81be01/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.0.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s=
 github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4=
+github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk=
+github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0=
+github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8=
+github.com/golangci/go-tools v0.0.0-20190318055746-e32c54105b7c/go.mod h1:unzUULGw35sjyOYjUt0jMTXqHlZPpPc6e+xfO4cd6mM=
+github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o=
+github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
+github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
+github.com/golangci/golangci-lint v1.18.0/go.mod h1:kaqo8l0OZKYPtjNmG4z4HrWLgcYNIJ9B9q3LWri9uLg=
+github.com/golangci/gosec v0.0.0-20190211064107-66fb7fc33547/go.mod h1:0qUabqiIQgfmlAmulqxyiGkkyF6/tOGSnY2cnPVwrzU=
+github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU=
+github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
+github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o=
+github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA=
+github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI=
+github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4=
+github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ=
 github.com/golangplus/bytes v0.0.0-20160111154220-45c989fe5450/go.mod h1:Bk6SMAONeMXrxql8uvOKuAZSu8aM5RUGv+1C6IJaEho=
 github.com/golangplus/fmt v0.0.0-20150411045040-2a5d6d7d2995/go.mod h1:lJgMEyOkYFkPcDKwRXegd+iM6E7matEszMG5HhwytU8=
 github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk=
-github.com/google/btree v0.0.0-20160524151835-7d79101e329e/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/cadvisor v0.33.2-0.20190411163913-9db8c7dee20a/go.mod h1:1nql6U13uTHaLYB8rLS5x9IJc2qT6Xd/Tr1sTX6NE48=
-github.com/google/certificate-transparency-go v1.0.21/go.mod h1:QeJfpSbVSfYc7RgB3gJFj9cbuQMMchQxrWXz8Ruopmg=
+github.com/google/cadvisor v0.35.0/go.mod h1:1nql6U13uTHaLYB8rLS5x9IJc2qT6Xd/Tr1sTX6NE48=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -212,18 +297,17 @@ github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4r
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
 github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
-github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
+github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
 github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/googleapis/gax-go v2.0.0+incompatible h1:j0GKcs05QVmm7yesiZq2+9cxHkNK9YM6zKx4D2qucQU=
-github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
@@ -231,19 +315,20 @@ github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1
 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
 github.com/googleapis/gnostic v0.3.0 h1:CcQijm0XKekKjP/YCz28LXVSpgguuB+nCxaSjCe09y0=
 github.com/googleapis/gnostic v0.3.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
-github.com/gophercloud/gophercloud v0.0.0-20190126172459-c818fa66e4c8/go.mod h1:3WdhXV3rUYy9p6AUW8d94kr+HS62Y4VL9mBnFxsD8q4=
+github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
 github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
 github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
-github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
+github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
+github.com/gostaticanalysis/analysisutil v0.0.3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
-github.com/grpc-ecosystem/go-grpc-middleware v0.0.0-20190222133341-cfaf5686ec79/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
-github.com/grpc-ecosystem/go-grpc-prometheus v0.0.0-20170330212424-2500245aa611/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
-github.com/grpc-ecosystem/grpc-gateway v1.3.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
-github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
+github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce h1:prjrVgOk2Yg6w+PflHoszQNLTUh4kaByUcEWM/9uin4=
 github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-cleanhttp v0.5.0 h1:wvCrVc9TjDls6+YGAF2hAifE1E5U1+b4tH6KdvN3Gig=
@@ -254,19 +339,19 @@ github.com/hashicorp/go-retryablehttp v0.5.4 h1:1BZvpawXoJCWX6pNtow9+rpEj+3itIlu
 github.com/hashicorp/go-retryablehttp v0.5.4/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
 github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo=
 github.com/hashicorp/go-safetemp v1.0.0/go.mod h1:oaerMy3BhqiTbVye6QuFhFtIceqFoDHxNAB65b+Rj1I=
+github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
 github.com/hashicorp/go-version v1.1.0 h1:bPIoEKD27tNdebFGGxxYwcL4nepeY4j1QP23PFRGzg0=
 github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/hcl v0.0.0-20160711231752-d8c773c4cba1/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w=
+github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
-github.com/heketi/heketi v0.0.0-20181109135656-558b29266ce0/go.mod h1:bB9ly3RchcQqsQ9CpyaQwvva7RS5ytVoSoholZQON6o=
-github.com/heketi/rest v0.0.0-20180404230133-aa6a65207413/go.mod h1:BeS3M108VzVlmAue3lv2WcGuPAX94/KN63MUURzbYSI=
+github.com/heketi/heketi v9.0.1-0.20190917153846-c2e2a4ab7ab9+incompatible/go.mod h1:bB9ly3RchcQqsQ9CpyaQwvva7RS5ytVoSoholZQON6o=
 github.com/heketi/tests v0.0.0-20151005000721-f3775cbcefd6/go.mod h1:xGMAM8JLi7UkZt1i4FQeQy0R2T8GLUwQhOP5M1gBhy4=
-github.com/heketi/utils v0.0.0-20170317161834-435bc5bdfa64/go.mod h1:RYlF4ghFZPPmk2TC5REt5OFwvfb6lzxFWrTWB+qs28s=
 github.com/hooklift/assert v0.0.0-20170704181755-9d1defd6d214 h1:WgfvpuKg42WVLkxNwzfFraXkTXPK36bMqXvMFN67clI=
 github.com/hooklift/assert v0.0.0-20170704181755-9d1defd6d214/go.mod h1:kj6hFWqfwSjFjLnYW5PK1DoxZ4O0uapwHRmd9jhln4E=
 github.com/hooklift/iso9660 v0.0.0-20170318115843-1cf07e5970d8 h1:ARl0RuGZTqBOMXQIfXen0twVSJ8kMojd7ThJf4EBcrc=
@@ -275,6 +360,8 @@ github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ=
+github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/intel-go/cpuid v0.0.0-20181003105527-1a4a6f06a1c6 h1:XboatR7lasl05yel5hNXF7kQBw2oFUGdMztcgisfhNU=
@@ -282,18 +369,21 @@ github.com/intel-go/cpuid v0.0.0-20181003105527-1a4a6f06a1c6/go.mod h1:RmeVYf9Xr
 github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
 github.com/jimmidyson/go-download v0.0.0-20161028105827-7f9a90c8c95b h1:3TknJxYSK1eDe21QorC3C2Yz8jylk6vlJG9YABnFzkU=
 github.com/jimmidyson/go-download v0.0.0-20161028105827-7f9a90c8c95b/go.mod h1:I3WsAhNNoG7a/d8HMlYUywJJlfOs/+/83NEUjuDp4lc=
+github.com/jimstudt/http-authentication v0.0.0-20140401203705-3eca13d6893a/go.mod h1:wK6yTYYcgjHE1Z1QtXACPDjcFJyBskHEdagmnq3vsP8=
+github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 h1:12VvqtR6Aowv3l/EQUlocDHW2Cp4G9WJVH7uyH8QFJE=
 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/johanneswuerbach/nfsexports v0.0.0-20181204082207-1aa528dcb345 h1:XP1VL9iOZu4yz/rq8zj+yvB23XEY5erXRzp8JYmkWu0=
 github.com/johanneswuerbach/nfsexports v0.0.0-20181204082207-1aa528dcb345/go.mod h1:+c1/kUpg2zlkoWqTOvzDs36Wpbm3Gd1nlmtXAEB0WGU=
-github.com/jonboulle/clockwork v0.0.0-20141017032234-72f9bd7c4e0c/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
-github.com/json-iterator/go v0.0.0-20180701071628-ab8a2e0c74be/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok=
+github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
-github.com/jteeuwen/go-bindata v0.0.0-20151023091102-a0ff2567cfb7/go.mod h1:JVvhzYOiGBnFSYRyV00iY8q7/0PThjIYav1p9h5dmKs=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/juju/clock v0.0.0-20190205081909-9c5c9712527c h1:3UvYABOQRhJAApj9MdCN+Ydv841ETSoy6xLzdmmr/9A=
 github.com/juju/clock v0.0.0-20190205081909-9c5c9712527c/go.mod h1:nD0vlnrUjcjJhqN5WuCWZyzfd5AHZAC9/ajvbSx69xA=
@@ -311,55 +401,79 @@ github.com/juju/utils v0.0.0-20180820210520-bf9cc5bdd62d h1:irPlN9z5VCe6BTsqVsxh
 github.com/juju/utils v0.0.0-20180820210520-bf9cc5bdd62d/go.mod h1:6/KLg8Wz/y2KVGWEpkK9vMNGkOnu4k/cqs8Z1fKjTOk=
 github.com/juju/version v0.0.0-20180108022336-b64dbd566305 h1:lQxPJ1URr2fjsKnJRt/BxiIxjLt9IKGvS+0injMHbag=
 github.com/juju/version v0.0.0-20180108022336-b64dbd566305/go.mod h1:kE8gK5X0CImdr7qpSKl3xB2PmpySSmfj7zVbkZFs81U=
-github.com/kardianos/osext v0.0.0-20150410034420-8fef92e41e22/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/karrick/godirwalk v1.7.5/go.mod h1:2c9FRhkDxdIbgkOnCEvnSWs71Bhugbl46shStcFDJ34=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
+github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
+github.com/kisielk/gotool v0.0.0-20161130080628-0de1eaf82fa3/go.mod h1:jxZFDH7ILpTPQTk+E2s+z4CUas9lVNjIuKR4c5/zKgM=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/kr/fs v0.0.0-20131111012553-2788f0dbd169/go.mod h1:glhvuHOU9Hy7/8PwwdtnarXqLagOX0b/TbZx2zLMqEg=
-github.com/kr/pretty v0.0.0-20140812000539-f31442d60e51/go.mod h1:Bvhd+E3laJ0AVkG0c9rmtZcnhV0HQ3+c3YxxqTvc/gA=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.0.0-20130911015532-6807e777504f/go.mod h1:sjUstKUATFIcff4qlB53Kml0wQPtJVc/3fWrmuUmcfA=
+github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/libopenstorage/openstorage v0.0.0-20170906232338-093a0c388875/go.mod h1:Sp1sIObHjat1BeXhfMqLZ14wnOzEhNx2YQedreMcUyc=
+github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
+github.com/libopenstorage/openstorage v1.0.0/go.mod h1:Sp1sIObHjat1BeXhfMqLZ14wnOzEhNx2YQedreMcUyc=
 github.com/libvirt/libvirt-go v3.4.0+incompatible h1:Cpyalgj1x8JIeTlL6SDYZBo7j8nY3+5XHqmi8DaunCk=
 github.com/libvirt/libvirt-go v3.4.0+incompatible/go.mod h1:34zsnB4iGeOv7Byj6qotuW8Ya4v4Tr43ttjz/F0wjLE=
 github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE=
 github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc=
+github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
 github.com/lpabon/godbc v0.1.1/go.mod h1:Jo9QV0cf3U6jZABgiJ2skINAXb9j8m51r07g4KI92ZA=
+github.com/lucas-clemente/aes12 v0.0.0-20171027163421-cd47fb39b79f/go.mod h1:JpH9J1c9oX6otFSgdUHwUBUizmKlrMjxWnIAjff4m04=
+github.com/lucas-clemente/quic-clients v0.1.0/go.mod h1:y5xVIEoObKqULIKivu+gD/LU90pL73bTdtQjPBvtCBk=
+github.com/lucas-clemente/quic-go v0.10.2/go.mod h1:hvaRS9IHjFLMq76puFJeWNfmn+H70QZ/CXoxqw9bzao=
+github.com/lucas-clemente/quic-go-certificates v0.0.0-20160823095156-d2f86524cced/go.mod h1:NCcRLrOTZbzhZvixZLlERbJtDtYsmMw8Jc4vS8Z0g58=
 github.com/machine-drivers/docker-machine-driver-vmware v0.1.1 h1:+E1IKKk+6kaQrCPg6edJZ/zISZijuZTPnzy6RE4C/Ho=
 github.com/machine-drivers/docker-machine-driver-vmware v0.1.1/go.mod h1:ej014C83EmSnxJeJ8PtVb8OLJ91PJKO1Q8Y7sM5CK0o=
 github.com/machine-drivers/machine v0.7.1-0.20191109154235-b39d5b50de51 h1:ra4e+hU8Ca02yNyF8WM89aOShgXEPWRqerGpsmfqgTA=
 github.com/machine-drivers/machine v0.7.1-0.20191109154235-b39d5b50de51/go.mod h1:79Uwa2hGd5S39LDJt58s8JZcIhGEK6pkq9bsuTbFWbk=
-github.com/magiconair/properties v0.0.0-20160816085511-61b492c03cf4/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/magiconair/properties v1.7.6/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
+github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/marstr/guid v0.0.0-20170427235115-8bdf7d1a087c/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho=
+github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
+github.com/marten-seemann/qtls v0.2.3/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwmjgmPuiQEcYk=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg=
+github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
+github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y=
 github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
-github.com/mattn/go-shellwords v0.0.0-20180605041737-f8471b0a71de/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
+github.com/mattn/go-shellwords v1.0.5/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
+github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mesos/mesos-go v0.0.9/go.mod h1:kPYCMQ9gsOXVAle1OsoY4I1+9kPu8GHkf88aV59fDr4=
-github.com/mholt/caddy v0.0.0-20180213163048-2de495001514/go.mod h1:Wb1PlT4DAYSqOEd03MsqkdkXnTxA8v9pKjdpxbqM1kY=
-github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
-github.com/miekg/dns v0.0.0-20160614162101-5d001d020961 h1:vX2vkMipgQZ8gfmAsFeZdcgmhHoB7jMo6chAtajG3AI=
-github.com/miekg/dns v0.0.0-20160614162101-5d001d020961/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
-github.com/mindprince/gonvml v0.0.0-20171110221305-fee913ce8fb2/go.mod h1:2eu9pRWp8mo84xCg6KswZ+USQHjwgRhNp06sozOdsTY=
-github.com/mistifyio/go-zfs v0.0.0-20151009155749-1b4ae6fb4e77/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4=
+github.com/mholt/certmagic v0.6.2-0.20190624175158-6a42ef9fe8c2/go.mod h1:g4cOPxcjV0oFq3qwpjSA30LReKD8AoIfwAY9VvG35NY=
+github.com/miekg/dns v1.1.3/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
+github.com/miekg/dns v1.1.4 h1:rCMZsU2ScVSYcAsOXgmC6+AKOK+6pmQTOcw03nfwYV0=
+github.com/miekg/dns v1.1.4/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
+github.com/mindprince/gonvml v0.0.0-20190828220739-9ebdce4bb989/go.mod h1:2eu9pRWp8mo84xCg6KswZ+USQHjwgRhNp06sozOdsTY=
+github.com/mistifyio/go-zfs v2.1.1+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4=
 github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
@@ -367,43 +481,55 @@ github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936 h1:kw1v0NlnN+GZcU8
 github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk=
 github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0=
 github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
-github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
 github.com/mitchellh/go-wordwrap v1.0.0 h1:6GlHJ/LTGMrIJbwgdqdl2eEH8o+Exx/0m8ir9Gns0u4=
 github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
+github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/moby/hyperkit v0.0.0-20171020124204-a12cd7250bcd h1:WDG9l//UGcGx4lqqEDY23+mRnQMKFY+8LD83OxQllRk=
 github.com/moby/hyperkit v0.0.0-20171020124204-a12cd7250bcd/go.mod h1:zGAVB/FkAf4ozkR8CCuj4LcVuErrNsj9APTDFvhOckw=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/mohae/deepcopy v0.0.0-20170603005431-491d3605edfb/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
 github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c h1:nXxl5PrvVm2L/wCy8dQu6DMTwH4oIuGN8GJDAlqDdVE=
 github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
-github.com/mrunalp/fileutils v0.0.0-20160930181131-4ee1cc9a8058/go.mod h1:x8F1gnqOkIEiO4rqoeEEEqQbo7HjGMTvyoq3gej4iT0=
+github.com/mozilla/tls-observatory v0.0.0-20180409132520-8791a200eb40/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk=
+github.com/mrunalp/fileutils v0.0.0-20171103030105-7d4729fb3618/go.mod h1:x8F1gnqOkIEiO4rqoeEEEqQbo7HjGMTvyoq3gej4iT0=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
-github.com/mvdan/xurls v0.0.0-20160110113200-1b768d7c393a/go.mod h1:tQlNn3BED8bE/15hnSL2HLkDeLWpNPAwtw7wkEq44oU=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/mvdan/xurls v1.1.0/go.mod h1:tQlNn3BED8bE/15hnSL2HLkDeLWpNPAwtw7wkEq44oU=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
-github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk=
-github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
-github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
-github.com/olekukonko/tablewriter v0.0.0-20160923125401-bdcc175572fd h1:nEatQ6JnwCT9iYD5uqYUiFqq8tJGX25to8KVKXqya7k=
-github.com/olekukonko/tablewriter v0.0.0-20160923125401-bdcc175572fd/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
+github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0=
+github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E=
+github.com/nbutton23/zxcvbn-go v0.0.0-20160627004424-a22cb81b2ecd/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
+github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
+github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5 h1:58+kh9C6jJVXYjt8IE48G2eWl6BjwU5Gj0gqY84fy78=
+github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
+github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
 github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/gomega v0.0.0-20190113212917-5533ce8a0da3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo=
+github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
+github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
 github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420 h1:Yu3681ykYHDfLoI6XVjL4JWmkE+3TX9yfIWwRCh1kFM=
-github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
-github.com/opencontainers/image-spec v0.0.0-20170604055404-372ad780f634 h1:BNgUWy7fCNMkfpyG05/9wWeDnIY4hqs9UpqkGIjAb68=
-github.com/opencontainers/image-spec v0.0.0-20170604055404-372ad780f634/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
-github.com/opencontainers/runc v0.0.0-20181113202123-f000fe11ece1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
+github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
+github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ=
+github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
+github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
+github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
 github.com/opencontainers/runtime-spec v1.0.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
-github.com/opencontainers/selinux v0.0.0-20170621221121-4a2974bf1ee9/go.mod h1:+BLncwf63G4dgOzykXAxcmnFlUaOlkDdmw/CqsW6pjs=
-github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
+github.com/opencontainers/selinux v1.3.1-0.20190929122143-5215b1806f52/go.mod h1:+BLncwf63G4dgOzykXAxcmnFlUaOlkDdmw/CqsW6pjs=
 github.com/otiai10/copy v1.0.2 h1:DDNipYy6RkIkjMwy+AWzgKiNTyj2RUI9yEMeETEpVyc=
 github.com/otiai10/copy v1.0.2/go.mod h1:c7RpqBkwMom4bYTSkLSym4VSJz/XtncWRAj/J4PEIMY=
 github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95 h1:+OLn68pqasWca0z5ryit9KGfp3sUsW4Lqg32iRMJyzs=
@@ -412,7 +538,7 @@ github.com/otiai10/mint v1.3.0 h1:Ady6MKVezQwHBkGzLFbrsywyp09Ah7rkmfjV3Bcr5uc=
 github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
 github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
-github.com/pelletier/go-toml v1.0.1/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/pelletier/go-toml v1.1.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
@@ -425,172 +551,198 @@ github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/profile v0.0.0-20161223203901-3a8809bd8a80 h1:DQFOykp5w+HOykOMzd2yOX5P6ty58Ggiu2rthHgcNQg=
 github.com/pkg/profile v0.0.0-20161223203901-3a8809bd8a80/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
-github.com/pkg/sftp v0.0.0-20160930220758-4d0e916071f6/go.mod h1:NxmoDg/QLVWluQDUYG7XBZTLUpKeFa8e3aMf1BfjyHk=
+github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=
 github.com/pquerna/ffjson v0.0.0-20180717144149-af8b230fcd20/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M=
-github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
-github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740=
-github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
-github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
-github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8=
-github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
-github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
-github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE=
-github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI=
 github.com/quobyte/api v0.1.2/go.mod h1:jL7lIHrmqQ7yh05OJ+eEEdHr0u/kmT1Ff9iHd+4H6VI=
 github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M=
-github.com/robfig/cron v0.0.0-20170309132418-df38d32658d8/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k=
+github.com/robfig/cron v1.1.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k=
+github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc=
+github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rubiojr/go-vhd v0.0.0-20160810183302-0bfd3b39853c/go.mod h1:DM5xW0nvfNNm2uytzsvhI3OnX8uzaRAg8UX/CnDqbto=
-github.com/russross/blackfriday v0.0.0-20151117072312-300106c228d5/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+github.com/russross/blackfriday v0.0.0-20170610170232-067529f716f4/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
 github.com/samalba/dockerclient v0.0.0-20160414174713-91d7393ff859 h1:XRl74t6xHKI5EVIjDI5nPlHRq0bHED9/TjQuD8/UMkE=
 github.com/samalba/dockerclient v0.0.0-20160414174713-91d7393ff859/go.mod h1:yeYR4SlaRZJct6lwNRKR+qd0CocnxxWDE7Vh5dxsn/w=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
-github.com/seccomp/libseccomp-golang v0.0.0-20150813023252-1b506fc7c24e/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=
+github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
 github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM=
 github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
 github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U=
 github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
-github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY=
-github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM=
-github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0=
 github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
 github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
-github.com/shurcooL/gofontwoff v0.0.0-20180329035133-29b52fc0a18d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw=
-github.com/shurcooL/gopherjslib v0.0.0-20160914041154-feb6d3990c2c/go.mod h1:8d3azKNyqcHP1GaQE/c6dDgjkgSx2BZ4IoEi4F1reUI=
-github.com/shurcooL/highlight_diff v0.0.0-20170515013008-09bb4053de1b/go.mod h1:ZpfEhSmds4ytuByIcDnOLkTHGUI6KNqRNPDLHDk+mUU=
-github.com/shurcooL/highlight_go v0.0.0-20181028180052-98c3abbbae20/go.mod h1:UDKB5a1T23gOMUJrI+uSuH0VRDStOiUVSjBTRDVBVag=
-github.com/shurcooL/home v0.0.0-20181020052607-80b7ffcb30f9/go.mod h1:+rgNQw2P9ARFAs37qieuu7ohDNQ3gds9msbT2yn85sg=
-github.com/shurcooL/htmlg v0.0.0-20170918183704-d01228ac9e50/go.mod h1:zPn1wHpTIePGnXSHpsVPWEktKXHr6+SS6x/IKRb7cpw=
-github.com/shurcooL/httperror v0.0.0-20170206035902-86b7830d14cc/go.mod h1:aYMfkZ6DWSJPJ6c4Wwz3QtW22G7mf/PEgaB9k/ik5+Y=
-github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg=
-github.com/shurcooL/httpgzip v0.0.0-20180522190206-b1c53ac65af9/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q=
-github.com/shurcooL/issues v0.0.0-20181008053335-6292fdc1e191/go.mod h1:e2qWDig5bLteJ4fwvDAc2NHzqFEthkqn7aOZAOpj+PQ=
-github.com/shurcooL/issuesapp v0.0.0-20180602232740-048589ce2241/go.mod h1:NPpHK2TI7iSaM0buivtFUc9offApnI0Alt/K8hcHy0I=
-github.com/shurcooL/notifications v0.0.0-20181007000457-627ab5aea122/go.mod h1:b5uSkrEVM1jQUspwbixRBhaIjIzL2xazXp6kntxYle0=
-github.com/shurcooL/octicon v0.0.0-20181028054416-fa4f57f9efb2/go.mod h1:eWdoE5JD4R5UVWDucdOPg1g2fqQRq78IQa9zlOV1vpQ=
-github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1lToEk4d2s07G3XGfz2QrgHXg4RJBvjrOozvoWfk=
-github.com/shurcooL/sanitized_anchor_name v0.0.0-20151028001915-10ef21a441db/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
-github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
-github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4=
-github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw=
-github.com/sigma/go-inotify v0.0.0-20181102212354-c87b6cf5033d/go.mod h1:stlh9OsqBQSdwxTxX73mu41BBtRbIpZLQ7flcAoxAfo=
+github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
+github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
-github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k=
-github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
+github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
-github.com/soheilhy/cmux v0.1.3/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
-github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE=
-github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA=
-github.com/spf13/afero v0.0.0-20160816080757-b28a7effac97/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
+github.com/sourcegraph/go-diff v0.5.1/go.mod h1:j2dHj3m8aZgQO8lMTcTnBcXkRRRqi34cd2MNlA9u1mE=
+github.com/spf13/afero v1.1.0/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
-github.com/spf13/cast v0.0.0-20160730092037-e31f36ffc91a/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
+github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
+github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
+github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
 github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cobra v0.0.0-20180319062004-c439c4fa0937/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
-github.com/spf13/cobra v0.0.4/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
+github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s=
 github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
-github.com/spf13/jwalterweatherman v0.0.0-20160311093646-33c24e77fb80/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
+github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
+github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/viper v0.0.0-20160820190039-7fb2782df3d8/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM=
 github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M=
 github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/storageos/go-api v0.0.0-20180912212459-343b3eff91fc/go.mod h1:ZrLn+e0ZuF3Y65PNF6dIwbJPZqfmtCXxFm9ckv0agOY=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
+github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/syndtr/gocapability v0.0.0-20160928074757-e7cb7fa329f4/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
+github.com/thecodeteam/goscaleio v0.1.0/go.mod h1:68sdkZAsK8bvEwBlbQnlLS+xU+hvLYM/iQ8KXej1AwM=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/timakin/bodyclose v0.0.0-20190721030226-87058b9bfcec/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/ulikunitz/xz v0.5.5 h1:pFrO0lVpTBXLpYw+pnLj6TbvHuyjXMfjGeCwSqCVwok=
 github.com/ulikunitz/xz v0.5.5/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
+github.com/ultraware/funlen v0.0.1/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA=
+github.com/ultraware/funlen v0.0.2/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA=
+github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4=
-github.com/vishvananda/netlink v0.0.0-20171020171820-b2de5d10e38e/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
+github.com/valyala/quicktemplate v1.1.1/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4=
+github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
+github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
+github.com/vishvananda/netlink v1.0.0/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
 github.com/vishvananda/netns v0.0.0-20171111001504-be1fbeda1936/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI=
-github.com/vmware/govmomi v0.20.1/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU=
-github.com/vmware/photon-controller-go-sdk v0.0.0-20170310013346-4a435daef6cc/go.mod h1:e6humHha1ekIwTCm+A5Qed5mG8V4JL+ChHcUOJ+L/8U=
-github.com/xanzy/go-cloudstack v0.0.0-20160728180336-1e2cbf647e57/go.mod h1:s3eL3z5pNXF5FVybcT+LIVdId8pYn709yv6v5mrkrQE=
+github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU=
 github.com/xeipuuv/gojsonpointer v0.0.0-20151027082146-e0fe6f683076 h1:KM4T3G70MiR+JtqplcYkNVoNz7pDwYaBxWBXQK804So=
 github.com/xeipuuv/gojsonpointer v0.0.0-20151027082146-e0fe6f683076/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xeipuuv/gojsonreference v0.0.0-20150808065054-e02fc20de94c h1:XZWnr3bsDQWAZg4Ne+cPoXRPILrNlPNQfxBuwLl43is=
 github.com/xeipuuv/gojsonreference v0.0.0-20150808065054-e02fc20de94c/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
 github.com/xeipuuv/gojsonschema v0.0.0-20160623135812-c539bca196be h1:sRGd3e18izj1hQgF1hSvDOA8RPPnA2t4p8YeLZ/GdBU=
 github.com/xeipuuv/gojsonschema v0.0.0-20160623135812-c539bca196be/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=
-github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSfTONNIgpN5RA8prR7fF8nkF6cTWTcNerRO8=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/zchee/go-vmnet v0.0.0-20161021174912-97ebf9174097 h1:Ucx5I1l1+TWXvqFmBigYu4Ub4MLvUuUU/whjoUvV95I=
 github.com/zchee/go-vmnet v0.0.0-20161021174912-97ebf9174097/go.mod h1:lFZSWRIpCfE/pt91hHBBpV6+x87YlCjsp+aIR2qCPPU=
-go.opencensus.io v0.18.0 h1:Mk5rgZcggtbvtAun5aJzAtjKKN/t0R3jJPlWILlv938=
-go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
+go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
+go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
+go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
+go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
-go.uber.org/atomic v0.0.0-20181018215023-8dc6146f7569/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
-go.uber.org/multierr v0.0.0-20180122172545-ddea229ff1df/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
-go.uber.org/zap v0.0.0-20180814183419-67bc79d13d15/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
-golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d h1:E2M5QgjZ/Jg+ObCQAudsXxuTsLj7Nl5RV/lZcQZmKSo=
-golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw=
+golang.org/x/build v0.0.0-20190927031335-2835ba2e683f h1:hXVePvSFG7tPGX4Pwk1d10ePFfoTCc0QmISfpKOHsS8=
+golang.org/x/build v0.0.0-20190927031335-2835ba2e683f/go.mod h1:fYw7AShPAhGMdXqA9gRadk/CcMsvLlClpE5oBwnS3dM=
+golang.org/x/crypto v0.0.0-20180426230345-b49d69b5da94/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20181025213731-e84da0312774/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190123085648-057139ce5d2b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190424203555-c05e17bb3b2d/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc=
-golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586 h1:7KByu05hhLed2MO29w7p1XfZvZ13m8mub3shuVftRs0=
+golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
-golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20181217174547-8f45f776aaf1/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20170915142106-8351a756f30f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181102091132-c10e9556a7bc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190206173232-65e2d4e15006/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190328230028-74de082e2cca/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190502183928-7f726cade0ab/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI=
+golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a h1:tImsplftrFpALCYumobsd0K86vlAs/eXGFms2txfJfA=
 golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -603,71 +755,94 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20171026204733-164713f0dfce/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181004145325-8469e314837c/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190122071731-054c452bb702/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190124100055-b90733256f2e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502175342-a43fa875dd82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k=
-golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg=
+golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.0.0-20170915090833-1cbadb444a80/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/time v0.0.0-20161028155119-f51c12702a4d h1:TnM+PKb3ylGmZvyPXmo9m/wktg7Jn/a/fNmr33HSj8g=
-golang.org/x/time v0.0.0-20161028155119-f51c12702a4d/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/tools v0.0.0-20170824195420-5d2fd3ccab98/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20170915040203-e531a2a1c15f/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190121143147-24cd39ecf745/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190122202912-9c309ee22fab/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190909030654-5b82db07426d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0=
 gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
 gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ=
-google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
-google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
-google.golang.org/api v0.0.0-20181220000619-583d854617af/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.6.1-0.20190607001116-5213b8090861/go.mod h1:btoxGiFvQNVUZQ8W08zLtrVS08CNpINPEfxXxgJL1Q4=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
 google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
 google.golang.org/api v0.9.0 h1:jbyannxz0XFD3zdjgrSUsaJbgpH4eTrkdhRChkHPfO8=
 google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
-google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I=
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
-google.golang.org/genproto v0.0.0-20170731182057-09f6ed296fc6/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
@@ -675,102 +850,116 @@ google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRn
 google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/grpc v1.13.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
-google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
-google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1 h1:j6XxA85m/6txkUCHvzlV5f+HBNl/1r5cZ2A/3IEFOO8=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.23.1 h1:q4XQuHFC6I28BKZpo6IYyb3mNO+l7lSOxRuYTCiDfXk=
+google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 gopkg.in/airbrake/gobrake.v2 v2.0.9 h1:7z2uVWwn7oVeeugY1DtlPAy5H+KYgB1KeKTnqjNatLo=
 gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
 gopkg.in/cheggaaa/pb.v1 v1.0.27 h1:kJdccidYzt3CaHD1crCFTS1hxyhSi059NhOFUf03YFo=
 gopkg.in/cheggaaa/pb.v1 v1.0.27/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/gcfg.v1 v1.2.0/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o=
 gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 h1:OAj3g0cR6Dx/R07QgQe8wkA9RNjB2u4i700xBkIT4e0=
 gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
-gopkg.in/inf.v0 v0.9.0 h1:3zYtXIO92bvsdS3ggAdA8Gb4Azj0YU+TVY1uGYNFA8o=
-gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+gopkg.in/mcuadros/go-syslog.v2 v2.2.1/go.mod h1:l5LPIyOOyIdQquNg+oU6Z3524YwrcqEm0aKH+5zpt2U=
 gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw=
 gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
-gopkg.in/square/go-jose.v2 v2.0.0-20180411045311-89060dee6a84/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
+gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/warnings.v0 v0.1.1/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
-gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg=
+gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gotest.tools v2.1.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
+gotest.tools/gotestsum v0.3.5/go.mod h1:Mnf3e5FUzXbkCfynWBGOwLssY7gTQgCHObK9tMpAriY=
 grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o=
-honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-k8s.io/gengo v0.0.0-20190116091435-f8a0810f38af/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.2/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+k8s.io/api v0.17.2 h1:NF1UFXcKN7/OOv1uxdRz3qfra8AHsPav5M93hlV9+Dc=
+k8s.io/api v0.17.2/go.mod h1:BS9fjjLc4CMuqfSO8vgbHPKMt5+SF0ET6u/RVDihTo4=
+k8s.io/apiextensions-apiserver v0.17.2/go.mod h1:4KdMpjkEjjDI2pPfBA15OscyNldHWdBCfsWMDWAmSTs=
+k8s.io/apimachinery v0.17.3-beta.0 h1:DeN0royOQ5+j3ytFWnDkxGiF5r9T7m6E9Ukzjg4vVHc=
+k8s.io/apimachinery v0.17.3-beta.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg=
+k8s.io/apiserver v0.17.2/go.mod h1:lBmw/TtQdtxvrTk0e2cgtOxHizXI+d0mmGQURIHQZlo=
+k8s.io/cli-runtime v0.17.2/go.mod h1:aa8t9ziyQdbkuizkNLAw3qe3srSyWh9zlSB7zTqRNPI=
+k8s.io/client-go v0.17.2 h1:ndIfkfXEGrNhLIgkr0+qhRguSD3u6DCmonepn1O6NYc=
+k8s.io/client-go v0.17.2/go.mod h1:QAzRgsa0C2xl4/eVpeVAZMvikCn8Nm81yqVx3Kk9XYI=
+k8s.io/cloud-provider v0.17.2/go.mod h1:9rEcGqEsUFHxC83oMUGBcsXTBpRVNVPX/U+nyQJTvHU=
+k8s.io/cluster-bootstrap v0.17.2 h1:KVjK1WviylwbBwC+3L51xKmGN3A+WmzW8rhtcfWdUqQ=
+k8s.io/cluster-bootstrap v0.17.2/go.mod h1:qiazpAM05fjAc+PEkrY8HSUhKlJSMBuLnVUSO6nvZL4=
+k8s.io/code-generator v0.17.3-beta.0/go.mod h1:DVmfPQgxQENqDIzVR2ddLXMH34qeszkKSdH/N+s+38s=
+k8s.io/component-base v0.17.2 h1:0XHf+cerTvL9I5Xwn9v+0jmqzGAZI7zNydv4tL6Cw6A=
+k8s.io/component-base v0.17.2/go.mod h1:zMPW3g5aH7cHJpKYQ/ZsGMcgbsA/VyhEugF3QT1awLs=
+k8s.io/cri-api v0.17.3-beta.0/go.mod h1:BzAkbBHHp81d+aXzbiIcUbilLkbXa40B8mUHOk6EX3s=
+k8s.io/csi-translation-lib v0.17.2/go.mod h1:NrhnhXJg/V6cHRTdPbmxvBuV3rJSqXsdLBE5JSRzcVI=
+k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
+k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/heapster v1.2.0-beta.1/go.mod h1:h1uhptVXMwC8xtZBYsPXKVi8fpdlYkTs6k949KozGrM=
-k8s.io/klog v0.3.1 h1:RVgyDHY/kFKtLqh67NvEWIgkMneNoIrdkN0CxDSQc68=
-k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
-k8s.io/klog v0.3.3 h1:niceAagH1tzskmaie/icWd7ci1wbG7Bf2c6YGcQv+3c=
-k8s.io/klog v0.3.3/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
-k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30 h1:TRb4wNWoBVrH9plmkp2q86FIDppkbrEXdXlxU3a3BMI=
-k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc=
-k8s.io/kubernetes v1.15.2 h1:RO9EuRw5vlN3oa/lnmPxmywOoJRtg9o40KcklHXNIAQ=
-k8s.io/kubernetes v1.15.2/go.mod h1:3RE5ikMc73WK+dSxk4pQuQ6ZaJcPXiZX2dj98RcdCuM=
-k8s.io/kubernetes/staging/src/k8s.io/api v0.0.0-20190623232353-8c3b7d7679cc h1:vZ5+77WP1yImZo23wc75vV5b5zCGq9gv484q8Yw5sBw=
-k8s.io/kubernetes/staging/src/k8s.io/api v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:pU9hbGZc8Z6+6HlNLEFY1GiNGzcCykU1Glsd4vEea2U=
-k8s.io/kubernetes/staging/src/k8s.io/apiextensions-apiserver v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:Q49J/iUBV6A9nn8loyV72DK2EXhN8sqCR8FyfxIFDA4=
-k8s.io/kubernetes/staging/src/k8s.io/apimachinery v0.0.0-20190623232353-8c3b7d7679cc h1:SHxaBZWgNouwsZCVg2+iffu0Um1ExSLPKgvO1drWShs=
-k8s.io/kubernetes/staging/src/k8s.io/apimachinery v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:rRBYbORqofLsn4/tsQWkeXkdKUoGrTfUwbI9s7NhU0Q=
-k8s.io/kubernetes/staging/src/k8s.io/apiserver v0.0.0-20190623232353-8c3b7d7679cc h1:2s4y/VeY+iATOZqZdNhr/sfPFL7+Tvu6wNl/u5WTBcI=
-k8s.io/kubernetes/staging/src/k8s.io/apiserver v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:0xlmg2F70Iaf7pdSDkfFQhWiyKh7eRqO0fBx4OvA4IA=
-k8s.io/kubernetes/staging/src/k8s.io/cli-runtime v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:YZQFNMZSNiRALChwxrRkzjVrmmJWNlSl4StwOETTIYI=
-k8s.io/kubernetes/staging/src/k8s.io/client-go v0.0.0-20190623232353-8c3b7d7679cc h1:NkZ6eBWe3aFn6wCHgMBPfcjVSji1Nz/EcwGpFQjcT0c=
-k8s.io/kubernetes/staging/src/k8s.io/client-go v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:O+HAJNvEkq6MA64VSkhBp2oKeuSVWtTgc9O3z7AgDfA=
-k8s.io/kubernetes/staging/src/k8s.io/cloud-provider v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:rQUOm/XLTGmz4tguTc3V1Ee8zCf5BZ2lCh7nv6dXTzA=
-k8s.io/kubernetes/staging/src/k8s.io/cluster-bootstrap v0.0.0-20190623232353-8c3b7d7679cc h1:RvTiGzEJ1/k72+BZMjwkGCF6gYMBxn0zeeiE8Yu6f/I=
-k8s.io/kubernetes/staging/src/k8s.io/cluster-bootstrap v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:8RJmS4bkGaaevegssxZX5h3eMJzW3smzE67LISltg+k=
-k8s.io/kubernetes/staging/src/k8s.io/code-generator v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:oFKBcitmDfPm+OpCoj4rfVSuHbWynjF3/L/sMV4tB0w=
-k8s.io/kubernetes/staging/src/k8s.io/component-base v0.0.0-20190623232353-8c3b7d7679cc h1:WLHw9cDk+CO4nOmbXsU4twzxlJymyKjncV96XDFwkck=
-k8s.io/kubernetes/staging/src/k8s.io/component-base v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:kUl6x7EkDsgIPSyB6dN6UIRqtofbZpoGp9RDYlqm1EY=
-k8s.io/kubernetes/staging/src/k8s.io/cri-api v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:YrwDlnLsMEkKgyPt/gJOHrXoCpYIdDpcikarkQ1hubo=
-k8s.io/kubernetes/staging/src/k8s.io/csi-translation-lib v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:5RWpGgZKzUcW9gCtmSVRq8maZkOetGv87HrohpTrnLI=
-k8s.io/kubernetes/staging/src/k8s.io/kube-aggregator v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:ogOX4l9UCMfFGIF+FZqmsln4NtCGPqf9zTMCIlm2YX4=
-k8s.io/kubernetes/staging/src/k8s.io/kube-controller-manager v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:o6aAFW1lCnr5CJm1riWnhQskrAHhyt8btyv5UHhgZ6c=
-k8s.io/kubernetes/staging/src/k8s.io/kube-proxy v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:y0hpsQGN8h3HcNqYbpSZEH4yC1ohi45N35c8ma9yg6M=
-k8s.io/kubernetes/staging/src/k8s.io/kube-scheduler v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:/hbCTKdfutEO2iTQv8NuYcnAxd8Tuu4mMEymYv/EZHk=
-k8s.io/kubernetes/staging/src/k8s.io/kubectl v0.0.0-20190623232353-8c3b7d7679cc h1:Bsljf/3UDy91qqLkevAiq6y+wl0qJrkLjWfBCQs9rss=
-k8s.io/kubernetes/staging/src/k8s.io/kubectl v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:Vg6Q3IDU3hfYMICKyb43lClOXWtCtOBh2o1FfuQw8mQ=
-k8s.io/kubernetes/staging/src/k8s.io/kubelet v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:9UInPlSttlDwZBFNMAsqhTtl7zH00dE2M88B9Z0Ennc=
-k8s.io/kubernetes/staging/src/k8s.io/legacy-cloud-providers v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:xlTRb77uaXbuT6evILwFescWPMENFKYGYj3a/kOjYQE=
-k8s.io/kubernetes/staging/src/k8s.io/metrics v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:6Cs3k9ccbWbJo3CQOrGDu2QEVLwsWbBlu9HitjPhuSk=
-k8s.io/kubernetes/staging/src/k8s.io/sample-apiserver v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:2LUaF4APkM/13Q3y9+NnAjBQrLPDy3IrJ0uKTS+zxP4=
-k8s.io/repo-infra v0.0.0-20181204233714-00fe14e3d1a3/go.mod h1:+G1xBfZDfVFsm1Tj/HNCvg4QqWx8rJ2Fxpqr1rqp/gQ=
-k8s.io/utils v0.0.0-20190221042446-c2654d5206da h1:ElyM7RPonbKnQqOcw7dG2IK5uvQQn3b/WPHqD5mBvP4=
-k8s.io/utils v0.0.0-20190221042446-c2654d5206da/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3oVhZz7sSzSiPnVZayjIX0=
+k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
+k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
+k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
+k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
+k8s.io/kube-aggregator v0.17.2/go.mod h1:8xQTzaH0GrcKPiSB4YYWwWbeQ0j/4zRsbQt8usEMbRg=
+k8s.io/kube-controller-manager v0.17.2/go.mod h1:xznSbCHdVODF5StxiBMh3s6HenyCBdsedazlsh6/J3M=
+k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a h1:UcxjrRMyNx/i/y8G7kPvLyy7rfbeuf1PYyBf973pgyU=
+k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E=
+k8s.io/kube-proxy v0.17.2 h1:DuZwJ9bdsANZ9DEa5dBkPU9KpR0HU4wFkDOAWQvOpIs=
+k8s.io/kube-proxy v0.17.2/go.mod h1:PVY+Cqds8qa/TLEqiSgDPgwWBiRHYjeS4kvp/C5dYjc=
+k8s.io/kube-scheduler v0.17.2/go.mod h1:BlP/p3YDLgsEIshEj4gbGjV11j4BQjNx7vbwRcLGnI8=
+k8s.io/kubectl v0.17.2 h1:QZR8Q6lWiVRjwKslekdbN5WPMp53dS/17j5e+oi5XVU=
+k8s.io/kubectl v0.17.2/go.mod h1:y4rfLV0n6aPmvbRCqZQjvOp3ezxsFgpqL+zF5jH/lxk=
+k8s.io/kubelet v0.17.2 h1:AggINdoeh58BjlM8oWy9XPLcWde8MIs44+/HJpyfPi0=
+k8s.io/kubelet v0.17.2/go.mod h1:XUOu5Fcnkx44FP13w5etBrn2GhK4D02CUcFA8tLtUKU=
+k8s.io/kubernetes v1.17.2 h1:g1UFZqFQsYx88xMUks4PKC6tsNcekxe0v06fcVGRwVE=
+k8s.io/kubernetes v1.17.2/go.mod h1:NbNV+69yL3eKiKDJ+ZEjqOplN3BFXKBeunzkoOy8WLo=
+k8s.io/legacy-cloud-providers v0.17.2/go.mod h1:a/qbE67VbTzWOemWfqH0wlcX31zxt4UOxFqZuBltY9Q=
+k8s.io/metrics v0.17.2/go.mod h1:3TkNHET4ROd+NfzNxkjoVfQ0Ob4iZnaHmSEA4vYpwLw=
+k8s.io/repo-infra v0.0.1-alpha.1/go.mod h1:wO1t9WaB99V80ljbeENTnayuEEwNZt7gECYh/CEyOJ8=
+k8s.io/sample-apiserver v0.17.2/go.mod h1:JLhi1DSBMlKAckfVdV1YNuz49EKNlfSflW+6LVDf1Mo=
+k8s.io/system-validators v1.0.4/go.mod h1:HgSgTg4NAGNoYYjKsUyk52gdNi2PVDswQ9Iyn66R7NI=
+k8s.io/utils v0.0.0-20191114184206-e782cd3c129f h1:GiPwtSzdP43eI1hpPCbROQCCIgCuiMMNF8YUVLF3vJo=
+k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
+k8s.io/utils v0.0.0-20200122174043-1e243dd1a584 h1:3tT5mBZNurtd5BoYrPBII3Sa8n7T2w405qdTQvr3vmY=
+k8s.io/utils v0.0.0-20200122174043-1e243dd1a584/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
 modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw=
 modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk=
 modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k=
 modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs=
 modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I=
+mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc=
+mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4=
+mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34/go.mod h1:H6SUd1XjIs+qQCyskXg5OFSrilMRUkD8ePJpHKDPaeY=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU=
 sigs.k8s.io/sig-storage-lib-external-provisioner v4.0.0+incompatible h1:qV3eFdgCp7Cp/ORjkJI9VBBEOntT+z385jLqdBtmgHA=
 sigs.k8s.io/sig-storage-lib-external-provisioner v4.0.0+incompatible/go.mod h1:qhqLyNwJC49PoUalmtzYb4s9fT8HOMBTLbTY1QoVOqI=
-sigs.k8s.io/structured-merge-diff v0.0.0-20190302045857-e85c7b244fd2/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI=
+sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI=
+sigs.k8s.io/structured-merge-diff v1.0.1-0.20191108220359-b1b620dd3f06/go.mod h1:/ULNhyfzRopfcjskuui0cTITekDduZ7ycKN3oUT9R18=
 sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
-sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck=
 sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=
 vbom.ml/util v0.0.0-20160121211510-db5cfe13f5cc/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj6uzU2+8OWDFv/HxUSs7kI=
diff --git a/pkg/drivers/none/none.go b/pkg/drivers/none/none.go
index 4d99fd8b8e..bbf3ab5971 100644
--- a/pkg/drivers/none/none.go
+++ b/pkg/drivers/none/none.go
@@ -94,7 +94,7 @@ func (d *Driver) DriverName() string {
 
 // GetIP returns an IP or hostname that this host is available at
 func (d *Driver) GetIP() (string, error) {
-	ip, err := net.ChooseBindAddress(nil)
+	ip, err := net.ChooseHostInterface()
 	if err != nil {
 		return "", err
 	}

From 1c30a2af8ed33042edacd964243fb457b894cfea Mon Sep 17 00:00:00 2001
From: tstromberg <tstromberg@google.com>
Date: Wed, 22 Jan 2020 14:13:05 -0800
Subject: [PATCH 090/250] ISO_VERSION should use VERSION_BUILD during beta

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index e3c9831d01..fb9e39a600 100755
--- a/Makefile
+++ b/Makefile
@@ -20,7 +20,7 @@ RAW_VERSION=$(VERSION_MAJOR).$(VERSION_MINOR).${VERSION_BUILD}
 VERSION ?= v$(RAW_VERSION)
 
 # Default to .0 for higher cache hit rates, as build increments typically don't require new ISO versions
-ISO_VERSION ?= v$(VERSION_MAJOR).$(VERSION_MINOR).0-beta.0
+ISO_VERSION ?= v$(VERSION_MAJOR).$(VERSION_MINOR).$(VERSION_BUILD)
 # Dashes are valid in semver, but not Linux packaging. Use ~ to delimit alpha/beta
 DEB_VERSION ?= $(subst -,~,$(RAW_VERSION))
 RPM_VERSION ?= $(DEB_VERSION)

From 7247a7dee614a563785c7cfcec602f6ddc21ba9c Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Wed, 22 Jan 2020 14:18:20 -0800
Subject: [PATCH 091/250] Switched to using kubectl apply and kubectl delete
 for addons

kubectl apply with --prune wasn't working because it requires the
--namespace flag to be passed in and we have multiple namespaces for
addons.

We tried to split up addons within /etc/kubernetes/addons by
namespace, but this didn't work when disabling dashboard addon because
there were no files left in the kubernetes-dashboard subdirectory. So,
kubectl apply complained when trying to prune dashboard because no files
were being passed in.

So, I ended up removing all dependencies on prune and instead running
`kubectl apply` when enabling an addon and running `kubectl delete` when
disabling an addon.
---
 pkg/addons/addons.go                          |  28 ++--
 pkg/addons/kubectl.go                         |  68 +++++++++
 .../{reconcile_test.go => kubectl_test.go}    |   0
 pkg/addons/reconcile.go                       | 141 ------------------
 pkg/minikube/assets/addons.go                 |   5 +-
 5 files changed, 89 insertions(+), 153 deletions(-)
 create mode 100644 pkg/addons/kubectl.go
 rename pkg/addons/{reconcile_test.go => kubectl_test.go} (100%)
 delete mode 100644 pkg/addons/reconcile.go

diff --git a/pkg/addons/addons.go b/pkg/addons/addons.go
index 184d7952ff..1380c6bf29 100644
--- a/pkg/addons/addons.go
+++ b/pkg/addons/addons.go
@@ -19,6 +19,7 @@ package addons
 import (
 	"fmt"
 	"os"
+	"path/filepath"
 	"strconv"
 
 	"github.com/pkg/errors"
@@ -174,15 +175,10 @@ func isAddonAlreadySet(addon *assets.Addon, enable bool) (bool, error) {
 }
 
 func enableOrDisableAddonInternal(addon *assets.Addon, cmd command.Runner, data interface{}, enable bool, profile string) error {
-	var err error
-
-	updateFile := cmd.Copy
-	if !enable {
-		updateFile = cmd.Remove
-	}
-
+	files := []string{}
 	for _, addon := range addon.Assets {
 		var addonFile assets.CopyableFile
+		var err error
 		if addon.IsTemplate() {
 			addonFile, err = addon.Evaluate(data)
 			if err != nil {
@@ -192,11 +188,23 @@ func enableOrDisableAddonInternal(addon *assets.Addon, cmd command.Runner, data
 		} else {
 			addonFile = addon
 		}
-		if err := updateFile(addonFile); err != nil {
-			return errors.Wrapf(err, "updating addon %s", addon.AssetName)
+		if enable {
+			if err := cmd.Copy(addonFile); err != nil {
+				return err
+			}
+		} else {
+			defer cmd.Remove(addonFile)
 		}
+		files = append(files, filepath.Join(addonFile.GetTargetDir(), addonFile.GetTargetName()))
 	}
-	return reconcile(cmd, profile)
+	command, err := kubectlCommand(profile, files, enable)
+	if err != nil {
+		return err
+	}
+	if result, err := cmd.RunCmd(command); err != nil {
+		return errors.Wrapf(err, "error enabling addon:\n%s", result.Output())
+	}
+	return nil
 }
 
 // enableOrDisableStorageClasses enables or disables storage classes
diff --git a/pkg/addons/kubectl.go b/pkg/addons/kubectl.go
new file mode 100644
index 0000000000..5cc96383f8
--- /dev/null
+++ b/pkg/addons/kubectl.go
@@ -0,0 +1,68 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package addons
+
+import (
+	"os"
+	"os/exec"
+	"path"
+
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/constants"
+)
+
+var (
+	// For testing
+	k8sVersion = kubernetesVersion
+)
+
+func kubectlCommand(profile string, files []string, enable bool) (*exec.Cmd, error) {
+	v, err := k8sVersion(profile)
+	if err != nil {
+		return nil, err
+	}
+	kubectlBinary := kubectlBinaryPath(v)
+
+	kubectlAction := "apply"
+	if !enable {
+		kubectlAction = "delete"
+	}
+
+	args := []string{"KUBECONFIG=/var/lib/minikube/kubeconfig", kubectlBinary, kubectlAction}
+	for _, f := range files {
+		args = append(args, []string{"-f", f}...)
+	}
+
+	cmd := exec.Command("sudo", args...)
+	return cmd, nil
+}
+
+func kubernetesVersion(profile string) (string, error) {
+	cc, err := config.Load(profile)
+	if err != nil && !os.IsNotExist(err) {
+		return "", err
+	}
+	version := constants.DefaultKubernetesVersion
+	if cc != nil {
+		version = cc.KubernetesConfig.KubernetesVersion
+	}
+	return version, nil
+}
+
+func kubectlBinaryPath(version string) string {
+	return path.Join("/var/lib/minikube/binaries", version, "kubectl")
+}
diff --git a/pkg/addons/reconcile_test.go b/pkg/addons/kubectl_test.go
similarity index 100%
rename from pkg/addons/reconcile_test.go
rename to pkg/addons/kubectl_test.go
diff --git a/pkg/addons/reconcile.go b/pkg/addons/reconcile.go
deleted file mode 100644
index e4096d71ff..0000000000
--- a/pkg/addons/reconcile.go
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
-Copyright 2016 The Kubernetes Authors All rights reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package addons
-
-import (
-	"fmt"
-	"io/ioutil"
-	"os"
-	"os/exec"
-	"path"
-
-	"github.com/blang/semver"
-	"github.com/golang/glog"
-	"github.com/pkg/errors"
-	"k8s.io/minikube/pkg/minikube/assets"
-	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil"
-	"k8s.io/minikube/pkg/minikube/command"
-	"k8s.io/minikube/pkg/minikube/config"
-	"k8s.io/minikube/pkg/minikube/constants"
-	"k8s.io/minikube/pkg/minikube/vmpath"
-)
-
-var (
-	// For testing
-	k8sVersion = kubernetesVersion
-)
-
-// taken from https://github.com/kubernetes/kubernetes/blob/master/cluster/addons/addon-manager/kube-addons.sh
-var kubectlPruneWhitelist = []string{
-	"core/v1/ConfigMap",
-	"core/v1/Endpoints",
-	"core/v1/Namespace",
-	"core/v1/PersistentVolumeClaim",
-	"core/v1/PersistentVolume",
-	"core/v1/Pod",
-	"core/v1/ReplicationController",
-	"core/v1/Secret",
-	"core/v1/Service",
-	"batch/v1/Job",
-	"batch/v1beta1/CronJob",
-	"apps/v1/DaemonSet",
-	"apps/v1/Deployment",
-	"apps/v1/ReplicaSet",
-	"apps/v1/StatefulSet",
-	"extensions/v1beta1/Ingress",
-}
-
-// reconcile runs kubectl apply -f on the addons directory
-// to reconcile addons state in all running profiles
-func reconcile(cmd command.Runner, profile string) error {
-	// Walk through all subdirs in /etc/kubernetes/addons and reconcile addons
-	dirs, err := ioutil.ReadDir(vmpath.GuestAddonsDir)
-	if err != nil {
-		return err
-	}
-
-	for _, d := range dirs {
-		if !d.IsDir() {
-			continue
-		}
-		namespace := d.Name()
-		glog.Infof("Reconciling addons in namespace %s", namespace)
-		c, err := kubectlCommand(profile, namespace)
-		if err != nil {
-			return err
-		}
-		if _, err := cmd.RunCmd(c); err != nil {
-			glog.Warningf("reconciling addons failed: %v", err)
-			return err
-		}
-	}
-
-	return nil
-}
-
-func kubectlCommand(profile, namespace string) (*exec.Cmd, error) {
-	v, err := k8sVersion(profile)
-	if err != nil {
-		return nil, err
-	}
-	kubectlBinary := kubectlBinaryPath(v)
-
-	// prune will delete any existing objects with the label specified by "-l" which don't appear in /etc/kubernetes/addons
-	// this is how we delete disabled addons
-	args := []string{"KUBECONFIG=/var/lib/minikube/kubeconfig", kubectlBinary, "apply", "-f", assets.TargetDirForAddon(namespace), "-l", "kubernetes.io/cluster-service!=true,addonmanager.kubernetes.io/mode=Reconcile", "--prune=true"}
-	for _, k := range kubectlPruneWhitelist {
-		args = append(args, []string{"--prune-whitelist", k}...)
-	}
-	args = append(args, "--recursive")
-
-	ok, err := shouldAppendNamespaceFlag(v)
-	if err != nil {
-		return nil, errors.Wrap(err, "appending namespace flag")
-	}
-	if ok {
-		args = append(args, fmt.Sprintf("--namespace=%s", namespace))
-	}
-
-	cmd := exec.Command("sudo", args...)
-	return cmd, nil
-}
-
-func kubernetesVersion(profile string) (string, error) {
-	cc, err := config.Load(profile)
-	if err != nil && !os.IsNotExist(err) {
-		return "", err
-	}
-	version := constants.DefaultKubernetesVersion
-	if cc != nil {
-		version = cc.KubernetesConfig.KubernetesVersion
-	}
-	return version, nil
-}
-
-// We need to append --namespace=kube-system for Kubernetes versions >=1.17
-// so that prune works as expected. See https://github.com/kubernetes/kubernetes/pull/83084/
-func shouldAppendNamespaceFlag(version string) (bool, error) {
-	v, err := bsutil.ParseKubernetesVersion(version)
-	if err != nil {
-		return false, err
-	}
-	return v.GTE(semver.MustParse("1.17.0")), nil
-}
-
-func kubectlBinaryPath(version string) string {
-	return path.Join("/var/lib/minikube/binaries", version, "kubectl")
-}
diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go
index 7be028bf8f..90eff0096a 100644
--- a/pkg/minikube/assets/addons.go
+++ b/pkg/minikube/assets/addons.go
@@ -66,11 +66,11 @@ func (a *Addon) IsEnabled() (bool, error) {
 // TODO: Make dynamically loadable: move this data to a .yaml file within each addon directory
 var Addons = map[string]*Addon{
 	"dashboard": NewAddon([]*BinAsset{
+		MustBinAsset("deploy/addons/dashboard/dashboard-ns.yaml", TargetDirForAddon("kubernetes-dashboard"), "dashboard-ns.yaml", "0640", false),
 		MustBinAsset("deploy/addons/dashboard/dashboard-clusterrole.yaml", TargetDirForAddon("kubernetes-dashboard"), "dashboard-clusterrole.yaml", "0640", false),
 		MustBinAsset("deploy/addons/dashboard/dashboard-clusterrolebinding.yaml", TargetDirForAddon("kubernetes-dashboard"), "dashboard-clusterrolebinding.yaml", "0640", false),
 		MustBinAsset("deploy/addons/dashboard/dashboard-configmap.yaml", TargetDirForAddon("kubernetes-dashboard"), "dashboard-configmap.yaml", "0640", false),
 		MustBinAsset("deploy/addons/dashboard/dashboard-dp.yaml", TargetDirForAddon("kubernetes-dashboard"), "dashboard-dp.yaml", "0640", false),
-		MustBinAsset("deploy/addons/dashboard/dashboard-ns.yaml", TargetDirForAddon("kubernetes-dashboard"), "dashboard-ns.yaml", "0640", false),
 		MustBinAsset("deploy/addons/dashboard/dashboard-role.yaml", TargetDirForAddon("kubernetes-dashboard"), "dashboard-role.yaml", "0640", false),
 		MustBinAsset("deploy/addons/dashboard/dashboard-rolebinding.yaml", TargetDirForAddon("kubernetes-dashboard"), "dashboard-rolebinding.yaml", "0640", false),
 		MustBinAsset("deploy/addons/dashboard/dashboard-sa.yaml", TargetDirForAddon("kubernetes-dashboard"), "dashboard-sa.yaml", "0640", false),
@@ -331,7 +331,8 @@ var Addons = map[string]*Addon{
 
 // TargetDirForAddon appends the namespace as a directory to /etc/kubernetes/addons
 func TargetDirForAddon(namespace string) string {
-	return filepath.Join(vmpath.GuestAddonsDir, namespace)
+	return vmpath.GuestAddonsDir
+	// return filepath.Join(vmpath.GuestAddonsDir, namespace)
 }
 
 // AddMinikubeDirAssets adds all addons and files to the list

From 62a72f44e89fb08b9032b1ec3b1cf361212a73dd Mon Sep 17 00:00:00 2001
From: tstromberg <tstromberg@google.com>
Date: Wed, 22 Jan 2020 14:21:34 -0800
Subject: [PATCH 092/250] Address feedback

---
 cmd/minikube/cmd/status.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/minikube/cmd/status.go b/cmd/minikube/cmd/status.go
index 171cf82e5a..471986c31f 100644
--- a/cmd/minikube/cmd/status.go
+++ b/cmd/minikube/cmd/status.go
@@ -48,7 +48,7 @@ const (
 	Misconfigured = "Misconfigured" // analagous to state.Error
 )
 
-// Status holds string representations of libmachine.state.State
+// Status holds string representations of component states
 type Status struct {
 	Host       string
 	Kubelet    string

From 15a52098414b0f9f00324770727e3c7935ab786e Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Wed, 22 Jan 2020 14:23:32 -0800
Subject: [PATCH 093/250] Update unit tests

---
 pkg/addons/kubectl_test.go    |  34 ++++-----
 pkg/minikube/assets/addons.go | 137 +++++++++++++++++-----------------
 2 files changed, 81 insertions(+), 90 deletions(-)

diff --git a/pkg/addons/kubectl_test.go b/pkg/addons/kubectl_test.go
index d53e87e698..493ff35412 100644
--- a/pkg/addons/kubectl_test.go
+++ b/pkg/addons/kubectl_test.go
@@ -17,31 +17,27 @@ limitations under the License.
 package addons
 
 import (
-	"fmt"
 	"strings"
 	"testing"
 )
 
 func TestKubectlCommand(t *testing.T) {
-	expectedCommand := "sudo KUBECONFIG=/var/lib/minikube/kubeconfig /var/lib/minikube/binaries/%s/kubectl apply -f /etc/kubernetes/addons -l kubernetes.io/cluster-service!=true,addonmanager.kubernetes.io/mode=Reconcile --prune=true --prune-whitelist core/v1/ConfigMap --prune-whitelist core/v1/Endpoints --prune-whitelist core/v1/Namespace --prune-whitelist core/v1/PersistentVolumeClaim --prune-whitelist core/v1/PersistentVolume --prune-whitelist core/v1/Pod --prune-whitelist core/v1/ReplicationController --prune-whitelist core/v1/Secret --prune-whitelist core/v1/Service --prune-whitelist batch/v1/Job --prune-whitelist batch/v1beta1/CronJob --prune-whitelist apps/v1/DaemonSet --prune-whitelist apps/v1/Deployment --prune-whitelist apps/v1/ReplicaSet --prune-whitelist apps/v1/StatefulSet --prune-whitelist extensions/v1beta1/Ingress --recursive"
-
 	tests := []struct {
 		description string
-		k8sVersion  string
+		files       []string
+		enable      bool
 		expected    string
 	}{
 		{
-			description: "k8s version < 1.17.0",
-			k8sVersion:  "v1.16.0",
-			expected:    expectedCommand,
+			description: "enable an addon",
+			files:       []string{"a", "b"},
+			enable:      true,
+			expected:    "sudo KUBECONFIG=/var/lib/minikube/kubeconfig /var/lib/minikube/binaries/v1.17.0/kubectl apply -f a -f b",
 		}, {
-			description: "k8s version == 1.17.0",
-			k8sVersion:  "v1.17.0",
-			expected:    expectedCommand + " --namespace=kube-system",
-		}, {
-			description: "k8s version > 1.17.0",
-			k8sVersion:  "v1.18.0",
-			expected:    expectedCommand + " --namespace=kube-system",
+			description: "disable an addon",
+			files:       []string{"a", "b"},
+			enable:      false,
+			expected:    "sudo KUBECONFIG=/var/lib/minikube/kubeconfig /var/lib/minikube/binaries/v1.17.0/kubectl delete -f a -f b",
 		},
 	}
 
@@ -50,19 +46,17 @@ func TestKubectlCommand(t *testing.T) {
 			originalK8sVersion := k8sVersion
 			defer func() { k8sVersion = originalK8sVersion }()
 			k8sVersion = func(_ string) (string, error) {
-				return test.k8sVersion, nil
+				return "v1.17.0", nil
 			}
 
-			command, err := kubectlCommand("")
+			command, err := kubectlCommand("", test.files, test.enable)
 			if err != nil {
 				t.Fatalf("error getting kubectl command: %v", err)
 			}
 			actual := strings.Join(command.Args, " ")
 
-			expected := fmt.Sprintf(test.expected, test.k8sVersion)
-
-			if actual != expected {
-				t.Fatalf("expected does not match actual\nExpected: %s\nActual: %s", expected, actual)
+			if actual != test.expected {
+				t.Fatalf("expected does not match actual\nExpected: %s\nActual: %s", test.expected, actual)
 			}
 		})
 	}
diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go
index 90eff0096a..1c8a0e0773 100644
--- a/pkg/minikube/assets/addons.go
+++ b/pkg/minikube/assets/addons.go
@@ -1,9 +1,12 @@
 /*
 Copyright 2016 The Kubernetes Authors All rights reserved.
+
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
+
     http://www.apache.org/licenses/LICENSE-2.0
+
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -37,7 +40,7 @@ type Addon struct {
 }
 
 // NewAddon creates a new Addon
-func NewAddon(assets []*BinAsset, enabled bool, addonName, namespace string) *Addon {
+func NewAddon(assets []*BinAsset, enabled bool, addonName string) *Addon {
 	a := &Addon{
 		Assets:    assets,
 		enabled:   enabled,
@@ -66,229 +69,229 @@ func (a *Addon) IsEnabled() (bool, error) {
 // TODO: Make dynamically loadable: move this data to a .yaml file within each addon directory
 var Addons = map[string]*Addon{
 	"dashboard": NewAddon([]*BinAsset{
-		MustBinAsset("deploy/addons/dashboard/dashboard-ns.yaml", TargetDirForAddon("kubernetes-dashboard"), "dashboard-ns.yaml", "0640", false),
-		MustBinAsset("deploy/addons/dashboard/dashboard-clusterrole.yaml", TargetDirForAddon("kubernetes-dashboard"), "dashboard-clusterrole.yaml", "0640", false),
-		MustBinAsset("deploy/addons/dashboard/dashboard-clusterrolebinding.yaml", TargetDirForAddon("kubernetes-dashboard"), "dashboard-clusterrolebinding.yaml", "0640", false),
-		MustBinAsset("deploy/addons/dashboard/dashboard-configmap.yaml", TargetDirForAddon("kubernetes-dashboard"), "dashboard-configmap.yaml", "0640", false),
-		MustBinAsset("deploy/addons/dashboard/dashboard-dp.yaml", TargetDirForAddon("kubernetes-dashboard"), "dashboard-dp.yaml", "0640", false),
-		MustBinAsset("deploy/addons/dashboard/dashboard-role.yaml", TargetDirForAddon("kubernetes-dashboard"), "dashboard-role.yaml", "0640", false),
-		MustBinAsset("deploy/addons/dashboard/dashboard-rolebinding.yaml", TargetDirForAddon("kubernetes-dashboard"), "dashboard-rolebinding.yaml", "0640", false),
-		MustBinAsset("deploy/addons/dashboard/dashboard-sa.yaml", TargetDirForAddon("kubernetes-dashboard"), "dashboard-sa.yaml", "0640", false),
-		MustBinAsset("deploy/addons/dashboard/dashboard-secret.yaml", TargetDirForAddon("kubernetes-dashboard"), "dashboard-secret.yaml", "0640", false),
-		MustBinAsset("deploy/addons/dashboard/dashboard-svc.yaml", TargetDirForAddon("kubernetes-dashboard"), "dashboard-svc.yaml", "0640", false),
-	}, false, "dashboard", "kubernetes-dashboard"),
+		MustBinAsset("deploy/addons/dashboard/dashboard-clusterrole.yaml", vmpath.GuestAddonsDir, "dashboard-clusterrole.yaml", "0640", false),
+		MustBinAsset("deploy/addons/dashboard/dashboard-clusterrolebinding.yaml", vmpath.GuestAddonsDir, "dashboard-clusterrolebinding.yaml", "0640", false),
+		MustBinAsset("deploy/addons/dashboard/dashboard-configmap.yaml", vmpath.GuestAddonsDir, "dashboard-configmap.yaml", "0640", false),
+		MustBinAsset("deploy/addons/dashboard/dashboard-dp.yaml", vmpath.GuestAddonsDir, "dashboard-dp.yaml", "0640", false),
+		MustBinAsset("deploy/addons/dashboard/dashboard-ns.yaml", vmpath.GuestAddonsDir, "dashboard-ns.yaml", "0640", false),
+		MustBinAsset("deploy/addons/dashboard/dashboard-role.yaml", vmpath.GuestAddonsDir, "dashboard-role.yaml", "0640", false),
+		MustBinAsset("deploy/addons/dashboard/dashboard-rolebinding.yaml", vmpath.GuestAddonsDir, "dashboard-rolebinding.yaml", "0640", false),
+		MustBinAsset("deploy/addons/dashboard/dashboard-sa.yaml", vmpath.GuestAddonsDir, "dashboard-sa.yaml", "0640", false),
+		MustBinAsset("deploy/addons/dashboard/dashboard-secret.yaml", vmpath.GuestAddonsDir, "dashboard-secret.yaml", "0640", false),
+		MustBinAsset("deploy/addons/dashboard/dashboard-svc.yaml", vmpath.GuestAddonsDir, "dashboard-svc.yaml", "0640", false),
+	}, false, "dashboard"),
 	"default-storageclass": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/storageclass/storageclass.yaml.tmpl",
-			TargetDirForAddon("kube-system"),
+			vmpath.GuestAddonsDir,
 			"storageclass.yaml",
 			"0640",
 			false),
-	}, true, "default-storageclass", "kube-system"),
+	}, true, "default-storageclass"),
 	"storage-provisioner": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/storage-provisioner/storage-provisioner.yaml.tmpl",
-			TargetDirForAddon("kube-system"),
+			vmpath.GuestAddonsDir,
 			"storage-provisioner.yaml",
 			"0640",
 			true),
-	}, true, "storage-provisioner", "kube-system"),
+	}, true, "storage-provisioner"),
 	"storage-provisioner-gluster": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/storage-provisioner-gluster/storage-gluster-ns.yaml.tmpl",
-			TargetDirForAddon("kube-system"),
+			vmpath.GuestAddonsDir,
 			"storage-gluster-ns.yaml",
 			"0640",
 			false),
 		MustBinAsset(
 			"deploy/addons/storage-provisioner-gluster/glusterfs-daemonset.yaml.tmpl",
-			TargetDirForAddon("kube-system"),
+			vmpath.GuestAddonsDir,
 			"glusterfs-daemonset.yaml",
 			"0640",
 			false),
 		MustBinAsset(
 			"deploy/addons/storage-provisioner-gluster/heketi-deployment.yaml.tmpl",
-			TargetDirForAddon("kube-system"),
+			vmpath.GuestAddonsDir,
 			"heketi-deployment.yaml",
 			"0640",
 			false),
 		MustBinAsset(
 			"deploy/addons/storage-provisioner-gluster/storage-provisioner-glusterfile.yaml.tmpl",
-			TargetDirForAddon("kube-system"),
+			vmpath.GuestAddonsDir,
 			"storage-privisioner-glusterfile.yaml",
 			"0640",
 			false),
-	}, false, "storage-provisioner-gluster", "kube-system"),
+	}, false, "storage-provisioner-gluster"),
 	"efk": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/efk/elasticsearch-rc.yaml.tmpl",
-			TargetDirForAddon("kube-system"),
+			vmpath.GuestAddonsDir,
 			"elasticsearch-rc.yaml",
 			"0640",
 			true),
 		MustBinAsset(
 			"deploy/addons/efk/elasticsearch-svc.yaml.tmpl",
-			TargetDirForAddon("kube-system"),
+			vmpath.GuestAddonsDir,
 			"elasticsearch-svc.yaml",
 			"0640",
 			false),
 		MustBinAsset(
 			"deploy/addons/efk/fluentd-es-rc.yaml.tmpl",
-			TargetDirForAddon("kube-system"),
+			vmpath.GuestAddonsDir,
 			"fluentd-es-rc.yaml",
 			"0640",
 			true),
 		MustBinAsset(
 			"deploy/addons/efk/fluentd-es-configmap.yaml.tmpl",
-			TargetDirForAddon("kube-system"),
+			vmpath.GuestAddonsDir,
 			"fluentd-es-configmap.yaml",
 			"0640",
 			false),
 		MustBinAsset(
 			"deploy/addons/efk/kibana-rc.yaml.tmpl",
-			TargetDirForAddon("kube-system"),
+			vmpath.GuestAddonsDir,
 			"kibana-rc.yaml",
 			"0640",
 			false),
 		MustBinAsset(
 			"deploy/addons/efk/kibana-svc.yaml.tmpl",
-			TargetDirForAddon("kube-system"),
+			vmpath.GuestAddonsDir,
 			"kibana-svc.yaml",
 			"0640",
 			false),
-	}, false, "efk", "kube-system"),
+	}, false, "efk"),
 	"ingress": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/ingress/ingress-configmap.yaml.tmpl",
-			TargetDirForAddon("kube-system"),
+			vmpath.GuestAddonsDir,
 			"ingress-configmap.yaml",
 			"0640",
 			false),
 		MustBinAsset(
 			"deploy/addons/ingress/ingress-rbac.yaml.tmpl",
-			TargetDirForAddon("kube-system"),
+			vmpath.GuestAddonsDir,
 			"ingress-rbac.yaml",
 			"0640",
 			false),
 		MustBinAsset(
 			"deploy/addons/ingress/ingress-dp.yaml.tmpl",
-			TargetDirForAddon("kube-system"),
+			vmpath.GuestAddonsDir,
 			"ingress-dp.yaml",
 			"0640",
 			true),
-	}, false, "ingress", "kube-system"),
+	}, false, "ingress"),
 	"istio-provisioner": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/istio-provisioner/istio-operator.yaml.tmpl",
-			TargetDirForAddon("istio-operator"),
+			vmpath.GuestAddonsDir,
 			"istio-operator.yaml",
 			"0640",
 			true),
-	}, false, "istio-provisioner", "istio-operator"),
+	}, false, "istio-provisioner"),
 	"istio": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/istio/istio-default-profile.yaml.tmpl",
-			TargetDirForAddon("istio-operator"),
+			vmpath.GuestAddonsDir,
 			"istio-default-profile.yaml",
 			"0640",
 			false),
-	}, false, "istio", "istio-operator"),
+	}, false, "istio"),
 	"metrics-server": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/metrics-server/metrics-apiservice.yaml.tmpl",
-			TargetDirForAddon("kube-system"),
+			vmpath.GuestAddonsDir,
 			"metrics-apiservice.yaml",
 			"0640",
 			false),
 		MustBinAsset(
 			"deploy/addons/metrics-server/metrics-server-deployment.yaml.tmpl",
-			TargetDirForAddon("kube-system"),
+			vmpath.GuestAddonsDir,
 			"metrics-server-deployment.yaml",
 			"0640",
 			true),
 		MustBinAsset(
 			"deploy/addons/metrics-server/metrics-server-service.yaml.tmpl",
-			TargetDirForAddon("kube-system"),
+			vmpath.GuestAddonsDir,
 			"metrics-server-service.yaml",
 			"0640",
 			false),
-	}, false, "metrics-server", "kube-system"),
+	}, false, "metrics-server"),
 	"registry": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/registry/registry-rc.yaml.tmpl",
-			TargetDirForAddon("kube-system"),
+			vmpath.GuestAddonsDir,
 			"registry-rc.yaml",
 			"0640",
 			false),
 		MustBinAsset(
 			"deploy/addons/registry/registry-svc.yaml.tmpl",
-			TargetDirForAddon("kube-system"),
+			vmpath.GuestAddonsDir,
 			"registry-svc.yaml",
 			"0640",
 			false),
 		MustBinAsset(
 			"deploy/addons/registry/registry-proxy.yaml.tmpl",
-			TargetDirForAddon("kube-system"),
+			vmpath.GuestAddonsDir,
 			"registry-proxy.yaml",
 			"0640",
 			false),
-	}, false, "registry", "kube-system"),
+	}, false, "registry"),
 	"registry-creds": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/registry-creds/registry-creds-rc.yaml.tmpl",
-			TargetDirForAddon("kube-system"),
+			vmpath.GuestAddonsDir,
 			"registry-creds-rc.yaml",
 			"0640",
 			false),
-	}, false, "registry-creds", "kube-system"),
+	}, false, "registry-creds"),
 	"freshpod": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/freshpod/freshpod-rc.yaml.tmpl",
-			TargetDirForAddon("kube-system"),
+			vmpath.GuestAddonsDir,
 			"freshpod-rc.yaml",
 			"0640",
 			true),
-	}, false, "freshpod", "kube-system"),
+	}, false, "freshpod"),
 	"nvidia-driver-installer": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/gpu/nvidia-driver-installer.yaml.tmpl",
-			TargetDirForAddon("kube-system"),
+			vmpath.GuestAddonsDir,
 			"nvidia-driver-installer.yaml",
 			"0640",
 			true),
-	}, false, "nvidia-driver-installer", "kube-system"),
+	}, false, "nvidia-driver-installer"),
 	"nvidia-gpu-device-plugin": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/gpu/nvidia-gpu-device-plugin.yaml.tmpl",
-			TargetDirForAddon("kube-system"),
+			vmpath.GuestAddonsDir,
 			"nvidia-gpu-device-plugin.yaml",
 			"0640",
 			true),
-	}, false, "nvidia-gpu-device-plugin", "kube-system"),
+	}, false, "nvidia-gpu-device-plugin"),
 	"logviewer": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/logviewer/logviewer-dp-and-svc.yaml.tmpl",
-			TargetDirForAddon("kube-system"),
+			vmpath.GuestAddonsDir,
 			"logviewer-dp-and-svc.yaml",
 			"0640",
 			false),
 		MustBinAsset(
 			"deploy/addons/logviewer/logviewer-rbac.yaml.tmpl",
-			TargetDirForAddon("kube-system"),
+			vmpath.GuestAddonsDir,
 			"logviewer-rbac.yaml",
 			"0640",
 			false),
-	}, false, "logviewer", "kube-system"),
+	}, false, "logviewer"),
 	"gvisor": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/gvisor/gvisor-pod.yaml.tmpl",
-			TargetDirForAddon("kube-system"),
+			vmpath.GuestAddonsDir,
 			"gvisor-pod.yaml",
 			"0640",
 			true),
 		MustBinAsset(
 			"deploy/addons/gvisor/gvisor-runtimeclass.yaml",
-			TargetDirForAddon("kube-system"),
+			vmpath.GuestAddonsDir,
 			"gvisor-runtimeclass.yaml",
 			"0640",
 			false),
@@ -298,41 +301,35 @@ var Addons = map[string]*Addon{
 			constants.GvisorConfigTomlTargetName,
 			"0640",
 			true),
-	}, false, "gvisor", "kube-system"),
+	}, false, "gvisor"),
 	"helm-tiller": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/helm-tiller/helm-tiller-dp.tmpl",
-			TargetDirForAddon("kube-system"),
+			vmpath.GuestAddonsDir,
 			"helm-tiller-dp.yaml",
 			"0640",
 			true),
 		MustBinAsset(
 			"deploy/addons/helm-tiller/helm-tiller-rbac.tmpl",
-			TargetDirForAddon("kube-system"),
+			vmpath.GuestAddonsDir,
 			"helm-tiller-rbac.yaml",
 			"0640",
 			true),
 		MustBinAsset(
 			"deploy/addons/helm-tiller/helm-tiller-svc.tmpl",
-			TargetDirForAddon("kube-system"),
+			vmpath.GuestAddonsDir,
 			"helm-tiller-svc.yaml",
 			"0640",
 			true),
-	}, false, "helm-tiller", "kube-system"),
+	}, false, "helm-tiller"),
 	"ingress-dns": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/ingress-dns/ingress-dns-pod.yaml",
-			TargetDirForAddon("kube-system"),
+			vmpath.GuestAddonsDir,
 			"ingress-dns-pod.yaml",
 			"0640",
 			false),
-	}, false, "ingress-dns", "kube-system"),
-}
-
-// TargetDirForAddon appends the namespace as a directory to /etc/kubernetes/addons
-func TargetDirForAddon(namespace string) string {
-	return vmpath.GuestAddonsDir
-	// return filepath.Join(vmpath.GuestAddonsDir, namespace)
+	}, false, "ingress-dns"),
 }
 
 // AddMinikubeDirAssets adds all addons and files to the list

From b81960e37c47f25bdb2b0f3e2b71883e24040e64 Mon Sep 17 00:00:00 2001
From: tstromberg <tstromberg@google.com>
Date: Wed, 22 Jan 2020 14:28:18 -0800
Subject: [PATCH 094/250] Fix none driver

---
 pkg/drivers/none/none.go | 64 +++++++++++++++++++++++++++++++++++-----
 1 file changed, 56 insertions(+), 8 deletions(-)

diff --git a/pkg/drivers/none/none.go b/pkg/drivers/none/none.go
index 9546cee2d6..c11a175ac6 100644
--- a/pkg/drivers/none/none.go
+++ b/pkg/drivers/none/none.go
@@ -19,17 +19,19 @@ package none
 import (
 	"fmt"
 	"os/exec"
+	"strings"
+	"time"
 
 	"github.com/docker/machine/libmachine/drivers"
 	"github.com/docker/machine/libmachine/state"
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/util/net"
-	"k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet"
 	pkgdrivers "k8s.io/minikube/pkg/drivers"
 	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/minikube/cruntime"
 	"k8s.io/minikube/pkg/minikube/vmpath"
+	"k8s.io/minikube/pkg/util/retry"
 )
 
 // cleanupPaths are paths to be removed by cleanup, and are used by both kubeadm and minikube.
@@ -121,7 +123,7 @@ func (d *Driver) GetURL() (string, error) {
 
 // GetState returns the state that the host is in (running, stopped, etc)
 func (d *Driver) GetState() (state.State, error) {
-	if err := kubelet.Check(d.exec); err != nil {
+	if err := checkKubelet(d.exec); err != nil {
 		glog.Infof("kubelet not running: %v", err)
 		return state.Stopped, nil
 	}
@@ -130,12 +132,12 @@ func (d *Driver) GetState() (state.State, error) {
 
 // Kill stops a host forcefully, including any containers that we are managing.
 func (d *Driver) Kill() error {
-	if err := kubelet.TryStopKubelet(d.exec); err != nil {
+	if err := stopKubelet(d.exec); err != nil {
 		return errors.Wrap(err, "kubelet")
 	}
 
 	// First try to gracefully stop containers
-	containers, err := d.runtime.ListContainers("")
+	containers, err := d.runtime.ListContainers(cruntime.ListOptions{})
 	if err != nil {
 		return errors.Wrap(err, "containers")
 	}
@@ -147,7 +149,7 @@ func (d *Driver) Kill() error {
 		return errors.Wrap(err, "stop")
 	}
 
-	containers, err = d.runtime.ListContainers("")
+	containers, err = d.runtime.ListContainers(cruntime.ListOptions{})
 	if err != nil {
 		return errors.Wrap(err, "containers")
 	}
@@ -175,7 +177,7 @@ func (d *Driver) Remove() error {
 
 // Restart a host
 func (d *Driver) Restart() error {
-	return kubelet.Restart(d.exec)
+	return restartKubelet(d.exec)
 }
 
 // Start a host
@@ -194,10 +196,10 @@ func (d *Driver) Start() error {
 
 // Stop a host gracefully, including any containers that we are managing.
 func (d *Driver) Stop() error {
-	if err := kubelet.Stop(d.exec); err != nil {
+	if err := stopKubelet(d.exec); err != nil {
 		return err
 	}
-	containers, err := d.runtime.ListContainers("")
+	containers, err := d.runtime.ListContainers(cruntime.ListOptions{})
 	if err != nil {
 		return errors.Wrap(err, "containers")
 	}
@@ -213,3 +215,49 @@ func (d *Driver) Stop() error {
 func (d *Driver) RunSSHCommandFromDriver() error {
 	return fmt.Errorf("driver does not support ssh commands")
 }
+
+// stopKubelet idempotently stops the kubelet
+func stopKubelet(cr command.Runner) error {
+	glog.Infof("stopping kubelet.service ...")
+	stop := func() error {
+		cmd := exec.Command("sudo", "systemctl", "stop", "kubelet.service")
+		if rr, err := cr.RunCmd(cmd); err != nil {
+			glog.Errorf("temporary error for %q : %v", rr.Command(), err)
+		}
+		cmd = exec.Command("sudo", "systemctl", "show", "-p", "SubState", "kubelet")
+		rr, err := cr.RunCmd(cmd)
+		if err != nil {
+			glog.Errorf("temporary error: for %q : %v", rr.Command(), err)
+		}
+		if !strings.Contains(rr.Stdout.String(), "dead") && !strings.Contains(rr.Stdout.String(), "failed") {
+			return fmt.Errorf("unexpected kubelet state: %q", rr.Stdout.String())
+		}
+		return nil
+	}
+
+	if err := retry.Expo(stop, 2*time.Second, time.Minute*3, 5); err != nil {
+		return errors.Wrapf(err, "error stopping kubelet")
+	}
+
+	return nil
+}
+
+// restartKubelet restarts the kubelet
+func restartKubelet(cr command.Runner) error {
+	glog.Infof("restarting kubelet.service ...")
+	c := exec.Command("sudo", "systemctl", "restart", "kubelet.service")
+	if _, err := cr.RunCmd(c); err != nil {
+		return err
+	}
+	return nil
+}
+
+// checkKubelet returns an error if the kubelet is not running.
+func checkKubelet(cr command.Runner) error {
+	glog.Infof("checking for running kubelet ...")
+	c := exec.Command("systemctl", "is-active", "--quiet", "service", "kubelet")
+	if _, err := cr.RunCmd(c); err != nil {
+		return errors.Wrap(err, "check kubelet")
+	}
+	return nil
+}

From 22de8c110ad7d75f6ff8f3584cbbffef4db27874 Mon Sep 17 00:00:00 2001
From: tstromberg <tstromberg@google.com>
Date: Wed, 22 Jan 2020 15:07:44 -0800
Subject: [PATCH 095/250] Return the number of containers paused

---
 cmd/minikube/cmd/pause.go                    |  6 ++---
 cmd/minikube/cmd/unpause.go                  | 12 ++++++---
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 11 +++++----
 pkg/minikube/cluster/pause.go                | 26 +++++++++++---------
 pkg/minikube/cruntime/docker.go              |  2 ++
 5 files changed, 34 insertions(+), 23 deletions(-)

diff --git a/cmd/minikube/cmd/pause.go b/cmd/minikube/cmd/pause.go
index dc9e3b61d8..ba805c0adb 100644
--- a/cmd/minikube/cmd/pause.go
+++ b/cmd/minikube/cmd/pause.go
@@ -88,15 +88,15 @@ func runPause(cmd *cobra.Command, args []string) {
 		}
 	}
 
-	err = cluster.Pause(cr, r, namespaces)
+	ids, err := cluster.Pause(cr, r, namespaces)
 	if err != nil {
 		exit.WithError("Pause", err)
 	}
 
 	if namespaces == nil {
-		out.T(out.Pause, "Paused all namespaces in the '{{.name}}' cluster", out.V{"name": cc.Name})
+		out.T(out.Unpause, "Paused kubelet and {{.count}} containers", out.V{"count": len(ids)})
 	} else {
-		out.T(out.Pause, "Paused the following namespaces in '{{.name}}': {{.namespaces}}", out.V{"name": cc.Name, "namespaces": strings.Join(namespaces, ", ")})
+		out.T(out.Unpause, "Paused kubelet and {{.count}} containers in: {{.namespaces}}", out.V{"count": len(ids), "namespaces": strings.Join(namespaces, ", ")})
 	}
 }
 
diff --git a/cmd/minikube/cmd/unpause.go b/cmd/minikube/cmd/unpause.go
index 7791c54e0b..6a38f0948b 100644
--- a/cmd/minikube/cmd/unpause.go
+++ b/cmd/minikube/cmd/unpause.go
@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"os"
+	"strings"
 
 	"github.com/golang/glog"
 	"github.com/spf13/cobra"
@@ -78,12 +79,17 @@ var unpauseCmd = &cobra.Command{
 			}
 		}
 
-		err = cluster.Unpause(cr, r, namespaces)
-
+		ids, err := cluster.Unpause(cr, r, namespaces)
 		if err != nil {
 			exit.WithError("Pause", err)
 		}
-		out.T(out.Unpause, "The '{{.name}}' cluster is now unpaused", out.V{"name": cc.Name})
+
+		if namespaces == nil {
+			out.T(out.Pause, "Unpaused kubelet and {{.count}} containers", out.V{"count": len(ids)})
+		} else {
+			out.T(out.Pause, "Unpaused kubelet and {{.count}} containers in: {{.namespaces}}", out.V{"count": len(ids), "namespaces": strings.Join(namespaces, ", ")})
+		}
+
 	},
 }
 
diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index 4582c6dbfa..ad470c47dd 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -107,16 +107,17 @@ func (k *Bootstrapper) GetAPIServerStatus(ip net.IP, apiserverPort int) (string,
 	pid := strings.TrimSpace(rr.Stdout.String())
 
 	// Get the freezer cgroup entry for this pid
-	rr, err = k.c.RunCmd(exec.Command("sudo", "egrep", "^[-09]+:freezer:", path.Join("/proc", pid, "cgroup")))
+	rr, err = k.c.RunCmd(exec.Command("sudo", "egrep", "^[0-9]+:freezer:", path.Join("/proc", pid, "cgroup")))
 	if err != nil {
-		return state.Error.String(), err
+		glog.Warningf("unable to find freezer cgroup: %v", err)
+		return kverify.APIServerStatus(ip, apiserverPort)
+
 	}
 	freezer := strings.TrimSpace(rr.Stdout.String())
 	glog.Infof("apiserver freezer: %q", freezer)
-
 	fparts := strings.Split(freezer, ":")
-	if len(fparts) == 3 {
-		glog.Warningf("unable to parse freezer: %s", freezer)
+	if len(fparts) != 3 {
+		glog.Warningf("unable to parse freezer - found %d parts: %s", len(fparts), freezer)
 		return kverify.APIServerStatus(ip, apiserverPort)
 	}
 
diff --git a/pkg/minikube/cluster/pause.go b/pkg/minikube/cluster/pause.go
index b963b4b002..6e4eaead45 100644
--- a/pkg/minikube/cluster/pause.go
+++ b/pkg/minikube/cluster/pause.go
@@ -32,45 +32,47 @@ var DefaultNamespaces = []string{
 }
 
 // Pause pauses a Kubernetes cluster
-func Pause(cr cruntime.Manager, r command.Runner, namespaces []string) error {
+func Pause(cr cruntime.Manager, r command.Runner, namespaces []string) ([]string, error) {
+	ids := []string{}
+	// Disable the kubelet so it does not attempt to restart paused pods
 	if err := kubelet.Disable(r); err != nil {
-		return errors.Wrap(err, "kubelet disable")
+		return ids, errors.Wrap(err, "kubelet disable")
 	}
 	if err := kubelet.Stop(r); err != nil {
-		return errors.Wrap(err, "kubelet stop")
+		return ids, errors.Wrap(err, "kubelet stop")
 	}
 	ids, err := cr.ListContainers(cruntime.ListOptions{State: cruntime.Running, Namespaces: namespaces})
 	if err != nil {
-		return errors.Wrap(err, "list running")
+		return ids, errors.Wrap(err, "list running")
 	}
 	if len(ids) == 0 {
 		glog.Warningf("no running containers to pause")
-		return nil
+		return ids, nil
 	}
-	return cr.PauseContainers(ids)
+	return ids, cr.PauseContainers(ids)
 
 }
 
 // Unpause unpauses a Kubernetes cluster
-func Unpause(cr cruntime.Manager, r command.Runner, namespaces []string) error {
+func Unpause(cr cruntime.Manager, r command.Runner, namespaces []string) ([]string, error) {
 	ids, err := cr.ListContainers(cruntime.ListOptions{State: cruntime.Paused, Namespaces: namespaces})
 	if err != nil {
-		return errors.Wrap(err, "list paused")
+		return ids, errors.Wrap(err, "list paused")
 	}
 
 	if len(ids) == 0 {
 		glog.Warningf("no paused containers found")
 	} else {
 		if err := cr.UnpauseContainers(ids); err != nil {
-			return errors.Wrap(err, "unpause")
+			return ids, errors.Wrap(err, "unpause")
 		}
 	}
 
 	if err := kubelet.Enable(r); err != nil {
-		return errors.Wrap(err, "kubelet enable")
+		return ids, errors.Wrap(err, "kubelet enable")
 	}
 	if err := kubelet.Start(r); err != nil {
-		return errors.Wrap(err, "kubelet start")
+		return ids, errors.Wrap(err, "kubelet start")
 	}
-	return nil
+	return ids, nil
 }
diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go
index a58cff35b1..24fb506c94 100644
--- a/pkg/minikube/cruntime/docker.go
+++ b/pkg/minikube/cruntime/docker.go
@@ -151,6 +151,8 @@ func (r *Docker) ListContainers(o ListOptions) ([]string, error) {
 	switch o.State {
 	case All:
 		args = append(args, "-a")
+	case Running:
+		args = append(args, "--filter", "status=running")
 	case Paused:
 		args = append(args, "--filter", "status=paused")
 	}

From 8d8ea98378654cf388cc7c1a3f4891c4651b6b45 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Wed, 22 Jan 2020 15:11:33 -0800
Subject: [PATCH 096/250] check cmd.Remove error for linting

---
 pkg/addons/addons.go | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/pkg/addons/addons.go b/pkg/addons/addons.go
index 1380c6bf29..0437793d04 100644
--- a/pkg/addons/addons.go
+++ b/pkg/addons/addons.go
@@ -22,6 +22,7 @@ import (
 	"path/filepath"
 	"strconv"
 
+	"github.com/golang/glog"
 	"github.com/pkg/errors"
 	"github.com/spf13/viper"
 	"k8s.io/minikube/pkg/minikube/assets"
@@ -193,7 +194,11 @@ func enableOrDisableAddonInternal(addon *assets.Addon, cmd command.Runner, data
 				return err
 			}
 		} else {
-			defer cmd.Remove(addonFile)
+			defer func() {
+				if err := cmd.Remove(addonFile); err != nil {
+					glog.Warningf("error removing %s; addon should still be disabled as expected", addonFile)
+				}
+			}()
 		}
 		files = append(files, filepath.Join(addonFile.GetTargetDir(), addonFile.GetTargetName()))
 	}
@@ -202,7 +207,7 @@ func enableOrDisableAddonInternal(addon *assets.Addon, cmd command.Runner, data
 		return err
 	}
 	if result, err := cmd.RunCmd(command); err != nil {
-		return errors.Wrapf(err, "error enabling addon:\n%s", result.Output())
+		return errors.Wrapf(err, "error updating addon:\n%s", result.Output())
 	}
 	return nil
 }

From 49b0cb1c9c2232dc58d1e27b853fa12755e2019c Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Wed, 22 Jan 2020 15:16:36 -0800
Subject: [PATCH 097/250] create dashboard ns first so that everything else can
 be created after

---
 pkg/minikube/assets/addons.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go
index 1c8a0e0773..67fd6b33df 100644
--- a/pkg/minikube/assets/addons.go
+++ b/pkg/minikube/assets/addons.go
@@ -69,11 +69,12 @@ func (a *Addon) IsEnabled() (bool, error) {
 // TODO: Make dynamically loadable: move this data to a .yaml file within each addon directory
 var Addons = map[string]*Addon{
 	"dashboard": NewAddon([]*BinAsset{
+		// We want to create the kubernetes-dashboard ns first so that every subsequent object can be created
+		MustBinAsset("deploy/addons/dashboard/dashboard-ns.yaml", vmpath.GuestAddonsDir, "dashboard-ns.yaml", "0640", false),
 		MustBinAsset("deploy/addons/dashboard/dashboard-clusterrole.yaml", vmpath.GuestAddonsDir, "dashboard-clusterrole.yaml", "0640", false),
 		MustBinAsset("deploy/addons/dashboard/dashboard-clusterrolebinding.yaml", vmpath.GuestAddonsDir, "dashboard-clusterrolebinding.yaml", "0640", false),
 		MustBinAsset("deploy/addons/dashboard/dashboard-configmap.yaml", vmpath.GuestAddonsDir, "dashboard-configmap.yaml", "0640", false),
 		MustBinAsset("deploy/addons/dashboard/dashboard-dp.yaml", vmpath.GuestAddonsDir, "dashboard-dp.yaml", "0640", false),
-		MustBinAsset("deploy/addons/dashboard/dashboard-ns.yaml", vmpath.GuestAddonsDir, "dashboard-ns.yaml", "0640", false),
 		MustBinAsset("deploy/addons/dashboard/dashboard-role.yaml", vmpath.GuestAddonsDir, "dashboard-role.yaml", "0640", false),
 		MustBinAsset("deploy/addons/dashboard/dashboard-rolebinding.yaml", vmpath.GuestAddonsDir, "dashboard-rolebinding.yaml", "0640", false),
 		MustBinAsset("deploy/addons/dashboard/dashboard-sa.yaml", vmpath.GuestAddonsDir, "dashboard-sa.yaml", "0640", false),

From 1a13055d0062a39576cdcc0e63b8f2f8d85322a4 Mon Sep 17 00:00:00 2001
From: tstromberg <tstromberg@google.com>
Date: Wed, 22 Jan 2020 15:50:09 -0800
Subject: [PATCH 098/250] Add integration tests

---
 cmd/minikube/cmd/pause.go                    |  6 +-
 cmd/minikube/cmd/status.go                   |  4 +-
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go |  2 +-
 pkg/minikube/cluster/pause.go                |  6 +-
 pkg/minikube/cruntime/cruntime_test.go       | 89 +++++++++++---------
 test/integration/helpers.go                  | 10 ---
 test/integration/start_stop_delete_test.go   | 47 ++++++++++-
 7 files changed, 98 insertions(+), 66 deletions(-)

diff --git a/cmd/minikube/cmd/pause.go b/cmd/minikube/cmd/pause.go
index ba805c0adb..54385a4d5c 100644
--- a/cmd/minikube/cmd/pause.go
+++ b/cmd/minikube/cmd/pause.go
@@ -82,10 +82,8 @@ func runPause(cmd *cobra.Command, args []string) {
 	glog.Infof("namespaces: %v keys: %v", namespaces, viper.AllSettings())
 	if allNamespaces {
 		namespaces = nil //all
-	} else {
-		if len(namespaces) == 0 {
-			exit.WithCodeT(exit.BadUsage, "Use -A to specify all namespaces")
-		}
+	} else if len(namespaces) == 0 {
+		exit.WithCodeT(exit.BadUsage, "Use -A to specify all namespaces")
 	}
 
 	ids, err := cluster.Pause(cr, r, namespaces)
diff --git a/cmd/minikube/cmd/status.go b/cmd/minikube/cmd/status.go
index 471986c31f..ba8e93c9e6 100644
--- a/cmd/minikube/cmd/status.go
+++ b/cmd/minikube/cmd/status.go
@@ -44,8 +44,8 @@ var output string
 
 const (
 	// Additional states used by kubeconfig
-	Configured    = "Configured"    // analagous to state.Saved
-	Misconfigured = "Misconfigured" // analagous to state.Error
+	Configured    = "Configured"    // analogous to state.Saved
+	Misconfigured = "Misconfigured" // analogous to state.Error
 )
 
 // Status holds string representations of component states
diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index ad470c47dd..5569dc67ba 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -123,7 +123,7 @@ func (k *Bootstrapper) GetAPIServerStatus(ip net.IP, apiserverPort int) (string,
 
 	rr, err = k.c.RunCmd(exec.Command("sudo", "cat", path.Join("/sys/fs/cgroup/freezer", fparts[2], "freezer.state")))
 	if err != nil {
-		glog.Errorf("unable to get freezer state: %s", rr.Stderr)
+		glog.Errorf("unable to get freezer state: %s", rr.Stderr.String())
 		return kverify.APIServerStatus(ip, apiserverPort)
 	}
 
diff --git a/pkg/minikube/cluster/pause.go b/pkg/minikube/cluster/pause.go
index 6e4eaead45..d2edcb1888 100644
--- a/pkg/minikube/cluster/pause.go
+++ b/pkg/minikube/cluster/pause.go
@@ -62,10 +62,8 @@ func Unpause(cr cruntime.Manager, r command.Runner, namespaces []string) ([]stri
 
 	if len(ids) == 0 {
 		glog.Warningf("no paused containers found")
-	} else {
-		if err := cr.UnpauseContainers(ids); err != nil {
-			return ids, errors.Wrap(err, "unpause")
-		}
+	} else if err := cr.UnpauseContainers(ids); err != nil {
+		return ids, errors.Wrap(err, "unpause")
 	}
 
 	if err := kubelet.Enable(r); err != nil {
diff --git a/pkg/minikube/cruntime/cruntime_test.go b/pkg/minikube/cruntime/cruntime_test.go
index cb92244ce3..7764e1b714 100644
--- a/pkg/minikube/cruntime/cruntime_test.go
+++ b/pkg/minikube/cruntime/cruntime_test.go
@@ -118,9 +118,9 @@ func TestKubeletOptions(t *testing.T) {
 type serviceState int
 
 const (
-	Exited serviceState = iota
-	Running
-	Restarted
+	SvcExited serviceState = iota
+	SvcRunning
+	SvcRestarted
 )
 
 // FakeRunner is a command runner that isn't very smart.
@@ -262,6 +262,7 @@ func (f *FakeRunner) containerd(args []string, _ bool) (string, error) {
 
 // crictl is a fake implementation of crictl
 func (f *FakeRunner) crictl(args []string, _ bool) (string, error) {
+	f.t.Logf("crictl args: %s", args)
 	switch cmd := args[0]; cmd {
 	case "info":
 		return `{
@@ -273,9 +274,21 @@ func (f *FakeRunner) crictl(args []string, _ bool) (string, error) {
 		  "golang": "go1.11.13"
 		}`, nil
 	case "ps":
+		fmt.Printf("args %d: %v\n", len(args), args)
+		if len(args) != 4 {
+			f.t.Logf("crictl all")
+			ids := []string{}
+			for id := range f.containers {
+				ids = append(ids, id)
+			}
+			f.t.Logf("fake crictl: Found containers: %v", ids)
+			return strings.Join(ids, "\n"), nil
+		}
+
 		// crictl ps -a --name=apiserver --state=Running --quiet
-		if args[1] == "-a" && strings.HasPrefix(args[2], "--name") {
-			fname := strings.Split(args[2], "=")[1]
+		if args[1] == "-a" && strings.HasPrefix(args[3], "--name") {
+			fname := strings.Split(args[3], "=")[1]
+			f.t.Logf("crictl filter for %s", fname)
 			ids := []string{}
 			f.t.Logf("fake crictl: Looking for containers matching %q", fname)
 			for id, cname := range f.containers {
@@ -285,14 +298,6 @@ func (f *FakeRunner) crictl(args []string, _ bool) (string, error) {
 			}
 			f.t.Logf("fake crictl: Found containers: %v", ids)
 			return strings.Join(ids, "\n"), nil
-		} else if args[1] == "-a" {
-			ids := []string{}
-			for id := range f.containers {
-				ids = append(ids, id)
-			}
-			f.t.Logf("fake crictl: Found containers: %v", ids)
-			return strings.Join(ids, "\n"), nil
-
 		}
 	case "stop":
 		for _, id := range args[1:] {
@@ -341,23 +346,23 @@ func (f *FakeRunner) systemctl(args []string, root bool) (string, error) { // no
 			if !root {
 				return out, fmt.Errorf("not root")
 			}
-			f.services[svc] = Exited
+			f.services[svc] = SvcExited
 			f.t.Logf("fake systemctl: stopped %s", svc)
 		case "start":
 			if !root {
 				return out, fmt.Errorf("not root")
 			}
-			f.services[svc] = Running
+			f.services[svc] = SvcRunning
 			f.t.Logf("fake systemctl: started %s", svc)
 		case "restart":
 			if !root {
 				return out, fmt.Errorf("not root")
 			}
-			f.services[svc] = Restarted
-			f.t.Logf("fake systemctl: restarted %s", svc)
+			f.services[svc] = SvcRestarted
+			f.t.Logf("fake systemctl: SvcRestarted %s", svc)
 		case "is-active":
 			f.t.Logf("fake systemctl: %s is-status: %v", svc, state)
-			if state == Running {
+			if state == SvcRunning {
 				return out, nil
 			}
 			return out, fmt.Errorf("%s in state: %v", svc, state)
@@ -403,11 +408,11 @@ func TestVersion(t *testing.T) {
 
 // defaultServices reflects the default boot state for the minikube VM
 var defaultServices = map[string]serviceState{
-	"docker":        Running,
-	"docker.socket": Running,
-	"crio":          Exited,
-	"crio-shutdown": Exited,
-	"containerd":    Exited,
+	"docker":        SvcRunning,
+	"docker.socket": SvcRunning,
+	"crio":          SvcExited,
+	"crio-shutdown": SvcExited,
+	"containerd":    SvcExited,
 }
 
 func TestDisable(t *testing.T) {
@@ -446,25 +451,25 @@ func TestEnable(t *testing.T) {
 		want    map[string]serviceState
 	}{
 		{"docker", map[string]serviceState{
-			"docker":        Running,
-			"docker.socket": Running,
-			"containerd":    Exited,
-			"crio":          Exited,
-			"crio-shutdown": Exited,
+			"docker":        SvcRunning,
+			"docker.socket": SvcRunning,
+			"containerd":    SvcExited,
+			"crio":          SvcExited,
+			"crio-shutdown": SvcExited,
 		}},
 		{"containerd", map[string]serviceState{
-			"docker":        Exited,
-			"docker.socket": Exited,
-			"containerd":    Restarted,
-			"crio":          Exited,
-			"crio-shutdown": Exited,
+			"docker":        SvcExited,
+			"docker.socket": SvcExited,
+			"containerd":    SvcRestarted,
+			"crio":          SvcExited,
+			"crio-shutdown": SvcExited,
 		}},
 		{"crio", map[string]serviceState{
-			"docker":        Exited,
-			"docker.socket": Exited,
-			"containerd":    Exited,
-			"crio":          Restarted,
-			"crio-shutdown": Exited,
+			"docker":        SvcExited,
+			"docker.socket": SvcExited,
+			"containerd":    SvcExited,
+			"crio":          SvcRestarted,
+			"crio-shutdown": SvcExited,
 		}},
 	}
 	for _, tc := range tests {
@@ -516,7 +521,7 @@ func TestContainerFunctions(t *testing.T) {
 			}
 
 			// Get the list of apiservers
-			got, err := cr.ListContainers("apiserver")
+			got, err := cr.ListContainers(ListOptions{Name: "apiserver"})
 			if err != nil {
 				t.Fatalf("ListContainers: %v", err)
 			}
@@ -529,7 +534,7 @@ func TestContainerFunctions(t *testing.T) {
 			if err := cr.StopContainers(got); err != nil {
 				t.Fatalf("stop failed: %v", err)
 			}
-			got, err = cr.ListContainers("apiserver")
+			got, err = cr.ListContainers(ListOptions{Name: "apiserver"})
 			if err != nil {
 				t.Fatalf("ListContainers: %v", err)
 			}
@@ -539,7 +544,7 @@ func TestContainerFunctions(t *testing.T) {
 			}
 
 			// Get the list of everything else.
-			got, err = cr.ListContainers("")
+			got, err = cr.ListContainers(ListOptions{})
 			if err != nil {
 				t.Fatalf("ListContainers: %v", err)
 			}
@@ -552,7 +557,7 @@ func TestContainerFunctions(t *testing.T) {
 			if err := cr.KillContainers(got); err != nil {
 				t.Errorf("KillContainers: %v", err)
 			}
-			got, err = cr.ListContainers("")
+			got, err = cr.ListContainers(ListOptions{})
 			if err != nil {
 				t.Fatalf("ListContainers: %v", err)
 			}
diff --git a/test/integration/helpers.go b/test/integration/helpers.go
index 8d145da091..1453f84cdb 100644
--- a/test/integration/helpers.go
+++ b/test/integration/helpers.go
@@ -321,16 +321,6 @@ func showPodLogs(ctx context.Context, t *testing.T, profile string, ns string, n
 	}
 }
 
-// Status returns the minikube cluster status as a string
-func Status(ctx context.Context, t *testing.T, path string, profile string) string {
-	t.Helper()
-	rr, err := Run(t, exec.CommandContext(ctx, path, "status", "--format={{.Host}}", "-p", profile))
-	if err != nil {
-		t.Logf("status error: %v (may be ok)", err)
-	}
-	return strings.TrimSpace(rr.Stdout.String())
-}
-
 // MaybeParallel sets that the test should run in parallel
 func MaybeParallel(t *testing.T) {
 	t.Helper()
diff --git a/test/integration/start_stop_delete_test.go b/test/integration/start_stop_delete_test.go
index ad1b6832b5..847e518fe9 100644
--- a/test/integration/start_stop_delete_test.go
+++ b/test/integration/start_stop_delete_test.go
@@ -36,6 +36,16 @@ import (
 	"k8s.io/minikube/pkg/minikube/constants"
 )
 
+// status returns a minikube component status as a string
+func status(ctx context.Context, t *testing.T, path string, profile string, key string) string {
+	t.Helper()
+	rr, err := Run(t, exec.CommandContext(ctx, path, "status", fmt.Sprintf("--format={{.%s}}", key), "-p", profile))
+	if err != nil {
+		t.Logf("status error: %v (may be ok)", err)
+	}
+	return strings.TrimSpace(rr.Stdout.String())
+}
+
 func TestStartStop(t *testing.T) {
 	MaybeParallel(t)
 
@@ -135,7 +145,7 @@ func TestStartStop(t *testing.T) {
 					t.Errorf("%s failed: %v", rr.Args, err)
 				}
 
-				got := Status(ctx, t, Target(), profile)
+				got := status(ctx, t, Target(), profile, "Host")
 				if got != state.Stopped.String() {
 					t.Errorf("status = %q; want = %q", got, state.Stopped)
 				}
@@ -187,9 +197,40 @@ func TestStartStop(t *testing.T) {
 					t.Fatalf("wait: %v", err)
 				}
 
-				got = Status(ctx, t, Target(), profile)
+				got = status(ctx, t, Target(), profile, "Host")
 				if got != state.Running.String() {
-					t.Errorf("status = %q; want = %q", got, state.Running)
+					t.Errorf("host status = %q; want = %q", got, state.Running)
+				}
+
+				// Can this runtime be paused and unpaused?
+				rr, err = Run(t, exec.CommandContext(ctx, Target(), "pause", "-p", profile, "--alsologtostderr", "-v=1"))
+				if err != nil {
+					t.Fatalf("%s failed: %v", rr.Args, err)
+				}
+
+				got = status(ctx, t, Target(), profile, "APIServer")
+				if got != state.Paused.String() {
+					t.Errorf("apiserver status = %q; want = %q", got, state.Paused)
+				}
+
+				got = status(ctx, t, Target(), profile, "Kubelet")
+				if got != state.Stopped.String() {
+					t.Errorf("kubelet status = %q; want = %q", got, state.Stopped)
+				}
+
+				rr, err = Run(t, exec.CommandContext(ctx, Target(), "unpause", "-p", profile, "--alsologtostderr", "-v=1"))
+				if err != nil {
+					t.Fatalf("%s failed: %v", rr.Args, err)
+				}
+
+				got = status(ctx, t, Target(), profile, "APIServer")
+				if got != state.Running.String() {
+					t.Errorf("apiserver status = %q; want = %q", got, state.Running)
+				}
+
+				got = status(ctx, t, Target(), profile, "Kubelet")
+				if got != state.Running.String() {
+					t.Errorf("kubelet status = %q; want = %q", got, state.Running)
 				}
 
 				if *cleanup {

From 9fc5db914c248244856034d4b9b7d6f6405250ba Mon Sep 17 00:00:00 2001
From: tstromberg <tstromberg@google.com>
Date: Wed, 22 Jan 2020 16:39:26 -0800
Subject: [PATCH 099/250] Decrease cylomatic complexity of TestStartStop

---
 test/integration/start_stop_delete_test.go | 209 +++++++++++----------
 1 file changed, 113 insertions(+), 96 deletions(-)

diff --git a/test/integration/start_stop_delete_test.go b/test/integration/start_stop_delete_test.go
index 847e518fe9..2ea0a44aa8 100644
--- a/test/integration/start_stop_delete_test.go
+++ b/test/integration/start_stop_delete_test.go
@@ -100,44 +100,14 @@ func TestStartStop(t *testing.T) {
 				startArgs := append([]string{"start", "-p", profile, "--alsologtostderr", "-v=3", "--wait=true"}, tc.args...)
 				startArgs = append(startArgs, StartArgs()...)
 				startArgs = append(startArgs, fmt.Sprintf("--kubernetes-version=%s", tc.version))
+
 				rr, err := Run(t, exec.CommandContext(ctx, Target(), startArgs...))
 				if err != nil {
-					// Fatal so that we may collect logs before stop/delete steps
 					t.Fatalf("%s failed: %v", rr.Args, err)
 				}
 
-				// SADNESS: 0/1 nodes are available: 1 node(s) had taints that the pod didn't tolerate.
-				if strings.Contains(tc.name, "cni") {
-					t.Logf("WARNING: cni mode requires additional setup before pods can schedule :(")
-				} else {
-					// schedule a pod to assert persistence
-					rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "create", "-f", filepath.Join(*testdataDir, "busybox.yaml")))
-					if err != nil {
-						t.Fatalf("%s failed: %v", rr.Args, err)
-					}
-
-					// 8 minutes, because 4 is not enough for images to pull in all cases.
-					names, err := PodWait(ctx, t, profile, "default", "integration-test=busybox", 8*time.Minute)
-					if err != nil {
-						t.Fatalf("wait: %v", err)
-					}
-
-					// Use this pod to confirm that the runtime resource limits are sane
-					rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "exec", names[0], "--", "/bin/sh", "-c", "ulimit -n"))
-					if err != nil {
-						t.Fatalf("ulimit: %v", err)
-					}
-
-					got, err := strconv.ParseInt(strings.TrimSpace(rr.Stdout.String()), 10, 64)
-					if err != nil {
-						t.Errorf("ParseInt(%q): %v", rr.Stdout.String(), err)
-					}
-
-					// Arbitrary value set by some container runtimes. If higher, apps like MySQL may make bad decisions.
-					expected := int64(1048576)
-					if got != expected {
-						t.Errorf("'ulimit -n' returned %d, expected %d", got, expected)
-					}
+				if !strings.Contains(tc.name, "cni") {
+					testPodScheduling(ctx, t, profile)
 				}
 
 				rr, err = Run(t, exec.CommandContext(ctx, Target(), "stop", "-p", profile, "--alsologtostderr", "-v=3"))
@@ -156,41 +126,6 @@ func TestStartStop(t *testing.T) {
 					t.Fatalf("%s failed: %v", rr.Args, err)
 				}
 
-				// Make sure that kubeadm did not need to pull in additional images
-				if !NoneDriver() {
-					rr, err = Run(t, exec.CommandContext(ctx, Target(), "ssh", "-p", profile, "sudo crictl images -o json"))
-					if err != nil {
-						t.Errorf("%s failed: %v", rr.Args, err)
-					}
-					jv := map[string][]struct {
-						Tags []string `json:"repoTags"`
-					}{}
-					err = json.Unmarshal(rr.Stdout.Bytes(), &jv)
-					if err != nil {
-						t.Errorf("images unmarshal: %v", err)
-					}
-					gotImages := []string{}
-					for _, img := range jv["images"] {
-						for _, i := range img.Tags {
-							if defaultImage(i) {
-								// Remove docker.io for naming consistency between container runtimes
-								gotImages = append(gotImages, strings.TrimPrefix(i, "docker.io/"))
-							} else {
-								t.Logf("Found non-minikube image: %s", i)
-							}
-						}
-					}
-					want, err := images.Kubeadm("", tc.version)
-					if err != nil {
-						t.Errorf("kubeadm images: %v", tc.version)
-					}
-					sort.Strings(want)
-					sort.Strings(gotImages)
-					if diff := cmp.Diff(want, gotImages); diff != "" {
-						t.Errorf("%s images mismatch (-want +got):\n%s", tc.version, diff)
-					}
-				}
-
 				if strings.Contains(tc.name, "cni") {
 					t.Logf("WARNING: cni mode requires additional setup before pods can schedule :(")
 				} else if _, err := PodWait(ctx, t, profile, "default", "integration-test=busybox", 4*time.Minute); err != nil {
@@ -202,36 +137,11 @@ func TestStartStop(t *testing.T) {
 					t.Errorf("host status = %q; want = %q", got, state.Running)
 				}
 
-				// Can this runtime be paused and unpaused?
-				rr, err = Run(t, exec.CommandContext(ctx, Target(), "pause", "-p", profile, "--alsologtostderr", "-v=1"))
-				if err != nil {
-					t.Fatalf("%s failed: %v", rr.Args, err)
+				if !NoneDriver() {
+					testPulledImages(ctx, t, profile, tc.version)
 				}
 
-				got = status(ctx, t, Target(), profile, "APIServer")
-				if got != state.Paused.String() {
-					t.Errorf("apiserver status = %q; want = %q", got, state.Paused)
-				}
-
-				got = status(ctx, t, Target(), profile, "Kubelet")
-				if got != state.Stopped.String() {
-					t.Errorf("kubelet status = %q; want = %q", got, state.Stopped)
-				}
-
-				rr, err = Run(t, exec.CommandContext(ctx, Target(), "unpause", "-p", profile, "--alsologtostderr", "-v=1"))
-				if err != nil {
-					t.Fatalf("%s failed: %v", rr.Args, err)
-				}
-
-				got = status(ctx, t, Target(), profile, "APIServer")
-				if got != state.Running.String() {
-					t.Errorf("apiserver status = %q; want = %q", got, state.Running)
-				}
-
-				got = status(ctx, t, Target(), profile, "Kubelet")
-				if got != state.Running.String() {
-					t.Errorf("kubelet status = %q; want = %q", got, state.Running)
-				}
+				testPause(ctx, t, profile)
 
 				if *cleanup {
 					// Normally handled by cleanuprofile, but not fatal there
@@ -245,6 +155,113 @@ func TestStartStop(t *testing.T) {
 	})
 }
 
+// testPodScheduling asserts that this configuration can schedule new pods
+func testPodScheduling(ctx context.Context, t *testing.T, profile string) {
+	t.Helper()
+
+	// schedule a pod to assert persistence
+	rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "create", "-f", filepath.Join(*testdataDir, "busybox.yaml")))
+	if err != nil {
+		t.Fatalf("%s failed: %v", rr.Args, err)
+	}
+
+	// 8 minutes, because 4 is not enough for images to pull in all cases.
+	names, err := PodWait(ctx, t, profile, "default", "integration-test=busybox", 8*time.Minute)
+	if err != nil {
+		t.Fatalf("wait: %v", err)
+	}
+
+	// Use this pod to confirm that the runtime resource limits are sane
+	rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "exec", names[0], "--", "/bin/sh", "-c", "ulimit -n"))
+	if err != nil {
+		t.Fatalf("ulimit: %v", err)
+	}
+
+	got, err := strconv.ParseInt(strings.TrimSpace(rr.Stdout.String()), 10, 64)
+	if err != nil {
+		t.Errorf("ParseInt(%q): %v", rr.Stdout.String(), err)
+	}
+
+	// Arbitrary value set by some container runtimes. If higher, apps like MySQL may make bad decisions.
+	expected := int64(1048576)
+	if got != expected {
+		t.Errorf("'ulimit -n' returned %d, expected %d", got, expected)
+	}
+}
+
+// testPulledImages asserts that this configuration pulls only expected images
+func testPulledImages(ctx context.Context, t *testing.T, profile string, version string) {
+	t.Helper()
+
+	rr, err := Run(t, exec.CommandContext(ctx, Target(), "ssh", "-p", profile, "sudo crictl images -o json"))
+	if err != nil {
+		t.Errorf("%s failed: %v", rr.Args, err)
+	}
+	jv := map[string][]struct {
+		Tags []string `json:"repoTags"`
+	}{}
+	err = json.Unmarshal(rr.Stdout.Bytes(), &jv)
+	if err != nil {
+		t.Errorf("images unmarshal: %v", err)
+	}
+	gotImages := []string{}
+	for _, img := range jv["images"] {
+		for _, i := range img.Tags {
+			if defaultImage(i) {
+				// Remove docker.io for naming consistency between container runtimes
+				gotImages = append(gotImages, strings.TrimPrefix(i, "docker.io/"))
+			} else {
+				t.Logf("Found non-minikube image: %s", i)
+			}
+		}
+	}
+	want, err := images.Kubeadm("", version)
+	if err != nil {
+		t.Errorf("kubeadm images: %v", version)
+	}
+	sort.Strings(want)
+	sort.Strings(gotImages)
+	if diff := cmp.Diff(want, gotImages); diff != "" {
+		t.Errorf("%s images mismatch (-want +got):\n%s", version, diff)
+	}
+}
+
+// testPause asserts that this configuration can be paused and unpaused
+func testPause(ctx context.Context, t *testing.T, profile string) {
+	t.Helper()
+
+	rr, err := Run(t, exec.CommandContext(ctx, Target(), "pause", "-p", profile, "--alsologtostderr", "-v=1"))
+	if err != nil {
+		t.Fatalf("%s failed: %v", rr.Args, err)
+	}
+
+	got := status(ctx, t, Target(), profile, "APIServer")
+	if got != state.Paused.String() {
+		t.Errorf("apiserver status = %q; want = %q", got, state.Paused)
+	}
+
+	got = status(ctx, t, Target(), profile, "Kubelet")
+	if got != state.Stopped.String() {
+		t.Errorf("kubelet status = %q; want = %q", got, state.Stopped)
+	}
+
+	rr, err = Run(t, exec.CommandContext(ctx, Target(), "unpause", "-p", profile, "--alsologtostderr", "-v=1"))
+	if err != nil {
+		t.Fatalf("%s failed: %v", rr.Args, err)
+	}
+
+	got = status(ctx, t, Target(), profile, "APIServer")
+	if got != state.Running.String() {
+		t.Errorf("apiserver status = %q; want = %q", got, state.Running)
+	}
+
+	got = status(ctx, t, Target(), profile, "Kubelet")
+	if got != state.Running.String() {
+		t.Errorf("kubelet status = %q; want = %q", got, state.Running)
+	}
+
+}
+
 // defaultImage returns true if this image is expected in a default minikube install
 func defaultImage(name string) bool {
 	if strings.Contains(name, ":latest") {

From d7ea60d61004a6ae04b4aa52842f439bc8dba7bc Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Wed, 22 Jan 2020 18:49:58 -0800
Subject: [PATCH 100/250] fix up tests for new config format

---
 pkg/minikube/bootstrapper/bsutil/kubeadm.go   | 10 +-
 .../bootstrapper/bsutil/kubeadm_test.go       | 50 ++++++----
 .../bootstrapper/bsutil/kubelet_test.go       | 94 ++++++++++++-------
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go  |  4 +-
 4 files changed, 103 insertions(+), 55 deletions(-)

diff --git a/pkg/minikube/bootstrapper/bsutil/kubeadm.go b/pkg/minikube/bootstrapper/bsutil/kubeadm.go
index 33b88e0834..1c3d0748e7 100644
--- a/pkg/minikube/bootstrapper/bsutil/kubeadm.go
+++ b/pkg/minikube/bootstrapper/bsutil/kubeadm.go
@@ -54,7 +54,11 @@ func GenerateKubeadmYAML(mc config.MachineConfig, r cruntime.Manager) ([]byte, e
 	}
 
 	// In case of no port assigned, use default
-	nodePort := mc.Nodes[0].Port
+	master, err := config.GetMasterNode(mc)
+	if err != nil {
+		return nil, errors.Wrap(err, "getting master node")
+	}
+	nodePort := master.Port
 	if nodePort <= 0 {
 		nodePort = constants.APIServerPort
 	}
@@ -78,11 +82,11 @@ func GenerateKubeadmYAML(mc config.MachineConfig, r cruntime.Manager) ([]byte, e
 		CertDir:           vmpath.GuestCertsDir,
 		ServiceCIDR:       constants.DefaultServiceCIDR,
 		PodSubnet:         k8s.ExtraOptions.Get("pod-network-cidr", Kubeadm),
-		AdvertiseAddress:  mc.Nodes[0].IP,
+		AdvertiseAddress:  master.IP,
 		APIServerPort:     nodePort,
 		KubernetesVersion: k8s.KubernetesVersion,
 		EtcdDataDir:       EtcdDataDir(),
-		NodeName:          mc.Nodes[0].Name,
+		NodeName:          master.Name,
 		CRISocket:         r.SocketPath(),
 		ImageRepository:   k8s.ImageRepository,
 		ExtraArgs:         extraComponentConfig,
diff --git a/pkg/minikube/bootstrapper/bsutil/kubeadm_test.go b/pkg/minikube/bootstrapper/bsutil/kubeadm_test.go
index b5fe8c7fe6..f6d08cf9b6 100644
--- a/pkg/minikube/bootstrapper/bsutil/kubeadm_test.go
+++ b/pkg/minikube/bootstrapper/bsutil/kubeadm_test.go
@@ -106,9 +106,9 @@ func TestGenerateKubeadmYAMLDNS(t *testing.T) {
 		name      string
 		runtime   string
 		shouldErr bool
-		cfg       config.KubernetesConfig
+		cfg       config.MachineConfig
 	}{
-		{"dns", "docker", false, config.KubernetesConfig{DNSDomain: "1.1.1.1"}},
+		{"dns", "docker", false, config.MachineConfig{KubernetesConfig: config.KubernetesConfig{DNSDomain: "1.1.1.1"}}},
 	}
 	for _, version := range versions {
 		for _, tc := range tests {
@@ -119,9 +119,13 @@ func TestGenerateKubeadmYAMLDNS(t *testing.T) {
 			tname := tc.name + "_" + version
 			t.Run(tname, func(t *testing.T) {
 				cfg := tc.cfg
-				cfg.NodeIP = "1.1.1.1"
-				cfg.NodeName = "mk"
-				cfg.KubernetesVersion = version + ".0"
+				cfg.Nodes = []config.Node{
+					config.Node{
+						IP:   "1.1.1.1",
+						Name: "mk",
+					},
+				}
+				cfg.KubernetesConfig.KubernetesVersion = version + ".0"
 
 				got, err := GenerateKubeadmYAML(cfg, runtime)
 				if err != nil && !tc.shouldErr {
@@ -166,17 +170,17 @@ func TestGenerateKubeadmYAML(t *testing.T) {
 		name      string
 		runtime   string
 		shouldErr bool
-		cfg       config.KubernetesConfig
+		cfg       config.MachineConfig
 	}{
-		{"default", "docker", false, config.KubernetesConfig{}},
-		{"containerd", "containerd", false, config.KubernetesConfig{}},
-		{"crio", "crio", false, config.KubernetesConfig{}},
-		{"options", "docker", false, config.KubernetesConfig{ExtraOptions: extraOpts}},
-		{"crio-options-gates", "crio", false, config.KubernetesConfig{ExtraOptions: extraOpts, FeatureGates: "a=b"}},
-		{"unknown-component", "docker", true, config.KubernetesConfig{ExtraOptions: config.ExtraOptionSlice{config.ExtraOption{Component: "not-a-real-component", Key: "killswitch", Value: "true"}}}},
-		{"containerd-api-port", "containerd", false, config.KubernetesConfig{NodePort: 12345}},
-		{"containerd-pod-network-cidr", "containerd", false, config.KubernetesConfig{ExtraOptions: extraOptsPodCidr}},
-		{"image-repository", "docker", false, config.KubernetesConfig{ImageRepository: "test/repo"}},
+		{"default", "docker", false, config.MachineConfig{}},
+		{"containerd", "containerd", false, config.MachineConfig{}},
+		{"crio", "crio", false, config.MachineConfig{}},
+		{"options", "docker", false, config.MachineConfig{KubernetesConfig: config.KubernetesConfig{ExtraOptions: extraOpts}}},
+		{"crio-options-gates", "crio", false, config.MachineConfig{KubernetesConfig: config.KubernetesConfig{ExtraOptions: extraOpts, FeatureGates: "a=b"}}},
+		{"unknown-component", "docker", true, config.MachineConfig{KubernetesConfig: config.KubernetesConfig{ExtraOptions: config.ExtraOptionSlice{config.ExtraOption{Component: "not-a-real-component", Key: "killswitch", Value: "true"}}}}},
+		{"containerd-api-port", "containerd", false, config.MachineConfig{Nodes: []config.Node{config.Node{Port: 12345}}}},
+		{"containerd-pod-network-cidr", "containerd", false, config.MachineConfig{KubernetesConfig: config.KubernetesConfig{ExtraOptions: extraOptsPodCidr}}},
+		{"image-repository", "docker", false, config.MachineConfig{KubernetesConfig: config.KubernetesConfig{ImageRepository: "test/repo"}}},
 	}
 	for _, version := range versions {
 		for _, tc := range tests {
@@ -187,9 +191,19 @@ func TestGenerateKubeadmYAML(t *testing.T) {
 			tname := tc.name + "_" + version
 			t.Run(tname, func(t *testing.T) {
 				cfg := tc.cfg
-				cfg.NodeIP = "1.1.1.1"
-				cfg.NodeName = "mk"
-				cfg.KubernetesVersion = version + ".0"
+
+				if len(cfg.Nodes) > 0 {
+					cfg.Nodes[0].IP = "1.1.1.1"
+					cfg.Nodes[0].Name = "mk"
+				} else {
+					cfg.Nodes = []config.Node{
+						config.Node{
+							IP:   "1.1.1.1",
+							Name: "mk",
+						},
+					}
+				}
+				cfg.KubernetesConfig.KubernetesVersion = version + ".0"
 
 				got, err := GenerateKubeadmYAML(cfg, runtime)
 				if err != nil && !tc.shouldErr {
diff --git a/pkg/minikube/bootstrapper/bsutil/kubelet_test.go b/pkg/minikube/bootstrapper/bsutil/kubelet_test.go
index 226f486996..0be1880f11 100644
--- a/pkg/minikube/bootstrapper/bsutil/kubelet_test.go
+++ b/pkg/minikube/bootstrapper/bsutil/kubelet_test.go
@@ -30,17 +30,23 @@ import (
 func TestGenerateKubeletConfig(t *testing.T) {
 	tests := []struct {
 		description string
-		cfg         config.KubernetesConfig
+		cfg         config.MachineConfig
 		expected    string
 		shouldErr   bool
 	}{
 		{
 			description: "old docker",
-			cfg: config.KubernetesConfig{
-				NodeIP:            "192.168.1.100",
-				KubernetesVersion: constants.OldestKubernetesVersion,
-				NodeName:          "minikube",
-				ContainerRuntime:  "docker",
+			cfg: config.MachineConfig{
+				KubernetesConfig: config.KubernetesConfig{
+					KubernetesVersion: constants.OldestKubernetesVersion,
+					ContainerRuntime:  "docker",
+				},
+				Nodes: []config.Node{
+					config.Node{
+						IP:   "192.168.1.100",
+						Name: "minikube",
+					},
+				},
 			},
 			expected: `[Unit]
 Wants=docker.socket
@@ -54,11 +60,17 @@ ExecStart=/var/lib/minikube/binaries/v1.11.10/kubelet --allow-privileged=true --
 		},
 		{
 			description: "newest cri runtime",
-			cfg: config.KubernetesConfig{
-				NodeIP:            "192.168.1.100",
-				KubernetesVersion: constants.NewestKubernetesVersion,
-				NodeName:          "minikube",
-				ContainerRuntime:  "cri-o",
+			cfg: config.MachineConfig{
+				KubernetesConfig: config.KubernetesConfig{
+					KubernetesVersion: constants.NewestKubernetesVersion,
+					ContainerRuntime:  "cri-o",
+				},
+				Nodes: []config.Node{
+					config.Node{
+						IP:   "192.168.1.100",
+						Name: "minikube",
+					},
+				},
 			},
 			expected: `[Unit]
 Wants=crio.service
@@ -72,11 +84,17 @@ ExecStart=/var/lib/minikube/binaries/v1.17.0/kubelet --authorization-mode=Webhoo
 		},
 		{
 			description: "default containerd runtime",
-			cfg: config.KubernetesConfig{
-				NodeIP:            "192.168.1.100",
-				KubernetesVersion: constants.DefaultKubernetesVersion,
-				NodeName:          "minikube",
-				ContainerRuntime:  "containerd",
+			cfg: config.MachineConfig{
+				KubernetesConfig: config.KubernetesConfig{
+					KubernetesVersion: constants.DefaultKubernetesVersion,
+					ContainerRuntime:  "containerd",
+				},
+				Nodes: []config.Node{
+					config.Node{
+						IP:   "192.168.1.100",
+						Name: "minikube",
+					},
+				},
 			},
 			expected: `[Unit]
 Wants=containerd.service
@@ -90,16 +108,22 @@ ExecStart=/var/lib/minikube/binaries/v1.17.0/kubelet --authorization-mode=Webhoo
 		},
 		{
 			description: "default containerd runtime",
-			cfg: config.KubernetesConfig{
-				NodeIP:            "192.168.1.100",
-				KubernetesVersion: constants.DefaultKubernetesVersion,
-				NodeName:          "minikube",
-				ContainerRuntime:  "containerd",
-				ExtraOptions: config.ExtraOptionSlice{
-					config.ExtraOption{
-						Component: Kubelet,
-						Key:       "node-ip",
-						Value:     "192.168.1.200",
+			cfg: config.MachineConfig{
+				KubernetesConfig: config.KubernetesConfig{
+					KubernetesVersion: constants.OldestKubernetesVersion,
+					ContainerRuntime:  "containerd",
+					ExtraOptions: config.ExtraOptionSlice{
+						config.ExtraOption{
+							Component: Kubelet,
+							Key:       "node-ip",
+							Value:     "192.168.1.200",
+						},
+					},
+				},
+				Nodes: []config.Node{
+					config.Node{
+						IP:   "192.168.1.100",
+						Name: "minikube",
 					},
 				},
 			},
@@ -115,12 +139,18 @@ ExecStart=/var/lib/minikube/binaries/v1.17.0/kubelet --authorization-mode=Webhoo
 		},
 		{
 			description: "docker with custom image repository",
-			cfg: config.KubernetesConfig{
-				NodeIP:            "192.168.1.100",
-				KubernetesVersion: constants.DefaultKubernetesVersion,
-				NodeName:          "minikube",
-				ContainerRuntime:  "docker",
-				ImageRepository:   "docker-proxy-image.io/google_containers",
+			cfg: config.MachineConfig{
+				KubernetesConfig: config.KubernetesConfig{
+					KubernetesVersion: constants.DefaultKubernetesVersion,
+					ContainerRuntime:  "docker",
+					ImageRepository:   "docker-proxy-image.io/google_containers",
+				},
+				Nodes: []config.Node{
+					config.Node{
+						IP:   "192.168.1.100",
+						Name: "minikube",
+					},
+				},
 			},
 			expected: `[Unit]
 Wants=docker.socket
diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index 2c28f74050..51c6b86c7a 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -272,7 +272,7 @@ func (k *Bootstrapper) WaitForCluster(cfg config.MachineConfig, timeout time.Dur
 		return errors.Wrap(err, "get k8s client")
 	}
 
-	return kverify.SystemPods(c, start, master.IP, master.Port, timeout)
+	return kverify.SystemPods(c, start, timeout)
 }
 
 // restartCluster restarts the Kubernetes cluster configured by kubeadm
@@ -327,7 +327,7 @@ func (k *Bootstrapper) restartCluster(cfg config.MachineConfig) error {
 			return errors.Wrap(err, "getting k8s client")
 		}
 
-		if err := kverify.SystemPods(client, time.Now(), n.IP, n.Port, kconst.DefaultControlPlaneTimeout); err != nil {
+		if err := kverify.SystemPods(client, time.Now(), kconst.DefaultControlPlaneTimeout); err != nil {
 			return errors.Wrap(err, "system pods")
 		}
 

From 412e78e54c6b5d1ae9c13bf55a8a4691f8566d01 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Thu, 23 Jan 2020 11:11:28 -0800
Subject: [PATCH 101/250] minimize the number of commands executed

---
 pkg/minikube/bootstrapper/bsutil/binaries.go |  9 +++-
 pkg/minikube/bootstrapper/certs.go           | 17 +++----
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 11 +++++
 pkg/minikube/cluster/cluster.go              | 52 ++++++++++++++++++++
 pkg/minikube/command/exec_runner.go          |  3 --
 pkg/minikube/command/kic_runner.go           | 23 +++++----
 pkg/minikube/command/ssh_runner.go           |  2 +-
 pkg/minikube/machine/cache_images.go         |  8 ++-
 pkg/provision/buildroot.go                   | 29 ++++++++++-
 9 files changed, 127 insertions(+), 27 deletions(-)

diff --git a/pkg/minikube/bootstrapper/bsutil/binaries.go b/pkg/minikube/bootstrapper/bsutil/binaries.go
index 183d217ae0..532be8a7ad 100644
--- a/pkg/minikube/bootstrapper/bsutil/binaries.go
+++ b/pkg/minikube/bootstrapper/bsutil/binaries.go
@@ -18,6 +18,7 @@ limitations under the License.
 package bsutil
 
 import (
+	"os/exec"
 	"path"
 	"runtime"
 
@@ -32,6 +33,12 @@ import (
 
 // TransferBinaries transfers all required Kubernetes binaries
 func TransferBinaries(cfg config.KubernetesConfig, c command.Runner) error {
+	dir := binRoot(cfg.KubernetesVersion)
+	_, err := c.RunCmd(exec.Command("sudo", "mkdir", "-p", dir))
+	if err != nil {
+		return err
+	}
+
 	var g errgroup.Group
 	for _, name := range constants.KubernetesReleaseBinaries {
 		name := name
@@ -41,7 +48,7 @@ func TransferBinaries(cfg config.KubernetesConfig, c command.Runner) error {
 				return errors.Wrapf(err, "downloading %s", name)
 			}
 
-			dst := path.Join(binRoot(cfg.KubernetesVersion), name)
+			dst := path.Join(dir, name)
 			if err := machine.CopyBinary(c, src, dst); err != nil {
 				return errors.Wrapf(err, "copybinary %s -> %s", src, dst)
 			}
diff --git a/pkg/minikube/bootstrapper/certs.go b/pkg/minikube/bootstrapper/certs.go
index 21c573454c..870fe43091 100644
--- a/pkg/minikube/bootstrapper/certs.go
+++ b/pkg/minikube/bootstrapper/certs.go
@@ -341,11 +341,9 @@ func configureCACerts(cr command.Runner, caCerts map[string]string) error {
 	for _, caCertFile := range caCerts {
 		dstFilename := path.Base(caCertFile)
 		certStorePath := path.Join(SSLCertStoreDir, dstFilename)
-		_, err := cr.RunCmd(exec.Command("sudo", "test", "-f", certStorePath))
-		if err != nil {
-			if _, err := cr.RunCmd(exec.Command("sudo", "ln", "-s", caCertFile, certStorePath)); err != nil {
-				return errors.Wrapf(err, "create symlink for %s", caCertFile)
-			}
+		cmd := fmt.Sprintf("test -f %s || ln -s %s %s", caCertFile, certStorePath, caCertFile)
+		if _, err := cr.RunCmd(exec.Command("sudo", "-s", "eval", cmd)); err != nil {
+			return errors.Wrapf(err, "create symlink for %s", caCertFile)
 		}
 		if hasSSLBinary {
 			subjectHash, err := getSubjectHash(cr, caCertFile)
@@ -353,11 +351,10 @@ func configureCACerts(cr command.Runner, caCerts map[string]string) error {
 				return errors.Wrapf(err, "calculate hash for cacert %s", caCertFile)
 			}
 			subjectHashLink := path.Join(SSLCertStoreDir, fmt.Sprintf("%s.0", subjectHash))
-			_, err = cr.RunCmd(exec.Command("sudo", "test", "-f", subjectHashLink))
-			if err != nil {
-				if _, err := cr.RunCmd(exec.Command("sudo", "ln", "-s", certStorePath, subjectHashLink)); err != nil {
-					return errors.Wrapf(err, "linking caCertFile %s", caCertFile)
-				}
+
+			cmd := fmt.Sprintf("test -f %s || ln -s %s %s", subjectHashLink, certStorePath, subjectHashLink)
+			if _, err := cr.RunCmd(exec.Command("sudo", "-s", "eval", cmd)); err != nil {
+				return errors.Wrapf(err, "create symlink for %s", caCertFile)
 			}
 		}
 	}
diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index 1f27f156cd..a00b3ed0d7 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -434,6 +434,17 @@ func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error {
 	if err := bsutil.AddAddons(&files, assets.GenerateTemplateData(cfg.KubernetesConfig)); err != nil {
 		return errors.Wrap(err, "adding addons")
 	}
+
+	// Combine mkdir request into a single call to reduce load
+	dirs := []string{}
+	for _, f := range files {
+		dirs = append(dirs, f.GetTargetDir())
+	}
+	args := append([]string{"mkdir", "-p"}, dirs...)
+	if _, err := k.c.RunCmd(exec.Command("sudo", args...)); err != nil {
+		return errors.Wrap(err, "mkdir")
+	}
+
 	for _, f := range files {
 		if err := k.c.Copy(f); err != nil {
 			return errors.Wrapf(err, "copy")
diff --git a/pkg/minikube/cluster/cluster.go b/pkg/minikube/cluster/cluster.go
index 97492a33cf..af8fb5f0ea 100644
--- a/pkg/minikube/cluster/cluster.go
+++ b/pkg/minikube/cluster/cluster.go
@@ -24,6 +24,7 @@ import (
 	"math"
 	"net"
 	"os/exec"
+	"path"
 	"path/filepath"
 	"regexp"
 	"strconv"
@@ -46,6 +47,7 @@ import (
 	"github.com/shirou/gopsutil/mem"
 	"github.com/spf13/viper"
 
+	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/minikube/config"
 	cfg "k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
@@ -54,6 +56,8 @@ import (
 	"k8s.io/minikube/pkg/minikube/localpath"
 	"k8s.io/minikube/pkg/minikube/out"
 	"k8s.io/minikube/pkg/minikube/registry"
+	"k8s.io/minikube/pkg/minikube/sshutil"
+	"k8s.io/minikube/pkg/minikube/vmpath"
 	"k8s.io/minikube/pkg/util/lock"
 	"k8s.io/minikube/pkg/util/retry"
 )
@@ -67,6 +71,17 @@ var (
 	// The maximum the guest VM clock is allowed to be ahead and behind. This value is intentionally
 	// large to allow for inaccurate methodology, but still small enough so that certificates are likely valid.
 	maxClockDesyncSeconds = 2.1
+
+	// requiredDirectories are directories to create on the host during setup
+	requiredDirectories = []string{
+		vmpath.GuestAddonsDir,
+		vmpath.GuestManifestsDir,
+		vmpath.GuestEphemeralDir,
+		vmpath.GuestPersistentDir,
+		vmpath.GuestCertsDir,
+		path.Join(vmpath.GuestPersistentDir, "images"),
+		path.Join(vmpath.GuestPersistentDir, "binaries"),
+	}
 )
 
 // This init function is used to set the logtostderr variable to false so that INFO level log info does not clutter the CLI
@@ -497,6 +512,10 @@ func createHost(api libmachine.API, config cfg.MachineConfig) (*host.Host, error
 		return nil, errors.Wrap(err, "create")
 	}
 
+	if err := createRequiredDirectories(h); err != nil {
+		return h, err
+	}
+
 	if driver.BareMetal(config.VMDriver) {
 		showLocalOsRelease()
 	} else if !driver.BareMetal(config.VMDriver) && !driver.IsKIC(config.VMDriver) {
@@ -646,3 +665,36 @@ func IsMinikubeRunning(api libmachine.API) bool {
 	}
 	return true
 }
+
+// createRequiredDirectories creates directories expected by minikube to exist
+func createRequiredDirectories(h *host.Host) error {
+	glog.Infof("creating required directories: %v", requiredDirectories)
+	r, err := commandRunner(h)
+	if err != nil {
+		return errors.Wrap(err, "command runner")
+	}
+
+	args := append([]string{"mkdir", "-p"}, requiredDirectories...)
+	if _, err := r.RunCmd(exec.Command("sudo", args...)); err != nil {
+		return errors.Wrap(err, "mkdir")
+	}
+	return nil
+}
+
+// commandRunner returns best available command runner for this host
+func commandRunner(h *host.Host) (command.Runner, error) {
+	if h.DriverName == driver.Mock {
+		return &command.FakeCommandRunner{}, nil
+	}
+	if driver.BareMetal(h.Driver.DriverName()) {
+		return &command.ExecRunner{}, nil
+	}
+	if h.Driver.DriverName() == driver.Docker {
+		return command.NewKICRunner(h.Name, "docker"), nil
+	}
+	client, err := sshutil.NewSSHClient(h.Driver)
+	if err != nil {
+		return nil, errors.Wrap(err, "getting ssh client for bootstrapper")
+	}
+	return command.NewSSHRunner(client), nil
+}
diff --git a/pkg/minikube/command/exec_runner.go b/pkg/minikube/command/exec_runner.go
index f9872db757..ed7740b5b6 100644
--- a/pkg/minikube/command/exec_runner.go
+++ b/pkg/minikube/command/exec_runner.go
@@ -80,9 +80,6 @@ func (*ExecRunner) RunCmd(cmd *exec.Cmd) (*RunResult, error) {
 
 // Copy copies a file and its permissions
 func (*ExecRunner) Copy(f assets.CopyableFile) error {
-	if err := os.MkdirAll(f.GetTargetDir(), os.ModePerm); err != nil {
-		return errors.Wrapf(err, "error making dirs for %s", f.GetTargetDir())
-	}
 	targetPath := path.Join(f.GetTargetDir(), f.GetTargetName())
 	if _, err := os.Stat(targetPath); err == nil {
 		if err := os.Remove(targetPath); err != nil {
diff --git a/pkg/minikube/command/kic_runner.go b/pkg/minikube/command/kic_runner.go
index bf357c356e..cc58035fd1 100644
--- a/pkg/minikube/command/kic_runner.go
+++ b/pkg/minikube/command/kic_runner.go
@@ -24,6 +24,7 @@ import (
 	"os"
 	"os/exec"
 	"path"
+	"strconv"
 	"time"
 
 	"github.com/golang/glog"
@@ -118,7 +119,7 @@ func (k *kicRunner) RunCmd(cmd *exec.Cmd) (*RunResult, error) {
 		if exitError, ok := err.(*exec.ExitError); ok {
 			rr.ExitCode = exitError.ExitCode()
 		}
-		err = errors.Wrapf(err, "command failed: %s", oc.Args)
+		err = fmt.Errorf("%s: %v\nstdout:\n%s\nstderr:\n%s", rr.Command(), err, rr.Stdout.String(), rr.Stderr.String())
 	}
 	return rr, err
 
@@ -148,22 +149,24 @@ func (k *kicRunner) Copy(f assets.CopyableFile) error {
 			return errors.Wrap(err, "close temporary file")
 		}
 		assetName = tmpFile.Name()
-
 	}
+
 	// based of format of "docker cp containerName:destination"
 	destination := fmt.Sprintf("%s:%s/%s", k.nameOrID, f.GetTargetDir(), f.GetTargetName())
-	// make sure dir exists inside the container
-	if _, err := k.RunCmd(exec.Command("mkdir", "-p", f.GetTargetDir())); err != nil {
-		return errors.Wrapf(err, "error making dir %s", f.GetTargetDir())
+
+	perms, err := strconv.ParseInt(f.GetPermissions(), 8, 0)
+	if err != nil {
+		return errors.Wrapf(err, "error converting permissions %s to integer", f.GetPermissions())
 	}
 
-	if out, err := exec.Command(k.ociBin, "cp", assetName, destination).CombinedOutput(); err != nil {
+	// Rely on cp -a to propagate permissions
+	if err := os.Chmod(assetName, os.FileMode(perms)); err != nil {
+		return errors.Wrapf(err, "chmod")
+	}
+	if out, err := exec.Command(k.ociBin, "cp", "-a", assetName, destination).CombinedOutput(); err != nil {
 		return errors.Wrapf(err, "error copying %s into node, output: %s", f.GetAssetName(), string(out))
 	}
-	fp := path.Join(f.GetTargetDir(), f.GetTargetName())
-	if _, err := k.RunCmd(exec.Command("sudo", "chmod", f.GetPermissions(), fp)); err != nil {
-		return errors.Wrapf(err, "failed to chmod file permissions %s", fp)
-	}
+
 	return nil
 }
 
diff --git a/pkg/minikube/command/ssh_runner.go b/pkg/minikube/command/ssh_runner.go
index b93e0e5eb4..41e4ba1323 100644
--- a/pkg/minikube/command/ssh_runner.go
+++ b/pkg/minikube/command/ssh_runner.go
@@ -196,7 +196,7 @@ func (s *SSHRunner) Copy(f assets.CopyableFile) error {
 		return nil
 	})
 
-	scp := fmt.Sprintf("sudo mkdir -p %s && sudo scp -t %s", f.GetTargetDir(), f.GetTargetDir())
+	scp := fmt.Sprintf("sudo test -d %s && sudo scp -t %s", f.GetTargetDir(), f.GetTargetDir())
 	mtime, err := f.GetModTime()
 	if err != nil {
 		glog.Infof("error getting modtime for %s: %v", dst, err)
diff --git a/pkg/minikube/machine/cache_images.go b/pkg/minikube/machine/cache_images.go
index 599270c90f..501e7519d7 100644
--- a/pkg/minikube/machine/cache_images.go
+++ b/pkg/minikube/machine/cache_images.go
@@ -22,6 +22,7 @@ import (
 	"path"
 	"path/filepath"
 	"sync"
+	"time"
 
 	"github.com/docker/docker/client"
 	"github.com/docker/machine/libmachine/state"
@@ -63,7 +64,12 @@ func CacheImagesForBootstrapper(imageRepository string, version string, clusterB
 // LoadImages loads previously cached images into the container runtime
 func LoadImages(cc *config.MachineConfig, runner command.Runner, images []string, cacheDir string) error {
 	glog.Infof("LoadImages start: %s", images)
-	defer glog.Infof("LoadImages end")
+	start := time.Now()
+
+	defer func() {
+		glog.Infof("LoadImages completed in %s", time.Since(start))
+	}()
+
 	var g errgroup.Group
 	cr, err := cruntime.New(cruntime.Config{Type: cc.ContainerRuntime, Runner: runner})
 	if err != nil {
diff --git a/pkg/provision/buildroot.go b/pkg/provision/buildroot.go
index b06f60ae92..21b1affebd 100644
--- a/pkg/provision/buildroot.go
+++ b/pkg/provision/buildroot.go
@@ -19,6 +19,7 @@ package provision
 import (
 	"bytes"
 	"fmt"
+	"os/exec"
 	"path"
 	"path/filepath"
 	"strings"
@@ -210,7 +211,7 @@ func (p *BuildrootProvisioner) Provision(swarmOptions swarm.Options, authOptions
 	p.AuthOptions = authOptions
 	p.EngineOptions = engineOptions
 
-	log.Debugf("setting hostname %q", p.Driver.GetMachineName())
+	log.Infof("provisioning hostname %q", p.Driver.GetMachineName())
 	if err := p.SetHostname(p.Driver.GetMachineName()); err != nil {
 		return err
 	}
@@ -221,6 +222,7 @@ func (p *BuildrootProvisioner) Provision(swarmOptions swarm.Options, authOptions
 	log.Debugf("setting up certificates")
 	configAuth := func() error {
 		if err := configureAuth(p); err != nil {
+			log.Warnf("configureAuth failed: %v", err)
 			return &retry.RetriableError{Err: err}
 		}
 		return nil
@@ -296,6 +298,12 @@ CRIO_MINIKUBE_OPTIONS='{{ range .EngineOptions.InsecureRegistry }}--insecure-reg
 }
 
 func configureAuth(p *BuildrootProvisioner) error {
+	log.Infof("configureAuth start")
+	start := time.Now()
+	defer func() {
+		log.Infof("configureAuth took %s", time.Since(start))
+	}()
+
 	driver := p.GetDriver()
 	machineName := driver.GetMachineName()
 	authOptions := p.GetAuthOptions()
@@ -345,6 +353,7 @@ func configureAuth(p *BuildrootProvisioner) error {
 }
 
 func copyHostCerts(authOptions auth.Options) error {
+	log.Infof("copyHostCerts")
 	execRunner := &command.ExecRunner{}
 	hostCerts := map[string]string{
 		authOptions.CaCertPath:     path.Join(authOptions.StorePath, "ca.pem"),
@@ -352,6 +361,10 @@ func copyHostCerts(authOptions auth.Options) error {
 		authOptions.ClientKeyPath:  path.Join(authOptions.StorePath, "key.pem"),
 	}
 
+	_, err := execRunner.RunCmd(exec.Command("mkdir", "-p", authOptions.StorePath))
+	if err != nil {
+		return err
+	}
 	for src, dst := range hostCerts {
 		f, err := assets.NewFileAsset(src, path.Dir(dst), filepath.Base(dst), "0777")
 		if err != nil {
@@ -366,6 +379,8 @@ func copyHostCerts(authOptions auth.Options) error {
 }
 
 func copyRemoteCerts(authOptions auth.Options, driver drivers.Driver) error {
+	log.Infof("copyRemoteCerts")
+
 	remoteCerts := map[string]string{
 		authOptions.CaCertPath:     authOptions.CaCertRemotePath,
 		authOptions.ServerCertPath: authOptions.ServerCertRemotePath,
@@ -377,6 +392,18 @@ func copyRemoteCerts(authOptions auth.Options, driver drivers.Driver) error {
 		return errors.Wrap(err, "provisioning: error getting ssh client")
 	}
 	sshRunner := command.NewSSHRunner(sshClient)
+
+	dirs := []string{}
+	for _, dst := range remoteCerts {
+		dirs = append(dirs, path.Dir(dst))
+	}
+
+	args := append([]string{"mkdir", "-p"}, dirs...)
+	_, err = sshRunner.RunCmd(exec.Command("sudo", args...))
+	if err != nil {
+		return err
+	}
+
 	for src, dst := range remoteCerts {
 		f, err := assets.NewFileAsset(src, path.Dir(dst), filepath.Base(dst), "0640")
 		if err != nil {

From e9f6aa106f037c78d8f2391e1c15472cefe8a948 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Thu, 23 Jan 2020 12:53:33 -0800
Subject: [PATCH 102/250] test fixing

---
 pkg/minikube/bootstrapper/certs.go      |  4 ++--
 pkg/minikube/bootstrapper/certs_test.go | 19 ++++---------------
 pkg/minikube/command/fake_runner.go     | 25 +++++++++++++++++--------
 3 files changed, 23 insertions(+), 25 deletions(-)

diff --git a/pkg/minikube/bootstrapper/certs.go b/pkg/minikube/bootstrapper/certs.go
index 870fe43091..95033b3a56 100644
--- a/pkg/minikube/bootstrapper/certs.go
+++ b/pkg/minikube/bootstrapper/certs.go
@@ -342,7 +342,7 @@ func configureCACerts(cr command.Runner, caCerts map[string]string) error {
 		dstFilename := path.Base(caCertFile)
 		certStorePath := path.Join(SSLCertStoreDir, dstFilename)
 		cmd := fmt.Sprintf("test -f %s || ln -s %s %s", caCertFile, certStorePath, caCertFile)
-		if _, err := cr.RunCmd(exec.Command("sudo", "-s", "eval", cmd)); err != nil {
+		if _, err := cr.RunCmd(exec.Command("sudo", "/bin/bash", "-c", cmd)); err != nil {
 			return errors.Wrapf(err, "create symlink for %s", caCertFile)
 		}
 		if hasSSLBinary {
@@ -353,7 +353,7 @@ func configureCACerts(cr command.Runner, caCerts map[string]string) error {
 			subjectHashLink := path.Join(SSLCertStoreDir, fmt.Sprintf("%s.0", subjectHash))
 
 			cmd := fmt.Sprintf("test -f %s || ln -s %s %s", subjectHashLink, certStorePath, subjectHashLink)
-			if _, err := cr.RunCmd(exec.Command("sudo", "-s", "eval", cmd)); err != nil {
+			if _, err := cr.RunCmd(exec.Command("sudo", "/bin/bash", "-c", cmd)); err != nil {
 				return errors.Wrapf(err, "create symlink for %s", caCertFile)
 			}
 		}
diff --git a/pkg/minikube/bootstrapper/certs_test.go b/pkg/minikube/bootstrapper/certs_test.go
index 167cf4084e..24b7ae6730 100644
--- a/pkg/minikube/bootstrapper/certs_test.go
+++ b/pkg/minikube/bootstrapper/certs_test.go
@@ -17,9 +17,7 @@ limitations under the License.
 package bootstrapper
 
 import (
-	"fmt"
 	"os"
-	"path"
 	"path/filepath"
 	"testing"
 
@@ -53,21 +51,12 @@ func TestSetupCerts(t *testing.T) {
 		t.Fatalf("error generating certificate: %v", err)
 	}
 
-	cmdMap := map[string]string{
-		"sudo mkdir -p  /var/lib/minikube/certs": "",
-	}
-	certFilenames := map[string]string{"ca.crt": "minikubeCA.pem", "mycert.pem": "mycert.pem"}
-	for _, dst := range certFilenames {
-		certFile := path.Join(CACertificatesDir, dst)
-		certStorePath := path.Join(SSLCertStoreDir, dst)
-		certNameHash := "abcdef"
-		remoteCertHashLink := path.Join(SSLCertStoreDir, fmt.Sprintf("%s.0", certNameHash))
-		cmdMap[fmt.Sprintf("sudo ln -s %s %s", certFile, certStorePath)] = "1"
-		cmdMap[fmt.Sprintf("openssl x509 -hash -noout -in %s", certFile)] = certNameHash
-		cmdMap[fmt.Sprintf("sudo ln -s %s %s", certStorePath, remoteCertHashLink)] = "1"
+	expected := map[string]string{
+		"fuck": "off",
+		`sudo /bin/bash -c "test -f /usr/share/ca-certificates/mycert.pem || ln -s /etc/ssl/certs/mycert.pem /usr/share/ca-certificates/mycert.pem"`: "wut",
 	}
 	f := command.NewFakeCommandRunner()
-	f.SetCommandToOutput(cmdMap)
+	f.SetCommandToOutput(expected)
 
 	var filesToBeTransferred []string
 	for _, cert := range certs {
diff --git a/pkg/minikube/command/fake_runner.go b/pkg/minikube/command/fake_runner.go
index fbdd40ca40..4ddcb0319b 100644
--- a/pkg/minikube/command/fake_runner.go
+++ b/pkg/minikube/command/fake_runner.go
@@ -55,6 +55,10 @@ func (f *FakeCommandRunner) RunCmd(cmd *exec.Cmd) (*RunResult, error) {
 	start := time.Now()
 
 	out, ok := f.cmdMap.Load(strings.Join(rr.Args, " "))
+	if !ok {
+		return rr, fmt.Errorf("unregistered command: `%s`\nexpected: %v\n", rr.Command(), f.commands())
+	}
+
 	var buf bytes.Buffer
 	outStr := ""
 	if out != nil {
@@ -69,14 +73,9 @@ func (f *FakeCommandRunner) RunCmd(cmd *exec.Cmd) (*RunResult, error) {
 
 	elapsed := time.Since(start)
 
-	if ok {
-		// Reduce log spam
-		if elapsed > (1 * time.Second) {
-			glog.Infof("(FakeCommandRunner) Done: %v: (%s)", rr.Command(), elapsed)
-		}
-	} else {
-		glog.Infof("(FakeCommandRunner) Non-zero exit: %v: (%s)\n%s", rr.Command(), elapsed, out)
-		return rr, fmt.Errorf("unavailable command: %s", rr.Command())
+	// Reduce log spam
+	if elapsed > (1 * time.Second) {
+		glog.Infof("(FakeCommandRunner) Done: %v: (%s)", rr.Command(), elapsed)
 	}
 	return rr, nil
 }
@@ -108,6 +107,7 @@ func (f *FakeCommandRunner) SetFileToContents(fileToContents map[string]string)
 // SetCommandToOutput stores the file to contents map for the FakeCommandRunner
 func (f *FakeCommandRunner) SetCommandToOutput(cmdToOutput map[string]string) {
 	for k, v := range cmdToOutput {
+		glog.Infof("fake command %q -> %q", k, v)
 		f.cmdMap.Store(k, v)
 	}
 }
@@ -121,6 +121,15 @@ func (f *FakeCommandRunner) GetFileToContents(filename string) (string, error) {
 	return contents.(string), nil
 }
 
+func (f *FakeCommandRunner) commands() []string {
+	cmds := []string{}
+	f.cmdMap.Range(func(k, v interface{}) bool {
+		cmds = append(cmds, fmt.Sprintf("`%s`", k))
+		return true
+	})
+	return cmds
+}
+
 // DumpMaps prints out the list of stored commands and stored filenames.
 func (f *FakeCommandRunner) DumpMaps(w io.Writer) {
 	fmt.Fprintln(w, "Commands:")

From 791f28ff4cf6ac6bd6eded023633e492c1f3df5a Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Thu, 23 Jan 2020 13:21:21 -0800
Subject: [PATCH 103/250] things work now

---
 cmd/minikube/cmd/start.go                     |  9 ++++----
 go.mod                                        |  3 +++
 pkg/minikube/bootstrapper/bsutil/kubelet.go   |  6 ++++-
 .../bootstrapper/bsutil/kubelet_test.go       |  6 ++---
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go  | 23 +++++++++++--------
 pkg/minikube/config/node.go                   |  4 ++--
 pkg/minikube/config/types.go                  |  1 -
 pkg/minikube/machine/cache_images.go          |  2 +-
 pkg/provision/buildroot.go                    |  2 +-
 test/integration/aaa_download_only_test.go    |  2 +-
 10 files changed, 33 insertions(+), 25 deletions(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 2c9ac06476..69074b6f9a 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -362,7 +362,7 @@ func runStart(cmd *cobra.Command, args []string) {
 	}
 
 	// setup kubeadm (must come after setupKubeconfig)
-	bs := setupKubeAdm(machineAPI, mRunner, mc, n)
+	bs := setupKubeAdm(machineAPI, mc, n)
 
 	// pull images or restart cluster
 	bootstrapCluster(bs, cr, mRunner, mc, preExists, isUpgrade)
@@ -953,7 +953,6 @@ func generateCfgFromFlags(cmd *cobra.Command, k8sVersion string, drvName string)
 		CPUs:                viper.GetInt(cpus),
 		DiskSize:            pkgutil.CalculateSizeInMB(viper.GetString(humanReadableDiskSize)),
 		VMDriver:            drvName,
-		ContainerRuntime:    viper.GetString(containerRuntime),
 		HyperkitVpnKitSock:  viper.GetString(vpnkitSock),
 		HyperkitVSockPorts:  viper.GetStringSlice(vsockPorts),
 		NFSShare:            viper.GetStringSlice(nfsShare),
@@ -1254,7 +1253,7 @@ func getKubernetesVersion(old *config.MachineConfig) (string, bool) {
 }
 
 // setupKubeAdm adds any requested files into the VM before Kubernetes is started
-func setupKubeAdm(mAPI libmachine.API, cr command.Runner, config config.MachineConfig, node config.Node) bootstrapper.Bootstrapper {
+func setupKubeAdm(mAPI libmachine.API, cfg config.MachineConfig, node config.Node) bootstrapper.Bootstrapper {
 	bs, err := getClusterBootstrapper(mAPI, viper.GetString(cmdcfg.Bootstrapper))
 	if err != nil {
 		exit.WithError("Failed to get bootstrapper", err)
@@ -1263,10 +1262,10 @@ func setupKubeAdm(mAPI libmachine.API, cr command.Runner, config config.MachineC
 		out.T(out.Option, "{{.extra_option_component_name}}.{{.key}}={{.value}}", out.V{"extra_option_component_name": eo.Component, "key": eo.Key, "value": eo.Value})
 	}
 	// Loads cached images, generates config files, download binaries
-	if err := bs.UpdateCluster(config); err != nil {
+	if err := bs.UpdateCluster(cfg); err != nil {
 		exit.WithError("Failed to update cluster", err)
 	}
-	if err := bs.SetupCerts(config.KubernetesConfig, node); err != nil {
+	if err := bs.SetupCerts(cfg.KubernetesConfig, node); err != nil {
 		exit.WithError("Failed to setup certs", err)
 	}
 	return bs
diff --git a/go.mod b/go.mod
index 0cd9aca2bf..28d8e10692 100644
--- a/go.mod
+++ b/go.mod
@@ -22,12 +22,15 @@ require (
 	github.com/elazarl/goproxy v0.0.0-20190421051319-9d40249d3c2f
 	github.com/elazarl/goproxy/ext v0.0.0-20190421051319-9d40249d3c2f // indirect
 	github.com/evanphx/json-patch v4.5.0+incompatible // indirect
+	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/go-ole/go-ole v1.2.4 // indirect
 	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
+	github.com/google/btree v1.0.0 // indirect
 	github.com/google/go-cmp v0.3.0
 	github.com/googleapis/gnostic v0.3.0 // indirect
 	github.com/gorilla/mux v1.7.3 // indirect
+	github.com/grpc-ecosystem/grpc-gateway v1.5.0 // indirect
 	github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce // indirect
 	github.com/hashicorp/go-getter v1.4.0
 	github.com/hashicorp/go-multierror v0.0.0-20160811015721-8c5f0ad93604 // indirect
diff --git a/pkg/minikube/bootstrapper/bsutil/kubelet.go b/pkg/minikube/bootstrapper/bsutil/kubelet.go
index abb452af39..7e5f215328 100644
--- a/pkg/minikube/bootstrapper/bsutil/kubelet.go
+++ b/pkg/minikube/bootstrapper/bsutil/kubelet.go
@@ -53,8 +53,12 @@ func NewKubeletConfig(mc config.MachineConfig, r cruntime.Manager) ([]byte, erro
 	if k8s.NetworkPlugin != "" {
 		extraOpts["network-plugin"] = k8s.NetworkPlugin
 	}
+	master, err := config.GetMasterNode(mc)
+	if err != nil {
+		return nil, errors.Wrap(err, "getting master node")
+	}
 	if _, ok := extraOpts["node-ip"]; !ok {
-		extraOpts["node-ip"] = mc.Nodes[0].IP
+		extraOpts["node-ip"] = master.IP
 	}
 
 	pauseImage := images.Pause(k8s.ImageRepository)
diff --git a/pkg/minikube/bootstrapper/bsutil/kubelet_test.go b/pkg/minikube/bootstrapper/bsutil/kubelet_test.go
index 0be1880f11..3861fbd7db 100644
--- a/pkg/minikube/bootstrapper/bsutil/kubelet_test.go
+++ b/pkg/minikube/bootstrapper/bsutil/kubelet_test.go
@@ -107,10 +107,10 @@ ExecStart=/var/lib/minikube/binaries/v1.17.0/kubelet --authorization-mode=Webhoo
 `,
 		},
 		{
-			description: "default containerd runtime",
+			description: "default containerd runtime with IP override",
 			cfg: config.MachineConfig{
 				KubernetesConfig: config.KubernetesConfig{
-					KubernetesVersion: constants.OldestKubernetesVersion,
+					KubernetesVersion: constants.DefaultKubernetesVersion,
 					ContainerRuntime:  "containerd",
 					ExtraOptions: config.ExtraOptionSlice{
 						config.ExtraOption{
@@ -166,7 +166,7 @@ ExecStart=/var/lib/minikube/binaries/v1.17.0/kubelet --authorization-mode=Webhoo
 
 	for _, tc := range tests {
 		t.Run(tc.description, func(t *testing.T) {
-			runtime, err := cruntime.New(cruntime.Config{Type: tc.cfg.ContainerRuntime,
+			runtime, err := cruntime.New(cruntime.Config{Type: tc.cfg.KubernetesConfig.ContainerRuntime,
 				Runner: command.NewFakeCommandRunner()})
 			if err != nil {
 				t.Fatalf("runtime: %v", err)
diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index 51c6b86c7a..bd0d63628f 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -213,7 +213,7 @@ func (k *Bootstrapper) StartCluster(cfg config.MachineConfig) error {
 	if !driver.IsKIC(cfg.VMDriver) { // TODO: skip for both after verifications https://github.com/kubernetes/minikube/issues/6239
 		glog.Infof("Configuring cluster permissions ...")
 		elevate := func() error {
-			client, err := k.client(cfg.KubernetesConfig, master)
+			client, err := k.client(master)
 			if err != nil {
 				return err
 			}
@@ -233,22 +233,22 @@ func (k *Bootstrapper) StartCluster(cfg config.MachineConfig) error {
 }
 
 // client sets and returns a Kubernetes client to use to speak to a kubeadm launched apiserver
-func (k *Bootstrapper) client(cfg config.KubernetesConfig, n config.Node) (*kubernetes.Clientset, error) {
+func (k *Bootstrapper) client(n config.Node) (*kubernetes.Clientset, error) {
 	if k.k8sClient != nil {
 		return k.k8sClient, nil
 	}
 
-	config, err := kapi.ClientConfig(k.contextName)
+	cc, err := kapi.ClientConfig(k.contextName)
 	if err != nil {
 		return nil, errors.Wrap(err, "client config")
 	}
 
 	endpoint := fmt.Sprintf("https://%s", net.JoinHostPort(n.IP, strconv.Itoa(n.Port)))
-	if config.Host != endpoint {
-		glog.Errorf("Overriding stale ClientConfig host %s with %s", config.Host, endpoint)
-		config.Host = endpoint
+	if cc.Host != endpoint {
+		glog.Errorf("Overriding stale ClientConfig host %s with %s", cc.Host, endpoint)
+		cc.Host = endpoint
 	}
-	c, err := kubernetes.NewForConfig(config)
+	c, err := kubernetes.NewForConfig(cc)
 	if err == nil {
 		k.k8sClient = c
 	}
@@ -260,6 +260,9 @@ func (k *Bootstrapper) WaitForCluster(cfg config.MachineConfig, timeout time.Dur
 	start := time.Now()
 	out.T(out.Waiting, "Waiting for cluster to come online ...")
 	master, err := config.GetMasterNode(cfg)
+	if err != nil {
+		return err
+	}
 	if err := kverify.APIServerProcess(k.c, start, timeout); err != nil {
 		return err
 	}
@@ -267,7 +270,7 @@ func (k *Bootstrapper) WaitForCluster(cfg config.MachineConfig, timeout time.Dur
 		return err
 	}
 
-	c, err := k.client(cfg.KubernetesConfig, master)
+	c, err := k.client(master)
 	if err != nil {
 		return errors.Wrap(err, "get k8s client")
 	}
@@ -322,7 +325,7 @@ func (k *Bootstrapper) restartCluster(cfg config.MachineConfig) error {
 	}
 
 	for _, n := range cfg.Nodes {
-		client, err := k.client(cfg.KubernetesConfig, n)
+		client, err := k.client(n)
 		if err != nil {
 			return errors.Wrap(err, "getting k8s client")
 		}
@@ -396,7 +399,7 @@ func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error {
 			out.FailureT("Unable to load cached images: {{.error}}", out.V{"error": err})
 		}
 	}
-	r, err := cruntime.New(cruntime.Config{Type: cfg.ContainerRuntime,
+	r, err := cruntime.New(cruntime.Config{Type: cfg.KubernetesConfig.ContainerRuntime,
 		Runner: k.c, Socket: cfg.KubernetesConfig.CRISocket})
 	if err != nil {
 		return errors.Wrap(err, "runtime")
diff --git a/pkg/minikube/config/node.go b/pkg/minikube/config/node.go
index 38a920f2d2..92af72f28f 100644
--- a/pkg/minikube/config/node.go
+++ b/pkg/minikube/config/node.go
@@ -17,7 +17,7 @@ limitations under the License.
 package config
 
 // AddNode adds a new node config to an existing cluster.
-func AddNode(cc *MachineConfig, name string, controlPlane bool, k8sVersion string, profileName string) {
+func AddNode(cc *MachineConfig, name string, controlPlane bool, k8sVersion string, profileName string) error {
 	t := Worker
 	if controlPlane {
 		t = Master
@@ -32,5 +32,5 @@ func AddNode(cc *MachineConfig, name string, controlPlane bool, k8sVersion strin
 	}
 
 	cc.Nodes = append(cc.Nodes, node)
-	SaveProfile(profileName, cc)
+	return SaveProfile(profileName, cc)
 }
diff --git a/pkg/minikube/config/types.go b/pkg/minikube/config/types.go
index 9884c41429..3e83c78414 100644
--- a/pkg/minikube/config/types.go
+++ b/pkg/minikube/config/types.go
@@ -40,7 +40,6 @@ type MachineConfig struct {
 	CPUs                int
 	DiskSize            int
 	VMDriver            string
-	ContainerRuntime    string
 	HyperkitVpnKitSock  string   // Only used by the Hyperkit driver
 	HyperkitVSockPorts  []string // Only used by the Hyperkit driver
 	DockerEnv           []string // Each entry is formatted as KEY=VALUE.
diff --git a/pkg/minikube/machine/cache_images.go b/pkg/minikube/machine/cache_images.go
index 599270c90f..952cb3f8dd 100644
--- a/pkg/minikube/machine/cache_images.go
+++ b/pkg/minikube/machine/cache_images.go
@@ -65,7 +65,7 @@ func LoadImages(cc *config.MachineConfig, runner command.Runner, images []string
 	glog.Infof("LoadImages start: %s", images)
 	defer glog.Infof("LoadImages end")
 	var g errgroup.Group
-	cr, err := cruntime.New(cruntime.Config{Type: cc.ContainerRuntime, Runner: runner})
+	cr, err := cruntime.New(cruntime.Config{Type: cc.KubernetesConfig.ContainerRuntime, Runner: runner})
 	if err != nil {
 		return errors.Wrap(err, "runtime")
 	}
diff --git a/pkg/provision/buildroot.go b/pkg/provision/buildroot.go
index b06f60ae92..bf02cdf6ed 100644
--- a/pkg/provision/buildroot.go
+++ b/pkg/provision/buildroot.go
@@ -260,7 +260,7 @@ func setContainerRuntimeOptions(p *BuildrootProvisioner) error {
 		return errors.Wrap(err, "getting cluster config")
 	}
 
-	switch c.ContainerRuntime {
+	switch c.KubernetesConfig.ContainerRuntime {
 	case "crio", "cri-o":
 		return p.setCrioOptions()
 	case "containerd":
diff --git a/test/integration/aaa_download_only_test.go b/test/integration/aaa_download_only_test.go
index d3e43ee77f..3232ba1fce 100644
--- a/test/integration/aaa_download_only_test.go
+++ b/test/integration/aaa_download_only_test.go
@@ -111,7 +111,7 @@ func TestDownloadOnly(t *testing.T) {
 			got := ""
 			for _, p := range ps["valid"] {
 				if p.Name == profile {
-					got = p.Config[0].VMDriver
+					got = p.Config.VMDriver
 				}
 			}
 

From 6a01d6b25b823bfc9f643663166d1fb25a68f108 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Thu, 23 Jan 2020 13:26:50 -0800
Subject: [PATCH 104/250] prefer getting master node over calling Nodes[0]

---
 cmd/minikube/cmd/config/profile_list.go     | 6 +++++-
 pkg/minikube/registry/drvs/docker/docker.go | 3 ++-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/cmd/minikube/cmd/config/profile_list.go b/cmd/minikube/cmd/config/profile_list.go
index 3fcb17610d..f407fddfa1 100644
--- a/cmd/minikube/cmd/config/profile_list.go
+++ b/cmd/minikube/cmd/config/profile_list.go
@@ -80,7 +80,11 @@ var printProfilesTable = func() {
 		if err != nil {
 			glog.Infof("error getting host status for %v", err)
 		}
-		validData = append(validData, []string{p.Name, p.Config.VMDriver, p.Config.Nodes[0].IP, strconv.Itoa(p.Config.Nodes[0].Port), p.Config.KubernetesConfig.KubernetesVersion, p.Status})
+		master, err := config.GetMasterNode(*p.Config)
+		if err != nil {
+			exit.WithError("profile has no master node", err)
+		}
+		validData = append(validData, []string{p.Name, p.Config.VMDriver, master.IP, strconv.Itoa(master.Port), p.Config.KubernetesConfig.KubernetesVersion, p.Status})
 	}
 
 	table.AppendBulk(validData)
diff --git a/pkg/minikube/registry/drvs/docker/docker.go b/pkg/minikube/registry/drvs/docker/docker.go
index eb9ed0fa7e..746bd58a24 100644
--- a/pkg/minikube/registry/drvs/docker/docker.go
+++ b/pkg/minikube/registry/drvs/docker/docker.go
@@ -42,13 +42,14 @@ func init() {
 }
 
 func configure(mc config.MachineConfig) interface{} {
+	master, _ := config.GetMasterNode(mc)
 	return kic.NewDriver(kic.Config{
 		MachineName:  mc.Name,
 		StorePath:    localpath.MiniPath(),
 		ImageDigest:  kic.BaseImage,
 		CPU:          mc.CPUs,
 		Memory:       mc.Memory,
-		HostBindPort: mc.Nodes[0].Port,
+		HostBindPort: master.Port,
 		OCIBinary:    oci.Docker,
 	})
 

From 5a89aa266d74e608071a1fa3430f84b01acf1df6 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Thu, 23 Jan 2020 13:27:44 -0800
Subject: [PATCH 105/250] go mod nonsense

---
 go.mod | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/go.mod b/go.mod
index 28d8e10692..0cd9aca2bf 100644
--- a/go.mod
+++ b/go.mod
@@ -22,15 +22,12 @@ require (
 	github.com/elazarl/goproxy v0.0.0-20190421051319-9d40249d3c2f
 	github.com/elazarl/goproxy/ext v0.0.0-20190421051319-9d40249d3c2f // indirect
 	github.com/evanphx/json-patch v4.5.0+incompatible // indirect
-	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/go-ole/go-ole v1.2.4 // indirect
 	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
-	github.com/google/btree v1.0.0 // indirect
 	github.com/google/go-cmp v0.3.0
 	github.com/googleapis/gnostic v0.3.0 // indirect
 	github.com/gorilla/mux v1.7.3 // indirect
-	github.com/grpc-ecosystem/grpc-gateway v1.5.0 // indirect
 	github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce // indirect
 	github.com/hashicorp/go-getter v1.4.0
 	github.com/hashicorp/go-multierror v0.0.0-20160811015721-8c5f0ad93604 // indirect

From fa4322e89a7cddda56e3af5d87040944f6363a93 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Thu, 23 Jan 2020 13:50:31 -0800
Subject: [PATCH 106/250] remove all references to
 MachineConfig.ContainerRuntime

---
 pkg/minikube/registry/drvs/none/none.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/minikube/registry/drvs/none/none.go b/pkg/minikube/registry/drvs/none/none.go
index 016ee42e90..371663d7d8 100644
--- a/pkg/minikube/registry/drvs/none/none.go
+++ b/pkg/minikube/registry/drvs/none/none.go
@@ -46,7 +46,7 @@ func configure(mc config.MachineConfig) interface{} {
 	return none.NewDriver(none.Config{
 		MachineName:      mc.Name,
 		StorePath:        localpath.MiniPath(),
-		ContainerRuntime: mc.ContainerRuntime,
+		ContainerRuntime: mc.KubernetesConfig.ContainerRuntime,
 	})
 }
 

From a9bc13b403dc03e4d5155288f4fcd6dbcdfa3d20 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Thu, 23 Jan 2020 15:44:30 -0800
Subject: [PATCH 107/250] Address pause code review feedback

---
 cmd/minikube/cmd/pause.go     |  3 +--
 cmd/minikube/cmd/status.go    | 32 +++++++++++++++++++-------------
 cmd/minikube/cmd/unpause.go   |  3 +--
 pkg/minikube/cluster/pause.go |  1 +
 4 files changed, 22 insertions(+), 17 deletions(-)

diff --git a/cmd/minikube/cmd/pause.go b/cmd/minikube/cmd/pause.go
index 54385a4d5c..e44cc79ed7 100644
--- a/cmd/minikube/cmd/pause.go
+++ b/cmd/minikube/cmd/pause.go
@@ -73,8 +73,7 @@ func runPause(cmd *cobra.Command, args []string) {
 		exit.WithError("Failed to get command runner", err)
 	}
 
-	config := cruntime.Config{Type: cc.ContainerRuntime, Runner: r}
-	cr, err := cruntime.New(config)
+	cr, err := cruntime.New(cruntime.Config{Type: cc.ContainerRuntime, Runner: r})
 	if err != nil {
 		exit.WithError("Failed runtime", err)
 	}
diff --git a/cmd/minikube/cmd/status.go b/cmd/minikube/cmd/status.go
index ba8e93c9e6..64a20bcddf 100644
--- a/cmd/minikube/cmd/status.go
+++ b/cmd/minikube/cmd/status.go
@@ -19,6 +19,7 @@ package cmd
 import (
 	"encoding/json"
 	"fmt"
+	"io"
 	"os"
 	"strings"
 	"text/template"
@@ -94,9 +95,13 @@ var statusCmd = &cobra.Command{
 
 		switch strings.ToLower(output) {
 		case "text":
-			printStatusText(st)
+			if err := statusText(st, os.Stdout); err != nil {
+				exit.WithError("status text failure", err)
+			}
 		case "json":
-			printStatusJSON(st)
+			if err := statusJSON(st, os.Stdout); err != nil {
+				exit.WithError("status json failure", err)
+			}
 		default:
 			exit.WithCodeT(exit.BadUsage, fmt.Sprintf("invalid output format: %s. Valid values: 'text', 'json'", output))
 		}
@@ -160,14 +165,13 @@ func status(api libmachine.API, name string) (*Status, error) {
 		st.APIServer = state.Error.String()
 	}
 
+	st.Kubeconfig = Misconfigured
 	ks, err := kubeconfig.IsClusterInConfig(ip, name)
 	if err != nil {
 		glog.Errorln("Error kubeconfig status:", err)
 	}
 	if ks {
 		st.Kubeconfig = Configured
-	} else {
-		st.Kubeconfig = Misconfigured
 	}
 	return st, nil
 }
@@ -180,24 +184,26 @@ For the list accessible variables for the template, see the struct values here:
 		`minikube status --output OUTPUT. json, text`)
 }
 
-var printStatusText = func(st *Status) {
+func statusText(st *Status, w io.Writer) error {
 	tmpl, err := template.New("status").Parse(statusFormat)
 	if err != nil {
-		exit.WithError("Error creating status template", err)
+		return err
 	}
-	err = tmpl.Execute(os.Stdout, st)
-	if err != nil {
-		exit.WithError("Error executing status template", err)
+	if err := tmpl.Execute(w, st); err != nil {
+		return err
 	}
 	if st.Kubeconfig == Misconfigured {
-		out.WarningT("Warning: Your kubectl is pointing to stale minikube-vm.\nTo fix the kubectl context, run `minikube update-context`")
+		_, err := w.Write([]byte("\nWARNING: Your kubectl is pointing to stale minikube-vm.\nTo fix the kubectl context, run `minikube update-context`\n"))
+		return err
 	}
+	return nil
 }
 
-var printStatusJSON = func(st *Status) {
+func statusJSON(st *Status, w io.Writer) error {
 	js, err := json.Marshal(st)
 	if err != nil {
-		exit.WithError("Error converting status to json", err)
+		return err
 	}
-	out.String(string(js))
+	_, err = w.Write(js)
+	return err
 }
diff --git a/cmd/minikube/cmd/unpause.go b/cmd/minikube/cmd/unpause.go
index 6a38f0948b..a0d2820721 100644
--- a/cmd/minikube/cmd/unpause.go
+++ b/cmd/minikube/cmd/unpause.go
@@ -64,8 +64,7 @@ var unpauseCmd = &cobra.Command{
 			exit.WithError("Failed to get command runner", err)
 		}
 
-		config := cruntime.Config{Type: cc.ContainerRuntime, Runner: r}
-		cr, err := cruntime.New(config)
+		cr, err := cruntime.New(cruntime.Config{Type: cc.ContainerRuntime, Runner: r})
 		if err != nil {
 			exit.WithError("Failed runtime", err)
 		}
diff --git a/pkg/minikube/cluster/pause.go b/pkg/minikube/cluster/pause.go
index d2edcb1888..2f98cf6de3 100644
--- a/pkg/minikube/cluster/pause.go
+++ b/pkg/minikube/cluster/pause.go
@@ -29,6 +29,7 @@ var DefaultNamespaces = []string{
 	"kube-system",
 	"kubernetes-dashboard",
 	"storage-gluster",
+	"istio-operator",
 }
 
 // Pause pauses a Kubernetes cluster

From 5191659ff21e5feca7f1a2773fed0a51767e4362 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Thu, 23 Jan 2020 17:28:30 -0800
Subject: [PATCH 108/250] bind ip use const instead of string

---
 pkg/drivers/kic/kic.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go
index bcd08b436c..eb04477d2f 100644
--- a/pkg/drivers/kic/kic.go
+++ b/pkg/drivers/kic/kic.go
@@ -97,7 +97,7 @@ func (d *Driver) Create() error {
 
 	// control plane specific options
 	params.PortMappings = append(params.PortMappings, oci.PortMapping{
-		ListenAddress: "127.0.0.1",
+		ListenAddress: DefaultBindIPV4,
 		HostPort:      int32(d.NodeConfig.HostBindPort),
 		ContainerPort: constants.APIServerPort,
 	})

From 773510b9c1fa86ef2ac8724ef7233826f251e247 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Thu, 23 Jan 2020 17:45:50 -0800
Subject: [PATCH 109/250] Add ssh to kic container image

---
 Makefile                            |  4 ++--
 hack/images/kicbase.Dockerfile      |  8 ++++++++
 pkg/drivers/kic/kic.go              | 12 ++++++++++--
 pkg/minikube/constants/constants.go |  2 ++
 4 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index fb9e39a600..22449c5a9d 100755
--- a/Makefile
+++ b/Makefile
@@ -485,8 +485,8 @@ endif
 
 .PHONY: kic-base-image
 kic-base-image: ## builds the base image used for kic.
-	docker rmi -f $(REGISTRY)/kicbase:v0.0.1-snapshot || true
-	docker build -f ./hack/images/kicbase.Dockerfile -t $(REGISTRY)/kicbase:v0.0.1-snapshot  --build-arg COMMIT_SHA=${VERSION}-$(COMMIT)  .
+	docker rmi -f $(REGISTRY)/kicbase:v0.0.2-snapshot || true
+	docker build -f ./hack/images/kicbase.Dockerfile -t $(REGISTRY)/kicbase:v0.0.2-snapshot  --build-arg COMMIT_SHA=${VERSION}-$(COMMIT)  .
 
 
 
diff --git a/hack/images/kicbase.Dockerfile b/hack/images/kicbase.Dockerfile
index 9d4aed5e88..0346b8cd27 100644
--- a/hack/images/kicbase.Dockerfile
+++ b/hack/images/kicbase.Dockerfile
@@ -1,10 +1,13 @@
 ARG COMMIT_SHA
+# using node image created by kind https://github.com/kubernetes-sigs/kind
+# could be changed to ubuntu.
 FROM kindest/node:v1.16.2
 USER root
 RUN apt-get update && apt-get install -y \
   sudo \
   dnsutils \
   && apt-get clean -y 
+# Deleting all kind related stuff from the image.
 RUN rm -rf \
     /var/cache/debconf/* \
     /var/lib/apt/lists/* \
@@ -16,3 +19,8 @@ RUN rm -rf \
     /usr/share/local/* \
     /kind/bin/kubeadm /kind/bin/kubelet /kind/systemd /kind/images /kind/manifests
 RUN echo "kic! Build: ${COMMIT_SHA} Time :$(date)" > "/kic.txt"
+# for minikube ssh. to match VM using docker username
+RUN useradd -ms /bin/bash docker 
+USER docker
+RUN mkdir /home/docker/.ssh
+
diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go
index eb04477d2f..4e66bb6277 100644
--- a/pkg/drivers/kic/kic.go
+++ b/pkg/drivers/kic/kic.go
@@ -39,7 +39,9 @@ const DefaultPodCIDR = "10.244.0.0/16"
 const DefaultBindIPV4 = "127.0.0.1"
 
 // BaseImage is the base image is used to spin up kic containers created by kind.
-const BaseImage = "gcr.io/k8s-minikube/kicbase:v0.0.1@sha256:c4ad2938877d2ae0d5b7248a5e7182ff58c0603165c3bedfe9d503e2d380a0db"
+// const BaseImage = "gcr.io/k8s-minikube/kicbase:v0.0.1@sha256:c4ad2938877d2ae0d5b7248a5e7182ff58c0603165c3bedfe9d503e2d380a0db"
+// BaseImage is the base image is used to spin up kic containers created by kind.
+const BaseImage = "gcr.io/k8s-minikube/kicbase:v0.0.2-snapshot"
 
 // OverlayImage is the cni plugin used for overlay image, created by kind.
 const OverlayImage = "kindest/kindnetd:0.5.3"
@@ -100,7 +102,13 @@ func (d *Driver) Create() error {
 		ListenAddress: DefaultBindIPV4,
 		HostPort:      int32(d.NodeConfig.HostBindPort),
 		ContainerPort: constants.APIServerPort,
-	})
+	},
+		oci.PortMapping{
+			ListenAddress: DefaultBindIPV4,
+			HostPort:      int32(d.NodeConfig.HostBindPort) + constants.SSHPort, // TODO: @medyagh: use github.com/phayes/freeport instead.
+			ContainerPort: constants.SSHPort,
+		},
+	)
 
 	_, err := node.CreateNode(params)
 	if err != nil {
diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go
index ad95e39a16..2d6276a099 100644
--- a/pkg/minikube/constants/constants.go
+++ b/pkg/minikube/constants/constants.go
@@ -27,6 +27,8 @@ import (
 )
 
 const (
+	// SSHPort is the SSH serviceport on the node vm and container
+	SSHPort = 22
 	// APIServerPort is the default API server port
 	APIServerPort = 8443
 	// APIServerName is the default API server name

From 1e05b1170343f515898b94efb749ed0a018d6c0c Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Thu, 23 Jan 2020 18:24:51 -0800
Subject: [PATCH 110/250] Add ssh to kic image

---
 hack/images/kicbase.Dockerfile | 20 +++++++++++++++-----
 pkg/drivers/kic/kic.go         |  2 +-
 2 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/hack/images/kicbase.Dockerfile b/hack/images/kicbase.Dockerfile
index 0346b8cd27..91e81bf891 100644
--- a/hack/images/kicbase.Dockerfile
+++ b/hack/images/kicbase.Dockerfile
@@ -1,13 +1,21 @@
 ARG COMMIT_SHA
-# using node image created by kind https://github.com/kubernetes-sigs/kind
-# could be changed to ubuntu.
+# for now using node image created by kind https://github.com/kubernetes-sigs/kind
+# could be changed to slim ubuntu with systemd. 
 FROM kindest/node:v1.16.2
 USER root
 RUN apt-get update && apt-get install -y \
   sudo \
   dnsutils \
+  openssh-server \
   && apt-get clean -y 
-# Deleting all kind related stuff from the image.
+# based on https://github.com/rastasheep/ubuntu-sshd/blob/master/18.04/Dockerfile
+# making SSH work for docker container 
+RUN mkdir /var/run/sshd
+RUN echo 'root:root' |chpasswd
+RUN sed -ri 's/^#?PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config
+RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config
+EXPOSE 22
+# Deleting all "kind" related stuff from the image.
 RUN rm -rf \
     /var/cache/debconf/* \
     /var/lib/apt/lists/* \
@@ -20,7 +28,9 @@ RUN rm -rf \
     /kind/bin/kubeadm /kind/bin/kubelet /kind/systemd /kind/images /kind/manifests
 RUN echo "kic! Build: ${COMMIT_SHA} Time :$(date)" > "/kic.txt"
 # for minikube ssh. to match VM using docker username
-RUN useradd -ms /bin/bash docker 
+RUN adduser --disabled-password --gecos '' docker
+RUN adduser docker sudo
+RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
 USER docker
 RUN mkdir /home/docker/.ssh
-
+USER root
diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go
index 4e66bb6277..533af1e234 100644
--- a/pkg/drivers/kic/kic.go
+++ b/pkg/drivers/kic/kic.go
@@ -41,7 +41,7 @@ const DefaultBindIPV4 = "127.0.0.1"
 // BaseImage is the base image is used to spin up kic containers created by kind.
 // const BaseImage = "gcr.io/k8s-minikube/kicbase:v0.0.1@sha256:c4ad2938877d2ae0d5b7248a5e7182ff58c0603165c3bedfe9d503e2d380a0db"
 // BaseImage is the base image is used to spin up kic containers created by kind.
-const BaseImage = "gcr.io/k8s-minikube/kicbase:v0.0.2-snapshot"
+const BaseImage = "kicbase:local"
 
 // OverlayImage is the cni plugin used for overlay image, created by kind.
 const OverlayImage = "kindest/kindnetd:0.5.3"

From 3d5c9ae870e8c9a5e158f6b34888ff247c25ecaa Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Thu, 23 Jan 2020 18:28:32 -0800
Subject: [PATCH 111/250] implement SSHPortName for kic

---
 pkg/drivers/kic/kic.go | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go
index 533af1e234..ea7336e050 100644
--- a/pkg/drivers/kic/kic.go
+++ b/pkg/drivers/kic/kic.go
@@ -58,16 +58,17 @@ type Driver struct {
 
 // Config is configuration for the kic driver used by registry
 type Config struct {
-	MachineName  string            // maps to the container name being created
-	CPU          int               // Number of CPU cores assigned to the container
-	Memory       int               // max memory in MB
-	StorePath    string            // libmachine store path
-	OCIBinary    string            // oci tool to use (docker, podman,...)
-	ImageDigest  string            // image name with sha to use for the node
-	HostBindPort int               // port to connect to forward from container to user's machine
-	Mounts       []oci.Mount       // mounts
-	PortMappings []oci.PortMapping // container port mappings
-	Envs         map[string]string // key,value of environment variables passed to the node
+	MachineName     string            // maps to the container name being created
+	CPU             int               // Number of CPU cores assigned to the container
+	Memory          int               // max memory in MB
+	StorePath       string            // libmachine store path
+	OCIBinary       string            // oci tool to use (docker, podman,...)
+	ImageDigest     string            // image name with sha to use for the node
+	APIHostBindPort int               // bind port for api server
+	SSHHostBindPort int               // bind port for ssh server
+	Mounts          []oci.Mount       // mounts
+	PortMappings    []oci.PortMapping // container port mappings
+	Envs            map[string]string // key,value of environment variables passed to the node
 }
 
 // NewDriver returns a fully configured Kic driver
@@ -137,12 +138,12 @@ func (d *Driver) GetIP() (string, error) {
 
 // GetSSHHostname returns hostname for use with ssh
 func (d *Driver) GetSSHHostname() (string, error) {
-	return "", fmt.Errorf("driver does not have SSHHostName")
+	return DefaultBindIPV4, nil
 }
 
 // GetSSHPort returns port for use with ssh
 func (d *Driver) GetSSHPort() (int, error) {
-	return 0, fmt.Errorf("driver does not support GetSSHPort")
+	return d.NodeConfig.SSHHostBindPort, nil
 }
 
 // GetURL returns ip of the container running kic control-panel

From 09d9d05c6b536e95db9e80b18fd5b3eeea247a73 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Thu, 23 Jan 2020 18:43:26 -0800
Subject: [PATCH 112/250] generate ssh certs for kic during creation

---
 pkg/drivers/kic/kic.go                      | 14 ++++++++++----
 pkg/minikube/registry/drvs/docker/docker.go | 14 +++++++-------
 2 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go
index ea7336e050..302f7f1cf5 100644
--- a/pkg/drivers/kic/kic.go
+++ b/pkg/drivers/kic/kic.go
@@ -23,8 +23,10 @@ import (
 	"strings"
 
 	"github.com/docker/machine/libmachine/drivers"
+	"github.com/docker/machine/libmachine/ssh"
 	"github.com/docker/machine/libmachine/state"
 	"github.com/pkg/errors"
+	"github.com/golang/glog"
 	pkgdrivers "k8s.io/minikube/pkg/drivers"
 	"k8s.io/minikube/pkg/drivers/kic/node"
 	"k8s.io/minikube/pkg/drivers/kic/oci"
@@ -94,23 +96,27 @@ func (d *Driver) Create() error {
 		CPUs:         strconv.Itoa(d.NodeConfig.CPU),
 		Memory:       strconv.Itoa(d.NodeConfig.Memory) + "mb",
 		Envs:         d.NodeConfig.Envs,
-		ExtraArgs:    []string{"--expose", fmt.Sprintf("%d", d.NodeConfig.HostBindPort)},
+		ExtraArgs:    []string{"--expose", fmt.Sprintf("%d", d.NodeConfig.APIHostBindPort)},
 		OCIBinary:    d.NodeConfig.OCIBinary,
 	}
 
 	// control plane specific options
 	params.PortMappings = append(params.PortMappings, oci.PortMapping{
 		ListenAddress: DefaultBindIPV4,
-		HostPort:      int32(d.NodeConfig.HostBindPort),
+		HostPort:      int32(d.NodeConfig.APIHostBindPort),
 		ContainerPort: constants.APIServerPort,
 	},
 		oci.PortMapping{
 			ListenAddress: DefaultBindIPV4,
-			HostPort:      int32(d.NodeConfig.HostBindPort) + constants.SSHPort, // TODO: @medyagh: use github.com/phayes/freeport instead.
+			HostPort:      int32(d.NodeConfig.APIHostBindPort) + constants.SSHPort, // TODO: @medyagh: use github.com/phayes/freeport instead.
 			ContainerPort: constants.SSHPort,
 		},
 	)
-
+	keyPath := d.GetSSHKeyPath()
+	glog.Infof("Creating ssh key for kic: %s...", keyPath)
+	if err := ssh.GenerateSSHKey(keyPath); err != nil {
+		return errors.Wrap(err, "generate ssh key")
+	}
 	_, err := node.CreateNode(params)
 	if err != nil {
 		return errors.Wrap(err, "create kic node")
diff --git a/pkg/minikube/registry/drvs/docker/docker.go b/pkg/minikube/registry/drvs/docker/docker.go
index bafcf1b093..fdb0c78ac6 100644
--- a/pkg/minikube/registry/drvs/docker/docker.go
+++ b/pkg/minikube/registry/drvs/docker/docker.go
@@ -43,13 +43,13 @@ func init() {
 
 func configure(mc config.MachineConfig) interface{} {
 	return kic.NewDriver(kic.Config{
-		MachineName:  mc.Name,
-		StorePath:    localpath.MiniPath(),
-		ImageDigest:  kic.BaseImage,
-		CPU:          mc.CPUs,
-		Memory:       mc.Memory,
-		HostBindPort: mc.KubernetesConfig.NodePort,
-		OCIBinary:    oci.Docker,
+		MachineName:     mc.Name,
+		StorePath:       localpath.MiniPath(),
+		ImageDigest:     kic.BaseImage,
+		CPU:             mc.CPUs,
+		Memory:          mc.Memory,
+		APIHostBindPort: mc.KubernetesConfig.NodePort,
+		OCIBinary:       oci.Docker,
 	})
 
 }

From ca0f4c7e2088f13b1b5e5a046540ec9489f8b08f Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Thu, 23 Jan 2020 19:20:44 -0800
Subject: [PATCH 113/250] generate ssh keys for kic container

---
 pkg/drivers/kic/kic.go | 29 ++++++++++++++++++++++++++---
 1 file changed, 26 insertions(+), 3 deletions(-)

diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go
index 302f7f1cf5..12880fd7e9 100644
--- a/pkg/drivers/kic/kic.go
+++ b/pkg/drivers/kic/kic.go
@@ -25,11 +25,12 @@ import (
 	"github.com/docker/machine/libmachine/drivers"
 	"github.com/docker/machine/libmachine/ssh"
 	"github.com/docker/machine/libmachine/state"
-	"github.com/pkg/errors"
 	"github.com/golang/glog"
+	"github.com/pkg/errors"
 	pkgdrivers "k8s.io/minikube/pkg/drivers"
 	"k8s.io/minikube/pkg/drivers/kic/node"
 	"k8s.io/minikube/pkg/drivers/kic/oci"
+	"k8s.io/minikube/pkg/minikube/assets"
 	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/minikube/constants"
 )
@@ -112,15 +113,37 @@ func (d *Driver) Create() error {
 			ContainerPort: constants.SSHPort,
 		},
 	)
+	_, err := node.CreateNode(params)
+	if err != nil {
+		return errors.Wrap(err, "create kic node")
+	}
+
+	if err := d.prepareSSH(); err != nil {
+		return errors.Wrap(err, "prepare kic ssh")
+	}
+	return nil
+}
+
+// prepareSSH will generate keys and copy to the container so minikube ssh works
+func (d *Driver) prepareSSH() error {
 	keyPath := d.GetSSHKeyPath()
 	glog.Infof("Creating ssh key for kic: %s...", keyPath)
 	if err := ssh.GenerateSSHKey(keyPath); err != nil {
 		return errors.Wrap(err, "generate ssh key")
 	}
-	_, err := node.CreateNode(params)
+
+	cmder := command.NewKICRunner(d.NodeConfig.MachineName, d.NodeConfig.OCIBinary)
+	f, err := assets.NewFileAsset(d.GetSSHKeyPath()+".pub", "/home/docker/.ssh/", "authorized_keys", "0644")
 	if err != nil {
-		return errors.Wrap(err, "create kic node")
+		return errors.Wrap(err, "create pubkey assetfile ")
 	}
+	if err := cmder.Copy(f); err != nil {
+		return errors.Wrap(err, "copying pub key")
+	}
+	if rr, err := cmder.RunCmd(exec.Command("chown", "docker:docker", "/home/docker/.ssh/authorized_keys")); err != nil {
+		return errors.Wrapf(err, "apply authorized_keys file ownership, output %s", rr.Output())
+	}
+
 	return nil
 }
 

From 2bd169f5081bd2703f51dd00a9bfc0aeba891956 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Thu, 23 Jan 2020 23:51:41 -0800
Subject: [PATCH 114/250] add config fields for kic SSH and API ports

---
 pkg/drivers/kic/kic.go       | 9 ++++++++-
 pkg/minikube/config/types.go | 3 ++-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go
index 12880fd7e9..7c7b79178b 100644
--- a/pkg/drivers/kic/kic.go
+++ b/pkg/drivers/kic/kic.go
@@ -27,11 +27,13 @@ import (
 	"github.com/docker/machine/libmachine/state"
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
+	"github.com/spf13/viper"
 	pkgdrivers "k8s.io/minikube/pkg/drivers"
 	"k8s.io/minikube/pkg/drivers/kic/node"
 	"k8s.io/minikube/pkg/drivers/kic/oci"
 	"k8s.io/minikube/pkg/minikube/assets"
 	"k8s.io/minikube/pkg/minikube/command"
+	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
 )
 
@@ -172,7 +174,12 @@ func (d *Driver) GetSSHHostname() (string, error) {
 
 // GetSSHPort returns port for use with ssh
 func (d *Driver) GetSSHPort() (int, error) {
-	return d.NodeConfig.SSHHostBindPort, nil
+	cc, err := config.Load(viper.GetString(config.MachineProfile))
+	if err != nil {
+		glog.Infof("error loading config file which may be okay on first run : %v ", err)
+		return 22, nil
+	}
+	return int(cc.SSHBindPort), nil
 }
 
 // GetURL returns ip of the container running kic control-panel
diff --git a/pkg/minikube/config/types.go b/pkg/minikube/config/types.go
index 18bd9a75b4..70d888e4f4 100644
--- a/pkg/minikube/config/types.go
+++ b/pkg/minikube/config/types.go
@@ -65,7 +65,8 @@ type MachineConfig struct {
 	HostOnlyNicType     string // Only used by virtualbox
 	NatNicType          string // Only used by virtualbox
 	Addons              map[string]bool
-	NodeBindPort        int32 // Only used by kic
+	APIBindPort         int32 // the host port to bind to apiserver inside the container only used by kic
+	SSHBindPort         int32 // the host port to bind to ssh service inside the container only used by kic
 }
 
 // KubernetesConfig contains the parameters used to configure the VM Kubernetes.

From b5d0253418613ade8f7473b2a3097d2914a8bf9d Mon Sep 17 00:00:00 2001
From: tstromberg <tstromberg@google.com>
Date: Fri, 24 Jan 2020 10:38:49 -0800
Subject: [PATCH 115/250] Change v0.17.3-beta.0 refs to v0.17.2

---
 go.mod | 6 +++---
 go.sum | 8 ++++----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/go.mod b/go.mod
index 30367be52c..c31b89c1c9 100644
--- a/go.mod
+++ b/go.mod
@@ -88,15 +88,15 @@ replace (
 	github.com/hashicorp/go-getter => github.com/afbjorklund/go-getter v1.4.1-0.20190910175809-eb9f6c26742c
 	k8s.io/api => k8s.io/api v0.17.2
 	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.17.2
-	k8s.io/apimachinery => k8s.io/apimachinery v0.17.3-beta.0
+	k8s.io/apimachinery => k8s.io/apimachinery v0.17.2
 	k8s.io/apiserver => k8s.io/apiserver v0.17.2
 	k8s.io/cli-runtime => k8s.io/cli-runtime v0.17.2
 	k8s.io/client-go => k8s.io/client-go v0.17.2
 	k8s.io/cloud-provider => k8s.io/cloud-provider v0.17.2
 	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.17.2
-	k8s.io/code-generator => k8s.io/code-generator v0.17.3-beta.0
+	k8s.io/code-generator => k8s.io/code-generator v0.17.2
 	k8s.io/component-base => k8s.io/component-base v0.17.2
-	k8s.io/cri-api => k8s.io/cri-api v0.17.3-beta.0
+	k8s.io/cri-api => k8s.io/cri-api v0.17.2
 	k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.17.2
 	k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.17.2
 	k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.17.2
diff --git a/go.sum b/go.sum
index d4cc8b5767..40b885f371 100644
--- a/go.sum
+++ b/go.sum
@@ -902,8 +902,8 @@ honnef.co/go/tools v0.0.1-2019.2.2/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt
 k8s.io/api v0.17.2 h1:NF1UFXcKN7/OOv1uxdRz3qfra8AHsPav5M93hlV9+Dc=
 k8s.io/api v0.17.2/go.mod h1:BS9fjjLc4CMuqfSO8vgbHPKMt5+SF0ET6u/RVDihTo4=
 k8s.io/apiextensions-apiserver v0.17.2/go.mod h1:4KdMpjkEjjDI2pPfBA15OscyNldHWdBCfsWMDWAmSTs=
-k8s.io/apimachinery v0.17.3-beta.0 h1:DeN0royOQ5+j3ytFWnDkxGiF5r9T7m6E9Ukzjg4vVHc=
-k8s.io/apimachinery v0.17.3-beta.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg=
+k8s.io/apimachinery v0.17.2 h1:hwDQQFbdRlpnnsR64Asdi55GyCaIP/3WQpMmbNBeWr4=
+k8s.io/apimachinery v0.17.2/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg=
 k8s.io/apiserver v0.17.2/go.mod h1:lBmw/TtQdtxvrTk0e2cgtOxHizXI+d0mmGQURIHQZlo=
 k8s.io/cli-runtime v0.17.2/go.mod h1:aa8t9ziyQdbkuizkNLAw3qe3srSyWh9zlSB7zTqRNPI=
 k8s.io/client-go v0.17.2 h1:ndIfkfXEGrNhLIgkr0+qhRguSD3u6DCmonepn1O6NYc=
@@ -911,10 +911,10 @@ k8s.io/client-go v0.17.2/go.mod h1:QAzRgsa0C2xl4/eVpeVAZMvikCn8Nm81yqVx3Kk9XYI=
 k8s.io/cloud-provider v0.17.2/go.mod h1:9rEcGqEsUFHxC83oMUGBcsXTBpRVNVPX/U+nyQJTvHU=
 k8s.io/cluster-bootstrap v0.17.2 h1:KVjK1WviylwbBwC+3L51xKmGN3A+WmzW8rhtcfWdUqQ=
 k8s.io/cluster-bootstrap v0.17.2/go.mod h1:qiazpAM05fjAc+PEkrY8HSUhKlJSMBuLnVUSO6nvZL4=
-k8s.io/code-generator v0.17.3-beta.0/go.mod h1:DVmfPQgxQENqDIzVR2ddLXMH34qeszkKSdH/N+s+38s=
+k8s.io/code-generator v0.17.2/go.mod h1:DVmfPQgxQENqDIzVR2ddLXMH34qeszkKSdH/N+s+38s=
 k8s.io/component-base v0.17.2 h1:0XHf+cerTvL9I5Xwn9v+0jmqzGAZI7zNydv4tL6Cw6A=
 k8s.io/component-base v0.17.2/go.mod h1:zMPW3g5aH7cHJpKYQ/ZsGMcgbsA/VyhEugF3QT1awLs=
-k8s.io/cri-api v0.17.3-beta.0/go.mod h1:BzAkbBHHp81d+aXzbiIcUbilLkbXa40B8mUHOk6EX3s=
+k8s.io/cri-api v0.17.2/go.mod h1:BzAkbBHHp81d+aXzbiIcUbilLkbXa40B8mUHOk6EX3s=
 k8s.io/csi-translation-lib v0.17.2/go.mod h1:NrhnhXJg/V6cHRTdPbmxvBuV3rJSqXsdLBE5JSRzcVI=
 k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=

From dbeac1b08c09087099a6b3a06664dfe3051a1415 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Fri, 24 Jan 2020 10:53:18 -0800
Subject: [PATCH 116/250] KubernetesConfig.ContainerRuntime

---
 cmd/minikube/cmd/pause.go   | 2 +-
 cmd/minikube/cmd/unpause.go | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cmd/minikube/cmd/pause.go b/cmd/minikube/cmd/pause.go
index e44cc79ed7..493c2c0990 100644
--- a/cmd/minikube/cmd/pause.go
+++ b/cmd/minikube/cmd/pause.go
@@ -73,7 +73,7 @@ func runPause(cmd *cobra.Command, args []string) {
 		exit.WithError("Failed to get command runner", err)
 	}
 
-	cr, err := cruntime.New(cruntime.Config{Type: cc.ContainerRuntime, Runner: r})
+	cr, err := cruntime.New(cruntime.Config{Type: cc.KubernetesConfig.ContainerRuntime, Runner: r})
 	if err != nil {
 		exit.WithError("Failed runtime", err)
 	}
diff --git a/cmd/minikube/cmd/unpause.go b/cmd/minikube/cmd/unpause.go
index a0d2820721..f2d86f8d9e 100644
--- a/cmd/minikube/cmd/unpause.go
+++ b/cmd/minikube/cmd/unpause.go
@@ -64,7 +64,7 @@ var unpauseCmd = &cobra.Command{
 			exit.WithError("Failed to get command runner", err)
 		}
 
-		cr, err := cruntime.New(cruntime.Config{Type: cc.ContainerRuntime, Runner: r})
+		cr, err := cruntime.New(cruntime.Config{Type: cc.KubernetesConfig.ContainerRuntime, Runner: r})
 		if err != nil {
 			exit.WithError("Failed runtime", err)
 		}

From 9953cc22500193a6bc3fd7ef86faf4e0a7a26199 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Fri, 24 Jan 2020 11:14:55 -0800
Subject: [PATCH 117/250] GetMasterNode -> MasterNode

---
 cmd/minikube/cmd/config/profile_list.go      | 2 +-
 pkg/minikube/bootstrapper/bsutil/kubeadm.go  | 2 +-
 pkg/minikube/bootstrapper/bsutil/kubelet.go  | 2 +-
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 4 ++--
 pkg/minikube/config/profile.go               | 4 ++--
 pkg/minikube/registry/drvs/docker/docker.go  | 2 +-
 6 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/cmd/minikube/cmd/config/profile_list.go b/cmd/minikube/cmd/config/profile_list.go
index f407fddfa1..bbca8091ac 100644
--- a/cmd/minikube/cmd/config/profile_list.go
+++ b/cmd/minikube/cmd/config/profile_list.go
@@ -80,7 +80,7 @@ var printProfilesTable = func() {
 		if err != nil {
 			glog.Infof("error getting host status for %v", err)
 		}
-		master, err := config.GetMasterNode(*p.Config)
+		master, err := config.MasterNode(*p.Config)
 		if err != nil {
 			exit.WithError("profile has no master node", err)
 		}
diff --git a/pkg/minikube/bootstrapper/bsutil/kubeadm.go b/pkg/minikube/bootstrapper/bsutil/kubeadm.go
index 1c3d0748e7..8d01ae0a4d 100644
--- a/pkg/minikube/bootstrapper/bsutil/kubeadm.go
+++ b/pkg/minikube/bootstrapper/bsutil/kubeadm.go
@@ -54,7 +54,7 @@ func GenerateKubeadmYAML(mc config.MachineConfig, r cruntime.Manager) ([]byte, e
 	}
 
 	// In case of no port assigned, use default
-	master, err := config.GetMasterNode(mc)
+	master, err := config.MasterNode(mc)
 	if err != nil {
 		return nil, errors.Wrap(err, "getting master node")
 	}
diff --git a/pkg/minikube/bootstrapper/bsutil/kubelet.go b/pkg/minikube/bootstrapper/bsutil/kubelet.go
index 7e5f215328..ecc20dd979 100644
--- a/pkg/minikube/bootstrapper/bsutil/kubelet.go
+++ b/pkg/minikube/bootstrapper/bsutil/kubelet.go
@@ -53,7 +53,7 @@ func NewKubeletConfig(mc config.MachineConfig, r cruntime.Manager) ([]byte, erro
 	if k8s.NetworkPlugin != "" {
 		extraOpts["network-plugin"] = k8s.NetworkPlugin
 	}
-	master, err := config.GetMasterNode(mc)
+	master, err := config.MasterNode(mc)
 	if err != nil {
 		return nil, errors.Wrap(err, "getting master node")
 	}
diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index 339abcf52f..7be9a085a7 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -203,7 +203,7 @@ func (k *Bootstrapper) StartCluster(cfg config.MachineConfig) error {
 		return err
 	}
 
-	master, err := config.GetMasterNode(cfg)
+	master, err := config.MasterNode(cfg)
 	if err != nil {
 		return err
 	}
@@ -295,7 +295,7 @@ func (k *Bootstrapper) client(n config.Node) (*kubernetes.Clientset, error) {
 func (k *Bootstrapper) WaitForCluster(cfg config.MachineConfig, timeout time.Duration) error {
 	start := time.Now()
 	out.T(out.Waiting, "Waiting for cluster to come online ...")
-	master, err := config.GetMasterNode(cfg)
+	master, err := config.MasterNode(cfg)
 	if err != nil {
 		return err
 	}
diff --git a/pkg/minikube/config/profile.go b/pkg/minikube/config/profile.go
index 75b5a29045..cca134d18e 100644
--- a/pkg/minikube/config/profile.go
+++ b/pkg/minikube/config/profile.go
@@ -50,8 +50,8 @@ func (p *Profile) IsValid() bool {
 	return true
 }
 
-// GetMasterNode gets the node specific config for the control plane
-func GetMasterNode(cc MachineConfig) (Node, error) {
+// MasterNode gets the node specific config for the control plane
+func MasterNode(cc MachineConfig) (Node, error) {
 	for _, n := range cc.Nodes {
 		if n.Type == Master {
 			return n, nil
diff --git a/pkg/minikube/registry/drvs/docker/docker.go b/pkg/minikube/registry/drvs/docker/docker.go
index 746bd58a24..7688be5935 100644
--- a/pkg/minikube/registry/drvs/docker/docker.go
+++ b/pkg/minikube/registry/drvs/docker/docker.go
@@ -42,7 +42,7 @@ func init() {
 }
 
 func configure(mc config.MachineConfig) interface{} {
-	master, _ := config.GetMasterNode(mc)
+	master, _ := config.MasterNode(mc)
 	return kic.NewDriver(kic.Config{
 		MachineName:  mc.Name,
 		StorePath:    localpath.MiniPath(),

From 0681830be4b763c9e800fd23aee5254a70cdfaf3 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Fri, 24 Jan 2020 11:55:59 -0800
Subject: [PATCH 118/250] io.ReadSeeker for all the Assets readers

---
 pkg/minikube/assets/vm_assets.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/pkg/minikube/assets/vm_assets.go b/pkg/minikube/assets/vm_assets.go
index 67c7fcf1a3..c3f30a667d 100644
--- a/pkg/minikube/assets/vm_assets.go
+++ b/pkg/minikube/assets/vm_assets.go
@@ -77,7 +77,7 @@ func (b *BaseAsset) GetModTime() (time.Time, error) {
 // FileAsset is an asset using a file
 type FileAsset struct {
 	BaseAsset
-	reader *io.SectionReader
+	reader io.ReadSeeker
 }
 
 // NewMemoryAssetTarget creates a new MemoryAsset, with target
@@ -139,7 +139,7 @@ func (f *FileAsset) Seek(offset int64, whence int) (int64, error) {
 // MemoryAsset is a memory-based asset
 type MemoryAsset struct {
 	BaseAsset
-	reader *bytes.Reader
+	reader io.ReadSeeker
 	length int
 }
 
@@ -174,7 +174,7 @@ func NewMemoryAsset(d []byte, targetDir, targetName, permissions string) *Memory
 // BinAsset is a bindata (binary data) asset
 type BinAsset struct {
 	BaseAsset
-	reader   *bytes.Reader
+	reader   io.ReadSeeker
 	template *template.Template
 	length   int
 }

From c43aea690dba471d53f185f46c8e7efd67aab5d2 Mon Sep 17 00:00:00 2001
From: tstromberg <tstromberg@google.com>
Date: Fri, 24 Jan 2020 11:24:32 -0800
Subject: [PATCH 119/250] Update DefaultKubernetesVersion to v1.17.2

---
 pkg/minikube/bootstrapper/bsutil/kubelet_test.go | 8 ++++----
 pkg/minikube/constants/constants.go              | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/pkg/minikube/bootstrapper/bsutil/kubelet_test.go b/pkg/minikube/bootstrapper/bsutil/kubelet_test.go
index 226f486996..6e75abaebf 100644
--- a/pkg/minikube/bootstrapper/bsutil/kubelet_test.go
+++ b/pkg/minikube/bootstrapper/bsutil/kubelet_test.go
@@ -65,7 +65,7 @@ Wants=crio.service
 
 [Service]
 ExecStart=
-ExecStart=/var/lib/minikube/binaries/v1.17.0/kubelet --authorization-mode=Webhook --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --cgroup-driver=cgroupfs --client-ca-file=/var/lib/minikube/certs/ca.crt --cluster-dns=10.96.0.10 --cluster-domain=cluster.local --config=/var/lib/kubelet/config.yaml --container-runtime=remote --container-runtime-endpoint=/var/run/crio/crio.sock --fail-swap-on=false --hostname-override=minikube --image-service-endpoint=/var/run/crio/crio.sock --kubeconfig=/etc/kubernetes/kubelet.conf --node-ip=192.168.1.100 --pod-manifest-path=/etc/kubernetes/manifests --runtime-request-timeout=15m
+ExecStart=/var/lib/minikube/binaries/v1.17.2/kubelet --authorization-mode=Webhook --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --cgroup-driver=cgroupfs --client-ca-file=/var/lib/minikube/certs/ca.crt --cluster-dns=10.96.0.10 --cluster-domain=cluster.local --config=/var/lib/kubelet/config.yaml --container-runtime=remote --container-runtime-endpoint=/var/run/crio/crio.sock --fail-swap-on=false --hostname-override=minikube --image-service-endpoint=/var/run/crio/crio.sock --kubeconfig=/etc/kubernetes/kubelet.conf --node-ip=192.168.1.100 --pod-manifest-path=/etc/kubernetes/manifests --runtime-request-timeout=15m
 
 [Install]
 `,
@@ -83,7 +83,7 @@ Wants=containerd.service
 
 [Service]
 ExecStart=
-ExecStart=/var/lib/minikube/binaries/v1.17.0/kubelet --authorization-mode=Webhook --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --cgroup-driver=cgroupfs --client-ca-file=/var/lib/minikube/certs/ca.crt --cluster-dns=10.96.0.10 --cluster-domain=cluster.local --config=/var/lib/kubelet/config.yaml --container-runtime=remote --container-runtime-endpoint=unix:///run/containerd/containerd.sock --fail-swap-on=false --hostname-override=minikube --image-service-endpoint=unix:///run/containerd/containerd.sock --kubeconfig=/etc/kubernetes/kubelet.conf --node-ip=192.168.1.100 --pod-manifest-path=/etc/kubernetes/manifests --runtime-request-timeout=15m
+ExecStart=/var/lib/minikube/binaries/v1.17.2/kubelet --authorization-mode=Webhook --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --cgroup-driver=cgroupfs --client-ca-file=/var/lib/minikube/certs/ca.crt --cluster-dns=10.96.0.10 --cluster-domain=cluster.local --config=/var/lib/kubelet/config.yaml --container-runtime=remote --container-runtime-endpoint=unix:///run/containerd/containerd.sock --fail-swap-on=false --hostname-override=minikube --image-service-endpoint=unix:///run/containerd/containerd.sock --kubeconfig=/etc/kubernetes/kubelet.conf --node-ip=192.168.1.100 --pod-manifest-path=/etc/kubernetes/manifests --runtime-request-timeout=15m
 
 [Install]
 `,
@@ -108,7 +108,7 @@ Wants=containerd.service
 
 [Service]
 ExecStart=
-ExecStart=/var/lib/minikube/binaries/v1.17.0/kubelet --authorization-mode=Webhook --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --cgroup-driver=cgroupfs --client-ca-file=/var/lib/minikube/certs/ca.crt --cluster-dns=10.96.0.10 --cluster-domain=cluster.local --config=/var/lib/kubelet/config.yaml --container-runtime=remote --container-runtime-endpoint=unix:///run/containerd/containerd.sock --fail-swap-on=false --hostname-override=minikube --image-service-endpoint=unix:///run/containerd/containerd.sock --kubeconfig=/etc/kubernetes/kubelet.conf --node-ip=192.168.1.200 --pod-manifest-path=/etc/kubernetes/manifests --runtime-request-timeout=15m
+ExecStart=/var/lib/minikube/binaries/v1.17.2/kubelet --authorization-mode=Webhook --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --cgroup-driver=cgroupfs --client-ca-file=/var/lib/minikube/certs/ca.crt --cluster-dns=10.96.0.10 --cluster-domain=cluster.local --config=/var/lib/kubelet/config.yaml --container-runtime=remote --container-runtime-endpoint=unix:///run/containerd/containerd.sock --fail-swap-on=false --hostname-override=minikube --image-service-endpoint=unix:///run/containerd/containerd.sock --kubeconfig=/etc/kubernetes/kubelet.conf --node-ip=192.168.1.200 --pod-manifest-path=/etc/kubernetes/manifests --runtime-request-timeout=15m
 
 [Install]
 `,
@@ -127,7 +127,7 @@ Wants=docker.socket
 
 [Service]
 ExecStart=
-ExecStart=/var/lib/minikube/binaries/v1.17.0/kubelet --authorization-mode=Webhook --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --cgroup-driver=cgroupfs --client-ca-file=/var/lib/minikube/certs/ca.crt --cluster-dns=10.96.0.10 --cluster-domain=cluster.local --config=/var/lib/kubelet/config.yaml --container-runtime=docker --fail-swap-on=false --hostname-override=minikube --kubeconfig=/etc/kubernetes/kubelet.conf --node-ip=192.168.1.100 --pod-infra-container-image=docker-proxy-image.io/google_containers/pause:3.1 --pod-manifest-path=/etc/kubernetes/manifests
+ExecStart=/var/lib/minikube/binaries/v1.17.2/kubelet --authorization-mode=Webhook --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --cgroup-driver=cgroupfs --client-ca-file=/var/lib/minikube/certs/ca.crt --cluster-dns=10.96.0.10 --cluster-domain=cluster.local --config=/var/lib/kubelet/config.yaml --container-runtime=docker --fail-swap-on=false --hostname-override=minikube --kubeconfig=/etc/kubernetes/kubelet.conf --node-ip=192.168.1.100 --pod-infra-container-image=docker-proxy-image.io/google_containers/pause:3.1 --pod-manifest-path=/etc/kubernetes/manifests
 
 [Install]
 `,
diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go
index ad95e39a16..43c359c884 100644
--- a/pkg/minikube/constants/constants.go
+++ b/pkg/minikube/constants/constants.go
@@ -67,10 +67,10 @@ var DefaultISOURL = fmt.Sprintf("https://storage.googleapis.com/%s/minikube-%s.i
 var DefaultISOSHAURL = DefaultISOURL + SHASuffix
 
 // DefaultKubernetesVersion is the default kubernetes version
-var DefaultKubernetesVersion = "v1.17.0"
+var DefaultKubernetesVersion = "v1.17.2"
 
 // NewestKubernetesVersion is the newest Kubernetes version to test against
-var NewestKubernetesVersion = "v1.17.0"
+var NewestKubernetesVersion = "v1.17.2"
 
 // OldestKubernetesVersion is the oldest Kubernetes version to test against
 var OldestKubernetesVersion = "v1.11.10"

From 2e24d7a12369ecebcb45f109355e1d5ea226ad51 Mon Sep 17 00:00:00 2001
From: tstromberg <tstromberg@google.com>
Date: Fri, 24 Jan 2020 13:08:26 -0800
Subject: [PATCH 120/250] Combine error conditions

---
 pkg/provision/buildroot.go | 16 ++++------------
 1 file changed, 4 insertions(+), 12 deletions(-)

diff --git a/pkg/provision/buildroot.go b/pkg/provision/buildroot.go
index 21b1affebd..e6080f1e1c 100644
--- a/pkg/provision/buildroot.go
+++ b/pkg/provision/buildroot.go
@@ -315,8 +315,7 @@ func configureAuth(p *BuildrootProvisioner) error {
 		return errors.Wrap(err, "error getting ip during provisioning")
 	}
 
-	err = copyHostCerts(authOptions)
-	if err != nil {
+	if err := copyHostCerts(authOptions); err != nil {
 		return err
 	}
 
@@ -344,12 +343,7 @@ func configureAuth(p *BuildrootProvisioner) error {
 		return fmt.Errorf("error generating server cert: %v", err)
 	}
 
-	err = copyRemoteCerts(authOptions, driver)
-	if err != nil {
-		return err
-	}
-
-	return nil
+	return copyRemoteCerts(authOptions, driver)
 }
 
 func copyHostCerts(authOptions auth.Options) error {
@@ -361,8 +355,7 @@ func copyHostCerts(authOptions auth.Options) error {
 		authOptions.ClientKeyPath:  path.Join(authOptions.StorePath, "key.pem"),
 	}
 
-	_, err := execRunner.RunCmd(exec.Command("mkdir", "-p", authOptions.StorePath))
-	if err != nil {
+	if _, err := execRunner.RunCmd(exec.Command("mkdir", "-p", authOptions.StorePath)); err != nil {
 		return err
 	}
 	for src, dst := range hostCerts {
@@ -399,8 +392,7 @@ func copyRemoteCerts(authOptions auth.Options, driver drivers.Driver) error {
 	}
 
 	args := append([]string{"mkdir", "-p"}, dirs...)
-	_, err = sshRunner.RunCmd(exec.Command("sudo", args...))
-	if err != nil {
+	if _, err = sshRunner.RunCmd(exec.Command("sudo", args...)); err != nil {
 		return err
 	}
 

From 0b460e6cb77cc60710f6816bb51cbe48fe039ec9 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Fri, 24 Jan 2020 13:24:38 -0800
Subject: [PATCH 121/250] master is no longer a thing

---
 cmd/minikube/cmd/config/profile_list.go      |  2 +-
 cmd/minikube/cmd/start.go                    |  9 +++++----
 pkg/minikube/bootstrapper/bsutil/kubeadm.go  | 10 +++++-----
 pkg/minikube/bootstrapper/bsutil/kubelet.go  |  4 ++--
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 10 +++++-----
 pkg/minikube/config/node.go                  |  2 +-
 pkg/minikube/config/profile.go               |  6 +++---
 pkg/minikube/config/types.go                 |  4 ++--
 pkg/minikube/registry/drvs/docker/docker.go  |  4 ++--
 9 files changed, 26 insertions(+), 25 deletions(-)

diff --git a/cmd/minikube/cmd/config/profile_list.go b/cmd/minikube/cmd/config/profile_list.go
index bbca8091ac..326501d32b 100644
--- a/cmd/minikube/cmd/config/profile_list.go
+++ b/cmd/minikube/cmd/config/profile_list.go
@@ -80,7 +80,7 @@ var printProfilesTable = func() {
 		if err != nil {
 			glog.Infof("error getting host status for %v", err)
 		}
-		master, err := config.MasterNode(*p.Config)
+		master, err := config.PrimaryControlPlane(*p.Config)
 		if err != nil {
 			exit.WithError("profile has no master node", err)
 		}
diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 69074b6f9a..4f1df41e00 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -937,11 +937,12 @@ func generateCfgFromFlags(cmd *cobra.Command, k8sVersion string, drvName string)
 		out.T(out.SuccessType, "Using image repository {{.name}}", out.V{"name": repository})
 	}
 
-	masterNode := config.Node{
+	// Create the initial node, which will necessarily be a control plane
+	cp := config.Node{
 		Port:              viper.GetInt(apiServerPort),
 		KubernetesVersion: k8sVersion,
 		Name:              constants.DefaultNodeName,
-		Type:              config.Master,
+		Type:              config.ControlPlane,
 	}
 
 	cfg := config.MachineConfig{
@@ -991,9 +992,9 @@ func generateCfgFromFlags(cmd *cobra.Command, k8sVersion string, drvName string)
 			ShouldLoadCachedImages: viper.GetBool(cacheImages),
 			EnableDefaultCNI:       selectedEnableDefaultCNI,
 		},
-		Nodes: []config.Node{masterNode},
+		Nodes: []config.Node{cp},
 	}
-	return cfg, masterNode, nil
+	return cfg, cp, nil
 }
 
 // setDockerProxy sets the proxy environment variables in the docker environment.
diff --git a/pkg/minikube/bootstrapper/bsutil/kubeadm.go b/pkg/minikube/bootstrapper/bsutil/kubeadm.go
index 8d01ae0a4d..22c0d368a3 100644
--- a/pkg/minikube/bootstrapper/bsutil/kubeadm.go
+++ b/pkg/minikube/bootstrapper/bsutil/kubeadm.go
@@ -54,11 +54,11 @@ func GenerateKubeadmYAML(mc config.MachineConfig, r cruntime.Manager) ([]byte, e
 	}
 
 	// In case of no port assigned, use default
-	master, err := config.MasterNode(mc)
+	cp, err := config.PrimaryControlPlane(mc)
 	if err != nil {
-		return nil, errors.Wrap(err, "getting master node")
+		return nil, errors.Wrap(err, "getting control plane")
 	}
-	nodePort := master.Port
+	nodePort := cp.Port
 	if nodePort <= 0 {
 		nodePort = constants.APIServerPort
 	}
@@ -82,11 +82,11 @@ func GenerateKubeadmYAML(mc config.MachineConfig, r cruntime.Manager) ([]byte, e
 		CertDir:           vmpath.GuestCertsDir,
 		ServiceCIDR:       constants.DefaultServiceCIDR,
 		PodSubnet:         k8s.ExtraOptions.Get("pod-network-cidr", Kubeadm),
-		AdvertiseAddress:  master.IP,
+		AdvertiseAddress:  cp.IP,
 		APIServerPort:     nodePort,
 		KubernetesVersion: k8s.KubernetesVersion,
 		EtcdDataDir:       EtcdDataDir(),
-		NodeName:          master.Name,
+		NodeName:          cp.Name,
 		CRISocket:         r.SocketPath(),
 		ImageRepository:   k8s.ImageRepository,
 		ExtraArgs:         extraComponentConfig,
diff --git a/pkg/minikube/bootstrapper/bsutil/kubelet.go b/pkg/minikube/bootstrapper/bsutil/kubelet.go
index ecc20dd979..7368ca2748 100644
--- a/pkg/minikube/bootstrapper/bsutil/kubelet.go
+++ b/pkg/minikube/bootstrapper/bsutil/kubelet.go
@@ -53,12 +53,12 @@ func NewKubeletConfig(mc config.MachineConfig, r cruntime.Manager) ([]byte, erro
 	if k8s.NetworkPlugin != "" {
 		extraOpts["network-plugin"] = k8s.NetworkPlugin
 	}
-	master, err := config.MasterNode(mc)
+	cp, err := config.PrimaryControlPlane(mc)
 	if err != nil {
 		return nil, errors.Wrap(err, "getting master node")
 	}
 	if _, ok := extraOpts["node-ip"]; !ok {
-		extraOpts["node-ip"] = master.IP
+		extraOpts["node-ip"] = cp.IP
 	}
 
 	pauseImage := images.Pause(k8s.ImageRepository)
diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index 7be9a085a7..f363aee8f9 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -203,7 +203,7 @@ func (k *Bootstrapper) StartCluster(cfg config.MachineConfig) error {
 		return err
 	}
 
-	master, err := config.MasterNode(cfg)
+	cp, err := config.PrimaryControlPlane(cfg)
 	if err != nil {
 		return err
 	}
@@ -249,7 +249,7 @@ func (k *Bootstrapper) StartCluster(cfg config.MachineConfig) error {
 	if !driver.IsKIC(cfg.VMDriver) { // TODO: skip for both after verifications https://github.com/kubernetes/minikube/issues/6239
 		glog.Infof("Configuring cluster permissions ...")
 		elevate := func() error {
-			client, err := k.client(master)
+			client, err := k.client(cp)
 			if err != nil {
 				return err
 			}
@@ -295,18 +295,18 @@ func (k *Bootstrapper) client(n config.Node) (*kubernetes.Clientset, error) {
 func (k *Bootstrapper) WaitForCluster(cfg config.MachineConfig, timeout time.Duration) error {
 	start := time.Now()
 	out.T(out.Waiting, "Waiting for cluster to come online ...")
-	master, err := config.MasterNode(cfg)
+	cp, err := config.PrimaryControlPlane(cfg)
 	if err != nil {
 		return err
 	}
 	if err := kverify.APIServerProcess(k.c, start, timeout); err != nil {
 		return err
 	}
-	if err := kverify.APIServerIsRunning(start, master.IP, master.Port, timeout); err != nil {
+	if err := kverify.APIServerIsRunning(start, cp.IP, cp.Port, timeout); err != nil {
 		return err
 	}
 
-	c, err := k.client(master)
+	c, err := k.client(cp)
 	if err != nil {
 		return errors.Wrap(err, "get k8s client")
 	}
diff --git a/pkg/minikube/config/node.go b/pkg/minikube/config/node.go
index 92af72f28f..edb7564904 100644
--- a/pkg/minikube/config/node.go
+++ b/pkg/minikube/config/node.go
@@ -20,7 +20,7 @@ package config
 func AddNode(cc *MachineConfig, name string, controlPlane bool, k8sVersion string, profileName string) error {
 	t := Worker
 	if controlPlane {
-		t = Master
+		t = ControlPlane
 	}
 	node := Node{
 		Name: name,
diff --git a/pkg/minikube/config/profile.go b/pkg/minikube/config/profile.go
index cca134d18e..f29f356432 100644
--- a/pkg/minikube/config/profile.go
+++ b/pkg/minikube/config/profile.go
@@ -50,10 +50,10 @@ func (p *Profile) IsValid() bool {
 	return true
 }
 
-// MasterNode gets the node specific config for the control plane
-func MasterNode(cc MachineConfig) (Node, error) {
+// PrimaryControlPlane gets the node specific config for the first created control plane
+func PrimaryControlPlane(cc MachineConfig) (Node, error) {
 	for _, n := range cc.Nodes {
-		if n.Type == Master {
+		if n.Type == ControlPlane {
 			return n, nil
 		}
 	}
diff --git a/pkg/minikube/config/types.go b/pkg/minikube/config/types.go
index 3e83c78414..919fe2a27e 100644
--- a/pkg/minikube/config/types.go
+++ b/pkg/minikube/config/types.go
@@ -100,8 +100,8 @@ type Node struct {
 type NodeType int
 
 const (
-	// Master is the control plane.
-	Master NodeType = iota
+	// ControlPlane is the control plane.
+	ControlPlane NodeType = iota
 	// Worker is a... worker.
 	Worker
 )
diff --git a/pkg/minikube/registry/drvs/docker/docker.go b/pkg/minikube/registry/drvs/docker/docker.go
index 7688be5935..3085c8eda9 100644
--- a/pkg/minikube/registry/drvs/docker/docker.go
+++ b/pkg/minikube/registry/drvs/docker/docker.go
@@ -42,14 +42,14 @@ func init() {
 }
 
 func configure(mc config.MachineConfig) interface{} {
-	master, _ := config.MasterNode(mc)
+	cp, _ := config.PrimaryControlPlane(mc)
 	return kic.NewDriver(kic.Config{
 		MachineName:  mc.Name,
 		StorePath:    localpath.MiniPath(),
 		ImageDigest:  kic.BaseImage,
 		CPU:          mc.CPUs,
 		Memory:       mc.Memory,
-		HostBindPort: master.Port,
+		HostBindPort: cp.Port,
 		OCIBinary:    oci.Docker,
 	})
 

From 58ba813a9ce21a69a2121ffd5b5de96b6f7667b7 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Fri, 24 Jan 2020 13:54:41 -0800
Subject: [PATCH 122/250] controlplane and worker are bools now magiiiiic

---
 cmd/minikube/cmd/start.go      |  3 ++-
 pkg/minikube/config/node.go    | 12 ++++++------
 pkg/minikube/config/profile.go |  2 +-
 pkg/minikube/config/types.go   | 13 ++-----------
 4 files changed, 11 insertions(+), 19 deletions(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 4f1df41e00..52f39eacce 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -942,7 +942,8 @@ func generateCfgFromFlags(cmd *cobra.Command, k8sVersion string, drvName string)
 		Port:              viper.GetInt(apiServerPort),
 		KubernetesVersion: k8sVersion,
 		Name:              constants.DefaultNodeName,
-		Type:              config.ControlPlane,
+		ControlPlane:      true,
+		Worker:            true,
 	}
 
 	cfg := config.MachineConfig{
diff --git a/pkg/minikube/config/node.go b/pkg/minikube/config/node.go
index edb7564904..219acb5e46 100644
--- a/pkg/minikube/config/node.go
+++ b/pkg/minikube/config/node.go
@@ -18,13 +18,13 @@ package config
 
 // AddNode adds a new node config to an existing cluster.
 func AddNode(cc *MachineConfig, name string, controlPlane bool, k8sVersion string, profileName string) error {
-	t := Worker
-	if controlPlane {
-		t = ControlPlane
-	}
 	node := Node{
-		Name: name,
-		Type: t,
+		Name:   name,
+		Worker: true,
+	}
+
+	if controlPlane {
+		node.ControlPlane = true
 	}
 
 	if k8sVersion != "" {
diff --git a/pkg/minikube/config/profile.go b/pkg/minikube/config/profile.go
index f29f356432..4b3ad2a9e9 100644
--- a/pkg/minikube/config/profile.go
+++ b/pkg/minikube/config/profile.go
@@ -53,7 +53,7 @@ func (p *Profile) IsValid() bool {
 // PrimaryControlPlane gets the node specific config for the first created control plane
 func PrimaryControlPlane(cc MachineConfig) (Node, error) {
 	for _, n := range cc.Nodes {
-		if n.Type == ControlPlane {
+		if n.ControlPlane {
 			return n, nil
 		}
 	}
diff --git a/pkg/minikube/config/types.go b/pkg/minikube/config/types.go
index 919fe2a27e..c1826a07b4 100644
--- a/pkg/minikube/config/types.go
+++ b/pkg/minikube/config/types.go
@@ -93,19 +93,10 @@ type Node struct {
 	IP                string
 	Port              int
 	KubernetesVersion string
-	Type              NodeType
+	ControlPlane      bool
+	Worker            bool
 }
 
-// NodeType is an enum to describe the node type.
-type NodeType int
-
-const (
-	// ControlPlane is the control plane.
-	ControlPlane NodeType = iota
-	// Worker is a... worker.
-	Worker
-)
-
 // VersionedExtraOption holds information on flags to apply to a specific range
 // of versions
 type VersionedExtraOption struct {

From 9aeae2df65ce4aa55990c57755cf34e1a0271dc9 Mon Sep 17 00:00:00 2001
From: tstromberg <tstromberg@google.com>
Date: Fri, 24 Jan 2020 13:57:07 -0800
Subject: [PATCH 123/250] Fix tests

---
 pkg/minikube/bootstrapper/certs_test.go |  4 ++--
 pkg/minikube/cluster/cluster.go         |  9 +++++++--
 pkg/minikube/cluster/cluster_test.go    |  2 ++
 pkg/minikube/command/fake_runner.go     | 16 +++++++++++++---
 4 files changed, 24 insertions(+), 7 deletions(-)

diff --git a/pkg/minikube/bootstrapper/certs_test.go b/pkg/minikube/bootstrapper/certs_test.go
index 24b7ae6730..ad36ff25b9 100644
--- a/pkg/minikube/bootstrapper/certs_test.go
+++ b/pkg/minikube/bootstrapper/certs_test.go
@@ -52,8 +52,8 @@ func TestSetupCerts(t *testing.T) {
 	}
 
 	expected := map[string]string{
-		"fuck": "off",
-		`sudo /bin/bash -c "test -f /usr/share/ca-certificates/mycert.pem || ln -s /etc/ssl/certs/mycert.pem /usr/share/ca-certificates/mycert.pem"`: "wut",
+		`sudo /bin/bash -c "test -f /usr/share/ca-certificates/mycert.pem || ln -s /etc/ssl/certs/mycert.pem /usr/share/ca-certificates/mycert.pem"`:             "-",
+		`sudo /bin/bash -c "test -f /usr/share/ca-certificates/minikubeCA.pem || ln -s /etc/ssl/certs/minikubeCA.pem /usr/share/ca-certificates/minikubeCA.pem"`: "-",
 	}
 	f := command.NewFakeCommandRunner()
 	f.SetCommandToOutput(expected)
diff --git a/pkg/minikube/cluster/cluster.go b/pkg/minikube/cluster/cluster.go
index af8fb5f0ea..8a93fd7a1a 100644
--- a/pkg/minikube/cluster/cluster.go
+++ b/pkg/minikube/cluster/cluster.go
@@ -513,7 +513,7 @@ func createHost(api libmachine.API, config cfg.MachineConfig) (*host.Host, error
 	}
 
 	if err := createRequiredDirectories(h); err != nil {
-		return h, err
+		return h, errors.Wrap(err, "required directories")
 	}
 
 	if driver.BareMetal(config.VMDriver) {
@@ -668,6 +668,10 @@ func IsMinikubeRunning(api libmachine.API) bool {
 
 // createRequiredDirectories creates directories expected by minikube to exist
 func createRequiredDirectories(h *host.Host) error {
+	if h.DriverName == driver.Mock {
+		glog.Infof("skipping createRequiredDirectories")
+		return nil
+	}
 	glog.Infof("creating required directories: %v", requiredDirectories)
 	r, err := commandRunner(h)
 	if err != nil {
@@ -676,7 +680,7 @@ func createRequiredDirectories(h *host.Host) error {
 
 	args := append([]string{"mkdir", "-p"}, requiredDirectories...)
 	if _, err := r.RunCmd(exec.Command("sudo", args...)); err != nil {
-		return errors.Wrap(err, "mkdir")
+		return errors.Wrapf(err, "sudo mkdir (%s)", h.DriverName)
 	}
 	return nil
 }
@@ -684,6 +688,7 @@ func createRequiredDirectories(h *host.Host) error {
 // commandRunner returns best available command runner for this host
 func commandRunner(h *host.Host) (command.Runner, error) {
 	if h.DriverName == driver.Mock {
+		glog.Errorf("commandRunner: returning unconfigured FakeCommandRunner, commands will fail!")
 		return &command.FakeCommandRunner{}, nil
 	}
 	if driver.BareMetal(h.Driver.DriverName()) {
diff --git a/pkg/minikube/cluster/cluster_test.go b/pkg/minikube/cluster/cluster_test.go
index 24cc6f7a2d..d91c469031 100644
--- a/pkg/minikube/cluster/cluster_test.go
+++ b/pkg/minikube/cluster/cluster_test.go
@@ -374,6 +374,8 @@ func TestGetHostDockerEnv(t *testing.T) {
 }
 
 func TestGetHostDockerEnvIPv6(t *testing.T) {
+	RegisterMockDriver(t)
+
 	tempDir := tests.MakeTempDir()
 	defer os.RemoveAll(tempDir)
 
diff --git a/pkg/minikube/command/fake_runner.go b/pkg/minikube/command/fake_runner.go
index 4ddcb0319b..82a6d833df 100644
--- a/pkg/minikube/command/fake_runner.go
+++ b/pkg/minikube/command/fake_runner.go
@@ -54,9 +54,19 @@ func (f *FakeCommandRunner) RunCmd(cmd *exec.Cmd) (*RunResult, error) {
 
 	start := time.Now()
 
-	out, ok := f.cmdMap.Load(strings.Join(rr.Args, " "))
+	key := rr.Command()
+	out, ok := f.cmdMap.Load(key)
 	if !ok {
-		return rr, fmt.Errorf("unregistered command: `%s`\nexpected: %v\n", rr.Command(), f.commands())
+		cmds := f.commands()
+		if len(cmds) == 0 {
+			return rr, fmt.Errorf("asked to execute %s, but FakeCommandRunner has no commands stored", rr.Command())
+		}
+
+		var txt strings.Builder
+		for _, c := range f.commands() {
+			txt.WriteString(fmt.Sprintf("  `%s`\n", c))
+		}
+		return rr, fmt.Errorf("unregistered command:\n  `%s`\nexpected one of:\n%s", key, txt.String())
 	}
 
 	var buf bytes.Buffer
@@ -124,7 +134,7 @@ func (f *FakeCommandRunner) GetFileToContents(filename string) (string, error) {
 func (f *FakeCommandRunner) commands() []string {
 	cmds := []string{}
 	f.cmdMap.Range(func(k, v interface{}) bool {
-		cmds = append(cmds, fmt.Sprintf("`%s`", k))
+		cmds = append(cmds, fmt.Sprintf("%s", k))
 		return true
 	})
 	return cmds

From c4e390f95c905ffe1118fc124d9b62a2883f9230 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Fri, 24 Jan 2020 13:57:47 -0800
Subject: [PATCH 124/250] missed one file

---
 cmd/minikube/cmd/config/profile_list.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cmd/minikube/cmd/config/profile_list.go b/cmd/minikube/cmd/config/profile_list.go
index 326501d32b..8df06f2d2d 100644
--- a/cmd/minikube/cmd/config/profile_list.go
+++ b/cmd/minikube/cmd/config/profile_list.go
@@ -80,11 +80,11 @@ var printProfilesTable = func() {
 		if err != nil {
 			glog.Infof("error getting host status for %v", err)
 		}
-		master, err := config.PrimaryControlPlane(*p.Config)
+		cp, err := config.PrimaryControlPlane(*p.Config)
 		if err != nil {
-			exit.WithError("profile has no master node", err)
+			exit.WithError("profile has no control plane", err)
 		}
-		validData = append(validData, []string{p.Name, p.Config.VMDriver, master.IP, strconv.Itoa(master.Port), p.Config.KubernetesConfig.KubernetesVersion, p.Status})
+		validData = append(validData, []string{p.Name, p.Config.VMDriver, cp.IP, strconv.Itoa(cp.Port), p.Config.KubernetesConfig.KubernetesVersion, p.Status})
 	}
 
 	table.AppendBulk(validData)

From c6c0046c5724c1411edf4b85a4640f6c020dfa6e Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Fri, 24 Jan 2020 14:15:48 -0800
Subject: [PATCH 125/250] let docker/podman choose hostbind port

---
 cmd/minikube/cmd/start.go                    | 14 ++---
 pkg/drivers/kic/kic.go                       | 65 +++++++++++---------
 pkg/drivers/kic/node/node.go                 | 23 +++----
 pkg/drivers/kic/oci/oci.go                   | 32 +++++++---
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 10 ++-
 pkg/minikube/config/types.go                 |  4 +-
 pkg/minikube/registry/drvs/docker/docker.go  | 14 ++---
 7 files changed, 93 insertions(+), 69 deletions(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 6cc275a858..7de4225c66 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -445,15 +445,11 @@ func displayEnviron(env []string) {
 }
 
 func setupKubeconfig(h *host.Host, c *cfg.MachineConfig, clusterName string) (*kubeconfig.Settings, error) {
-	addr := ""
-	var err error
-	if driver.IsKIC(h.DriverName) {
-		addr = fmt.Sprintf("https://%s", net.JoinHostPort("127.0.0.1", fmt.Sprint(c.KubernetesConfig.NodePort)))
-	} else {
-		addr, err = h.Driver.GetURL()
-		if err != nil {
-			exit.WithError("Failed to get driver URL", err)
-		}
+	addr, err := h.Driver.GetURL()
+	if err != nil {
+		exit.WithError("Failed to get driver URL", err)
+	}
+	if !driver.IsKIC(h.DriverName) {
 		addr = strings.Replace(addr, "tcp://", "https://", -1)
 		addr = strings.Replace(addr, ":2376", ":"+strconv.Itoa(c.KubernetesConfig.NodePort), -1)
 	}
diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go
index 7c7b79178b..38d21d63c1 100644
--- a/pkg/drivers/kic/kic.go
+++ b/pkg/drivers/kic/kic.go
@@ -18,6 +18,7 @@ package kic
 
 import (
 	"fmt"
+	"net"
 	"os/exec"
 	"strconv"
 	"strings"
@@ -27,13 +28,11 @@ import (
 	"github.com/docker/machine/libmachine/state"
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
-	"github.com/spf13/viper"
 	pkgdrivers "k8s.io/minikube/pkg/drivers"
 	"k8s.io/minikube/pkg/drivers/kic/node"
 	"k8s.io/minikube/pkg/drivers/kic/oci"
 	"k8s.io/minikube/pkg/minikube/assets"
 	"k8s.io/minikube/pkg/minikube/command"
-	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
 )
 
@@ -63,17 +62,16 @@ type Driver struct {
 
 // Config is configuration for the kic driver used by registry
 type Config struct {
-	MachineName     string            // maps to the container name being created
-	CPU             int               // Number of CPU cores assigned to the container
-	Memory          int               // max memory in MB
-	StorePath       string            // libmachine store path
-	OCIBinary       string            // oci tool to use (docker, podman,...)
-	ImageDigest     string            // image name with sha to use for the node
-	APIHostBindPort int               // bind port for api server
-	SSHHostBindPort int               // bind port for ssh server
-	Mounts          []oci.Mount       // mounts
-	PortMappings    []oci.PortMapping // container port mappings
-	Envs            map[string]string // key,value of environment variables passed to the node
+	MachineName   string            // maps to the container name being created
+	CPU           int               // Number of CPU cores assigned to the container
+	Memory        int               // max memory in MB
+	StorePath     string            // libmachine store path
+	OCIBinary     string            // oci tool to use (docker, podman,...)
+	ImageDigest   string            // image name with sha to use for the node
+	Mounts        []oci.Mount       // mounts
+	APIServerPort int               // kubernetes api server port inside the container
+	PortMappings  []oci.PortMapping // container port mappings
+	Envs          map[string]string // key,value of environment variables passed to the node
 }
 
 // NewDriver returns a fully configured Kic driver
@@ -93,25 +91,24 @@ func NewDriver(c Config) *Driver {
 // Create a host using the driver's config
 func (d *Driver) Create() error {
 	params := node.CreateConfig{
-		Name:         d.NodeConfig.MachineName,
-		Image:        d.NodeConfig.ImageDigest,
-		ClusterLabel: node.ClusterLabelKey + "=" + d.MachineName,
-		CPUs:         strconv.Itoa(d.NodeConfig.CPU),
-		Memory:       strconv.Itoa(d.NodeConfig.Memory) + "mb",
-		Envs:         d.NodeConfig.Envs,
-		ExtraArgs:    []string{"--expose", fmt.Sprintf("%d", d.NodeConfig.APIHostBindPort)},
-		OCIBinary:    d.NodeConfig.OCIBinary,
+		Name:          d.NodeConfig.MachineName,
+		Image:         d.NodeConfig.ImageDigest,
+		ClusterLabel:  node.ClusterLabelKey + "=" + d.MachineName,
+		CPUs:          strconv.Itoa(d.NodeConfig.CPU),
+		Memory:        strconv.Itoa(d.NodeConfig.Memory) + "mb",
+		Envs:          d.NodeConfig.Envs,
+		ExtraArgs:     []string{"--expose", fmt.Sprintf("%d", d.NodeConfig.APIServerPort)},
+		OCIBinary:     d.NodeConfig.OCIBinary,
+		APIServerPort: d.NodeConfig.APIServerPort,
 	}
 
 	// control plane specific options
 	params.PortMappings = append(params.PortMappings, oci.PortMapping{
 		ListenAddress: DefaultBindIPV4,
-		HostPort:      int32(d.NodeConfig.APIHostBindPort),
 		ContainerPort: constants.APIServerPort,
 	},
 		oci.PortMapping{
 			ListenAddress: DefaultBindIPV4,
-			HostPort:      int32(d.NodeConfig.APIHostBindPort) + constants.SSHPort, // TODO: @medyagh: use github.com/phayes/freeport instead.
 			ContainerPort: constants.SSHPort,
 		},
 	)
@@ -174,17 +171,29 @@ func (d *Driver) GetSSHHostname() (string, error) {
 
 // GetSSHPort returns port for use with ssh
 func (d *Driver) GetSSHPort() (int, error) {
-	cc, err := config.Load(viper.GetString(config.MachineProfile))
+	p, err := oci.HostPortBinding(d.OCIBinary, d.MachineName, constants.SSHPort)
 	if err != nil {
-		glog.Infof("error loading config file which may be okay on first run : %v ", err)
-		return 22, nil
+		return p, errors.Wrap(err, "get ssh host-port")
 	}
-	return int(cc.SSHBindPort), nil
+	return p, nil
+}
+
+// GetSSHKeyPath returns the ssh key path
+func (d *Driver) GetSSHKeyPath() string {
+	if d.SSHKeyPath == "" {
+		d.SSHKeyPath = d.ResolveStorePath("id_rsa")
+	}
+	return d.SSHKeyPath
 }
 
 // GetURL returns ip of the container running kic control-panel
 func (d *Driver) GetURL() (string, error) {
-	return d.GetIP()
+	p, err := oci.HostPortBinding(d.NodeConfig.OCIBinary, d.MachineName, d.NodeConfig.APIServerPort)
+	url := fmt.Sprintf("https://%s", net.JoinHostPort("127.0.0.1", fmt.Sprint(p)))
+	if err != nil {
+		return url, errors.Wrap(err, "api host port binding")
+	}
+	return url, nil
 }
 
 // GetState returns the state that the host is in (running, stopped, etc)
diff --git a/pkg/drivers/kic/node/node.go b/pkg/drivers/kic/node/node.go
index 6f1aa60c56..aa3f0d3084 100644
--- a/pkg/drivers/kic/node/node.go
+++ b/pkg/drivers/kic/node/node.go
@@ -46,17 +46,18 @@ type Node struct {
 }
 
 type CreateConfig struct {
-	Name         string            // used for container name and hostname
-	Image        string            // container image to use to create the node.
-	ClusterLabel string            // label the containers we create using minikube so we can clean up
-	Role         string            // currently only role supported is control-plane
-	Mounts       []oci.Mount       // volume mounts
-	PortMappings []oci.PortMapping // ports to map to container from host
-	CPUs         string            // number of cpu cores assign to container
-	Memory       string            // memory (mbs) to assign to the container
-	Envs         map[string]string // environment variables to pass to the container
-	ExtraArgs    []string          // a list of any extra option to pass to oci binary during creation time, for example --expose 8080...
-	OCIBinary    string            // docker or podman
+	Name          string            // used for container name and hostname
+	Image         string            // container image to use to create the node.
+	ClusterLabel  string            // label the containers we create using minikube so we can clean up
+	Role          string            // currently only role supported is control-plane
+	Mounts        []oci.Mount       // volume mounts
+	APIServerPort int               // kubernetes api server port
+	PortMappings  []oci.PortMapping // ports to map to container from host
+	CPUs          string            // number of cpu cores assign to container
+	Memory        string            // memory (mbs) to assign to the container
+	Envs          map[string]string // environment variables to pass to the container
+	ExtraArgs     []string          // a list of any extra option to pass to oci binary during creation time, for example --expose 8080...
+	OCIBinary     string            // docker or podman
 }
 
 // CreateNode creates a new container node
diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go
index 2c4e899055..50db161f58 100644
--- a/pkg/drivers/kic/oci/oci.go
+++ b/pkg/drivers/kic/oci/oci.go
@@ -18,6 +18,7 @@ package oci
 
 import (
 	"os"
+	"strconv"
 
 	"github.com/docker/machine/libmachine/state"
 	"k8s.io/minikube/pkg/minikube/assets"
@@ -28,7 +29,6 @@ import (
 	"github.com/pkg/errors"
 
 	"fmt"
-	"net"
 	"os/exec"
 	"strings"
 	"time"
@@ -279,13 +279,9 @@ func UsernsRemap(ociBinary string) bool {
 func generatePortMappings(portMappings ...PortMapping) []string {
 	result := make([]string, 0, len(portMappings))
 	for _, pm := range portMappings {
-		var hostPortBinding string
-		if pm.ListenAddress != "" {
-			hostPortBinding = net.JoinHostPort(pm.ListenAddress, fmt.Sprintf("%d", pm.HostPort))
-		} else {
-			hostPortBinding = fmt.Sprintf("%d", pm.HostPort)
-		}
-		publish := fmt.Sprintf("--publish=%s:%d", hostPortBinding, pm.ContainerPort)
+		// let docker pick a host port by leaving it as ::
+		// example --publish=127.0.0.17::8443 will get a random host port for 8443
+		publish := fmt.Sprintf("--publish=%s::%d", pm.ListenAddress, pm.ContainerPort)
 		result = append(result, publish)
 	}
 	return result
@@ -394,3 +390,23 @@ func Copy(ociBinary string, ociID string, asset assets.CopyableFile) error {
 	}
 	return nil
 }
+
+// HostPortBinding will return port mapping for a container using cli.
+// example : HostPortBinding("docker", "minikube", "22")
+// will return the docker assigned port:
+// 32769, nil
+// only supports TCP ports
+func HostPortBinding(ociBinary string, ociID string, contPort int) (int, error) {
+	cmd := exec.Command(ociBinary, "inspect", "-f", fmt.Sprintf("'{{(index (index .NetworkSettings.Ports \"%d/tcp\") 0).HostPort}}'", contPort), ociID)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return 0, errors.Wrapf(err, "getting host-bind port %q for container ID %q", contPort, ociID)
+	}
+	o := strings.Trim(string(out), "\n")
+	o = strings.Trim(o, "'")
+	p, err := strconv.Atoi(o)
+	if err != nil {
+		return p, errors.Wrapf(err, "convert host-port %q to number", p)
+	}
+	return p, nil
+}
diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index 1f27f156cd..0a99a575f7 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -39,6 +39,7 @@ import (
 	"k8s.io/client-go/kubernetes"
 	kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/minikube/pkg/drivers/kic"
+	"k8s.io/minikube/pkg/drivers/kic/oci"
 	"k8s.io/minikube/pkg/kapi"
 	"k8s.io/minikube/pkg/minikube/assets"
 	"k8s.io/minikube/pkg/minikube/bootstrapper"
@@ -464,9 +465,12 @@ func (k *Bootstrapper) applyKicOverlay(cfg config.MachineConfig) error {
 
 // clientEndpointAddr returns ip and port accessible for the kubernetes clients to talk to the cluster
 func (k *Bootstrapper) clientEndpointAddr(cfg config.MachineConfig) (string, int) {
-	if driver.IsKIC(cfg.VMDriver) {
-		// because docker container ip on non-linux is not accesible
-		return kic.DefaultBindIPV4, cfg.KubernetesConfig.NodePort
+	if driver.IsKIC(cfg.VMDriver) { // for kic we ask docker/podman what port it assigned to node port
+		p, err := oci.HostPortBinding(cfg.VMDriver, cfg.Name, cfg.KubernetesConfig.NodePort)
+		if err != nil {
+			glog.Warningf("Error getting host bind port %s for api server for %s driver: %v ", p, cfg.VMDriver, err)
+		}
+		return kic.DefaultBindIPV4, p
 	}
 	return cfg.KubernetesConfig.NodeIP, cfg.KubernetesConfig.NodePort
 }
diff --git a/pkg/minikube/config/types.go b/pkg/minikube/config/types.go
index 70d888e4f4..80dd0bf81a 100644
--- a/pkg/minikube/config/types.go
+++ b/pkg/minikube/config/types.go
@@ -65,15 +65,13 @@ type MachineConfig struct {
 	HostOnlyNicType     string // Only used by virtualbox
 	NatNicType          string // Only used by virtualbox
 	Addons              map[string]bool
-	APIBindPort         int32 // the host port to bind to apiserver inside the container only used by kic
-	SSHBindPort         int32 // the host port to bind to ssh service inside the container only used by kic
 }
 
 // KubernetesConfig contains the parameters used to configure the VM Kubernetes.
 type KubernetesConfig struct {
 	KubernetesVersion string
 	NodeIP            string
-	NodePort          int
+	NodePort          int // kubernetes api server port
 	NodeName          string
 	APIServerName     string
 	APIServerNames    []string
diff --git a/pkg/minikube/registry/drvs/docker/docker.go b/pkg/minikube/registry/drvs/docker/docker.go
index fdb0c78ac6..f99143db1e 100644
--- a/pkg/minikube/registry/drvs/docker/docker.go
+++ b/pkg/minikube/registry/drvs/docker/docker.go
@@ -43,13 +43,13 @@ func init() {
 
 func configure(mc config.MachineConfig) interface{} {
 	return kic.NewDriver(kic.Config{
-		MachineName:     mc.Name,
-		StorePath:       localpath.MiniPath(),
-		ImageDigest:     kic.BaseImage,
-		CPU:             mc.CPUs,
-		Memory:          mc.Memory,
-		APIHostBindPort: mc.KubernetesConfig.NodePort,
-		OCIBinary:       oci.Docker,
+		MachineName:   mc.Name,
+		StorePath:     localpath.MiniPath(),
+		ImageDigest:   kic.BaseImage,
+		CPU:           mc.CPUs,
+		Memory:        mc.Memory,
+		APIServerPort: mc.KubernetesConfig.NodePort,
+		OCIBinary:     oci.Docker,
 	})
 
 }

From 83dd9fc4ffbf876cb1736f51f53170a4fa1ea9bc Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Fri, 24 Jan 2020 14:20:49 -0800
Subject: [PATCH 126/250] Bump kic base image for ssh

---
 pkg/drivers/kic/kic.go | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go
index 38d21d63c1..e4c7d1efe3 100644
--- a/pkg/drivers/kic/kic.go
+++ b/pkg/drivers/kic/kic.go
@@ -43,9 +43,7 @@ const DefaultPodCIDR = "10.244.0.0/16"
 const DefaultBindIPV4 = "127.0.0.1"
 
 // BaseImage is the base image is used to spin up kic containers created by kind.
-// const BaseImage = "gcr.io/k8s-minikube/kicbase:v0.0.1@sha256:c4ad2938877d2ae0d5b7248a5e7182ff58c0603165c3bedfe9d503e2d380a0db"
-// BaseImage is the base image is used to spin up kic containers created by kind.
-const BaseImage = "kicbase:local"
+const BaseImage = "gcr.io/k8s-minikube/kicbase:v0.0.2@sha256:8f531b90901721a7bd4e67ceffbbc7ee6c4292b0e6d1a9d6eb59f117d57bc4e9"
 
 // OverlayImage is the cni plugin used for overlay image, created by kind.
 const OverlayImage = "kindest/kindnetd:0.5.3"

From 711dab3848b1e6c111284aeb029843d9a6a9899f Mon Sep 17 00:00:00 2001
From: tstromberg <tstromberg@google.com>
Date: Fri, 24 Jan 2020 14:39:53 -0800
Subject: [PATCH 127/250] Update CRIO test to v1.15.7 to fix cgroups bug

---
 test/integration/start_stop_delete_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/integration/start_stop_delete_test.go b/test/integration/start_stop_delete_test.go
index 2ea0a44aa8..832e231aba 100644
--- a/test/integration/start_stop_delete_test.go
+++ b/test/integration/start_stop_delete_test.go
@@ -77,7 +77,7 @@ func TestStartStop(t *testing.T) {
 				"containerd=/var/run/containerd/containerd.sock",
 				"--apiserver-port=8444",
 			}},
-			{"crio", "v1.15.0", []string{
+			{"crio", "v1.15.7", []string{
 				"--container-runtime=crio",
 				"--disable-driver-mounts",
 				"--extra-config=kubeadm.ignore-preflight-errors=SystemVerification",

From ba3965b076ff5135e29a875bbecb61887c81ab37 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Fri, 24 Jan 2020 14:42:28 -0800
Subject: [PATCH 128/250] specify SSH username

---
 pkg/drivers/kic/kic.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go
index e4c7d1efe3..38a5d9cdcd 100644
--- a/pkg/drivers/kic/kic.go
+++ b/pkg/drivers/kic/kic.go
@@ -176,6 +176,11 @@ func (d *Driver) GetSSHPort() (int, error) {
 	return p, nil
 }
 
+// GetSSHUsername returns the ssh username
+func (d *Driver) GetSSHUsername() string {
+	return "docker"
+}
+
 // GetSSHKeyPath returns the ssh key path
 func (d *Driver) GetSSHKeyPath() string {
 	if d.SSHKeyPath == "" {

From 7cf8ed1edc903d03ef7c020ef5e921218e09d1b0 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Fri, 24 Jan 2020 14:52:27 -0800
Subject: [PATCH 129/250] lint

---
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index 0a99a575f7..aa7c571b1c 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -468,7 +468,7 @@ func (k *Bootstrapper) clientEndpointAddr(cfg config.MachineConfig) (string, int
 	if driver.IsKIC(cfg.VMDriver) { // for kic we ask docker/podman what port it assigned to node port
 		p, err := oci.HostPortBinding(cfg.VMDriver, cfg.Name, cfg.KubernetesConfig.NodePort)
 		if err != nil {
-			glog.Warningf("Error getting host bind port %s for api server for %s driver: %v ", p, cfg.VMDriver, err)
+			glog.Warningf("Error getting host bind port %q for api server for %q driver: %v ", p, cfg.VMDriver, err)
 		}
 		return kic.DefaultBindIPV4, p
 	}

From 5c906d33f31a1f550a0ef4c1fc98e79af3f149d7 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Fri, 24 Jan 2020 15:03:27 -0800
Subject: [PATCH 130/250] fix unit tests

---
 .../bootstrapper/bsutil/kubeadm_test.go       | 11 +++++---
 .../bootstrapper/bsutil/kubelet_test.go       | 25 +++++++++++--------
 2 files changed, 22 insertions(+), 14 deletions(-)

diff --git a/pkg/minikube/bootstrapper/bsutil/kubeadm_test.go b/pkg/minikube/bootstrapper/bsutil/kubeadm_test.go
index f6d08cf9b6..a5aef997bd 100644
--- a/pkg/minikube/bootstrapper/bsutil/kubeadm_test.go
+++ b/pkg/minikube/bootstrapper/bsutil/kubeadm_test.go
@@ -121,8 +121,9 @@ func TestGenerateKubeadmYAMLDNS(t *testing.T) {
 				cfg := tc.cfg
 				cfg.Nodes = []config.Node{
 					config.Node{
-						IP:   "1.1.1.1",
-						Name: "mk",
+						IP:           "1.1.1.1",
+						Name:         "mk",
+						ControlPlane: true,
 					},
 				}
 				cfg.KubernetesConfig.KubernetesVersion = version + ".0"
@@ -195,11 +196,13 @@ func TestGenerateKubeadmYAML(t *testing.T) {
 				if len(cfg.Nodes) > 0 {
 					cfg.Nodes[0].IP = "1.1.1.1"
 					cfg.Nodes[0].Name = "mk"
+					cfg.Nodes[0].ControlPlane = true
 				} else {
 					cfg.Nodes = []config.Node{
 						config.Node{
-							IP:   "1.1.1.1",
-							Name: "mk",
+							IP:           "1.1.1.1",
+							Name:         "mk",
+							ControlPlane: true,
 						},
 					}
 				}
diff --git a/pkg/minikube/bootstrapper/bsutil/kubelet_test.go b/pkg/minikube/bootstrapper/bsutil/kubelet_test.go
index 3861fbd7db..b0cca057c8 100644
--- a/pkg/minikube/bootstrapper/bsutil/kubelet_test.go
+++ b/pkg/minikube/bootstrapper/bsutil/kubelet_test.go
@@ -43,8 +43,9 @@ func TestGenerateKubeletConfig(t *testing.T) {
 				},
 				Nodes: []config.Node{
 					config.Node{
-						IP:   "192.168.1.100",
-						Name: "minikube",
+						IP:           "192.168.1.100",
+						Name:         "minikube",
+						ControlPlane: true,
 					},
 				},
 			},
@@ -67,8 +68,9 @@ ExecStart=/var/lib/minikube/binaries/v1.11.10/kubelet --allow-privileged=true --
 				},
 				Nodes: []config.Node{
 					config.Node{
-						IP:   "192.168.1.100",
-						Name: "minikube",
+						IP:           "192.168.1.100",
+						Name:         "minikube",
+						ControlPlane: true,
 					},
 				},
 			},
@@ -91,8 +93,9 @@ ExecStart=/var/lib/minikube/binaries/v1.17.0/kubelet --authorization-mode=Webhoo
 				},
 				Nodes: []config.Node{
 					config.Node{
-						IP:   "192.168.1.100",
-						Name: "minikube",
+						IP:           "192.168.1.100",
+						Name:         "minikube",
+						ControlPlane: true,
 					},
 				},
 			},
@@ -122,8 +125,9 @@ ExecStart=/var/lib/minikube/binaries/v1.17.0/kubelet --authorization-mode=Webhoo
 				},
 				Nodes: []config.Node{
 					config.Node{
-						IP:   "192.168.1.100",
-						Name: "minikube",
+						IP:           "192.168.1.100",
+						Name:         "minikube",
+						ControlPlane: true,
 					},
 				},
 			},
@@ -147,8 +151,9 @@ ExecStart=/var/lib/minikube/binaries/v1.17.0/kubelet --authorization-mode=Webhoo
 				},
 				Nodes: []config.Node{
 					config.Node{
-						IP:   "192.168.1.100",
-						Name: "minikube",
+						IP:           "192.168.1.100",
+						Name:         "minikube",
+						ControlPlane: true,
 					},
 				},
 			},

From de0573784ed17dee5933bf0587714e19eec525f5 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Fri, 24 Jan 2020 16:01:14 -0800
Subject: [PATCH 131/250] ok it builds again

---
 .../kubeconfig/testdata/kubeconfig/config1         |  6 +++---
 pkg/minikube/registry/drvs/docker/docker.go        | 14 ++++++--------
 2 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/pkg/minikube/kubeconfig/testdata/kubeconfig/config1 b/pkg/minikube/kubeconfig/testdata/kubeconfig/config1
index 8c497d4ccb..7e39058ec1 100644
--- a/pkg/minikube/kubeconfig/testdata/kubeconfig/config1
+++ b/pkg/minikube/kubeconfig/testdata/kubeconfig/config1
@@ -1,7 +1,7 @@
 apiVersion: v1
-clusters: []
-contexts: []
+clusters: null
+contexts: null
 current-context: minikube
 kind: Config
 preferences: {}
-users: []
+users: null
diff --git a/pkg/minikube/registry/drvs/docker/docker.go b/pkg/minikube/registry/drvs/docker/docker.go
index 3085c8eda9..e9cb5309c7 100644
--- a/pkg/minikube/registry/drvs/docker/docker.go
+++ b/pkg/minikube/registry/drvs/docker/docker.go
@@ -42,15 +42,13 @@ func init() {
 }
 
 func configure(mc config.MachineConfig) interface{} {
-	cp, _ := config.PrimaryControlPlane(mc)
 	return kic.NewDriver(kic.Config{
-		MachineName:  mc.Name,
-		StorePath:    localpath.MiniPath(),
-		ImageDigest:  kic.BaseImage,
-		CPU:          mc.CPUs,
-		Memory:       mc.Memory,
-		HostBindPort: cp.Port,
-		OCIBinary:    oci.Docker,
+		MachineName: mc.Name,
+		StorePath:   localpath.MiniPath(),
+		ImageDigest: kic.BaseImage,
+		CPU:         mc.CPUs,
+		Memory:      mc.Memory,
+		OCIBinary:   oci.Docker,
 	})
 
 }

From 5a5003965d04f084a122f0b76f0441868f204f17 Mon Sep 17 00:00:00 2001
From: tstromberg <tstromberg@google.com>
Date: Fri, 24 Jan 2020 16:06:37 -0800
Subject: [PATCH 132/250] Update CHANGELOG for v1.7.0-beta.1

---
 CHANGELOG.md | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 06955e8881..3c360769bb 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,34 @@
 # Release Notes
 
+## Version 1.7.0-beta.1 - 2020-01-24
+
+* Add 'pause' command to freeze Kubernetes cluster [#5962](https://github.com/kubernetes/minikube/pull/5962)
+* kic driver: add multiple profiles and ssh [#6390](https://github.com/kubernetes/minikube/pull/6390)
+* Update DefaultKubernetesVersion to v1.17.2 [#6392](https://github.com/kubernetes/minikube/pull/6392)
+* Add varlink program for using with podman-remote [#6349](https://github.com/kubernetes/minikube/pull/6349)
+* Batch file manipulation calls to reduce number of redundant commands [#6385](https://github.com/kubernetes/minikube/pull/6385)
+* Update Kubernetes libraries to v1.17.2 [#6374](https://github.com/kubernetes/minikube/pull/6374)
+* Fix dashboard by using kubectl apply/delete instead of prune for addons [#6376](https://github.com/kubernetes/minikube/pull/6376)
+* Remove addon manager [#6334](https://github.com/kubernetes/minikube/pull/6334)
+* Remove unnecessary crio restart to improve start latency [#6369](https://github.com/kubernetes/minikube/pull/6369)
+* Check for nil ref and img before passing them into go-containerregistry [#6236](https://github.com/kubernetes/minikube/pull/6236)
+* Change the compression methods used on the iso [#6341](https://github.com/kubernetes/minikube/pull/6341)
+* Update the crio.conf instead of overwriting it [#6219](https://github.com/kubernetes/minikube/pull/6219)
+* Update Japanese translation [#6339](https://github.com/kubernetes/minikube/pull/6339)
+* Fix syntax of minikube kubectl example command [#6255](https://github.com/kubernetes/minikube/pull/6255)
+* Stop minikube dashboard from crashing at start [#6325](https://github.com/kubernetes/minikube/pull/6325)
+
+Thanks you to the following contributors:
+
+- Anders F Björklund
+- inductor
+- Medya Ghazizadeh
+- Naoki Oketani
+- Priya Wadhwa
+- Sharif Elgamal
+- sshukun
+- Thomas Strömberg
+
 ## Version 1.7.0-beta.0 - 2020-01-15
 
 * add the ability to mark drivers as experimental  [#6326](https://github.com/kubernetes/minikube/pull/6326)

From b3404a70e6e11d4cb96aeceb766dc00342b8ca62 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Fri, 24 Jan 2020 19:03:50 -0800
Subject: [PATCH 133/250] patch node refactor for kic

---
 pkg/drivers/kic/oci/oci.go                   |  2 +-
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 23 +++++++++++++++-----
 pkg/minikube/registry/drvs/docker/docker.go  | 13 ++++++-----
 3 files changed, 25 insertions(+), 13 deletions(-)

diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go
index 50db161f58..54bf60f1d1 100644
--- a/pkg/drivers/kic/oci/oci.go
+++ b/pkg/drivers/kic/oci/oci.go
@@ -400,7 +400,7 @@ func HostPortBinding(ociBinary string, ociID string, contPort int) (int, error)
 	cmd := exec.Command(ociBinary, "inspect", "-f", fmt.Sprintf("'{{(index (index .NetworkSettings.Ports \"%d/tcp\") 0).HostPort}}'", contPort), ociID)
 	out, err := cmd.CombinedOutput()
 	if err != nil {
-		return 0, errors.Wrapf(err, "getting host-bind port %q for container ID %q", contPort, ociID)
+		return 0, errors.Wrapf(err, "getting host-bind port %d for container ID %q, output %s", contPort, ociID, out)
 	}
 	o := strings.Trim(string(out), "\n")
 	o = strings.Trim(o, "'")
diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index ea6c524849..ad2f481dd5 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -39,6 +39,7 @@ import (
 	"k8s.io/client-go/kubernetes"
 	kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/minikube/pkg/drivers/kic"
+	"k8s.io/minikube/pkg/drivers/kic/oci"
 	"k8s.io/minikube/pkg/kapi"
 	"k8s.io/minikube/pkg/minikube/assets"
 	"k8s.io/minikube/pkg/minikube/bootstrapper"
@@ -249,7 +250,7 @@ func (k *Bootstrapper) StartCluster(cfg config.MachineConfig) error {
 	if !driver.IsKIC(cfg.VMDriver) { // TODO: skip for both after verifications https://github.com/kubernetes/minikube/issues/6239
 		glog.Infof("Configuring cluster permissions ...")
 		elevate := func() error {
-			client, err := k.client(cp)
+			client, err := k.client(cp.IP, cp.Port)
 			if err != nil {
 				return err
 			}
@@ -269,7 +270,7 @@ func (k *Bootstrapper) StartCluster(cfg config.MachineConfig) error {
 }
 
 // client sets and returns a Kubernetes client to use to speak to a kubeadm launched apiserver
-func (k *Bootstrapper) client(n config.Node) (*kubernetes.Clientset, error) {
+func (k *Bootstrapper) client(ip string, port int) (*kubernetes.Clientset, error) {
 	if k.k8sClient != nil {
 		return k.k8sClient, nil
 	}
@@ -279,7 +280,7 @@ func (k *Bootstrapper) client(n config.Node) (*kubernetes.Clientset, error) {
 		return nil, errors.Wrap(err, "client config")
 	}
 
-	endpoint := fmt.Sprintf("https://%s", net.JoinHostPort(n.IP, strconv.Itoa(n.Port)))
+	endpoint := fmt.Sprintf("https://%s", net.JoinHostPort(ip, strconv.Itoa(port)))
 	if cc.Host != endpoint {
 		glog.Errorf("Overriding stale ClientConfig host %s with %s", cc.Host, endpoint)
 		cc.Host = endpoint
@@ -302,11 +303,21 @@ func (k *Bootstrapper) WaitForCluster(cfg config.MachineConfig, timeout time.Dur
 	if err := kverify.APIServerProcess(k.c, start, timeout); err != nil {
 		return err
 	}
-	if err := kverify.APIServerIsRunning(start, cp.IP, cp.Port, timeout); err != nil {
+
+	ip := cp.IP
+	port := cp.Port
+	if driver.IsKIC(cfg.VMDriver) {
+		ip = kic.DefaultBindIPV4
+		port, err = oci.HostPortBinding(cfg.VMDriver, cfg.Name, port)
+		if err != nil {
+			return errors.Wrapf(err, "get host-bind port %d for container %s", port, cfg.Name)
+		}
+	}
+	if err := kverify.APIServerIsRunning(start, ip, port, timeout); err != nil {
 		return err
 	}
 
-	c, err := k.client(cp)
+	c, err := k.client(ip, port)
 	if err != nil {
 		return errors.Wrap(err, "get k8s client")
 	}
@@ -361,7 +372,7 @@ func (k *Bootstrapper) restartCluster(cfg config.MachineConfig) error {
 	}
 
 	for _, n := range cfg.Nodes {
-		client, err := k.client(n)
+		client, err := k.client(n.IP, n.Port)
 		if err != nil {
 			return errors.Wrap(err, "getting k8s client")
 		}
diff --git a/pkg/minikube/registry/drvs/docker/docker.go b/pkg/minikube/registry/drvs/docker/docker.go
index e9cb5309c7..bb7df86abd 100644
--- a/pkg/minikube/registry/drvs/docker/docker.go
+++ b/pkg/minikube/registry/drvs/docker/docker.go
@@ -43,12 +43,13 @@ func init() {
 
 func configure(mc config.MachineConfig) interface{} {
 	return kic.NewDriver(kic.Config{
-		MachineName: mc.Name,
-		StorePath:   localpath.MiniPath(),
-		ImageDigest: kic.BaseImage,
-		CPU:         mc.CPUs,
-		Memory:      mc.Memory,
-		OCIBinary:   oci.Docker,
+		MachineName:   mc.Name,
+		StorePath:     localpath.MiniPath(),
+		ImageDigest:   kic.BaseImage,
+		CPU:           mc.CPUs,
+		Memory:        mc.Memory,
+		OCIBinary:     oci.Docker,
+		APIServerPort: mc.Nodes[0].Port,
 	})
 
 }

From 2ba31162543cb03a362e1235452b6716af193468 Mon Sep 17 00:00:00 2001
From: akshay <akshay.sub12@gmail.com>
Date: Wed, 22 Jan 2020 23:04:12 -0800
Subject: [PATCH 134/250] Update create secret in service so registry creds add
 on does not fail when a secret does not exist.

---
 pkg/minikube/service/service.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/pkg/minikube/service/service.go b/pkg/minikube/service/service.go
index b26fa0d6d0..7f6c64663f 100644
--- a/pkg/minikube/service/service.go
+++ b/pkg/minikube/service/service.go
@@ -331,10 +331,11 @@ func CreateSecret(namespace, name string, dataValues map[string]string, labels m
 	if err != nil {
 		return &retry.RetriableError{Err: err}
 	}
+
 	secrets := client.Secrets(namespace)
 	secret, err := secrets.Get(name, meta.GetOptions{})
 	if err != nil {
-		return &retry.RetriableError{Err: err}
+		glog.Infof("Failed to retrieve existing secret: %v", err)
 	}
 
 	// Delete existing secret

From d8a348c0c82b39be6ea39fbb167c0972c20f1d2c Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Tue, 28 Jan 2020 16:05:49 -0800
Subject: [PATCH 135/250] add integration test script for docker driver

---
 .../jenkins/linux_integration_tests_docker.sh | 36 +++++++++++++++++++
 1 file changed, 36 insertions(+)
 create mode 100755 hack/jenkins/linux_integration_tests_docker.sh

diff --git a/hack/jenkins/linux_integration_tests_docker.sh b/hack/jenkins/linux_integration_tests_docker.sh
new file mode 100755
index 0000000000..b5e8b3b36e
--- /dev/null
+++ b/hack/jenkins/linux_integration_tests_docker.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# Copyright 2019 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# This script runs the integration tests on a Linux machine for the KVM Driver
+
+# The script expects the following env variables:
+# MINIKUBE_LOCATION: GIT_COMMIT from upstream build.
+# COMMIT: Actual commit ID from upstream build
+# EXTRA_BUILD_ARGS (optional): Extra args to be passed into the minikube integrations tests
+# access_token: The Github API access token. Injected by the Jenkins credential provider. 
+
+set -e
+
+OS_ARCH="linux-amd64"
+VM_DRIVER="docker"
+JOB_NAME="docker_Linux"
+EXPECTED_DEFAULT_DRIVER="kvm2"
+
+mkdir -p cron && gsutil -qm rsync "gs://minikube-builds/${MINIKUBE_LOCATION}/cron" cron || echo "FAILED TO GET CRON FILES"
+sudo install cron/cleanup_and_reboot_Linux.sh /etc/cron.hourly/cleanup_and_reboot || echo "FAILED TO INSTALL CLEANUP"
+
+source ./common.sh

From 178a3a256f8592adf0e744a821af6564e0197657 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Tue, 28 Jan 2020 16:40:53 -0800
Subject: [PATCH 136/250] add container runtime info to profile list

---
 cmd/minikube/cmd/config/profile_list.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cmd/minikube/cmd/config/profile_list.go b/cmd/minikube/cmd/config/profile_list.go
index 8df06f2d2d..a9d2dcfe36 100644
--- a/cmd/minikube/cmd/config/profile_list.go
+++ b/cmd/minikube/cmd/config/profile_list.go
@@ -60,7 +60,7 @@ var printProfilesTable = func() {
 
 	var validData [][]string
 	table := tablewriter.NewWriter(os.Stdout)
-	table.SetHeader([]string{"Profile", "VM Driver", "NodeIP", "Node Port", "Kubernetes Version", "Status"})
+	table.SetHeader([]string{"Profile", "VM Driver", "Container Runtime", "NodeIP", "API Port", "Kubernetes Version", "Status"})
 	table.SetAutoFormatHeaders(false)
 	table.SetBorders(tablewriter.Border{Left: true, Top: true, Right: true, Bottom: true})
 	table.SetCenterSeparator("|")
@@ -84,7 +84,7 @@ var printProfilesTable = func() {
 		if err != nil {
 			exit.WithError("profile has no control plane", err)
 		}
-		validData = append(validData, []string{p.Name, p.Config.VMDriver, cp.IP, strconv.Itoa(cp.Port), p.Config.KubernetesConfig.KubernetesVersion, p.Status})
+		validData = append(validData, []string{p.Name, p.Config.VMDriver, p.Config.KubernetesConfig.ContainerRuntime, cp.IP, strconv.Itoa(cp.Port), p.Config.KubernetesConfig.KubernetesVersion, p.Status})
 	}
 
 	table.AppendBulk(validData)

From 987e5dc91e185048fa775587cf9e6796f5303bb1 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Tue, 28 Jan 2020 17:08:02 -0800
Subject: [PATCH 137/250] set pending for docker driver too

---
 hack/jenkins/linux_integration_tests_docker.sh | 2 +-
 hack/jenkins/minikube_set_pending.sh           | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/hack/jenkins/linux_integration_tests_docker.sh b/hack/jenkins/linux_integration_tests_docker.sh
index b5e8b3b36e..62f0a17045 100755
--- a/hack/jenkins/linux_integration_tests_docker.sh
+++ b/hack/jenkins/linux_integration_tests_docker.sh
@@ -27,7 +27,7 @@ set -e
 
 OS_ARCH="linux-amd64"
 VM_DRIVER="docker"
-JOB_NAME="docker_Linux"
+JOB_NAME="Docker_Linux"
 EXPECTED_DEFAULT_DRIVER="kvm2"
 
 mkdir -p cron && gsutil -qm rsync "gs://minikube-builds/${MINIKUBE_LOCATION}/cron" cron || echo "FAILED TO GET CRON FILES"
diff --git a/hack/jenkins/minikube_set_pending.sh b/hack/jenkins/minikube_set_pending.sh
index 18c3f04248..ffffc5e72c 100755
--- a/hack/jenkins/minikube_set_pending.sh
+++ b/hack/jenkins/minikube_set_pending.sh
@@ -40,6 +40,7 @@ jobs=(
      # 'KVM-GPU_Linux' - Disabled
      'KVM_Linux'
      'none_Linux'
+     'Docker_Linux'
 )
 
 # retry_github_status provides reliable github status updates

From 0760859e62ed2d1c1096b9c56ea3e99cba3bc44f Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Tue, 28 Jan 2020 18:27:35 -0800
Subject: [PATCH 138/250] remove not needed check

---
 hack/jenkins/linux_integration_tests_docker.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/hack/jenkins/linux_integration_tests_docker.sh b/hack/jenkins/linux_integration_tests_docker.sh
index 62f0a17045..6adcccb4b9 100755
--- a/hack/jenkins/linux_integration_tests_docker.sh
+++ b/hack/jenkins/linux_integration_tests_docker.sh
@@ -28,7 +28,6 @@ set -e
 OS_ARCH="linux-amd64"
 VM_DRIVER="docker"
 JOB_NAME="Docker_Linux"
-EXPECTED_DEFAULT_DRIVER="kvm2"
 
 mkdir -p cron && gsutil -qm rsync "gs://minikube-builds/${MINIKUBE_LOCATION}/cron" cron || echo "FAILED TO GET CRON FILES"
 sudo install cron/cleanup_and_reboot_Linux.sh /etc/cron.hourly/cleanup_and_reboot || echo "FAILED TO INSTALL CLEANUP"

From 541c67d45a450efd6f08572203a1077fb01493f6 Mon Sep 17 00:00:00 2001
From: Medya Ghazizadeh <medyagh@users.noreply.github.com>
Date: Wed, 29 Jan 2020 10:38:51 -0800
Subject: [PATCH 139/250] add github actions integration tests (#6421)

---
 .github/workflows/main.yml | 121 +++++++++++++++++++++++++++++++++++++
 1 file changed, 121 insertions(+)
 create mode 100644 .github/workflows/main.yml

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
new file mode 100644
index 0000000000..e7ba04525b
--- /dev/null
+++ b/.github/workflows/main.yml
@@ -0,0 +1,121 @@
+name: CI
+
+on: [push]
+
+jobs:
+  docker_ubuntu:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: build binaries
+      run : |
+        make minikube-linux-amd64
+        make e2e-linux-amd64
+        mkdir -p report
+    - name: run integration test
+      run: |
+        echo running docker driver intergration test on ubuntu
+        ./out/e2e-linux-amd64 -minikube-start-args=--vm-driver=docker -expected-default-driver= -test.timeout=70m -test.v -binary=out/minikube-linux-amd64 2>&1 | tee ./report/testout.txt
+    - name: install gopogh
+      run: |
+        cd /tmp
+        GO111MODULE="on" go get github.com/medyagh/gopogh@v0.0.17 || true
+        cd -
+    - name: generate gopogh report
+      run: |
+        export PATH=${PATH}:`go env GOPATH`/bin
+        go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
+        gopogh -in ./report/testout.json -out ./report/testout.html -name "docker ubuntu" -repo github.com/kubernetes/minikube/  || true
+    - uses: actions/upload-artifact@v1
+      with:
+        name: docker_on_ubuntu_report
+        path: report
+
+  docker_macos:
+
+    runs-on: macos-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: build binaries
+      run : |
+        make minikube-darwin-amd64
+        make e2e-darwin-amd64
+        mkdir -p report
+    - name: run integration test
+      run: |
+        echo running docker driver intergration test on ubuntu
+        ./out/e2e-darwin-amd64 -minikube-start-args=--vm-driver=docker -expected-default-driver= -test.timeout=70m -test.v -binary=out/minikube-darwin-amd64 2>&1 | tee ./report/testout.txt
+    - name: install gopogh
+      run: |
+        cd /tmp
+        GO111MODULE="on" go get github.com/medyagh/gopogh@v0.0.17 || true
+        cd -
+    - name: generate gopogh report
+      run: |
+        export PATH=${PATH}:`go env GOPATH`/bin
+        go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
+        gopogh -in ./report/testout.json -out ./report/testout.html -name "docker macos" -repo github.com/kubernetes/minikube/  || true
+    - uses: actions/upload-artifact@v1
+      with:
+        name: docker_on_macos_report
+        path: ./report
+
+  none_ubuntu:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: build binaries
+      run : |
+        make minikube-linux-amd64
+        make e2e-linux-amd64
+        mkdir -p report
+    - name: run integration test
+      run: |
+        echo running docker driver intergration test on ubuntu
+        ./out/e2e-linux-amd64 -minikube-start-args=--vm-driver=none -expected-default-driver= -test.timeout=70m -test.v -binary=out/minikube-linux-amd64 2>&1 | tee ./report/testout.txt
+    - name: install gopogh
+      run: |
+        cd /tmp
+        GO111MODULE="on" go get github.com/medyagh/gopogh@v0.0.17 || true
+        cd -
+    - name: generate gopogh report
+      run: |
+        export PATH=${PATH}:`go env GOPATH`/bin
+        go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
+        gopogh -in ./report/testout.json -out ./report/testout.html -name "docker ubuntu" -repo github.com/kubernetes/minikube/  || true
+    - uses: actions/upload-artifact@v1
+      with:
+        name: none_on_ubuntu_latest_report
+        path: report
+
+  none_ubuntu16_04:
+    runs-on: ubuntu-16.04
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: build binaries
+      run : |
+        make minikube-linux-amd64
+        make e2e-linux-amd64
+        mkdir -p report
+    - name: run integration test
+      run: |
+        echo running docker driver intergration test on ubuntu
+        ./out/e2e-linux-amd64 -minikube-start-args=--vm-driver=none -expected-default-driver= -test.timeout=70m -test.v -binary=out/minikube-linux-amd64 2>&1 | tee ./report/testout.txt
+    - name: install gopogh
+      run: |
+        cd /tmp
+        GO111MODULE="on" go get github.com/medyagh/gopogh@v0.0.17 || true
+        cd -
+    - name: generate gopogh report
+      run: |
+        export PATH=${PATH}:`go env GOPATH`/bin
+        go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
+        gopogh -in ./report/testout.json -out ./report/testout.html -name "docker ubuntu" -repo github.com/kubernetes/minikube/  || true
+    - uses: actions/upload-artifact@v1
+      with:
+        name: none_on_ubuntu_16_04
+        path: report

From e725104e2f1c29d70c07fe6fb2ceb8559102aa0d Mon Sep 17 00:00:00 2001
From: Medya Ghazizadeh <medyagh@users.noreply.github.com>
Date: Wed, 29 Jan 2020 14:47:23 -0800
Subject: [PATCH 140/250] improve checking modprob netfilter (#6427)

---
 pkg/minikube/command/kic_runner.go | 11 +++++------
 pkg/minikube/cruntime/cruntime.go  | 11 +++++++----
 2 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/pkg/minikube/command/kic_runner.go b/pkg/minikube/command/kic_runner.go
index cc58035fd1..3fe57bb17d 100644
--- a/pkg/minikube/command/kic_runner.go
+++ b/pkg/minikube/command/kic_runner.go
@@ -115,13 +115,12 @@ func (k *kicRunner) RunCmd(cmd *exec.Cmd) (*RunResult, error) {
 		if elapsed > (1 * time.Second) {
 			glog.Infof("Done: %v: (%s)", oc.Args, elapsed)
 		}
-	} else {
-		if exitError, ok := err.(*exec.ExitError); ok {
-			rr.ExitCode = exitError.ExitCode()
-		}
-		err = fmt.Errorf("%s: %v\nstdout:\n%s\nstderr:\n%s", rr.Command(), err, rr.Stdout.String(), rr.Stderr.String())
+		return rr, nil
 	}
-	return rr, err
+	if exitError, ok := err.(*exec.ExitError); ok {
+		rr.ExitCode = exitError.ExitCode()
+	}
+	return rr, fmt.Errorf("%s: %v\nstdout:\n%s\nstderr:\n%s", rr.Command(), err, rr.Stdout.String(), rr.Stderr.String())
 
 }
 
diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go
index 2705a012ce..812d469000 100644
--- a/pkg/minikube/cruntime/cruntime.go
+++ b/pkg/minikube/cruntime/cruntime.go
@@ -167,11 +167,14 @@ func disableOthers(me Manager, cr CommandRunner) error {
 // enableIPForwarding configures IP forwarding, which is handled normally by Docker
 // Context: https://github.com/kubernetes/kubeadm/issues/1062
 func enableIPForwarding(cr CommandRunner) error {
-	c := exec.Command("sudo", "modprobe", "br_netfilter")
-	if _, err := cr.RunCmd(c); err != nil {
-		return errors.Wrap(err, "br_netfilter")
+	c := exec.Command("sudo", "sysctl", "net.netfilter.nf_conntrack_count")
+	if rr, err := cr.RunCmd(c); err != nil {
+		glog.Infof("couldn't verify netfilter by %q which might be okay. error: %v", rr.Command(), err)
+		c = exec.Command("sudo", "modprobe", "br_netfilter")
+		if _, err := cr.RunCmd(c); err != nil {
+			return errors.Wrapf(err, "br_netfilter")
+		}
 	}
-
 	c = exec.Command("sudo", "sh", "-c", "echo 1 > /proc/sys/net/ipv4/ip_forward")
 	if _, err := cr.RunCmd(c); err != nil {
 		return errors.Wrapf(err, "ip_forward")

From 775186b3ef64907b6d8b567ebe59de122e9a6d8d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Str=C3=B6mberg?= <tstromberg@google.com>
Date: Wed, 29 Jan 2020 16:12:18 -0800
Subject: [PATCH 141/250] Add a 2-second timeout to driver detection commands
 (#6422)

---
 pkg/minikube/registry/drvs/docker/docker.go         |  7 ++++++-
 pkg/minikube/registry/drvs/hyperkit/hyperkit.go     |  7 ++++++-
 pkg/minikube/registry/drvs/hyperv/hyperv.go         |  8 +++++++-
 pkg/minikube/registry/drvs/kvm2/kvm2.go             | 10 ++++++++--
 pkg/minikube/registry/drvs/virtualbox/virtualbox.go |  8 +++++++-
 5 files changed, 34 insertions(+), 6 deletions(-)

diff --git a/pkg/minikube/registry/drvs/docker/docker.go b/pkg/minikube/registry/drvs/docker/docker.go
index bb7df86abd..0d912150de 100644
--- a/pkg/minikube/registry/drvs/docker/docker.go
+++ b/pkg/minikube/registry/drvs/docker/docker.go
@@ -17,8 +17,10 @@ limitations under the License.
 package docker
 
 import (
+	"context"
 	"fmt"
 	"os/exec"
+	"time"
 
 	"github.com/docker/machine/libmachine/drivers"
 	"k8s.io/minikube/pkg/drivers/kic"
@@ -59,8 +61,11 @@ func status() registry.State {
 	if err != nil {
 		return registry.State{Error: err, Installed: false, Healthy: false, Fix: "Docker is required.", Doc: "https://minikube.sigs.k8s.io/docs/reference/drivers/kic/"}
 	}
+	// Allow no more than 2 seconds for querying state
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel()
 
-	err = exec.Command("docker", "info").Run()
+	err = exec.CommandContext(ctx, "docker", "info").Run()
 	if err != nil {
 		return registry.State{Error: err, Installed: true, Healthy: false, Fix: "Docker is not running. Try: restarting docker desktop."}
 	}
diff --git a/pkg/minikube/registry/drvs/hyperkit/hyperkit.go b/pkg/minikube/registry/drvs/hyperkit/hyperkit.go
index f6be3f789a..cc544488f6 100644
--- a/pkg/minikube/registry/drvs/hyperkit/hyperkit.go
+++ b/pkg/minikube/registry/drvs/hyperkit/hyperkit.go
@@ -19,6 +19,7 @@ limitations under the License.
 package hyperkit
 
 import (
+	"context"
 	"fmt"
 	"os/exec"
 	"strings"
@@ -87,7 +88,11 @@ func status() registry.State {
 		return registry.State{Error: err, Fix: "Run 'brew install hyperkit'", Doc: docURL}
 	}
 
-	cmd := exec.Command(path, "-v")
+	// Allow no more than 2 seconds for querying state
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel()
+
+	cmd := exec.CommandContext(ctx, path, "-v")
 	out, err := cmd.CombinedOutput()
 	if err != nil {
 		return registry.State{Installed: true, Error: fmt.Errorf("%s failed:\n%s", strings.Join(cmd.Args, " "), out), Fix: "Run 'brew install hyperkit'", Doc: docURL}
diff --git a/pkg/minikube/registry/drvs/hyperv/hyperv.go b/pkg/minikube/registry/drvs/hyperv/hyperv.go
index 7bafd1403a..877c81555d 100644
--- a/pkg/minikube/registry/drvs/hyperv/hyperv.go
+++ b/pkg/minikube/registry/drvs/hyperv/hyperv.go
@@ -19,9 +19,11 @@ limitations under the License.
 package hyperv
 
 import (
+	"context"
 	"fmt"
 	"os/exec"
 	"strings"
+	"time"
 
 	"github.com/docker/machine/drivers/hyperv"
 	"github.com/docker/machine/libmachine/drivers"
@@ -66,7 +68,11 @@ func status() registry.State {
 		return registry.State{Error: err}
 	}
 
-	cmd := exec.Command(path, "Get-WindowsOptionalFeature", "-FeatureName", "Microsoft-Hyper-V-All", "-Online")
+	// Allow no more than 2 seconds for querying state
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel()
+
+	cmd := exec.CommandContext(ctx, path, "Get-WindowsOptionalFeature", "-FeatureName", "Microsoft-Hyper-V-All", "-Online")
 	out, err := cmd.CombinedOutput()
 	if err != nil {
 		return registry.State{Installed: false, Error: fmt.Errorf("%s failed:\n%s", strings.Join(cmd.Args, " "), out), Fix: "Start PowerShell as Administrator, and run: 'Enable-WindowsOptionalFeature -Online -FeatureName Microsoft-Hyper-V -All'", Doc: docURL}
diff --git a/pkg/minikube/registry/drvs/kvm2/kvm2.go b/pkg/minikube/registry/drvs/kvm2/kvm2.go
index 89c5e7781b..52c5766b2d 100644
--- a/pkg/minikube/registry/drvs/kvm2/kvm2.go
+++ b/pkg/minikube/registry/drvs/kvm2/kvm2.go
@@ -19,11 +19,13 @@ limitations under the License.
 package kvm2
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/docker/machine/libmachine/drivers"
 
@@ -97,13 +99,17 @@ func defaultURI() string {
 }
 
 func status() registry.State {
+	// Allow no more than 2 seconds for querying state
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel()
+
 	path, err := exec.LookPath("virsh")
 	if err != nil {
 		return registry.State{Error: err, Fix: "Install libvirt", Doc: docURL}
 	}
 
 	// On Ubuntu 19.10 (libvirt 5.4), this fails if LIBVIRT_DEFAULT_URI is unset
-	cmd := exec.Command(path, "domcapabilities", "--virttype", "kvm")
+	cmd := exec.CommandContext(ctx, path, "domcapabilities", "--virttype", "kvm")
 	cmd.Env = append(os.Environ(), fmt.Sprintf("LIBVIRT_DEFAULT_URI=%s", defaultURI()))
 
 	out, err := cmd.CombinedOutput()
@@ -116,7 +122,7 @@ func status() registry.State {
 		}
 	}
 
-	cmd = exec.Command("virsh", "list")
+	cmd = exec.CommandContext(ctx, "virsh", "list")
 	cmd.Env = append(os.Environ(), fmt.Sprintf("LIBVIRT_DEFAULT_URI=%s", defaultURI()))
 	out, err = cmd.CombinedOutput()
 	if err != nil {
diff --git a/pkg/minikube/registry/drvs/virtualbox/virtualbox.go b/pkg/minikube/registry/drvs/virtualbox/virtualbox.go
index 4f6edf70c5..60931a38f7 100644
--- a/pkg/minikube/registry/drvs/virtualbox/virtualbox.go
+++ b/pkg/minikube/registry/drvs/virtualbox/virtualbox.go
@@ -17,9 +17,11 @@ limitations under the License.
 package virtualbox
 
 import (
+	"context"
 	"fmt"
 	"os/exec"
 	"strings"
+	"time"
 
 	"github.com/docker/machine/drivers/virtualbox"
 	"github.com/docker/machine/libmachine/drivers"
@@ -75,7 +77,11 @@ func status() registry.State {
 		}
 	}
 
-	cmd := exec.Command(path, "list", "hostinfo")
+	// Allow no more than 2 seconds for querying state
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel()
+
+	cmd := exec.CommandContext(ctx, path, "list", "hostinfo")
 	out, err := cmd.CombinedOutput()
 	if err != nil {
 		return registry.State{

From a9af9d0bd3763d90a29dccb1133cc7bae008d7b5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Anders=20Bj=C3=B6rklund?= <anders.f.bjorklund@gmail.com>
Date: Thu, 30 Jan 2020 01:15:32 +0100
Subject: [PATCH 142/250] Use the complete filenames for the packages (#6283)

Follow standard file naming practices, for each package manager.

This opens up for allowing additional architectures, in future.
---
 hack/jenkins/release_build_and_upload.sh | 8 ++++----
 hack/jenkins/release_github_page.sh      | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/hack/jenkins/release_build_and_upload.sh b/hack/jenkins/release_build_and_upload.sh
index 6640fbfd79..0f5fc8339d 100755
--- a/hack/jenkins/release_build_and_upload.sh
+++ b/hack/jenkins/release_build_and_upload.sh
@@ -43,10 +43,10 @@ env GOPATH=$HOME/go BUILD_IN_DOCKER=y \
   make -j 16 \
   all \
   out/minikube-installer.exe \
-  "out/minikube_${DEB_VERSION}.deb" \
-  "out/minikube-${RPM_VERSION}.rpm" \
-  "out/docker-machine-driver-kvm2_${DEB_VERSION}.deb" \
-  "out/docker-machine-driver-kvm2-${RPM_VERSION}.rpm"
+  "out/minikube_${DEB_VERSION}-0_amd64.deb" \
+  "out/minikube-${RPM_VERSION}-0.x86_64.rpm" \
+  "out/docker-machine-driver-kvm2_${DEB_VERSION}-0_amd64.deb" \
+  "out/docker-machine-driver-kvm2-${RPM_VERSION}-0.x86_64.rpm"
 
 make checksum
 
diff --git a/hack/jenkins/release_github_page.sh b/hack/jenkins/release_github_page.sh
index 02784c881c..3ce137911b 100755
--- a/hack/jenkins/release_github_page.sh
+++ b/hack/jenkins/release_github_page.sh
@@ -84,8 +84,8 @@ FILES_TO_UPLOAD=(
     'minikube-windows-amd64.exe'
     'minikube-windows-amd64.exe.sha256'
     'minikube-installer.exe'
-    "minikube_${DEB_VERSION}.deb"
-    "minikube-${RPM_VERSION}.rpm"
+    "minikube_${DEB_VERSION}-0_amd64.deb"
+    "minikube-${RPM_VERSION}-0.x86_64.rpm"
     'docker-machine-driver-kvm2'
     'docker-machine-driver-kvm2.sha256'
     'docker-machine-driver-hyperkit'

From d45414899084f7eea9d6f4c93f0a18f8f9a4765b Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Thu, 30 Jan 2020 10:16:43 -0800
Subject: [PATCH 143/250] Add missing status test

---
 cmd/minikube/cmd/status_test.go | 106 ++++++++++++++++++++++++++++++++
 1 file changed, 106 insertions(+)
 create mode 100644 cmd/minikube/cmd/status_test.go

diff --git a/cmd/minikube/cmd/status_test.go b/cmd/minikube/cmd/status_test.go
new file mode 100644
index 0000000000..015e5bd483
--- /dev/null
+++ b/cmd/minikube/cmd/status_test.go
@@ -0,0 +1,106 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cmd
+
+import (
+	"bytes"
+	"encoding/json"
+	"testing"
+)
+
+func TestExitCode(t *testing.T) {
+	var tests = []struct {
+		name  string
+		want  int
+		state *Status
+	}{
+		{"ok", 0, &Status{Host: "Running", Kubelet: "Running", APIServer: "Running", Kubeconfig: Configured}},
+		{"paused", 2, &Status{Host: "Running", Kubelet: "Stopped", APIServer: "Paused", Kubeconfig: Configured}},
+		{"down", 7, &Status{Host: "Stopped", Kubelet: "Stopped", APIServer: "Stopped", Kubeconfig: Misconfigured}},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			got := exitCode(tc.state)
+			if got != tc.want {
+				t.Errorf("exitcode(%+v) = %d, want: %d", tc.state, got, tc.want)
+			}
+		})
+	}
+}
+
+func TestStatusText(t *testing.T) {
+	var tests = []struct {
+		name  string
+		state *Status
+		want  string
+	}{
+		{
+			name:  "ok",
+			state: &Status{Host: "Running", Kubelet: "Running", APIServer: "Running", Kubeconfig: Configured},
+			want:  "host: Running\nkubelet: Running\napiserver: Running\nkubeconfig: Configured\n",
+		},
+		{
+			name:  "paused",
+			state: &Status{Host: "Running", Kubelet: "Stopped", APIServer: "Paused", Kubeconfig: Configured},
+			want:  "host: Running\nkubelet: Stopped\napiserver: Paused\nkubeconfig: Configured\n",
+		},
+		{
+			name:  "down",
+			state: &Status{Host: "Stopped", Kubelet: "Stopped", APIServer: "Stopped", Kubeconfig: Misconfigured},
+			want:  "host: Stopped\nkubelet: Stopped\napiserver: Stopped\nkubeconfig: Misconfigured\n\nWARNING: Your kubectl is pointing to stale minikube-vm.\nTo fix the kubectl context, run `minikube update-context`\n",
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			var b bytes.Buffer
+			err := statusText(tc.state, &b)
+			if err != nil {
+				t.Errorf("text(%+v) error: %v", tc.state, err)
+			}
+
+			got := b.String()
+			if got != tc.want {
+				t.Errorf("text(%+v) = %q, want: %q", tc.state, got, tc.want)
+			}
+		})
+	}
+}
+
+func TestStatusJSON(t *testing.T) {
+	var tests = []struct {
+		name  string
+		state *Status
+	}{
+		{"ok", &Status{Host: "Running", Kubelet: "Running", APIServer: "Running", Kubeconfig: Configured}},
+		{"paused", &Status{Host: "Running", Kubelet: "Stopped", APIServer: "Paused", Kubeconfig: Configured}},
+		{"down", &Status{Host: "Stopped", Kubelet: "Stopped", APIServer: "Stopped", Kubeconfig: Misconfigured}},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			var b bytes.Buffer
+			err := statusJSON(tc.state, &b)
+			if err != nil {
+				t.Errorf("json(%+v) error: %v", tc.state, err)
+			}
+
+			st := &Status{}
+			if err := json.Unmarshal(b.Bytes(), st); err != nil {
+				t.Errorf("json(%+v) unmarshal error: %v", tc.state, err)
+			}
+		})
+	}
+}

From a1d291fd0d25fc7899dc201ddb3df39aef03d4b9 Mon Sep 17 00:00:00 2001
From: tstromberg <tstromberg@google.com>
Date: Thu, 30 Jan 2020 11:59:47 -0800
Subject: [PATCH 144/250] Handle cert symlinks that point to a nonexistent file

---
 pkg/minikube/bootstrapper/certs.go      | 5 +++--
 pkg/minikube/bootstrapper/certs_test.go | 4 ++--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/pkg/minikube/bootstrapper/certs.go b/pkg/minikube/bootstrapper/certs.go
index d03eea13ee..f7b3d3a7fe 100644
--- a/pkg/minikube/bootstrapper/certs.go
+++ b/pkg/minikube/bootstrapper/certs.go
@@ -341,7 +341,7 @@ func configureCACerts(cr command.Runner, caCerts map[string]string) error {
 	for _, caCertFile := range caCerts {
 		dstFilename := path.Base(caCertFile)
 		certStorePath := path.Join(SSLCertStoreDir, dstFilename)
-		cmd := fmt.Sprintf("test -f %s || ln -s %s %s", caCertFile, certStorePath, caCertFile)
+		cmd := fmt.Sprintf("test -f %s || ln -fs %s %s", caCertFile, certStorePath, caCertFile)
 		if _, err := cr.RunCmd(exec.Command("sudo", "/bin/bash", "-c", cmd)); err != nil {
 			return errors.Wrapf(err, "create symlink for %s", caCertFile)
 		}
@@ -352,7 +352,8 @@ func configureCACerts(cr command.Runner, caCerts map[string]string) error {
 			}
 			subjectHashLink := path.Join(SSLCertStoreDir, fmt.Sprintf("%s.0", subjectHash))
 
-			cmd := fmt.Sprintf("test -f %s || ln -s %s %s", subjectHashLink, certStorePath, subjectHashLink)
+			// NOTE: This symlink may exist, but point to a missing file
+			cmd := fmt.Sprintf("test -L %s || ln -fs %s %s", subjectHashLink, certStorePath, subjectHashLink)
 			if _, err := cr.RunCmd(exec.Command("sudo", "/bin/bash", "-c", cmd)); err != nil {
 				return errors.Wrapf(err, "create symlink for %s", caCertFile)
 			}
diff --git a/pkg/minikube/bootstrapper/certs_test.go b/pkg/minikube/bootstrapper/certs_test.go
index f70cf65c10..a3448657d1 100644
--- a/pkg/minikube/bootstrapper/certs_test.go
+++ b/pkg/minikube/bootstrapper/certs_test.go
@@ -52,8 +52,8 @@ func TestSetupCerts(t *testing.T) {
 	}
 
 	expected := map[string]string{
-		`sudo /bin/bash -c "test -f /usr/share/ca-certificates/mycert.pem || ln -s /etc/ssl/certs/mycert.pem /usr/share/ca-certificates/mycert.pem"`:             "-",
-		`sudo /bin/bash -c "test -f /usr/share/ca-certificates/minikubeCA.pem || ln -s /etc/ssl/certs/minikubeCA.pem /usr/share/ca-certificates/minikubeCA.pem"`: "-",
+		`sudo /bin/bash -c "test -f /usr/share/ca-certificates/mycert.pem || ln -fs /etc/ssl/certs/mycert.pem /usr/share/ca-certificates/mycert.pem"`:             "-",
+		`sudo /bin/bash -c "test -f /usr/share/ca-certificates/minikubeCA.pem || ln -fs /etc/ssl/certs/minikubeCA.pem /usr/share/ca-certificates/minikubeCA.pem"`: "-",
 	}
 	f := command.NewFakeCommandRunner()
 	f.SetCommandToOutput(expected)

From 4bf3fc422a7e4fc08ae301a710ebd551276bd6dc Mon Sep 17 00:00:00 2001
From: Medya Ghazizadeh <medyagh@users.noreply.github.com>
Date: Thu, 30 Jan 2020 14:30:04 -0800
Subject: [PATCH 145/250] Add docker run-time for kic driver (#6436)

* Add docker container run time for kic
---
 Makefile                            |  4 ++--
 hack/images/kicbase.Dockerfile      |  3 ++-
 pkg/drivers/kic/kic.go              |  6 +++++-
 pkg/minikube/cluster/cluster.go     | 26 ++++++++++++++++++++------
 pkg/minikube/constants/constants.go |  2 ++
 pkg/minikube/driver/driver.go       |  1 -
 6 files changed, 31 insertions(+), 11 deletions(-)

diff --git a/Makefile b/Makefile
index a4ba320757..e460ac2886 100755
--- a/Makefile
+++ b/Makefile
@@ -494,8 +494,8 @@ storage-provisioner-image: out/storage-provisioner-$(GOARCH) ## Build storage-pr
 
 .PHONY: kic-base-image
 kic-base-image: ## builds the base image used for kic.
-	docker rmi -f $(REGISTRY)/kicbase:v0.0.2-snapshot || true
-	docker build -f ./hack/images/kicbase.Dockerfile -t $(REGISTRY)/kicbase:v0.0.2-snapshot  --build-arg COMMIT_SHA=${VERSION}-$(COMMIT)  .
+	docker rmi -f $(REGISTRY)/kicbase:v0.0.3-snapshot || true
+	docker build -f ./hack/images/kicbase.Dockerfile -t $(REGISTRY)/kicbase:v0.0.3-snapshot  --build-arg COMMIT_SHA=${VERSION}-$(COMMIT)  .
 
 
 
diff --git a/hack/images/kicbase.Dockerfile b/hack/images/kicbase.Dockerfile
index 91e81bf891..aeafd04403 100644
--- a/hack/images/kicbase.Dockerfile
+++ b/hack/images/kicbase.Dockerfile
@@ -7,6 +7,7 @@ RUN apt-get update && apt-get install -y \
   sudo \
   dnsutils \
   openssh-server \
+  docker.io \
   && apt-get clean -y 
 # based on https://github.com/rastasheep/ubuntu-sshd/blob/master/18.04/Dockerfile
 # making SSH work for docker container 
@@ -28,7 +29,7 @@ RUN rm -rf \
     /kind/bin/kubeadm /kind/bin/kubelet /kind/systemd /kind/images /kind/manifests
 RUN echo "kic! Build: ${COMMIT_SHA} Time :$(date)" > "/kic.txt"
 # for minikube ssh. to match VM using docker username
-RUN adduser --disabled-password --gecos '' docker
+RUN adduser --ingroup docker --disabled-password --gecos '' docker 
 RUN adduser docker sudo
 RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
 USER docker
diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go
index 38a5d9cdcd..194daa1581 100644
--- a/pkg/drivers/kic/kic.go
+++ b/pkg/drivers/kic/kic.go
@@ -43,7 +43,7 @@ const DefaultPodCIDR = "10.244.0.0/16"
 const DefaultBindIPV4 = "127.0.0.1"
 
 // BaseImage is the base image is used to spin up kic containers created by kind.
-const BaseImage = "gcr.io/k8s-minikube/kicbase:v0.0.2@sha256:8f531b90901721a7bd4e67ceffbbc7ee6c4292b0e6d1a9d6eb59f117d57bc4e9"
+const BaseImage = "gcr.io/k8s-minikube/kicbase:v0.0.3@sha256:34db5e30f8830c0d5e49b62f3ea6b2844f805980592fe0084cbea799bfb12664"
 
 // OverlayImage is the cni plugin used for overlay image, created by kind.
 const OverlayImage = "kindest/kindnetd:0.5.3"
@@ -109,6 +109,10 @@ func (d *Driver) Create() error {
 			ListenAddress: DefaultBindIPV4,
 			ContainerPort: constants.SSHPort,
 		},
+		oci.PortMapping{
+			ListenAddress: DefaultBindIPV4,
+			ContainerPort: constants.DockerDaemonPort,
+		},
 	)
 	_, err := node.CreateNode(params)
 	if err != nil {
diff --git a/pkg/minikube/cluster/cluster.go b/pkg/minikube/cluster/cluster.go
index 664172e7b1..af3f6cc184 100644
--- a/pkg/minikube/cluster/cluster.go
+++ b/pkg/minikube/cluster/cluster.go
@@ -47,6 +47,8 @@ import (
 	"github.com/shirou/gopsutil/mem"
 	"github.com/spf13/viper"
 
+	"k8s.io/minikube/pkg/drivers/kic"
+	"k8s.io/minikube/pkg/drivers/kic/oci"
 	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
@@ -534,21 +536,33 @@ func createHost(api libmachine.API, cfg config.MachineConfig) (*host.Host, error
 
 // GetHostDockerEnv gets the necessary docker env variables to allow the use of docker through minikube's vm
 func GetHostDockerEnv(api libmachine.API) (map[string]string, error) {
-	host, err := CheckIfHostExistsAndLoad(api, viper.GetString(config.MachineProfile))
+	pName := viper.GetString(config.MachineProfile)
+	host, err := CheckIfHostExistsAndLoad(api, pName)
 	if err != nil {
 		return nil, errors.Wrap(err, "Error checking that api exists and loading it")
 	}
-	ip, err := host.Driver.GetIP()
-	if err != nil {
-		return nil, errors.Wrap(err, "Error getting ip from host")
+
+	ip := kic.DefaultBindIPV4
+	if !driver.IsKIC(host.Driver.DriverName()) { // kic externally accessible ip is different that node ip
+		ip, err = host.Driver.GetIP()
+		if err != nil {
+			return nil, errors.Wrap(err, "Error getting ip from host")
+		}
+
 	}
 
 	tcpPrefix := "tcp://"
-	port := "2376"
+	port := constants.DockerDaemonPort
+	if driver.IsKIC(host.Driver.DriverName()) { // for kic we need to find out what port docker allocated during creation
+		port, err = oci.HostPortBinding(host.Driver.DriverName(), pName, constants.DockerDaemonPort)
+		if err != nil {
+			return nil, errors.Wrapf(err, "get hostbind port for %d", constants.DockerDaemonPort)
+		}
+	}
 
 	envMap := map[string]string{
 		"DOCKER_TLS_VERIFY": "1",
-		"DOCKER_HOST":       tcpPrefix + net.JoinHostPort(ip, port),
+		"DOCKER_HOST":       tcpPrefix + net.JoinHostPort(ip, fmt.Sprint(port)),
 		"DOCKER_CERT_PATH":  localpath.MakeMiniPath("certs"),
 	}
 	return envMap, nil
diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go
index 48850cfc8f..a05042939a 100644
--- a/pkg/minikube/constants/constants.go
+++ b/pkg/minikube/constants/constants.go
@@ -27,6 +27,8 @@ import (
 )
 
 const (
+	// DockerDaemonPort is the port Docker daemon listening inside a minikube node (vm or container).
+	DockerDaemonPort = 2376
 	// SSHPort is the SSH serviceport on the node vm and container
 	SSHPort = 22
 	// APIServerPort is the default API server port
diff --git a/pkg/minikube/driver/driver.go b/pkg/minikube/driver/driver.go
index 0dd832a229..6af4933423 100644
--- a/pkg/minikube/driver/driver.go
+++ b/pkg/minikube/driver/driver.go
@@ -108,7 +108,6 @@ func FlagDefaults(name string) FlagHints {
 		fh.CacheImages = true
 		// only for kic, till other run-times are available we auto-set containerd.
 		if name == Docker {
-			fh.ContainerRuntime = "containerd"
 			fh.ExtraOptions = append(fh.ExtraOptions, fmt.Sprintf("kubeadm.pod-network-cidr=%s", kic.DefaultPodCIDR))
 		}
 		return fh

From f283a576c1a2dc2cd75d87ce2d3db7c436daae1c Mon Sep 17 00:00:00 2001
From: tstromberg <tstromberg@google.com>
Date: Thu, 30 Jan 2020 14:36:39 -0800
Subject: [PATCH 146/250] Promote status function to one that should be used
 across tests

---
 test/integration/helpers.go                | 28 ++++++++++++++++++++--
 test/integration/start_stop_delete_test.go | 22 +++++------------
 2 files changed, 32 insertions(+), 18 deletions(-)

diff --git a/test/integration/helpers.go b/test/integration/helpers.go
index c525cf99f1..bac683ea87 100644
--- a/test/integration/helpers.go
+++ b/test/integration/helpers.go
@@ -185,9 +185,9 @@ func CleanupWithLogs(t *testing.T, profile string, cancel context.CancelFunc) {
 
 // clusterLogs shows logs for debugging a failed cluster
 func clusterLogs(t *testing.T, profile string) {
-	st := Status(context.Background(), t, Target(), profile)
+	st := Status(context.Background(), t, Target(), profile, "Host")
 	if st != state.Running.String() {
-		t.Logf("%q is not running, skipping log retrieval (state=%q)", profile, st)
+		t.Logf("%q host is not running, skipping log retrieval (state=%q)", profile, st)
 		return
 	}
 
@@ -199,6 +199,12 @@ func clusterLogs(t *testing.T, profile string) {
 	}
 	t.Logf("%s logs: %s", t.Name(), rr.Stdout)
 
+	st = Status(context.Background(), t, Target(), profile, "APIServer")
+	if st != state.Running.String() {
+		t.Logf("%q apiserver is not running, skipping kubectl commands (state=%q)", profile, st)
+		return
+	}
+
 	rr, rerr := Run(t, exec.Command("kubectl", "--context", profile, "get", "po", "-A", "--show-labels"))
 	if rerr != nil {
 		t.Logf("%s: %v", rr.Command(), rerr)
@@ -313,8 +319,26 @@ func PodWait(ctx context.Context, t *testing.T, profile string, ns string, selec
 	return names, fmt.Errorf("%s: %v", fmt.Sprintf("%s within %s", selector, timeout), err)
 }
 
+// Status returns a minikube component status as a string
+func Status(ctx context.Context, t *testing.T, path string, profile string, key string) string {
+	t.Helper()
+	// Reminder of useful keys: "Host", "Kubelet", "APIServer"
+	rr, err := Run(t, exec.CommandContext(ctx, path, "status", fmt.Sprintf("--format={{.%s}}", key), "-p", profile))
+	if err != nil {
+		t.Logf("status error: %v (may be ok)", err)
+	}
+	return strings.TrimSpace(rr.Stdout.String())
+}
+
 // showPodLogs logs debug info for pods
 func showPodLogs(ctx context.Context, t *testing.T, profile string, ns string, names []string) {
+	t.Helper()
+	st := Status(context.Background(), t, Target(), profile, "APIServer")
+	if st != state.Running.String() {
+		t.Logf("%q apiserver is not running, skipping kubectl commands (state=%q)", profile, st)
+		return
+	}
+
 	t.Logf("%s: showing logs for failed pods as of %s", t.Name(), time.Now())
 
 	for _, name := range names {
diff --git a/test/integration/start_stop_delete_test.go b/test/integration/start_stop_delete_test.go
index 832e231aba..f71c78e6b2 100644
--- a/test/integration/start_stop_delete_test.go
+++ b/test/integration/start_stop_delete_test.go
@@ -36,16 +36,6 @@ import (
 	"k8s.io/minikube/pkg/minikube/constants"
 )
 
-// status returns a minikube component status as a string
-func status(ctx context.Context, t *testing.T, path string, profile string, key string) string {
-	t.Helper()
-	rr, err := Run(t, exec.CommandContext(ctx, path, "status", fmt.Sprintf("--format={{.%s}}", key), "-p", profile))
-	if err != nil {
-		t.Logf("status error: %v (may be ok)", err)
-	}
-	return strings.TrimSpace(rr.Stdout.String())
-}
-
 func TestStartStop(t *testing.T) {
 	MaybeParallel(t)
 
@@ -115,7 +105,7 @@ func TestStartStop(t *testing.T) {
 					t.Errorf("%s failed: %v", rr.Args, err)
 				}
 
-				got := status(ctx, t, Target(), profile, "Host")
+				got := Status(ctx, t, Target(), profile, "Host")
 				if got != state.Stopped.String() {
 					t.Errorf("status = %q; want = %q", got, state.Stopped)
 				}
@@ -132,7 +122,7 @@ func TestStartStop(t *testing.T) {
 					t.Fatalf("wait: %v", err)
 				}
 
-				got = status(ctx, t, Target(), profile, "Host")
+				got = Status(ctx, t, Target(), profile, "Host")
 				if got != state.Running.String() {
 					t.Errorf("host status = %q; want = %q", got, state.Running)
 				}
@@ -235,12 +225,12 @@ func testPause(ctx context.Context, t *testing.T, profile string) {
 		t.Fatalf("%s failed: %v", rr.Args, err)
 	}
 
-	got := status(ctx, t, Target(), profile, "APIServer")
+	got := Status(ctx, t, Target(), profile, "APIServer")
 	if got != state.Paused.String() {
 		t.Errorf("apiserver status = %q; want = %q", got, state.Paused)
 	}
 
-	got = status(ctx, t, Target(), profile, "Kubelet")
+	got = Status(ctx, t, Target(), profile, "Kubelet")
 	if got != state.Stopped.String() {
 		t.Errorf("kubelet status = %q; want = %q", got, state.Stopped)
 	}
@@ -250,12 +240,12 @@ func testPause(ctx context.Context, t *testing.T, profile string) {
 		t.Fatalf("%s failed: %v", rr.Args, err)
 	}
 
-	got = status(ctx, t, Target(), profile, "APIServer")
+	got = Status(ctx, t, Target(), profile, "APIServer")
 	if got != state.Running.String() {
 		t.Errorf("apiserver status = %q; want = %q", got, state.Running)
 	}
 
-	got = status(ctx, t, Target(), profile, "Kubelet")
+	got = Status(ctx, t, Target(), profile, "Kubelet")
 	if got != state.Running.String() {
 		t.Errorf("kubelet status = %q; want = %q", got, state.Running)
 	}

From 6bd48a86c9525e95e27874fb7c62acac3952063e Mon Sep 17 00:00:00 2001
From: tstromberg <tstromberg@google.com>
Date: Thu, 30 Jan 2020 14:41:45 -0800
Subject: [PATCH 147/250] Explicitly say that a cluster is nonexistent

---
 cmd/minikube/cmd/status.go      | 22 +++++++++++++++++++---
 cmd/minikube/cmd/status_test.go |  1 +
 2 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/cmd/minikube/cmd/status.go b/cmd/minikube/cmd/status.go
index 64a20bcddf..ad050cf199 100644
--- a/cmd/minikube/cmd/status.go
+++ b/cmd/minikube/cmd/status.go
@@ -45,8 +45,10 @@ var output string
 
 const (
 	// Additional states used by kubeconfig
-	Configured    = "Configured"    // analogous to state.Saved
-	Misconfigured = "Misconfigured" // analogous to state.Error
+	Configured    = "Configured"    // ~state.Saved
+	Misconfigured = "Misconfigured" // ~state.Error
+	// Additional states used for clarity
+	Nonexistent = "Nonexistent" // ~state.None
 )
 
 // Status holds string representations of component states
@@ -92,6 +94,9 @@ var statusCmd = &cobra.Command{
 		if err != nil {
 			glog.Errorf("status error: %v", err)
 		}
+		if st.Host == Nonexistent {
+			glog.Errorf("The %q cluster does not exist!", machineName)
+		}
 
 		switch strings.ToLower(output) {
 		case "text":
@@ -125,11 +130,22 @@ func exitCode(st *Status) int {
 }
 
 func status(api libmachine.API, name string) (*Status, error) {
-	st := &Status{}
+	st := &Status{
+		Host:       Nonexistent,
+		APIServer:  Nonexistent,
+		Kubelet:    Nonexistent,
+		Kubeconfig: Nonexistent,
+	}
+
 	hs, err := cluster.GetHostStatus(api, name)
 	if err != nil {
 		return st, errors.Wrap(err, "host")
 	}
+
+	// Nonexistent it is!
+	if hs == state.None.String() {
+		return st, nil
+	}
 	st.Host = hs
 	if st.Host != state.Running.String() {
 		return st, nil
diff --git a/cmd/minikube/cmd/status_test.go b/cmd/minikube/cmd/status_test.go
index 015e5bd483..ef414631f8 100644
--- a/cmd/minikube/cmd/status_test.go
+++ b/cmd/minikube/cmd/status_test.go
@@ -31,6 +31,7 @@ func TestExitCode(t *testing.T) {
 		{"ok", 0, &Status{Host: "Running", Kubelet: "Running", APIServer: "Running", Kubeconfig: Configured}},
 		{"paused", 2, &Status{Host: "Running", Kubelet: "Stopped", APIServer: "Paused", Kubeconfig: Configured}},
 		{"down", 7, &Status{Host: "Stopped", Kubelet: "Stopped", APIServer: "Stopped", Kubeconfig: Misconfigured}},
+		{"missing", 7, &Status{Host: "Nonexistent", Kubelet: "Nonexistent", APIServer: "Nonexistent", Kubeconfig: "Nonexistent"}},
 	}
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {

From d5a83c7b244add0eb178edf7ce1628f5da7ec20d Mon Sep 17 00:00:00 2001
From: tstromberg <tstromberg@google.com>
Date: Thu, 30 Jan 2020 15:54:04 -0800
Subject: [PATCH 148/250] Add addon enablement to start

---
 cmd/minikube/cmd/config/addons_list.go       |  8 ++---
 cmd/minikube/cmd/config/disable.go           |  2 +-
 cmd/minikube/cmd/config/enable.go            |  3 +-
 cmd/minikube/cmd/start.go                    | 31 +++++++++++++++--
 pkg/addons/addons.go                         | 35 +++++++++++++-------
 pkg/minikube/assets/addons.go                | 16 ++++++---
 pkg/minikube/bootstrapper/bsutil/files.go    | 31 -----------------
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go |  5 ---
 pkg/minikube/out/style.go                    |  2 ++
 pkg/minikube/out/style_enum.go               |  2 ++
 10 files changed, 74 insertions(+), 61 deletions(-)

diff --git a/cmd/minikube/cmd/config/addons_list.go b/cmd/minikube/cmd/config/addons_list.go
index 18a5ff8016..8a66f4c753 100644
--- a/cmd/minikube/cmd/config/addons_list.go
+++ b/cmd/minikube/cmd/config/addons_list.go
@@ -104,7 +104,7 @@ var printAddonsList = func() {
 		addonBundle := assets.Addons[addonName]
 		addonStatus, err := addonBundle.IsEnabled()
 		if err != nil {
-			exit.WithError("Error getting addons status", err)
+			out.WarningT("Unable to get addon status for {{.name}}: {{.error}}", out.V{"name": addonName, "error": err})
 		}
 		tData = append(tData, []string{addonName, pName, fmt.Sprintf("%s %s", stringFromStatus(addonStatus), iconFromStatus(addonStatus))})
 	}
@@ -114,12 +114,11 @@ var printAddonsList = func() {
 
 	v, _, err := config.ListProfiles()
 	if err != nil {
-		glog.Infof("error getting list of porfiles: %v", err)
+		glog.Errorf("list profiles returned error: %v", err)
 	}
 	if len(v) > 1 {
 		out.T(out.Tip, "To see addons list for other profiles use: `minikube addons -p name list`")
 	}
-
 }
 
 var printAddonsJSON = func() {
@@ -137,7 +136,8 @@ var printAddonsJSON = func() {
 
 		addonStatus, err := addonBundle.IsEnabled()
 		if err != nil {
-			exit.WithError("Error getting addons status", err)
+			glog.Errorf("Unable to get addon status for {{.name}}: {{.error}}", err)
+			continue
 		}
 
 		addonsMap[addonName] = map[string]interface{}{
diff --git a/cmd/minikube/cmd/config/disable.go b/cmd/minikube/cmd/config/disable.go
index c6c538f4b1..e9916a8929 100644
--- a/cmd/minikube/cmd/config/disable.go
+++ b/cmd/minikube/cmd/config/disable.go
@@ -39,7 +39,7 @@ var addonsDisableCmd = &cobra.Command{
 		if err != nil {
 			exit.WithError("disable failed", err)
 		}
-		out.SuccessT(`"{{.minikube_addon}}" was successfully disabled`, out.V{"minikube_addon": addon})
+		out.T(out.AddonDisable, `"The '{{.minikube_addon}}' addon is disabled`, out.V{"minikube_addon": addon})
 	},
 }
 
diff --git a/cmd/minikube/cmd/config/enable.go b/cmd/minikube/cmd/config/enable.go
index a920b9be0d..8bef60235a 100644
--- a/cmd/minikube/cmd/config/enable.go
+++ b/cmd/minikube/cmd/config/enable.go
@@ -33,13 +33,12 @@ var addonsEnableCmd = &cobra.Command{
 		if len(args) != 1 {
 			exit.UsageT("usage: minikube addons enable ADDON_NAME")
 		}
-
 		addon := args[0]
 		err := addons.Set(addon, "true", viper.GetString(config.MachineProfile))
 		if err != nil {
 			exit.WithError("enable failed", err)
 		}
-		out.SuccessT("{{.addonName}} was successfully enabled", out.V{"addonName": addon})
+		out.T(out.AddonEnable, "The '{{.addonName}}' addon is enabled", out.V{"addonName": addon})
 	},
 }
 
diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 43976eef18..3469742d45 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -47,6 +47,7 @@ import (
 	"golang.org/x/sync/errgroup"
 	cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config"
 	pkgaddons "k8s.io/minikube/pkg/addons"
+	"k8s.io/minikube/pkg/minikube/assets"
 	"k8s.io/minikube/pkg/minikube/bootstrapper"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
@@ -409,10 +410,36 @@ func cacheISO(cfg *config.MachineConfig, driverName string) {
 }
 
 func enableAddons() {
-	for _, a := range addonList {
+	start := time.Now()
+	glog.Infof("enableAddons")
+	defer func() {
+		glog.Infof("enableAddons completed in %s", time.Since(start))
+	}()
+	toEnable := []string{}
+
+	// Apply addons that are enabled by default
+	for name, a := range assets.Addons {
+		enabled, err := a.IsEnabled()
+		if err != nil {
+			glog.Errorf("is-enabled failed for %q: %v", a.Name(), err)
+			continue
+		}
+		if enabled {
+			toEnable = append(toEnable, name)
+		}
+	}
+
+	toEnable = append(toEnable, addonList...)
+	if len(toEnable) == 0 {
+		return
+	}
+
+	out.T(out.AddonEnable, "Enabling addons: {{.addons}}", out.V{"addons": strings.Join(toEnable, ", ")})
+	for _, a := range toEnable {
 		err := pkgaddons.Set(a, "true", viper.GetString(config.MachineProfile))
 		if err != nil {
-			exit.WithError("addon enable failed", err)
+			// Intentionally non-fatal
+			out.WarningT("Enabling '{{.name}}' returned an error: {{.error}}", out.V{"name": a, "error": err})
 		}
 	}
 }
diff --git a/pkg/addons/addons.go b/pkg/addons/addons.go
index 0437793d04..dae1ef0c6c 100644
--- a/pkg/addons/addons.go
+++ b/pkg/addons/addons.go
@@ -19,7 +19,7 @@ package addons
 import (
 	"fmt"
 	"os"
-	"path/filepath"
+	"path"
 	"strconv"
 
 	"github.com/golang/glog"
@@ -41,6 +41,8 @@ const defaultStorageClassProvisioner = "standard"
 
 // Set sets a value
 func Set(name, value, profile string) error {
+	glog.Infof("Setting %s=%s in profile %q", name, value, profile)
+
 	a, valid := isAddonValid(name)
 	if !valid {
 		return errors.Errorf("%s is not a valid addon", name)
@@ -100,6 +102,7 @@ func SetBool(m *config.MachineConfig, name string, val string) error {
 
 // enableOrDisableAddon updates addon status executing any commands necessary
 func enableOrDisableAddon(name, val, profile string) error {
+	glog.Infof("Setting addon %s=%s in %q", name, val, profile)
 	enable, err := strconv.ParseBool(val)
 	if err != nil {
 		return errors.Wrapf(err, "parsing bool: %s", name)
@@ -112,9 +115,9 @@ func enableOrDisableAddon(name, val, profile string) error {
 		out.ErrT(out.Conflict, "{{.error}}", out.V{"error": err})
 		return err
 	}
-	//if addon is already enabled or disabled, do nothing
+
 	if alreadySet {
-		return nil
+		glog.Warningf("addon %s should already be in state %v", name, val)
 	}
 
 	if name == "istio" && enable {
@@ -137,6 +140,7 @@ func enableOrDisableAddon(name, val, profile string) error {
 	//if minikube is not running, we return and simply update the value in the addon
 	//config and rewrite the file
 	if !cluster.IsMinikubeRunning(api) {
+		glog.Warningf("minikube is not running, writing to disk and doing nothing")
 		return nil
 	}
 
@@ -178,37 +182,44 @@ func isAddonAlreadySet(addon *assets.Addon, enable bool) (bool, error) {
 func enableOrDisableAddonInternal(addon *assets.Addon, cmd command.Runner, data interface{}, enable bool, profile string) error {
 	files := []string{}
 	for _, addon := range addon.Assets {
-		var addonFile assets.CopyableFile
+		var f assets.CopyableFile
 		var err error
 		if addon.IsTemplate() {
-			addonFile, err = addon.Evaluate(data)
+			f, err = addon.Evaluate(data)
 			if err != nil {
 				return errors.Wrapf(err, "evaluate bundled addon %s asset", addon.GetAssetName())
 			}
 
 		} else {
-			addonFile = addon
+			f = addon
 		}
+		fPath := path.Join(f.GetTargetDir(), f.GetTargetName())
+
 		if enable {
-			if err := cmd.Copy(addonFile); err != nil {
+			glog.Infof("installing %s", fPath)
+			if err := cmd.Copy(f); err != nil {
 				return err
 			}
 		} else {
+			glog.Infof("Removing %+v", fPath)
 			defer func() {
-				if err := cmd.Remove(addonFile); err != nil {
-					glog.Warningf("error removing %s; addon should still be disabled as expected", addonFile)
+				if err := cmd.Remove(f); err != nil {
+					glog.Warningf("error removing %s; addon should still be disabled as expected", fPath)
 				}
 			}()
 		}
-		files = append(files, filepath.Join(addonFile.GetTargetDir(), addonFile.GetTargetName()))
+		files = append(files, fPath)
 	}
 	command, err := kubectlCommand(profile, files, enable)
 	if err != nil {
 		return err
 	}
-	if result, err := cmd.RunCmd(command); err != nil {
-		return errors.Wrapf(err, "error updating addon:\n%s", result.Output())
+	glog.Infof("Running: %s", command)
+	rr, err := cmd.RunCmd(command)
+	if err != nil {
+		return errors.Wrapf(err, "addon apply")
 	}
+	glog.Infof("output:\n%s", rr.Output())
 	return nil
 }
 
diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go
index 67fd6b33df..6b18b810f6 100644
--- a/pkg/minikube/assets/addons.go
+++ b/pkg/minikube/assets/addons.go
@@ -23,6 +23,7 @@ import (
 	"path/filepath"
 	"runtime"
 
+	"github.com/golang/glog"
 	"github.com/pkg/errors"
 	"github.com/spf13/viper"
 	"k8s.io/minikube/pkg/minikube/config"
@@ -57,11 +58,18 @@ func (a *Addon) Name() string {
 // IsEnabled checks if an Addon is enabled for the current profile
 func (a *Addon) IsEnabled() (bool, error) {
 	c, err := config.Load(viper.GetString(config.MachineProfile))
-	if err == nil {
-		if status, ok := c.Addons[a.Name()]; ok {
-			return status, nil
-		}
+	if err != nil {
+		return false, err
 	}
+
+	// Is this addon explicitly listed in their configuration?
+	status, ok := c.Addons[a.Name()]
+	glog.Infof("IsEnabled %q = %v (listed in config=%v)", a.Name(), status, ok)
+	if ok {
+		return status, nil
+	}
+
+	// Return the default unconfigured state of the addon
 	return a.enabled, nil
 }
 
diff --git a/pkg/minikube/bootstrapper/bsutil/files.go b/pkg/minikube/bootstrapper/bsutil/files.go
index decb2fed55..df6cb7472f 100644
--- a/pkg/minikube/bootstrapper/bsutil/files.go
+++ b/pkg/minikube/bootstrapper/bsutil/files.go
@@ -20,7 +20,6 @@ package bsutil
 import (
 	"path"
 
-	"github.com/pkg/errors"
 	"k8s.io/minikube/pkg/minikube/assets"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/vmpath"
@@ -52,33 +51,3 @@ func ConfigFileAssets(cfg config.KubernetesConfig, kubeadm []byte, kubelet []byt
 	}
 	return fs
 }
-
-// AddAddons adds addons to list of files
-func AddAddons(files *[]assets.CopyableFile, data interface{}) error {
-	// add addons to file list
-	// custom addons
-	if err := assets.AddMinikubeDirAssets(files); err != nil {
-		return errors.Wrap(err, "adding minikube dir assets")
-	}
-	// bundled addons
-	for _, addonBundle := range assets.Addons {
-		if isEnabled, err := addonBundle.IsEnabled(); err == nil && isEnabled {
-			for _, addon := range addonBundle.Assets {
-				if addon.IsTemplate() {
-					addonFile, err := addon.Evaluate(data)
-					if err != nil {
-						return errors.Wrapf(err, "evaluate bundled addon %s asset", addon.GetAssetName())
-					}
-
-					*files = append(*files, addonFile)
-				} else {
-					*files = append(*files, addon)
-				}
-			}
-		} else if err != nil {
-			return nil
-		}
-	}
-
-	return nil
-}
diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index ad2f481dd5..23dcf30ead 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -41,7 +41,6 @@ import (
 	"k8s.io/minikube/pkg/drivers/kic"
 	"k8s.io/minikube/pkg/drivers/kic/oci"
 	"k8s.io/minikube/pkg/kapi"
-	"k8s.io/minikube/pkg/minikube/assets"
 	"k8s.io/minikube/pkg/minikube/bootstrapper"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/kverify"
@@ -484,10 +483,6 @@ func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error {
 	}
 	files := bsutil.ConfigFileAssets(cfg.KubernetesConfig, kubeadmCfg, kubeletCfg, kubeletService, cniFile)
 
-	if err := bsutil.AddAddons(&files, assets.GenerateTemplateData(cfg.KubernetesConfig)); err != nil {
-		return errors.Wrap(err, "adding addons")
-	}
-
 	// Combine mkdir request into a single call to reduce load
 	dirs := []string{}
 	for _, f := range files {
diff --git a/pkg/minikube/out/style.go b/pkg/minikube/out/style.go
index 682ca2fc0c..56f6bf8b35 100644
--- a/pkg/minikube/out/style.go
+++ b/pkg/minikube/out/style.go
@@ -116,6 +116,8 @@ var styles = map[StyleEnum]style{
 	MountOptions:     {Prefix: "💾  "},
 	Fileserver:       {Prefix: "🚀  ", OmitNewline: true},
 	DryRun:           {Prefix: "🏜️   "},
+	AddonEnable:      {Prefix: "🌟  "},
+	AddonDisable:     {Prefix: "🌑  "},
 }
 
 // Add a prefix to a string
diff --git a/pkg/minikube/out/style_enum.go b/pkg/minikube/out/style_enum.go
index 0d0c6e9ae1..150b19fa37 100644
--- a/pkg/minikube/out/style_enum.go
+++ b/pkg/minikube/out/style_enum.go
@@ -87,4 +87,6 @@ const (
 	Pause
 	Unpause
 	DryRun
+	AddonEnable
+	AddonDisable
 )

From 41687346e383ea1b06174fca7d9a7d95bfc3dd78 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Thu, 30 Jan 2020 16:06:27 -0800
Subject: [PATCH 149/250] make it less than 80

---
 cmd/minikube/cmd/config/profile_list.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/minikube/cmd/config/profile_list.go b/cmd/minikube/cmd/config/profile_list.go
index a9d2dcfe36..6671f3fd34 100644
--- a/cmd/minikube/cmd/config/profile_list.go
+++ b/cmd/minikube/cmd/config/profile_list.go
@@ -60,7 +60,7 @@ var printProfilesTable = func() {
 
 	var validData [][]string
 	table := tablewriter.NewWriter(os.Stdout)
-	table.SetHeader([]string{"Profile", "VM Driver", "Container Runtime", "NodeIP", "API Port", "Kubernetes Version", "Status"})
+	table.SetHeader([]string{"Profile", "VM Driver", "Runtime", "IP", "Port", "Version", "Status"})
 	table.SetAutoFormatHeaders(false)
 	table.SetBorders(tablewriter.Border{Left: true, Top: true, Right: true, Bottom: true})
 	table.SetCenterSeparator("|")

From 8e0ffdf1cf2e3a188e9047953a657795461bd00c Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Thu, 30 Jan 2020 16:20:55 -0800
Subject: [PATCH 150/250] resolve conflict

---
 pkg/drivers/kic/kic.go         |  85 ++++++++++----
 pkg/drivers/kic/node/node.go   | 196 ---------------------------------
 pkg/drivers/kic/oci/oci.go     |  68 ++++++++++--
 pkg/drivers/kic/types.go       |  65 +++++++++++
 pkg/minikube/config/profile.go |   8 ++
 5 files changed, 196 insertions(+), 226 deletions(-)
 delete mode 100644 pkg/drivers/kic/node/node.go
 create mode 100644 pkg/drivers/kic/types.go

diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go
index 194daa1581..973c1e9566 100644
--- a/pkg/drivers/kic/kic.go
+++ b/pkg/drivers/kic/kic.go
@@ -29,7 +29,6 @@ import (
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
 	pkgdrivers "k8s.io/minikube/pkg/drivers"
-	"k8s.io/minikube/pkg/drivers/kic/node"
 	"k8s.io/minikube/pkg/drivers/kic/oci"
 	"k8s.io/minikube/pkg/minikube/assets"
 	"k8s.io/minikube/pkg/minikube/command"
@@ -58,20 +57,6 @@ type Driver struct {
 	OCIBinary  string // docker,podman
 }
 
-// Config is configuration for the kic driver used by registry
-type Config struct {
-	MachineName   string            // maps to the container name being created
-	CPU           int               // Number of CPU cores assigned to the container
-	Memory        int               // max memory in MB
-	StorePath     string            // libmachine store path
-	OCIBinary     string            // oci tool to use (docker, podman,...)
-	ImageDigest   string            // image name with sha to use for the node
-	Mounts        []oci.Mount       // mounts
-	APIServerPort int               // kubernetes api server port inside the container
-	PortMappings  []oci.PortMapping // container port mappings
-	Envs          map[string]string // key,value of environment variables passed to the node
-}
-
 // NewDriver returns a fully configured Kic driver
 func NewDriver(c Config) *Driver {
 	d := &Driver{
@@ -88,10 +73,10 @@ func NewDriver(c Config) *Driver {
 
 // Create a host using the driver's config
 func (d *Driver) Create() error {
-	params := node.CreateConfig{
+	params := createConfig{
 		Name:          d.NodeConfig.MachineName,
 		Image:         d.NodeConfig.ImageDigest,
-		ClusterLabel:  node.ClusterLabelKey + "=" + d.MachineName,
+		ClusterLabel:  ClusterLabelKey + "=" + d.MachineName,
 		CPUs:          strconv.Itoa(d.NodeConfig.CPU),
 		Memory:        strconv.Itoa(d.NodeConfig.Memory) + "mb",
 		Envs:          d.NodeConfig.Envs,
@@ -114,7 +99,7 @@ func (d *Driver) Create() error {
 			ContainerPort: constants.DockerDaemonPort,
 		},
 	)
-	_, err := node.CreateNode(params)
+	err := createNode(params)
 	if err != nil {
 		return errors.Wrap(err, "create kic node")
 	}
@@ -158,14 +143,15 @@ func (d *Driver) DriverName() string {
 
 // GetIP returns an IP or hostname that this host is available at
 func (d *Driver) GetIP() (string, error) {
-	node, err := node.Find(d.OCIBinary, d.MachineName, d.exec)
-	if err != nil {
-		return "", fmt.Errorf("ip not found for nil node")
-	}
-	ip, _, err := node.IP()
+	ip, _, err := oci.ContainerIPs(d.OCIBinary, d.MachineName)
 	return ip, err
 }
 
+// GetExternalIP returns an IP which is accissble from outside
+func (d *Driver) GetExternalIP() (string, error) {
+	return DefaultBindIPV4, nil
+}
+
 // GetSSHHostname returns hostname for use with ssh
 func (d *Driver) GetSSHHostname() (string, error) {
 	return DefaultBindIPV4, nil
@@ -322,3 +308,56 @@ func (d *Driver) nodeID(nameOrID string) (string, error) {
 	}
 	return string(id), err
 }
+
+// createNode creates a new container node
+func createNode(p createConfig) error {
+	runArgs := []string{
+		fmt.Sprintf("--cpus=%s", p.CPUs),
+		fmt.Sprintf("--memory=%s", p.Memory),
+		"-d", // run the container detached
+		"-t", // allocate a tty for entrypoint logs
+		// running containers in a container requires privileged
+		// NOTE: we could try to replicate this with --cap-add, and use less
+		// privileges, but this flag also changes some mounts that are necessary
+		// including some ones docker would otherwise do by default.
+		// for now this is what we want. in the future we may revisit this.
+		"--privileged",
+		"--security-opt", "seccomp=unconfined", // also ignore seccomp
+		"--tmpfs", "/tmp", // various things depend on working /tmp
+		"--tmpfs", "/run", // systemd wants a writable /run
+		// logs,pods be stroed on  filesystem vs inside container,
+		"--volume", "/var",
+		// some k8s things want /lib/modules
+		"-v", "/lib/modules:/lib/modules:ro",
+		"--hostname", p.Name, // make hostname match container name
+		"--name", p.Name, // ... and set the container name
+		// label the node with the cluster ID
+		"--label", p.ClusterLabel,
+		// label the node with the role ID
+		"--label", fmt.Sprintf("%s=%s", NodeRoleKey, p.Role),
+	}
+
+	for key, val := range p.Envs {
+		runArgs = append(runArgs, "-e", fmt.Sprintf("%s=%s", key, val))
+	}
+
+	// adds node specific args
+	runArgs = append(runArgs, p.ExtraArgs...)
+
+	if oci.UsernsRemap(p.OCIBinary) {
+		// We need this argument in order to make this command work
+		// in systems that have userns-remap enabled on the docker daemon
+		runArgs = append(runArgs, "--userns=host")
+	}
+
+	_, err := oci.CreateContainer(p.OCIBinary,
+		p.Image,
+		oci.WithRunArgs(runArgs...),
+		oci.WithMounts(p.Mounts),
+		oci.WithPortMappings(p.PortMappings),
+	)
+	if err != nil {
+		return errors.Wrap(err, "create a kic node")
+	}
+	return nil
+}
diff --git a/pkg/drivers/kic/node/node.go b/pkg/drivers/kic/node/node.go
deleted file mode 100644
index aa3f0d3084..0000000000
--- a/pkg/drivers/kic/node/node.go
+++ /dev/null
@@ -1,196 +0,0 @@
-/*
-Copyright 2019 The Kubernetes Authors All rights reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package node
-
-import (
-	"fmt"
-	"os/exec"
-	"path/filepath"
-	"strings"
-
-	"k8s.io/minikube/pkg/drivers/kic/oci"
-	"k8s.io/minikube/pkg/minikube/assets"
-	"k8s.io/minikube/pkg/minikube/command"
-
-	"github.com/pkg/errors"
-)
-
-const (
-	// Docker default bridge network is named "bridge" (https://docs.docker.com/network/bridge/#use-the-default-bridge-network)
-	DefaultNetwork  = "bridge"
-	ClusterLabelKey = "io.x-k8s.kic.cluster" // ClusterLabelKey is applied to each node docker container for identification
-	NodeRoleKey     = "io.k8s.sigs.kic.role"
-)
-
-// Node represents a handle to a kic node
-// This struct must be created by one of: CreateControlPlane
-type Node struct {
-	id        string         // container id
-	name      string         // container name
-	r         command.Runner // Runner
-	ociBinary string
-}
-
-type CreateConfig struct {
-	Name          string            // used for container name and hostname
-	Image         string            // container image to use to create the node.
-	ClusterLabel  string            // label the containers we create using minikube so we can clean up
-	Role          string            // currently only role supported is control-plane
-	Mounts        []oci.Mount       // volume mounts
-	APIServerPort int               // kubernetes api server port
-	PortMappings  []oci.PortMapping // ports to map to container from host
-	CPUs          string            // number of cpu cores assign to container
-	Memory        string            // memory (mbs) to assign to the container
-	Envs          map[string]string // environment variables to pass to the container
-	ExtraArgs     []string          // a list of any extra option to pass to oci binary during creation time, for example --expose 8080...
-	OCIBinary     string            // docker or podman
-}
-
-// CreateNode creates a new container node
-func CreateNode(p CreateConfig) (*Node, error) {
-	cmder := command.NewKICRunner(p.Name, p.OCIBinary)
-	runArgs := []string{
-		fmt.Sprintf("--cpus=%s", p.CPUs),
-		fmt.Sprintf("--memory=%s", p.Memory),
-		"-d", // run the container detached
-		"-t", // allocate a tty for entrypoint logs
-		// running containers in a container requires privileged
-		// NOTE: we could try to replicate this with --cap-add, and use less
-		// privileges, but this flag also changes some mounts that are necessary
-		// including some ones docker would otherwise do by default.
-		// for now this is what we want. in the future we may revisit this.
-		"--privileged",
-		"--security-opt", "seccomp=unconfined", // also ignore seccomp
-		"--tmpfs", "/tmp", // various things depend on working /tmp
-		"--tmpfs", "/run", // systemd wants a writable /run
-		// logs,pods be stroed on  filesystem vs inside container,
-		"--volume", "/var",
-		// some k8s things want /lib/modules
-		"-v", "/lib/modules:/lib/modules:ro",
-		"--hostname", p.Name, // make hostname match container name
-		"--name", p.Name, // ... and set the container name
-		// label the node with the cluster ID
-		"--label", p.ClusterLabel,
-		// label the node with the role ID
-		"--label", fmt.Sprintf("%s=%s", NodeRoleKey, p.Role),
-	}
-
-	for key, val := range p.Envs {
-		runArgs = append(runArgs, "-e", fmt.Sprintf("%s=%s", key, val))
-	}
-
-	// adds node specific args
-	runArgs = append(runArgs, p.ExtraArgs...)
-
-	if oci.UsernsRemap(p.OCIBinary) {
-		// We need this argument in order to make this command work
-		// in systems that have userns-remap enabled on the docker daemon
-		runArgs = append(runArgs, "--userns=host")
-	}
-
-	_, err := oci.CreateContainer(p.OCIBinary,
-		p.Image,
-		oci.WithRunArgs(runArgs...),
-		oci.WithMounts(p.Mounts),
-		oci.WithPortMappings(p.PortMappings),
-	)
-
-	if err != nil {
-		return nil, errors.Wrap(err, "oci create ")
-	}
-
-	// we should return a handle so the caller can clean it up
-	node, err := Find(p.OCIBinary, p.Name, cmder)
-	if err != nil {
-		return node, errors.Wrap(err, "find node")
-	}
-
-	return node, nil
-}
-
-// Find finds a node
-func Find(ociBinary string, name string, cmder command.Runner) (*Node, error) {
-	n, err := oci.Inspect(ociBinary, name, "{{.Id}}")
-	if err != nil {
-		return nil, fmt.Errorf("can't find node %v", err)
-	}
-	return &Node{
-		ociBinary: ociBinary,
-		id:        n[0],
-		name:      name,
-		r:         cmder,
-	}, nil
-}
-
-// WriteFile writes content to dest on the node
-func (n *Node) WriteFile(dest, content string, perm string) error {
-	// create destination directory
-	cmd := exec.Command("mkdir", "-p", filepath.Dir(dest))
-	rr, err := n.r.RunCmd(cmd)
-	if err != nil {
-		return errors.Wrapf(err, "failed to create directory %s cmd: %v output:%q", cmd.Args, dest, rr.Output())
-	}
-
-	cmd = exec.Command("cp", "/dev/stdin", dest)
-	cmd.Stdin = strings.NewReader(content)
-
-	if rr, err := n.r.RunCmd(cmd); err != nil {
-		return errors.Wrapf(err, "failed to run: cp /dev/stdin %s cmd: %v output:%q", dest, cmd.Args, rr.Output())
-	}
-
-	cmd = exec.Command("chmod", perm, dest)
-	_, err = n.r.RunCmd(cmd)
-	if err != nil {
-		return errors.Wrapf(err, "failed to run: chmod %s %s", perm, dest)
-	}
-	return nil
-}
-
-// IP returns the IP address of the node
-func (n *Node) IP() (ipv4 string, ipv6 string, err error) {
-	// retrieve the IP address of the node using docker inspect
-	lines, err := oci.Inspect(n.ociBinary, n.name, "{{range .NetworkSettings.Networks}}{{.IPAddress}},{{.GlobalIPv6Address}}{{end}}")
-	if err != nil {
-		return "", "", errors.Wrap(err, "node ips")
-	}
-	if len(lines) != 1 {
-		return "", "", errors.Errorf("file should only be one line, got %d lines", len(lines))
-	}
-	ips := strings.Split(lines[0], ",")
-	if len(ips) != 2 {
-		return "", "", errors.Errorf("container addresses should have 2 values, got %d values: %+v", len(ips), ips)
-	}
-	return ips[0], ips[1], nil
-}
-
-// Copy copies a local asset into the node
-func (n *Node) Copy(ociBinary string, asset assets.CopyableFile) error {
-	if err := oci.Copy(ociBinary, n.name, asset); err != nil {
-		return errors.Wrap(err, "failed to copy file/folder")
-	}
-
-	cmd := exec.Command("chmod", asset.GetPermissions(), asset.GetTargetName())
-	if _, err := n.r.RunCmd(cmd); err != nil {
-		return errors.Wrap(err, "failed to chmod file permissions")
-	}
-	return nil
-}
-
-// Remove removes the node
-func (n *Node) Remove() error {
-	return oci.Remove(n.ociBinary, n.name)
-}
diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go
index 54bf60f1d1..7b0651f868 100644
--- a/pkg/drivers/kic/oci/oci.go
+++ b/pkg/drivers/kic/oci/oci.go
@@ -21,7 +21,6 @@ import (
 	"strconv"
 
 	"github.com/docker/machine/libmachine/state"
-	"k8s.io/minikube/pkg/minikube/assets"
 
 	"bufio"
 	"bytes"
@@ -378,15 +377,15 @@ func WithPortMappings(portMappings []PortMapping) CreateOpt {
 }
 
 // Copy copies a local asset into the container
-func Copy(ociBinary string, ociID string, asset assets.CopyableFile) error {
-	if _, err := os.Stat(asset.GetAssetName()); os.IsNotExist(err) {
-		return errors.Wrapf(err, "error source %s does not exist", asset.GetAssetName())
+func Copy(ociBinary string, ociID string, targetDir string, fName string) error {
+	if _, err := os.Stat(fName); os.IsNotExist(err) {
+		return errors.Wrapf(err, "error source %s does not exist", fName)
 	}
-	destination := fmt.Sprintf("%s:%s", ociID, asset.GetTargetDir())
-	cmd := exec.Command(ociBinary, "cp", asset.GetAssetName(), destination)
+	destination := fmt.Sprintf("%s:%s", ociID, targetDir)
+	cmd := exec.Command(ociBinary, "cp", fName, destination)
 	err := cmd.Run()
 	if err != nil {
-		return errors.Wrapf(err, "error copying %s into node", asset.GetAssetName())
+		return errors.Wrapf(err, "error copying %s into node", fName)
 	}
 	return nil
 }
@@ -410,3 +409,58 @@ func HostPortBinding(ociBinary string, ociID string, contPort int) (int, error)
 	}
 	return p, nil
 }
+
+// ListContainers lists all the containres that kic driver created on user's machine using a label
+// io.x-k8s.kic.cluster
+func ListContainersByLabel(ociBinary string, label string) (error, []string) {
+	cmd := exec.Command(ociBinary, "ps", "--filter", fmt.Sprintf("label=%s", label), "--format", "{{.Names}")
+	var b bytes.Buffer
+	cmd.Stdout = &b
+	cmd.Stderr = &b
+	err := cmd.Run()
+	var lines []string
+	sc := bufio.NewScanner(&b)
+	for sc.Scan() {
+		lines = append(lines, sc.Text())
+	}
+	return err, lines
+}
+
+// ContainerID returns a container ID from its name
+func ContainerID(ociBinary string, name string) (string, error) {
+	ids, err := Inspect(ociBinary, name, "{{.Id}}")
+	return ids[0], err
+}
+
+// ContainerIPs returns ipv4,ipv6, error of a container by their name
+func ContainerIPs(ociBinary string, name string) (string, string, error) {
+	// retrieve the IP address of the node using docker inspect
+	lines, err := Inspect(ociBinary, name, "{{range .NetworkSettings.Networks}}{{.IPAddress}},{{.GlobalIPv6Address}}{{end}}")
+	if err != nil {
+		return "", "", errors.Wrap(err, "inspecting NetworkSettings.Networks")
+	}
+	if len(lines) != 1 {
+		return "", "", errors.Errorf("IPs output should only be one line, got %d lines", len(lines))
+	}
+	ips := strings.Split(lines[0], ",")
+	if len(ips) != 2 {
+		return "", "", errors.Errorf("container addresses should have 2 values, got %d values: %+v", len(ips), ips)
+	}
+	return ips[0], ips[1], nil
+
+}
+
+// ListOwnedContainers lists all the containres that kic driver created on user's machine using a label
+func ListOwnedContainers(ociBinary string) ([]string, error) {
+	cmd := exec.Command(ociBinary, "ps", "-a", "--filter", "label=io.x-k8s.kic.cluster", "--format", "{{.Names}}")
+	var b bytes.Buffer
+	cmd.Stdout = &b
+	cmd.Stderr = &b
+	err := cmd.Run()
+	var lines []string
+	sc := bufio.NewScanner(&b)
+	for sc.Scan() {
+		lines = append(lines, sc.Text())
+	}
+	return lines, err
+}
diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go
new file mode 100644
index 0000000000..3582c9200b
--- /dev/null
+++ b/pkg/drivers/kic/types.go
@@ -0,0 +1,65 @@
+/*
+Copyright 2019 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kic
+
+import "k8s.io/minikube/pkg/drivers/kic/oci"
+
+const (
+	// Docker default bridge network is named "bridge" (https://docs.docker.com/network/bridge/#use-the-default-bridge-network)
+	DefaultNetwork = "bridge"
+	// ClusterLabelKey is applied to each node docker container for identification
+	ClusterLabelKey = "io.x-k8s.kic.cluster"
+	// NodeRoleKey is used to identify if it is control plane or worker
+	NodeRoleKey = "io.k8s.sigs.kic.role"
+	// DefaultPodCIDR is The CIDR to be used for pods inside the node.
+	DefaultPodCIDR = "10.244.0.0/16"
+	// DefaultBindIPV4 is The default IP the container will bind to.
+	DefaultBindIPV4 = "127.0.0.1"
+	// BaseImage is the base image is used to spin up kic containers created by kind.
+	BaseImage = "gcr.io/k8s-minikube/kicbase:v0.0.2@sha256:8f531b90901721a7bd4e67ceffbbc7ee6c4292b0e6d1a9d6eb59f117d57bc4e9"
+	// OverlayImage is the cni plugin used for overlay image, created by kind.
+	OverlayImage = "kindest/kindnetd:0.5.3"
+)
+
+// Config is configuration for the kic driver used by registry
+type Config struct {
+	MachineName   string            // maps to the container name being created
+	CPU           int               // Number of CPU cores assigned to the container
+	Memory        int               // max memory in MB
+	StorePath     string            // libmachine store path
+	OCIBinary     string            // oci tool to use (docker, podman,...)
+	ImageDigest   string            // image name with sha to use for the node
+	Mounts        []oci.Mount       // mounts
+	APIServerPort int               // kubernetes api server port inside the container
+	PortMappings  []oci.PortMapping // container port mappings
+	Envs          map[string]string // key,value of environment variables passed to the node
+}
+
+type createConfig struct {
+	Name          string            // used for container name and hostname
+	Image         string            // container image to use to create the node.
+	ClusterLabel  string            // label the containers we create using minikube so we can clean up
+	Role          string            // currently only role supported is control-plane
+	Mounts        []oci.Mount       // volume mounts
+	APIServerPort int               // kubernetes api server port
+	PortMappings  []oci.PortMapping // ports to map to container from host
+	CPUs          string            // number of cpu cores assign to container
+	Memory        string            // memory (mbs) to assign to the container
+	Envs          map[string]string // environment variables to pass to the container
+	ExtraArgs     []string          // a list of any extra option to pass to oci binary during creation time, for example --expose 8080...
+	OCIBinary     string            // docker or podman
+}
diff --git a/pkg/minikube/config/profile.go b/pkg/minikube/config/profile.go
index 4b3ad2a9e9..ee5c1e2f06 100644
--- a/pkg/minikube/config/profile.go
+++ b/pkg/minikube/config/profile.go
@@ -25,6 +25,7 @@ import (
 	"strings"
 
 	"github.com/golang/glog"
+	"k8s.io/minikube/pkg/drivers/kic/oci"
 	"k8s.io/minikube/pkg/minikube/localpath"
 	"k8s.io/minikube/pkg/util/lock"
 )
@@ -146,10 +147,17 @@ func DeleteProfile(profile string, miniHome ...string) error {
 // invalidPs are the profiles that have a directory or config file but not usable
 // invalidPs would be suggested to be deleted
 func ListProfiles(miniHome ...string) (validPs []*Profile, inValidPs []*Profile, err error) {
+
+	// try to get profiles list based on left over evidences such as directory
 	pDirs, err := profileDirs(miniHome...)
 	if err != nil {
 		return nil, nil, err
 	}
+	// try to get profiles list based on all contrainers created by docker driver
+	cs, err := oci.ListOwnedContainers("docker")
+	if err == nil {
+		pDirs = append(pDirs, cs...)
+	}
 	for _, n := range pDirs {
 		p, err := LoadProfile(n, miniHome...)
 		if err != nil {

From 58b637b05fe3485ecbf50834fda84beeedfd044a Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Thu, 30 Jan 2020 16:42:14 -0800
Subject: [PATCH 151/250] delete all cleans up stale kic profiles

---
 pkg/drivers/kic/kic.go         | 16 ++--------------
 pkg/drivers/kic/oci/oci.go     | 20 +++++---------------
 pkg/drivers/kic/oci/types.go   |  4 ++++
 pkg/drivers/kic/types.go       |  4 ----
 pkg/minikube/config/profile.go |  2 +-
 5 files changed, 12 insertions(+), 34 deletions(-)

diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go
index 973c1e9566..a7d6f29fea 100644
--- a/pkg/drivers/kic/kic.go
+++ b/pkg/drivers/kic/kic.go
@@ -35,18 +35,6 @@ import (
 	"k8s.io/minikube/pkg/minikube/constants"
 )
 
-// DefaultPodCIDR is The CIDR to be used for pods inside the node.
-const DefaultPodCIDR = "10.244.0.0/16"
-
-// DefaultBindIPV4 is The default IP the container will bind to.
-const DefaultBindIPV4 = "127.0.0.1"
-
-// BaseImage is the base image is used to spin up kic containers created by kind.
-const BaseImage = "gcr.io/k8s-minikube/kicbase:v0.0.3@sha256:34db5e30f8830c0d5e49b62f3ea6b2844f805980592fe0084cbea799bfb12664"
-
-// OverlayImage is the cni plugin used for overlay image, created by kind.
-const OverlayImage = "kindest/kindnetd:0.5.3"
-
 // Driver represents a kic driver https://minikube.sigs.k8s.io/docs/reference/drivers/kic/
 type Driver struct {
 	*drivers.BaseDriver
@@ -76,7 +64,7 @@ func (d *Driver) Create() error {
 	params := createConfig{
 		Name:          d.NodeConfig.MachineName,
 		Image:         d.NodeConfig.ImageDigest,
-		ClusterLabel:  ClusterLabelKey + "=" + d.MachineName,
+		ClusterLabel:  oci.ClusterLabelKey + "=" + d.MachineName,
 		CPUs:          strconv.Itoa(d.NodeConfig.CPU),
 		Memory:        strconv.Itoa(d.NodeConfig.Memory) + "mb",
 		Envs:          d.NodeConfig.Envs,
@@ -334,7 +322,7 @@ func createNode(p createConfig) error {
 		// label the node with the cluster ID
 		"--label", p.ClusterLabel,
 		// label the node with the role ID
-		"--label", fmt.Sprintf("%s=%s", NodeRoleKey, p.Role),
+		"--label", fmt.Sprintf("%s=%s", oci.NodeRoleKey, p.Role),
 	}
 
 	for key, val := range p.Envs {
diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go
index 7b0651f868..acc608706e 100644
--- a/pkg/drivers/kic/oci/oci.go
+++ b/pkg/drivers/kic/oci/oci.go
@@ -410,10 +410,10 @@ func HostPortBinding(ociBinary string, ociID string, contPort int) (int, error)
 	return p, nil
 }
 
-// ListContainers lists all the containres that kic driver created on user's machine using a label
+// listContainersByLabel lists all the containres that kic driver created on user's machine using a label
 // io.x-k8s.kic.cluster
-func ListContainersByLabel(ociBinary string, label string) (error, []string) {
-	cmd := exec.Command(ociBinary, "ps", "--filter", fmt.Sprintf("label=%s", label), "--format", "{{.Names}")
+func listContainersByLabel(ociBinary string, label string) ([]string, error) {
+	cmd := exec.Command(ociBinary, "ps", "-a", "--filter", fmt.Sprintf("label=%s", label), "--format", "{{.Names}}")
 	var b bytes.Buffer
 	cmd.Stdout = &b
 	cmd.Stderr = &b
@@ -423,7 +423,7 @@ func ListContainersByLabel(ociBinary string, label string) (error, []string) {
 	for sc.Scan() {
 		lines = append(lines, sc.Text())
 	}
-	return err, lines
+	return lines, err
 }
 
 // ContainerID returns a container ID from its name
@@ -452,15 +452,5 @@ func ContainerIPs(ociBinary string, name string) (string, string, error) {
 
 // ListOwnedContainers lists all the containres that kic driver created on user's machine using a label
 func ListOwnedContainers(ociBinary string) ([]string, error) {
-	cmd := exec.Command(ociBinary, "ps", "-a", "--filter", "label=io.x-k8s.kic.cluster", "--format", "{{.Names}}")
-	var b bytes.Buffer
-	cmd.Stdout = &b
-	cmd.Stderr = &b
-	err := cmd.Run()
-	var lines []string
-	sc := bufio.NewScanner(&b)
-	for sc.Scan() {
-		lines = append(lines, sc.Text())
-	}
-	return lines, err
+	return listContainersByLabel(ociBinary, ClusterLabelKey)
 }
diff --git a/pkg/drivers/kic/oci/types.go b/pkg/drivers/kic/oci/types.go
index 6e35a79dc3..49694f4e69 100644
--- a/pkg/drivers/kic/oci/types.go
+++ b/pkg/drivers/kic/oci/types.go
@@ -19,6 +19,10 @@ package oci
 const (
 	Docker = "docker"
 	Podman = "podman"
+	// ClusterLabelKey is applied to each node docker container for identification
+	ClusterLabelKey = "io.x-k8s.kic.cluster"
+	// NodeRoleKey is used to identify if it is control plane or worker
+	NodeRoleKey = "io.k8s.sigs.kic.role"
 )
 
 /*
diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go
index 3582c9200b..48e26c8eb9 100644
--- a/pkg/drivers/kic/types.go
+++ b/pkg/drivers/kic/types.go
@@ -21,10 +21,6 @@ import "k8s.io/minikube/pkg/drivers/kic/oci"
 const (
 	// Docker default bridge network is named "bridge" (https://docs.docker.com/network/bridge/#use-the-default-bridge-network)
 	DefaultNetwork = "bridge"
-	// ClusterLabelKey is applied to each node docker container for identification
-	ClusterLabelKey = "io.x-k8s.kic.cluster"
-	// NodeRoleKey is used to identify if it is control plane or worker
-	NodeRoleKey = "io.k8s.sigs.kic.role"
 	// DefaultPodCIDR is The CIDR to be used for pods inside the node.
 	DefaultPodCIDR = "10.244.0.0/16"
 	// DefaultBindIPV4 is The default IP the container will bind to.
diff --git a/pkg/minikube/config/profile.go b/pkg/minikube/config/profile.go
index ee5c1e2f06..66688b93cb 100644
--- a/pkg/minikube/config/profile.go
+++ b/pkg/minikube/config/profile.go
@@ -154,7 +154,7 @@ func ListProfiles(miniHome ...string) (validPs []*Profile, inValidPs []*Profile,
 		return nil, nil, err
 	}
 	// try to get profiles list based on all contrainers created by docker driver
-	cs, err := oci.ListOwnedContainers("docker")
+	cs, err := oci.ListOwnedContainers(oci.Docker)
 	if err == nil {
 		pDirs = append(pDirs, cs...)
 	}

From 5b055d7ff889c769e70efc81c8bec10202c35490 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Thu, 30 Jan 2020 16:57:53 -0800
Subject: [PATCH 152/250] add comment

---
 pkg/drivers/kic/types.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go
index 48e26c8eb9..9cc1a1aa92 100644
--- a/pkg/drivers/kic/types.go
+++ b/pkg/drivers/kic/types.go
@@ -26,8 +26,8 @@ const (
 	// DefaultBindIPV4 is The default IP the container will bind to.
 	DefaultBindIPV4 = "127.0.0.1"
 	// BaseImage is the base image is used to spin up kic containers created by kind.
-	BaseImage = "gcr.io/k8s-minikube/kicbase:v0.0.2@sha256:8f531b90901721a7bd4e67ceffbbc7ee6c4292b0e6d1a9d6eb59f117d57bc4e9"
-	// OverlayImage is the cni plugin used for overlay image, created by kind.
+	BaseImage = "gcr.io/k8s-minikube/kicbase:v0.0.3@sha256:34db5e30f8830c0d5e49b62f3ea6b2844f805980592fe0084cbea799bfb12664" // OverlayImage is the cni plugin used for overlay image, created by kind.
+	// CNI plugin image used for kic drivers created by kind.
 	OverlayImage = "kindest/kindnetd:0.5.3"
 )
 

From e7fa3e0448134bcd985011eecf56a5447600fb8f Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Thu, 30 Jan 2020 17:11:11 -0800
Subject: [PATCH 153/250] remove duplicate profile names

---
 pkg/minikube/config/profile.go | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/pkg/minikube/config/profile.go b/pkg/minikube/config/profile.go
index 66688b93cb..c75916d980 100644
--- a/pkg/minikube/config/profile.go
+++ b/pkg/minikube/config/profile.go
@@ -158,6 +158,7 @@ func ListProfiles(miniHome ...string) (validPs []*Profile, inValidPs []*Profile,
 	if err == nil {
 		pDirs = append(pDirs, cs...)
 	}
+	pDirs = removeDups(pDirs)
 	for _, n := range pDirs {
 		p, err := LoadProfile(n, miniHome...)
 		if err != nil {
@@ -173,6 +174,26 @@ func ListProfiles(miniHome ...string) (validPs []*Profile, inValidPs []*Profile,
 	return validPs, inValidPs, nil
 }
 
+// removeDups removes duplicates
+func removeDups(profiles []string) []string {
+	// Use map to record duplicates as we find them.
+	seen := map[string]bool{}
+	result := []string{}
+
+	for n := range profiles {
+		if seen[profiles[n]] {
+			// Do not add duplicate.
+		} else {
+			// Record this element as an encountered element.
+			seen[profiles[n]] = true
+			// Append to result slice.
+			result = append(result, profiles[n])
+		}
+	}
+	// Return the new slice.
+	return result
+}
+
 // LoadProfile loads type Profile based on its name
 func LoadProfile(name string, miniHome ...string) (*Profile, error) {
 	cfg, err := DefaultLoader.LoadConfigFromFile(name, miniHome...)

From fefa13e9dac02751f0cbe759f694eae540d7546b Mon Sep 17 00:00:00 2001
From: Kim Bao Long <longkb@vn.fujitsu.com>
Date: Fri, 31 Jan 2020 08:25:27 +0700
Subject: [PATCH 154/250] Remove some redundance words

Signed-off-by: Kim Bao Long <longkb@vn.fujitsu.com>
---
 cmd/minikube/cmd/start.go | 2 +-
 translations/ja.json      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 43976eef18..4ccd39cf2f 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -159,7 +159,7 @@ func initMinikubeFlags() {
 
 	startCmd.Flags().Bool(force, false, "Force minikube to perform possibly dangerous operations")
 	startCmd.Flags().Bool(interactive, true, "Allow user prompts for more information")
-	startCmd.Flags().Bool(dryRun, false, "dry-run mode. Validates configuration, but does does not mutate system state")
+	startCmd.Flags().Bool(dryRun, false, "dry-run mode. Validates configuration, but does not mutate system state")
 
 	startCmd.Flags().Int(cpus, 2, "Number of CPUs allocated to the minikube VM.")
 	startCmd.Flags().String(memory, defaultMemorySize, "Amount of RAM allocated to the minikube VM (format: <number>[<unit>], where unit = b, k, m or g).")
diff --git a/translations/ja.json b/translations/ja.json
index 0854956f60..9e6637b253 100644
--- a/translations/ja.json
+++ b/translations/ja.json
@@ -516,7 +516,7 @@
 	"config view failed": "",
 	"dashboard service is not running: {{.error}}": "",
 	"disable failed": "",
-	"dry-run mode. Validates configuration, but does does not mutate system state": "",
+	"dry-run mode. Validates configuration, but does not mutate system state": "",
 	"dry-run validation complete!": "",
 	"enable failed": "",
 	"error creating clientset": "",

From b97124207cad7d7eb7d728a656e1e64b5052cf4d Mon Sep 17 00:00:00 2001
From: tstromberg <tstromberg@google.com>
Date: Thu, 30 Jan 2020 17:46:25 -0800
Subject: [PATCH 155/250] Move addon enablement to addons module + lint fixes

---
 cmd/minikube/cmd/config/addons_list.go |  2 +-
 cmd/minikube/cmd/start.go              | 45 +++-----------------------
 pkg/addons/addons.go                   | 40 ++++++++++++++++++++++-
 pkg/minikube/assets/addons.go          |  2 +-
 4 files changed, 45 insertions(+), 44 deletions(-)

diff --git a/cmd/minikube/cmd/config/addons_list.go b/cmd/minikube/cmd/config/addons_list.go
index 8a66f4c753..c73c914c96 100644
--- a/cmd/minikube/cmd/config/addons_list.go
+++ b/cmd/minikube/cmd/config/addons_list.go
@@ -136,7 +136,7 @@ var printAddonsJSON = func() {
 
 		addonStatus, err := addonBundle.IsEnabled()
 		if err != nil {
-			glog.Errorf("Unable to get addon status for {{.name}}: {{.error}}", err)
+			glog.Errorf("Unable to get addon status for %s: %v", addonName, err)
 			continue
 		}
 
diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 3469742d45..5643b7be20 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -46,8 +46,6 @@ import (
 	"github.com/spf13/viper"
 	"golang.org/x/sync/errgroup"
 	cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config"
-	pkgaddons "k8s.io/minikube/pkg/addons"
-	"k8s.io/minikube/pkg/minikube/assets"
 	"k8s.io/minikube/pkg/minikube/bootstrapper"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
@@ -58,6 +56,7 @@ import (
 	"k8s.io/minikube/pkg/minikube/cruntime"
 	"k8s.io/minikube/pkg/minikube/driver"
 	"k8s.io/minikube/pkg/minikube/exit"
+	"k8s.io/minikube/pkg/addons"
 	"k8s.io/minikube/pkg/minikube/kubeconfig"
 	"k8s.io/minikube/pkg/minikube/localpath"
 	"k8s.io/minikube/pkg/minikube/logs"
@@ -105,7 +104,6 @@ const (
 	imageMirrorCountry    = "image-mirror-country"
 	mountString           = "mount-string"
 	disableDriverMounts   = "disable-driver-mounts"
-	addons                = "addons"
 	cacheImages           = "cache-images"
 	uuid                  = "uuid"
 	vpnkitSock            = "hyperkit-vpnkit-sock"
@@ -173,7 +171,7 @@ func initMinikubeFlags() {
 	startCmd.Flags().String(containerRuntime, "docker", "The container runtime to be used (docker, crio, containerd).")
 	startCmd.Flags().Bool(createMount, false, "This will start the mount daemon and automatically mount files into minikube.")
 	startCmd.Flags().String(mountString, constants.DefaultMountDir+":/minikube-host", "The argument to pass the minikube mount command on start.")
-	startCmd.Flags().StringArrayVar(&addonList, addons, nil, "Enable addons. see `minikube addons list` for a list of valid addon names.")
+	startCmd.Flags().StringArrayVar(&addonList,"addons", nil, "Enable addons. see `minikube addons list` for a list of valid addon names.")
 	startCmd.Flags().String(criSocket, "", "The cri socket path to be used.")
 	startCmd.Flags().String(networkPlugin, "", "The name of the network plugin.")
 	startCmd.Flags().Bool(enableDefaultCNI, false, "Enable the default CNI plugin (/etc/cni/net.d/k8s.conf). Used in conjunction with \"--network-plugin=cni\".")
@@ -369,8 +367,8 @@ func runStart(cmd *cobra.Command, args []string) {
 	bootstrapCluster(bs, cr, mRunner, mc, preExists, isUpgrade)
 	configureMounts()
 
-	// enable addons with start command
-	enableAddons()
+	// enable addons
+	addons.Start(viper.GetString(config.MachineProfile), addonList)
 
 	if err = cacheAndLoadImagesInConfig(); err != nil {
 		out.T(out.FailureType, "Unable to load cached images from config file.")
@@ -409,41 +407,6 @@ func cacheISO(cfg *config.MachineConfig, driverName string) {
 	}
 }
 
-func enableAddons() {
-	start := time.Now()
-	glog.Infof("enableAddons")
-	defer func() {
-		glog.Infof("enableAddons completed in %s", time.Since(start))
-	}()
-	toEnable := []string{}
-
-	// Apply addons that are enabled by default
-	for name, a := range assets.Addons {
-		enabled, err := a.IsEnabled()
-		if err != nil {
-			glog.Errorf("is-enabled failed for %q: %v", a.Name(), err)
-			continue
-		}
-		if enabled {
-			toEnable = append(toEnable, name)
-		}
-	}
-
-	toEnable = append(toEnable, addonList...)
-	if len(toEnable) == 0 {
-		return
-	}
-
-	out.T(out.AddonEnable, "Enabling addons: {{.addons}}", out.V{"addons": strings.Join(toEnable, ", ")})
-	for _, a := range toEnable {
-		err := pkgaddons.Set(a, "true", viper.GetString(config.MachineProfile))
-		if err != nil {
-			// Intentionally non-fatal
-			out.WarningT("Enabling '{{.name}}' returned an error: {{.error}}", out.V{"name": a, "error": err})
-		}
-	}
-}
-
 func displayVersion(version string) {
 	prefix := ""
 	if viper.GetString(config.MachineProfile) != constants.DefaultMachineName {
diff --git a/pkg/addons/addons.go b/pkg/addons/addons.go
index dae1ef0c6c..2070da2ca2 100644
--- a/pkg/addons/addons.go
+++ b/pkg/addons/addons.go
@@ -21,6 +21,8 @@ import (
 	"os"
 	"path"
 	"strconv"
+	"strings"
+	"time"
 
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
@@ -167,7 +169,7 @@ func isAddonAlreadySet(addon *assets.Addon, enable bool) (bool, error) {
 	addonStatus, err := addon.IsEnabled()
 
 	if err != nil {
-		return false, errors.Wrap(err, "get the addon status")
+		return false, errors.Wrap(err, "addon is enabled")
 	}
 
 	if addonStatus && enable {
@@ -255,3 +257,39 @@ func enableOrDisableStorageClasses(name, val, profile string) error {
 
 	return enableOrDisableAddon(name, val, profile)
 }
+
+// Start enables the default addons for a profile, plus any additional
+func Start(profile string, additional []string) {
+	start := time.Now()
+	glog.Infof("enableAddons")
+	defer func() {
+		glog.Infof("enableAddons completed in %s", time.Since(start))
+	}()
+	toEnable := []string{}
+
+	// Apply addons that are enabled by default
+	for name, a := range assets.Addons {
+		enabled, err := a.IsEnabled()
+		if err != nil {
+			glog.Errorf("is-enabled failed for %q: %v", a.Name(), err)
+			continue
+		}
+		if enabled {
+			toEnable = append(toEnable, name)
+		}
+	}
+
+	toEnable = append(toEnable, additional...)
+	if len(toEnable) == 0 {
+		return
+	}
+
+	out.T(out.AddonEnable, "Enabling addons: {{.addons}}", out.V{"addons": strings.Join(toEnable, ", ")})
+	for _, a := range toEnable {
+		err := Set(a, "true", profile)
+		if err != nil {
+			// Intentionally non-fatal
+			out.WarningT("Enabling '{{.name}}' returned an error: {{.error}}", out.V{"name": a, "error": err})
+		}
+	}
+}
diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go
index 6b18b810f6..14959a28ca 100644
--- a/pkg/minikube/assets/addons.go
+++ b/pkg/minikube/assets/addons.go
@@ -59,7 +59,7 @@ func (a *Addon) Name() string {
 func (a *Addon) IsEnabled() (bool, error) {
 	c, err := config.Load(viper.GetString(config.MachineProfile))
 	if err != nil {
-		return false, err
+		return false, errors.Wrap(err, "load")
 	}
 
 	// Is this addon explicitly listed in their configuration?

From 80b4121c0652a3aa9f7fe901d2304747a7526942 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Thu, 30 Jan 2020 17:46:40 -0800
Subject: [PATCH 156/250] delete stale

---
 cmd/minikube/cmd/delete.go      |  1 -
 pkg/minikube/cluster/cluster.go | 26 ++++++++++++++++++--------
 2 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/cmd/minikube/cmd/delete.go b/cmd/minikube/cmd/delete.go
index 5efe7e2ee3..cf75aa10a1 100644
--- a/cmd/minikube/cmd/delete.go
+++ b/cmd/minikube/cmd/delete.go
@@ -98,7 +98,6 @@ func runDelete(cmd *cobra.Command, args []string) {
 
 	validProfiles, invalidProfiles, err := pkg_config.ListProfiles()
 	profilesToDelete := append(validProfiles, invalidProfiles...)
-
 	// If the purge flag is set, go ahead and delete the .minikube directory.
 	if purge && len(profilesToDelete) > 1 && !deleteAll {
 		out.ErrT(out.Notice, "Multiple minikube profiles were found - ")
diff --git a/pkg/minikube/cluster/cluster.go b/pkg/minikube/cluster/cluster.go
index af3f6cc184..b6839c482c 100644
--- a/pkg/minikube/cluster/cluster.go
+++ b/pkg/minikube/cluster/cluster.go
@@ -310,8 +310,18 @@ func StopHost(api libmachine.API) error {
 
 // DeleteHost deletes the host VM.
 func DeleteHost(api libmachine.API, machineName string) error {
-	host, err := api.Load(machineName)
-	if err != nil {
+	h, err := api.Load(machineName)
+	if err != nil && h.DriverName == "" {
+		// before we give up on deleting the host
+		//  we try to kill the possible orphan kic driver
+		// this case will happen if the user deleted both profile and machine folder
+		// inside minikube home
+		cmd := exec.Command(oci.Docker, "rm", "-f", "-v", machineName)
+		err := cmd.Run()
+		if err == nil {
+			glog.Infof("Found stale kic container and successfully cleaned it up.")
+			return nil
+		}
 		return errors.Wrap(err, "load")
 	}
 
@@ -323,19 +333,19 @@ func DeleteHost(api libmachine.API, machineName string) error {
 	}
 
 	if status == state.None.String() {
-		return mcnerror.ErrHostDoesNotExist{Name: host.Name}
+		return mcnerror.ErrHostDoesNotExist{Name: h.Name}
 	}
 
 	// This is slow if SSH is not responding, but HyperV hangs otherwise, See issue #2914
-	if host.Driver.DriverName() == driver.HyperV {
-		if err := trySSHPowerOff(host); err != nil {
+	if h.Driver.DriverName() == driver.HyperV {
+		if err := trySSHPowerOff(h); err != nil {
 			glog.Infof("Unable to power off minikube because the host was not found.")
 		}
-		out.T(out.DeletingHost, "Successfully powered off Hyper-V. minikube driver -- {{.driver}}", out.V{"driver": host.Driver.DriverName()})
+		out.T(out.DeletingHost, "Successfully powered off Hyper-V. minikube driver -- {{.driver}}", out.V{"driver": h.Driver.DriverName()})
 	}
 
-	out.T(out.DeletingHost, `Deleting "{{.profile_name}}" in {{.driver_name}} ...`, out.V{"profile_name": machineName, "driver_name": host.DriverName})
-	if err := host.Driver.Remove(); err != nil {
+	out.T(out.DeletingHost, `Deleting "{{.profile_name}}" in {{.driver_name}} ...`, out.V{"profile_name": machineName, "driver_name": h.DriverName})
+	if err := h.Driver.Remove(); err != nil {
 		return errors.Wrap(err, "host remove")
 	}
 	if err := api.Remove(machineName); err != nil {

From a709fd5d37f4b70204fe4e486eb0957ebae3d7d6 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Thu, 30 Jan 2020 18:10:36 -0800
Subject: [PATCH 157/250] rename funcs

---
 cmd/minikube/cmd/delete.go      |  1 +
 pkg/minikube/cluster/cluster.go | 16 ++++++++--------
 pkg/minikube/config/profile.go  |  6 +++---
 3 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/cmd/minikube/cmd/delete.go b/cmd/minikube/cmd/delete.go
index cf75aa10a1..5efe7e2ee3 100644
--- a/cmd/minikube/cmd/delete.go
+++ b/cmd/minikube/cmd/delete.go
@@ -98,6 +98,7 @@ func runDelete(cmd *cobra.Command, args []string) {
 
 	validProfiles, invalidProfiles, err := pkg_config.ListProfiles()
 	profilesToDelete := append(validProfiles, invalidProfiles...)
+
 	// If the purge flag is set, go ahead and delete the .minikube directory.
 	if purge && len(profilesToDelete) > 1 && !deleteAll {
 		out.ErrT(out.Notice, "Multiple minikube profiles were found - ")
diff --git a/pkg/minikube/cluster/cluster.go b/pkg/minikube/cluster/cluster.go
index b6839c482c..a45df13020 100644
--- a/pkg/minikube/cluster/cluster.go
+++ b/pkg/minikube/cluster/cluster.go
@@ -310,8 +310,8 @@ func StopHost(api libmachine.API) error {
 
 // DeleteHost deletes the host VM.
 func DeleteHost(api libmachine.API, machineName string) error {
-	h, err := api.Load(machineName)
-	if err != nil && h.DriverName == "" {
+	host, err := api.Load(machineName)
+	if err != nil && host.DriverName == "" {
 		// before we give up on deleting the host
 		//  we try to kill the possible orphan kic driver
 		// this case will happen if the user deleted both profile and machine folder
@@ -333,19 +333,19 @@ func DeleteHost(api libmachine.API, machineName string) error {
 	}
 
 	if status == state.None.String() {
-		return mcnerror.ErrHostDoesNotExist{Name: h.Name}
+		return mcnerror.ErrHostDoesNotExist{Name: host.Name}
 	}
 
 	// This is slow if SSH is not responding, but HyperV hangs otherwise, See issue #2914
-	if h.Driver.DriverName() == driver.HyperV {
-		if err := trySSHPowerOff(h); err != nil {
+	if host.Driver.DriverName() == driver.HyperV {
+		if err := trySSHPowerOff(host); err != nil {
 			glog.Infof("Unable to power off minikube because the host was not found.")
 		}
-		out.T(out.DeletingHost, "Successfully powered off Hyper-V. minikube driver -- {{.driver}}", out.V{"driver": h.Driver.DriverName()})
+		out.T(out.DeletingHost, "Successfully powered off Hyper-V. minikube driver -- {{.driver}}", out.V{"driver": host.Driver.DriverName()})
 	}
 
-	out.T(out.DeletingHost, `Deleting "{{.profile_name}}" in {{.driver_name}} ...`, out.V{"profile_name": machineName, "driver_name": h.DriverName})
-	if err := h.Driver.Remove(); err != nil {
+	out.T(out.DeletingHost, `Deleting "{{.profile_name}}" in {{.driver_name}} ...`, out.V{"profile_name": machineName, "driver_name": host.DriverName})
+	if err := host.Driver.Remove(); err != nil {
 		return errors.Wrap(err, "host remove")
 	}
 	if err := api.Remove(machineName); err != nil {
diff --git a/pkg/minikube/config/profile.go b/pkg/minikube/config/profile.go
index c75916d980..bb74a88dba 100644
--- a/pkg/minikube/config/profile.go
+++ b/pkg/minikube/config/profile.go
@@ -158,7 +158,7 @@ func ListProfiles(miniHome ...string) (validPs []*Profile, inValidPs []*Profile,
 	if err == nil {
 		pDirs = append(pDirs, cs...)
 	}
-	pDirs = removeDups(pDirs)
+	pDirs = removeDupes(pDirs)
 	for _, n := range pDirs {
 		p, err := LoadProfile(n, miniHome...)
 		if err != nil {
@@ -174,8 +174,8 @@ func ListProfiles(miniHome ...string) (validPs []*Profile, inValidPs []*Profile,
 	return validPs, inValidPs, nil
 }
 
-// removeDups removes duplicates
-func removeDups(profiles []string) []string {
+// removeDupes removes duplicates
+func removeDupes(profiles []string) []string {
 	// Use map to record duplicates as we find them.
 	seen := map[string]bool{}
 	result := []string{}

From b20322a8c29a3e032ef7838efda5e32df663e5b2 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Thu, 30 Jan 2020 18:12:04 -0800
Subject: [PATCH 158/250] rename funcs

---
 pkg/drivers/kic/kic.go   | 2 +-
 pkg/drivers/kic/types.go | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go
index a7d6f29fea..ed27eedb59 100644
--- a/pkg/drivers/kic/kic.go
+++ b/pkg/drivers/kic/kic.go
@@ -61,7 +61,7 @@ func NewDriver(c Config) *Driver {
 
 // Create a host using the driver's config
 func (d *Driver) Create() error {
-	params := createConfig{
+	params := createParams{
 		Name:          d.NodeConfig.MachineName,
 		Image:         d.NodeConfig.ImageDigest,
 		ClusterLabel:  oci.ClusterLabelKey + "=" + d.MachineName,
diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go
index 9cc1a1aa92..590f0f8127 100644
--- a/pkg/drivers/kic/types.go
+++ b/pkg/drivers/kic/types.go
@@ -45,7 +45,7 @@ type Config struct {
 	Envs          map[string]string // key,value of environment variables passed to the node
 }
 
-type createConfig struct {
+type createParams struct {
 	Name          string            // used for container name and hostname
 	Image         string            // container image to use to create the node.
 	ClusterLabel  string            // label the containers we create using minikube so we can clean up

From 23e125649dc9599a22bfe57c4bf9a32a86433ad5 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Thu, 30 Jan 2020 18:13:11 -0800
Subject: [PATCH 159/250] rename createConfig

---
 pkg/drivers/kic/kic.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go
index ed27eedb59..51d4568164 100644
--- a/pkg/drivers/kic/kic.go
+++ b/pkg/drivers/kic/kic.go
@@ -298,7 +298,7 @@ func (d *Driver) nodeID(nameOrID string) (string, error) {
 }
 
 // createNode creates a new container node
-func createNode(p createConfig) error {
+func createNode(p createParams) error {
 	runArgs := []string{
 		fmt.Sprintf("--cpus=%s", p.CPUs),
 		fmt.Sprintf("--memory=%s", p.Memory),

From be38a021cbc9983dfb8ba3f8690ef099bcec5ca1 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Thu, 30 Jan 2020 18:24:16 -0800
Subject: [PATCH 160/250] fix kic restart use external ip

---
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index ad2f481dd5..73aea4464e 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -372,7 +372,16 @@ func (k *Bootstrapper) restartCluster(cfg config.MachineConfig) error {
 	}
 
 	for _, n := range cfg.Nodes {
-		client, err := k.client(n.IP, n.Port)
+		ip := n.IP
+		port := n.Port
+		if driver.IsKIC(cfg.VMDriver) {
+			ip = kic.DefaultBindIPV4
+			port, err = oci.HostPortBinding(cfg.VMDriver, cfg.Name, port)
+			if err != nil {
+				return errors.Wrapf(err, "get host-bind port %d for container %s", port, cfg.Name)
+			}
+		}
+		client, err := k.client(ip, port)
 		if err != nil {
 			return errors.Wrap(err, "getting k8s client")
 		}

From 4f9df4596e74c1ff94ecef452f4fd938051383c3 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Thu, 30 Jan 2020 18:30:19 -0800
Subject: [PATCH 161/250] fix kic minikube status

---
 pkg/minikube/cluster/cluster.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/pkg/minikube/cluster/cluster.go b/pkg/minikube/cluster/cluster.go
index a45df13020..7ec7ecd7dc 100644
--- a/pkg/minikube/cluster/cluster.go
+++ b/pkg/minikube/cluster/cluster.go
@@ -387,6 +387,9 @@ func GetHostDriverIP(api libmachine.API, machineName string) (net.IP, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "getting IP")
 	}
+	if driver.IsKIC(host.DriverName) {
+		ipStr = kic.DefaultBindIPV4
+	}
 	ip := net.ParseIP(ipStr)
 	if ip == nil {
 		return nil, fmt.Errorf("parsing IP: %s", ipStr)

From 7b3fa71dbf684c971b093c91235ac8b24a736224 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Thu, 30 Jan 2020 19:21:55 -0800
Subject: [PATCH 162/250] fix nil logic

---
 pkg/minikube/cluster/cluster.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/minikube/cluster/cluster.go b/pkg/minikube/cluster/cluster.go
index 7ec7ecd7dc..b809937174 100644
--- a/pkg/minikube/cluster/cluster.go
+++ b/pkg/minikube/cluster/cluster.go
@@ -311,7 +311,7 @@ func StopHost(api libmachine.API) error {
 // DeleteHost deletes the host VM.
 func DeleteHost(api libmachine.API, machineName string) error {
 	host, err := api.Load(machineName)
-	if err != nil && host.DriverName == "" {
+	if err != nil && host == nil {
 		// before we give up on deleting the host
 		//  we try to kill the possible orphan kic driver
 		// this case will happen if the user deleted both profile and machine folder

From dbb3e92a5eaf5bde32437a255be646e04c9b4df8 Mon Sep 17 00:00:00 2001
From: Medya Ghazizadeh <medyagh@users.noreply.github.com>
Date: Thu, 30 Jan 2020 19:28:33 -0800
Subject: [PATCH 163/250] add more tests to github actions (#6423)

---
 .github/workflows/main.yml | 123 ++++++++++++++++++++++---------------
 1 file changed, 72 insertions(+), 51 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index e7ba04525b..be95831c0a 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -1,11 +1,10 @@
 name: CI
 
-on: [push]
+on: [pull_request]
 
 jobs:
-  docker_ubuntu:
-    runs-on: ubuntu-latest
-
+  docker_ubuntu_16_04:
+    runs-on: ubuntu-16.04
     steps:
     - uses: actions/checkout@v2
     - name: build binaries
@@ -13,15 +12,14 @@ jobs:
         make minikube-linux-amd64
         make e2e-linux-amd64
         mkdir -p report
-    - name: run integration test
-      run: |
-        echo running docker driver intergration test on ubuntu
-        ./out/e2e-linux-amd64 -minikube-start-args=--vm-driver=docker -expected-default-driver= -test.timeout=70m -test.v -binary=out/minikube-linux-amd64 2>&1 | tee ./report/testout.txt
     - name: install gopogh
       run: |
         cd /tmp
         GO111MODULE="on" go get github.com/medyagh/gopogh@v0.0.17 || true
         cd -
+    - name: run integration test
+      run: |
+        ./out/e2e-linux-amd64 -minikube-start-args=--vm-driver=docker -expected-default-driver= -test.timeout=70m -test.v -binary=out/minikube-linux-amd64 2>&1 | tee ./report/testout.txt
     - name: generate gopogh report
       run: |
         export PATH=${PATH}:`go env GOPATH`/bin
@@ -29,13 +27,37 @@ jobs:
         gopogh -in ./report/testout.json -out ./report/testout.html -name "docker ubuntu" -repo github.com/kubernetes/minikube/  || true
     - uses: actions/upload-artifact@v1
       with:
-        name: docker_on_ubuntu_report
+        name: docker_on_ubuntu_16_04_report
+        path: report
+  docker_ubuntu_18_04:
+    runs-on: ubuntu-18.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: build binaries
+      run : |
+        make minikube-linux-amd64
+        make e2e-linux-amd64
+        mkdir -p report
+    - name: install gopogh
+      run: |
+        cd /tmp
+        GO111MODULE="on" go get github.com/medyagh/gopogh@v0.0.17 || true
+        cd -
+    - name: run integration test
+      run: |
+        mkdir -p /tmp/testhome 
+        MINIKUBE_HOME=/tmp/testhome ./out/e2e-linux-amd64 -minikube-start-args=--vm-driver=docker -expected-default-driver= -test.timeout=70m -test.v -binary=out/minikube-linux-amd64 2>&1 | tee ./report/testout.txt
+    - name: generate gopogh report
+      run: |
+        export PATH=${PATH}:`go env GOPATH`/bin
+        go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
+        gopogh -in ./report/testout.json -out ./report/testout.html -name "docker ubuntu" -repo github.com/kubernetes/minikube/  || true
+    - uses: actions/upload-artifact@v1
+      with:
+        name: docker_on_ubuntu_18_04_report
         path: report
-
   docker_macos:
-
     runs-on: macos-latest
-
     steps:
     - uses: actions/checkout@v2
     - name: build binaries
@@ -43,15 +65,19 @@ jobs:
         make minikube-darwin-amd64
         make e2e-darwin-amd64
         mkdir -p report
-    - name: run integration test
-      run: |
-        echo running docker driver intergration test on ubuntu
-        ./out/e2e-darwin-amd64 -minikube-start-args=--vm-driver=docker -expected-default-driver= -test.timeout=70m -test.v -binary=out/minikube-darwin-amd64 2>&1 | tee ./report/testout.txt
+    - name: install docker
+        run: |
+          brew install docker-machine docker
+          sudo docker –version
     - name: install gopogh
       run: |
         cd /tmp
         GO111MODULE="on" go get github.com/medyagh/gopogh@v0.0.17 || true
         cd -
+    - name: run integration test
+      run: |
+        mkdir -p /tmp/testhome
+        MINIKUBE_HOME=/tmp/testhome ./out/e2e-darwin-amd64 -minikube-start-args=--vm-driver=docker -expected-default-driver= -test.timeout=70m -test.v -binary=./out/minikube-darwin-amd64 2>&1 | tee ./report/testout.txt
     - name: generate gopogh report
       run: |
         export PATH=${PATH}:`go env GOPATH`/bin
@@ -61,39 +87,8 @@ jobs:
       with:
         name: docker_on_macos_report
         path: ./report
-
-  none_ubuntu:
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v2
-    - name: build binaries
-      run : |
-        make minikube-linux-amd64
-        make e2e-linux-amd64
-        mkdir -p report
-    - name: run integration test
-      run: |
-        echo running docker driver intergration test on ubuntu
-        ./out/e2e-linux-amd64 -minikube-start-args=--vm-driver=none -expected-default-driver= -test.timeout=70m -test.v -binary=out/minikube-linux-amd64 2>&1 | tee ./report/testout.txt
-    - name: install gopogh
-      run: |
-        cd /tmp
-        GO111MODULE="on" go get github.com/medyagh/gopogh@v0.0.17 || true
-        cd -
-    - name: generate gopogh report
-      run: |
-        export PATH=${PATH}:`go env GOPATH`/bin
-        go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
-        gopogh -in ./report/testout.json -out ./report/testout.html -name "docker ubuntu" -repo github.com/kubernetes/minikube/  || true
-    - uses: actions/upload-artifact@v1
-      with:
-        name: none_on_ubuntu_latest_report
-        path: report
-
   none_ubuntu16_04:
     runs-on: ubuntu-16.04
-
     steps:
     - uses: actions/checkout@v2
     - name: build binaries
@@ -101,15 +96,15 @@ jobs:
         make minikube-linux-amd64
         make e2e-linux-amd64
         mkdir -p report
-    - name: run integration test
-      run: |
-        echo running docker driver intergration test on ubuntu
-        ./out/e2e-linux-amd64 -minikube-start-args=--vm-driver=none -expected-default-driver= -test.timeout=70m -test.v -binary=out/minikube-linux-amd64 2>&1 | tee ./report/testout.txt
     - name: install gopogh
       run: |
         cd /tmp
         GO111MODULE="on" go get github.com/medyagh/gopogh@v0.0.17 || true
         cd -
+    - name: run integration test
+      run: |
+        mkdir -p /tmp/testhome
+        MINIKUBE_HOME=/tmp/testhome sudo -E ./out/e2e-linux-amd64 -minikube-start-args=--vm-driver=none -expected-default-driver= -test.timeout=70m -test.v -binary=out/minikube-linux-amd64 2>&1 | tee ./report/testout.txt
     - name: generate gopogh report
       run: |
         export PATH=${PATH}:`go env GOPATH`/bin
@@ -119,3 +114,29 @@ jobs:
       with:
         name: none_on_ubuntu_16_04
         path: report
+  none_ubuntu_18_04:
+    runs-on: ubuntu-18.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: build binaries
+      run : |
+        make minikube-linux-amd64
+        make e2e-linux-amd64
+    - name: install gopogh
+      run: |
+        cd /tmp
+        GO111MODULE="on" go get github.com/medyagh/gopogh@v0.0.17 || true
+        cd -
+    - name: run integration test
+      run: |
+        mkdir -p /tmp/testhome
+        MINIKUBE_HOME=/tmp/testhome sudo -E ./out/e2e-linux-amd64 -minikube-start-args=--vm-driver=none -expected-default-driver= -test.timeout=70m -test.v -binary=out/minikube-linux-amd64 2>&1 | tee ./report/testout.txt
+    - name: generate gopogh report
+      run: |
+        export PATH=${PATH}:`go env GOPATH`/bin
+        go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
+        gopogh -in ./report/testout.json -out ./report/testout.html -name "docker ubuntu" -repo github.com/kubernetes/minikube/  || true
+    - uses: actions/upload-artifact@v1
+      with:
+        name: none_on_ubuntu_latest_report
+        path: report

From 79b3cc5643d57f985f073eec6fc49afbc196fc9a Mon Sep 17 00:00:00 2001
From: Medya Ghazizadeh <medyagh@users.noreply.github.com>
Date: Thu, 30 Jan 2020 20:34:23 -0800
Subject: [PATCH 164/250] Update github actions (#6444)

* Update main.yml
---
 .github/workflows/main.yml | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index be95831c0a..2ec7b9b32c 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -19,7 +19,8 @@ jobs:
         cd -
     - name: run integration test
       run: |
-        ./out/e2e-linux-amd64 -minikube-start-args=--vm-driver=docker -expected-default-driver= -test.timeout=70m -test.v -binary=out/minikube-linux-amd64 2>&1 | tee ./report/testout.txt
+        mkdir -p /tmp/testhome
+        MINIKUBE_HOME=/tmp/testhome ./out/e2e-linux-amd64 -minikube-start-args=--vm-driver=docker -expected-default-driver= -test.timeout=70m -test.v -binary=out/minikube-linux-amd64 2>&1 | tee ./report/testout.txt
     - name: generate gopogh report
       run: |
         export PATH=${PATH}:`go env GOPATH`/bin
@@ -66,9 +67,10 @@ jobs:
         make e2e-darwin-amd64
         mkdir -p report
     - name: install docker
-        run: |
-          brew install docker-machine docker
-          sudo docker –version
+      run: |
+          brew install docker-machine docker || true
+          brew services start docker-machine || true
+          docker version || true
     - name: install gopogh
       run: |
         cd /tmp

From 006c781fbcbe0753bab90f5a6bb1c838862e7e88 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Thu, 30 Jan 2020 20:44:32 -0800
Subject: [PATCH 165/250] remove unused kic code

---
 pkg/drivers/kic/oci/oci.go | 195 ++-----------------------------------
 1 file changed, 6 insertions(+), 189 deletions(-)

diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go
index acc608706e..ee6525449d 100644
--- a/pkg/drivers/kic/oci/oci.go
+++ b/pkg/drivers/kic/oci/oci.go
@@ -20,8 +20,6 @@ import (
 	"os"
 	"strconv"
 
-	"github.com/docker/machine/libmachine/state"
-
 	"bufio"
 	"bytes"
 
@@ -30,81 +28,8 @@ import (
 	"fmt"
 	"os/exec"
 	"strings"
-	"time"
-
-	"github.com/cenkalti/backoff"
 )
 
-// Stop stops a container
-func Stop(ociBinary, ociID string) error {
-	cmd := exec.Command(ociBinary, "stop", ociID)
-	err := cmd.Run()
-	if err != nil {
-		return errors.Wrapf(err, "error stop node %s", ociID)
-	}
-
-	return nil
-}
-
-// Status returns the status of the container
-func Status(ociBinary string, ociID string) (state.State, error) {
-	cmd := exec.Command(ociBinary, "inspect", "-f", "{{.State.Status}}", ociID)
-	out, err := cmd.CombinedOutput()
-	o := strings.Trim(string(out), "\n")
-	s := state.Error
-	switch o {
-	case "running":
-		s = state.Running
-	case "exited":
-		s = state.Stopped
-	case "paused":
-		s = state.Paused
-	case "restaring":
-		s = state.Starting
-	}
-
-	if err != nil {
-		return state.Error, errors.Wrapf(err, "error getting node %s status", ociID)
-	}
-	return s, nil
-}
-
-// SystemStatus checks if the oci container engine is running
-func SystemStatus(ociBinary string, ociID string) (state.State, error) {
-	_, err := exec.LookPath(ociBinary)
-	if err != nil {
-		return state.Error, err
-	}
-
-	err = exec.Command("docker", "info").Run()
-	if err != nil {
-		return state.Error, err
-	}
-
-	return state.Running, nil
-}
-
-// Remove removes a container
-func Remove(ociBinary string, ociID string) error {
-	// TODO: force remove should be an option
-	cmd := exec.Command(ociBinary, "rm", "-f", "-v", ociID)
-	if err := cmd.Run(); err != nil {
-		return errors.Wrapf(err, "error removing node %s", ociID)
-	}
-
-	return nil
-}
-
-// Pause pauses a container
-func Pause(ociBinary string, ociID string) error {
-	cmd := exec.Command(ociBinary, "pause", ociID)
-	if err := cmd.Run(); err != nil {
-		return errors.Wrapf(err, "error pausing node %s", ociID)
-	}
-
-	return nil
-}
-
 // Inspect return low-level information on containers
 func Inspect(ociBinary string, containerNameOrID, format string) ([]string, error) {
 	cmd := exec.Command(ociBinary, "inspect",
@@ -122,65 +47,6 @@ func Inspect(ociBinary string, containerNameOrID, format string) ([]string, erro
 	return lines, err
 }
 
-// NetworkInspect displays detailed information on one or more networks
-func NetworkInspect(networkNames []string, format string) ([]string, error) {
-	cmd := exec.Command("docker", "network", "inspect",
-		"-f", format,
-		strings.Join(networkNames, " "))
-	var buff bytes.Buffer
-	cmd.Stdout = &buff
-	cmd.Stderr = &buff
-	err := cmd.Run()
-	scanner := bufio.NewScanner(&buff)
-	var lines []string
-	for scanner.Scan() {
-		lines = append(lines, scanner.Text())
-	}
-	return lines, err
-}
-
-// GetSubnets returns a slice of subnets for a specified network name
-// For example the command : docker network inspect -f '{{range (index (index . "IPAM") "Config")}}{{index . "Subnet"}} {{end}}' bridge
-// returns 172.17.0.0/16
-func GetSubnets(networkName string) ([]string, error) {
-	format := `{{range (index (index . "IPAM") "Config")}}{{index . "Subnet"}} {{end}}`
-	lines, err := NetworkInspect([]string{networkName}, format)
-	if err != nil {
-		return nil, err
-	}
-	return strings.Split(lines[0], " "), nil
-}
-
-// ImageInspect return low-level information on containers images
-func ImageInspect(containerNameOrID, format string) ([]string, error) {
-	cmd := exec.Command("docker", "image", "inspect",
-		"-f", format,
-		containerNameOrID,
-	)
-	var buff bytes.Buffer
-	cmd.Stdout = &buff
-	cmd.Stderr = &buff
-	err := cmd.Run()
-	scanner := bufio.NewScanner(&buff)
-	var lines []string
-	for scanner.Scan() {
-		lines = append(lines, scanner.Text())
-	}
-	return lines, err
-}
-
-// ImageID return the Id of the container image
-func ImageID(containerNameOrID string) (string, error) {
-	lines, err := ImageInspect(containerNameOrID, "{{ .Id }}")
-	if err != nil {
-		return "", err
-	}
-	if len(lines) != 1 {
-		return "", fmt.Errorf("docker image ID should only be one line, got %d lines", len(lines))
-	}
-	return lines[0], nil
-}
-
 /*
 This is adapated from:
 https://github.com/kubernetes/kubernetes/blob/07a5488b2a8f67add543da72e8819407d8314204/pkg/kubelet/dockershim/helpers.go#L115-L155
@@ -227,31 +93,6 @@ func generateMountBindings(mounts ...Mount) []string {
 	return result
 }
 
-// PullIfNotPresent pulls docker image if not present back off exponentially
-func PullIfNotPresent(ociBinary string, image string, forceUpdate bool, maxWait time.Duration) error {
-	cmd := exec.Command(ociBinary, "inspect", "--type=image", image)
-	err := cmd.Run()
-	if err == nil && !forceUpdate {
-		return nil // if presents locally and not force
-	}
-	b := backoff.NewExponentialBackOff()
-	b.MaxElapsedTime = maxWait
-	f := func() error {
-		return pull(ociBinary, image)
-	}
-	return backoff.Retry(f, b)
-}
-
-// Pull pulls an image, retrying up to retries times
-func pull(ociBinary string, image string) error {
-	cmd := exec.Command(ociBinary, "pull", image)
-	err := cmd.Run()
-	if err != nil {
-		return fmt.Errorf("error pull image %s : %v", image, err)
-	}
-	return err
-}
-
 // UsernsRemap checks if userns-remap is enabled in dockerd
 func UsernsRemap(ociBinary string) bool {
 	cmd := exec.Command(ociBinary, "info", "--format", "'{{json .SecurityOptions}}'")
@@ -286,26 +127,8 @@ func generatePortMappings(portMappings ...PortMapping) []string {
 	return result
 }
 
-// Save saves an image archive "docker/podman save"
-func Save(ociBinary string, image, dest string) error {
-	cmd := exec.Command(ociBinary, "save", "-o", dest, image)
-	var buff bytes.Buffer
-	cmd.Stdout = &buff
-	cmd.Stderr = &buff
-	err := cmd.Run()
-	scanner := bufio.NewScanner(&buff)
-	var lines []string
-	for scanner.Scan() {
-		lines = append(lines, scanner.Text())
-	}
-	if err != nil {
-		return errors.Wrapf(err, "saving image to tar failed, output %s", lines[0])
-	}
-	return nil
-}
-
-// CreateOpt is an option for Create
-type CreateOpt func(*createOpts) *createOpts
+// createOpt is an option for Create
+type createOpt func(*createOpts) *createOpts
 
 // actual options struct
 type createOpts struct {
@@ -316,7 +139,7 @@ type createOpts struct {
 }
 
 // CreateContainer creates a container with "docker/podman run"
-func CreateContainer(ociBinary string, image string, opts ...CreateOpt) ([]string, error) {
+func CreateContainer(ociBinary string, image string, opts ...createOpt) ([]string, error) {
 	o := &createOpts{}
 	for _, opt := range opts {
 		o = opt(o)
@@ -353,7 +176,7 @@ func CreateContainer(ociBinary string, image string, opts ...CreateOpt) ([]strin
 
 // WithRunArgs sets the args for docker run
 // as in the args portion of `docker run args... image containerArgs...`
-func WithRunArgs(args ...string) CreateOpt {
+func WithRunArgs(args ...string) createOpt {
 	return func(r *createOpts) *createOpts {
 		r.RunArgs = args
 		return r
@@ -361,7 +184,7 @@ func WithRunArgs(args ...string) CreateOpt {
 }
 
 // WithMounts sets the container mounts
-func WithMounts(mounts []Mount) CreateOpt {
+func WithMounts(mounts []Mount) createOpt {
 	return func(r *createOpts) *createOpts {
 		r.Mounts = mounts
 		return r
@@ -369,7 +192,7 @@ func WithMounts(mounts []Mount) CreateOpt {
 }
 
 // WithPortMappings sets the container port mappings to the host
-func WithPortMappings(portMappings []PortMapping) CreateOpt {
+func WithPortMappings(portMappings []PortMapping) createOpt {
 	return func(r *createOpts) *createOpts {
 		r.PortMappings = portMappings
 		return r
@@ -426,12 +249,6 @@ func listContainersByLabel(ociBinary string, label string) ([]string, error) {
 	return lines, err
 }
 
-// ContainerID returns a container ID from its name
-func ContainerID(ociBinary string, name string) (string, error) {
-	ids, err := Inspect(ociBinary, name, "{{.Id}}")
-	return ids[0], err
-}
-
 // ContainerIPs returns ipv4,ipv6, error of a container by their name
 func ContainerIPs(ociBinary string, name string) (string, string, error) {
 	// retrieve the IP address of the node using docker inspect

From 968fe86869244ee3a3c6a6b269a97a7968d52e4a Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Thu, 30 Jan 2020 21:01:27 -0800
Subject: [PATCH 166/250] unexport more funcs

---
 pkg/drivers/kic/kic.go       |  57 +-------
 pkg/drivers/kic/oci/oci.go   | 270 ++++++++++++++++++++---------------
 pkg/drivers/kic/oci/types.go |  26 ++++
 pkg/drivers/kic/types.go     |  15 --
 4 files changed, 184 insertions(+), 184 deletions(-)

diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go
index 51d4568164..c461d2c3d6 100644
--- a/pkg/drivers/kic/kic.go
+++ b/pkg/drivers/kic/kic.go
@@ -61,7 +61,7 @@ func NewDriver(c Config) *Driver {
 
 // Create a host using the driver's config
 func (d *Driver) Create() error {
-	params := createParams{
+	params := oci.CreateParams{
 		Name:          d.NodeConfig.MachineName,
 		Image:         d.NodeConfig.ImageDigest,
 		ClusterLabel:  oci.ClusterLabelKey + "=" + d.MachineName,
@@ -87,7 +87,7 @@ func (d *Driver) Create() error {
 			ContainerPort: constants.DockerDaemonPort,
 		},
 	)
-	err := createNode(params)
+	err := oci.CreateContainerNode(params)
 	if err != nil {
 		return errors.Wrap(err, "create kic node")
 	}
@@ -296,56 +296,3 @@ func (d *Driver) nodeID(nameOrID string) (string, error) {
 	}
 	return string(id), err
 }
-
-// createNode creates a new container node
-func createNode(p createParams) error {
-	runArgs := []string{
-		fmt.Sprintf("--cpus=%s", p.CPUs),
-		fmt.Sprintf("--memory=%s", p.Memory),
-		"-d", // run the container detached
-		"-t", // allocate a tty for entrypoint logs
-		// running containers in a container requires privileged
-		// NOTE: we could try to replicate this with --cap-add, and use less
-		// privileges, but this flag also changes some mounts that are necessary
-		// including some ones docker would otherwise do by default.
-		// for now this is what we want. in the future we may revisit this.
-		"--privileged",
-		"--security-opt", "seccomp=unconfined", // also ignore seccomp
-		"--tmpfs", "/tmp", // various things depend on working /tmp
-		"--tmpfs", "/run", // systemd wants a writable /run
-		// logs,pods be stroed on  filesystem vs inside container,
-		"--volume", "/var",
-		// some k8s things want /lib/modules
-		"-v", "/lib/modules:/lib/modules:ro",
-		"--hostname", p.Name, // make hostname match container name
-		"--name", p.Name, // ... and set the container name
-		// label the node with the cluster ID
-		"--label", p.ClusterLabel,
-		// label the node with the role ID
-		"--label", fmt.Sprintf("%s=%s", oci.NodeRoleKey, p.Role),
-	}
-
-	for key, val := range p.Envs {
-		runArgs = append(runArgs, "-e", fmt.Sprintf("%s=%s", key, val))
-	}
-
-	// adds node specific args
-	runArgs = append(runArgs, p.ExtraArgs...)
-
-	if oci.UsernsRemap(p.OCIBinary) {
-		// We need this argument in order to make this command work
-		// in systems that have userns-remap enabled on the docker daemon
-		runArgs = append(runArgs, "--userns=host")
-	}
-
-	_, err := oci.CreateContainer(p.OCIBinary,
-		p.Image,
-		oci.WithRunArgs(runArgs...),
-		oci.WithMounts(p.Mounts),
-		oci.WithPortMappings(p.PortMappings),
-	)
-	if err != nil {
-		return errors.Wrap(err, "create a kic node")
-	}
-	return nil
-}
diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go
index ee6525449d..9b0f68e6bd 100644
--- a/pkg/drivers/kic/oci/oci.go
+++ b/pkg/drivers/kic/oci/oci.go
@@ -30,8 +30,154 @@ import (
 	"strings"
 )
 
-// Inspect return low-level information on containers
-func Inspect(ociBinary string, containerNameOrID, format string) ([]string, error) {
+// CreateContainerNode creates a new container node
+func CreateContainerNode(p CreateParams) error {
+	runArgs := []string{
+		fmt.Sprintf("--cpus=%s", p.CPUs),
+		fmt.Sprintf("--memory=%s", p.Memory),
+		"-d", // run the container detached
+		"-t", // allocate a tty for entrypoint logs
+		// running containers in a container requires privileged
+		// NOTE: we could try to replicate this with --cap-add, and use less
+		// privileges, but this flag also changes some mounts that are necessary
+		// including some ones docker would otherwise do by default.
+		// for now this is what we want. in the future we may revisit this.
+		"--privileged",
+		"--security-opt", "seccomp=unconfined", // also ignore seccomp
+		"--tmpfs", "/tmp", // various things depend on working /tmp
+		"--tmpfs", "/run", // systemd wants a writable /run
+		// logs,pods be stroed on  filesystem vs inside container,
+		"--volume", "/var",
+		// some k8s things want /lib/modules
+		"-v", "/lib/modules:/lib/modules:ro",
+		"--hostname", p.Name, // make hostname match container name
+		"--name", p.Name, // ... and set the container name
+		// label the node with the cluster ID
+		"--label", p.ClusterLabel,
+		// label the node with the role ID
+		"--label", fmt.Sprintf("%s=%s", NodeRoleKey, p.Role),
+	}
+
+	for key, val := range p.Envs {
+		runArgs = append(runArgs, "-e", fmt.Sprintf("%s=%s", key, val))
+	}
+
+	// adds node specific args
+	runArgs = append(runArgs, p.ExtraArgs...)
+
+	if isUsernsRemapEnabled(p.OCIBinary) {
+		// We need this argument in order to make this command work
+		// in systems that have userns-remap enabled on the docker daemon
+		runArgs = append(runArgs, "--userns=host")
+	}
+
+	_, err := createContainer(p.OCIBinary,
+		p.Image,
+		withRunArgs(runArgs...),
+		withMounts(p.Mounts),
+		withPortMappings(p.PortMappings),
+	)
+	if err != nil {
+		return errors.Wrap(err, "create a kic node")
+	}
+	return nil
+}
+
+// CreateContainer creates a container with "docker/podman run"
+func createContainer(ociBinary string, image string, opts ...createOpt) ([]string, error) {
+	o := &createOpts{}
+	for _, opt := range opts {
+		o = opt(o)
+	}
+	// convert mounts to container run args
+	runArgs := o.RunArgs
+	for _, mount := range o.Mounts {
+		runArgs = append(runArgs, generateMountBindings(mount)...)
+	}
+	for _, portMapping := range o.PortMappings {
+		runArgs = append(runArgs, generatePortMappings(portMapping)...)
+	}
+	// construct the actual docker run argv
+	args := []string{"run"}
+	args = append(args, runArgs...)
+	args = append(args, image)
+	args = append(args, o.ContainerArgs...)
+	cmd := exec.Command(ociBinary, args...)
+	var buff bytes.Buffer
+	cmd.Stdout = &buff
+	cmd.Stderr = &buff
+	err := cmd.Run()
+	scanner := bufio.NewScanner(&buff)
+	var output []string
+	for scanner.Scan() {
+		output = append(output, scanner.Text())
+	}
+
+	if err != nil {
+		return output, errors.Wrapf(err, "args: %v  output: %s ", args, output)
+	}
+	return output, nil
+}
+
+// Copy copies a local asset into the container
+func Copy(ociBinary string, ociID string, targetDir string, fName string) error {
+	if _, err := os.Stat(fName); os.IsNotExist(err) {
+		return errors.Wrapf(err, "error source %s does not exist", fName)
+	}
+	destination := fmt.Sprintf("%s:%s", ociID, targetDir)
+	cmd := exec.Command(ociBinary, "cp", fName, destination)
+	err := cmd.Run()
+	if err != nil {
+		return errors.Wrapf(err, "error copying %s into node", fName)
+	}
+	return nil
+}
+
+// HostPortBinding will return port mapping for a container using cli.
+// example : HostPortBinding("docker", "minikube", "22")
+// will return the docker assigned port:
+// 32769, nil
+// only supports TCP ports
+func HostPortBinding(ociBinary string, ociID string, contPort int) (int, error) {
+	cmd := exec.Command(ociBinary, "inspect", "-f", fmt.Sprintf("'{{(index (index .NetworkSettings.Ports \"%d/tcp\") 0).HostPort}}'", contPort), ociID)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return 0, errors.Wrapf(err, "getting host-bind port %d for container ID %q, output %s", contPort, ociID, out)
+	}
+	o := strings.Trim(string(out), "\n")
+	o = strings.Trim(o, "'")
+	p, err := strconv.Atoi(o)
+	if err != nil {
+		return p, errors.Wrapf(err, "convert host-port %q to number", p)
+	}
+	return p, nil
+}
+
+// ContainerIPs returns ipv4,ipv6, error of a container by their name
+func ContainerIPs(ociBinary string, name string) (string, string, error) {
+	// retrieve the IP address of the node using docker inspect
+	lines, err := inspect(ociBinary, name, "{{range .NetworkSettings.Networks}}{{.IPAddress}},{{.GlobalIPv6Address}}{{end}}")
+	if err != nil {
+		return "", "", errors.Wrap(err, "inspecting NetworkSettings.Networks")
+	}
+	if len(lines) != 1 {
+		return "", "", errors.Errorf("IPs output should only be one line, got %d lines", len(lines))
+	}
+	ips := strings.Split(lines[0], ",")
+	if len(ips) != 2 {
+		return "", "", errors.Errorf("container addresses should have 2 values, got %d values: %+v", len(ips), ips)
+	}
+	return ips[0], ips[1], nil
+
+}
+
+// ListOwnedContainers lists all the containres that kic driver created on user's machine using a label
+func ListOwnedContainers(ociBinary string) ([]string, error) {
+	return listContainersByLabel(ociBinary, ClusterLabelKey)
+}
+
+// inspect return low-level information on containers
+func inspect(ociBinary string, containerNameOrID, format string) ([]string, error) {
 	cmd := exec.Command(ociBinary, "inspect",
 		"-f", format,
 		containerNameOrID) // ... against the "node" container
@@ -93,8 +239,8 @@ func generateMountBindings(mounts ...Mount) []string {
 	return result
 }
 
-// UsernsRemap checks if userns-remap is enabled in dockerd
-func UsernsRemap(ociBinary string) bool {
+// isUsernsRemapEnabled checks if userns-remap is enabled in docker
+func isUsernsRemapEnabled(ociBinary string) bool {
 	cmd := exec.Command(ociBinary, "info", "--format", "'{{json .SecurityOptions}}'")
 	var buff bytes.Buffer
 	cmd.Stdout = &buff
@@ -127,112 +273,31 @@ func generatePortMappings(portMappings ...PortMapping) []string {
 	return result
 }
 
-// createOpt is an option for Create
-type createOpt func(*createOpts) *createOpts
-
-// actual options struct
-type createOpts struct {
-	RunArgs       []string
-	ContainerArgs []string
-	Mounts        []Mount
-	PortMappings  []PortMapping
-}
-
-// CreateContainer creates a container with "docker/podman run"
-func CreateContainer(ociBinary string, image string, opts ...createOpt) ([]string, error) {
-	o := &createOpts{}
-	for _, opt := range opts {
-		o = opt(o)
-	}
-	// convert mounts to container run args
-	runArgs := o.RunArgs
-	for _, mount := range o.Mounts {
-		runArgs = append(runArgs, generateMountBindings(mount)...)
-	}
-	for _, portMapping := range o.PortMappings {
-		runArgs = append(runArgs, generatePortMappings(portMapping)...)
-	}
-	// construct the actual docker run argv
-	args := []string{"run"}
-	args = append(args, runArgs...)
-	args = append(args, image)
-	args = append(args, o.ContainerArgs...)
-	cmd := exec.Command(ociBinary, args...)
-	var buff bytes.Buffer
-	cmd.Stdout = &buff
-	cmd.Stderr = &buff
-	err := cmd.Run()
-	scanner := bufio.NewScanner(&buff)
-	var output []string
-	for scanner.Scan() {
-		output = append(output, scanner.Text())
-	}
-
-	if err != nil {
-		return output, errors.Wrapf(err, "args: %v  output: %s ", args, output)
-	}
-	return output, nil
-}
-
-// WithRunArgs sets the args for docker run
+// withRunArgs sets the args for docker run
 // as in the args portion of `docker run args... image containerArgs...`
-func WithRunArgs(args ...string) createOpt {
+func withRunArgs(args ...string) createOpt {
 	return func(r *createOpts) *createOpts {
 		r.RunArgs = args
 		return r
 	}
 }
 
-// WithMounts sets the container mounts
-func WithMounts(mounts []Mount) createOpt {
+// withMounts sets the container mounts
+func withMounts(mounts []Mount) createOpt {
 	return func(r *createOpts) *createOpts {
 		r.Mounts = mounts
 		return r
 	}
 }
 
-// WithPortMappings sets the container port mappings to the host
-func WithPortMappings(portMappings []PortMapping) createOpt {
+// withPortMappings sets the container port mappings to the host
+func withPortMappings(portMappings []PortMapping) createOpt {
 	return func(r *createOpts) *createOpts {
 		r.PortMappings = portMappings
 		return r
 	}
 }
 
-// Copy copies a local asset into the container
-func Copy(ociBinary string, ociID string, targetDir string, fName string) error {
-	if _, err := os.Stat(fName); os.IsNotExist(err) {
-		return errors.Wrapf(err, "error source %s does not exist", fName)
-	}
-	destination := fmt.Sprintf("%s:%s", ociID, targetDir)
-	cmd := exec.Command(ociBinary, "cp", fName, destination)
-	err := cmd.Run()
-	if err != nil {
-		return errors.Wrapf(err, "error copying %s into node", fName)
-	}
-	return nil
-}
-
-// HostPortBinding will return port mapping for a container using cli.
-// example : HostPortBinding("docker", "minikube", "22")
-// will return the docker assigned port:
-// 32769, nil
-// only supports TCP ports
-func HostPortBinding(ociBinary string, ociID string, contPort int) (int, error) {
-	cmd := exec.Command(ociBinary, "inspect", "-f", fmt.Sprintf("'{{(index (index .NetworkSettings.Ports \"%d/tcp\") 0).HostPort}}'", contPort), ociID)
-	out, err := cmd.CombinedOutput()
-	if err != nil {
-		return 0, errors.Wrapf(err, "getting host-bind port %d for container ID %q, output %s", contPort, ociID, out)
-	}
-	o := strings.Trim(string(out), "\n")
-	o = strings.Trim(o, "'")
-	p, err := strconv.Atoi(o)
-	if err != nil {
-		return p, errors.Wrapf(err, "convert host-port %q to number", p)
-	}
-	return p, nil
-}
-
 // listContainersByLabel lists all the containres that kic driver created on user's machine using a label
 // io.x-k8s.kic.cluster
 func listContainersByLabel(ociBinary string, label string) ([]string, error) {
@@ -248,26 +313,3 @@ func listContainersByLabel(ociBinary string, label string) ([]string, error) {
 	}
 	return lines, err
 }
-
-// ContainerIPs returns ipv4,ipv6, error of a container by their name
-func ContainerIPs(ociBinary string, name string) (string, string, error) {
-	// retrieve the IP address of the node using docker inspect
-	lines, err := Inspect(ociBinary, name, "{{range .NetworkSettings.Networks}}{{.IPAddress}},{{.GlobalIPv6Address}}{{end}}")
-	if err != nil {
-		return "", "", errors.Wrap(err, "inspecting NetworkSettings.Networks")
-	}
-	if len(lines) != 1 {
-		return "", "", errors.Errorf("IPs output should only be one line, got %d lines", len(lines))
-	}
-	ips := strings.Split(lines[0], ",")
-	if len(ips) != 2 {
-		return "", "", errors.Errorf("container addresses should have 2 values, got %d values: %+v", len(ips), ips)
-	}
-	return ips[0], ips[1], nil
-
-}
-
-// ListOwnedContainers lists all the containres that kic driver created on user's machine using a label
-func ListOwnedContainers(ociBinary string) ([]string, error) {
-	return listContainersByLabel(ociBinary, ClusterLabelKey)
-}
diff --git a/pkg/drivers/kic/oci/types.go b/pkg/drivers/kic/oci/types.go
index 49694f4e69..872b57bf6a 100644
--- a/pkg/drivers/kic/oci/types.go
+++ b/pkg/drivers/kic/oci/types.go
@@ -25,6 +25,32 @@ const (
 	NodeRoleKey = "io.k8s.sigs.kic.role"
 )
 
+type CreateParams struct {
+	Name          string            // used for container name and hostname
+	Image         string            // container image to use to create the node.
+	ClusterLabel  string            // label the containers we create using minikube so we can clean up
+	Role          string            // currently only role supported is control-plane
+	Mounts        []Mount           // volume mounts
+	APIServerPort int               // kubernetes api server port
+	PortMappings  []PortMapping     // ports to map to container from host
+	CPUs          string            // number of cpu cores assign to container
+	Memory        string            // memory (mbs) to assign to the container
+	Envs          map[string]string // environment variables to pass to the container
+	ExtraArgs     []string          // a list of any extra option to pass to oci binary during creation time, for example --expose 8080...
+	OCIBinary     string            // docker or podman
+}
+
+// createOpt is an option for Create
+type createOpt func(*createOpts) *createOpts
+
+// actual options struct
+type createOpts struct {
+	RunArgs       []string
+	ContainerArgs []string
+	Mounts        []Mount
+	PortMappings  []PortMapping
+}
+
 /*
 These types are from
 https://github.com/kubernetes/kubernetes/blob/063e7ff358fdc8b0916e6f39beedc0d025734cb1/pkg/kubelet/apis/cri/runtime/v1alpha2/api.pb.go#L183
diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go
index 590f0f8127..32f2a254ba 100644
--- a/pkg/drivers/kic/types.go
+++ b/pkg/drivers/kic/types.go
@@ -44,18 +44,3 @@ type Config struct {
 	PortMappings  []oci.PortMapping // container port mappings
 	Envs          map[string]string // key,value of environment variables passed to the node
 }
-
-type createParams struct {
-	Name          string            // used for container name and hostname
-	Image         string            // container image to use to create the node.
-	ClusterLabel  string            // label the containers we create using minikube so we can clean up
-	Role          string            // currently only role supported is control-plane
-	Mounts        []oci.Mount       // volume mounts
-	APIServerPort int               // kubernetes api server port
-	PortMappings  []oci.PortMapping // ports to map to container from host
-	CPUs          string            // number of cpu cores assign to container
-	Memory        string            // memory (mbs) to assign to the container
-	Envs          map[string]string // environment variables to pass to the container
-	ExtraArgs     []string          // a list of any extra option to pass to oci binary during creation time, for example --expose 8080...
-	OCIBinary     string            // docker or podman
-}

From a6ec05e35a946528c83dc19fa4aa2d112710d148 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Thu, 30 Jan 2020 21:05:51 -0800
Subject: [PATCH 167/250] unexport noderolekey

---
 pkg/drivers/kic/oci/oci.go   | 2 +-
 pkg/drivers/kic/oci/types.go | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go
index 9b0f68e6bd..29805c1e9c 100644
--- a/pkg/drivers/kic/oci/oci.go
+++ b/pkg/drivers/kic/oci/oci.go
@@ -55,7 +55,7 @@ func CreateContainerNode(p CreateParams) error {
 		// label the node with the cluster ID
 		"--label", p.ClusterLabel,
 		// label the node with the role ID
-		"--label", fmt.Sprintf("%s=%s", NodeRoleKey, p.Role),
+		"--label", fmt.Sprintf("%s=%s", nodeRoleKey, p.Role),
 	}
 
 	for key, val := range p.Envs {
diff --git a/pkg/drivers/kic/oci/types.go b/pkg/drivers/kic/oci/types.go
index 872b57bf6a..786cf3980b 100644
--- a/pkg/drivers/kic/oci/types.go
+++ b/pkg/drivers/kic/oci/types.go
@@ -22,7 +22,7 @@ const (
 	// ClusterLabelKey is applied to each node docker container for identification
 	ClusterLabelKey = "io.x-k8s.kic.cluster"
 	// NodeRoleKey is used to identify if it is control plane or worker
-	NodeRoleKey = "io.k8s.sigs.kic.role"
+	nodeRoleKey = "io.k8s.sigs.kic.role"
 )
 
 type CreateParams struct {

From 0e5059ddd05fb8c179c41b07c82e9f3c2bdc244a Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Thu, 30 Jan 2020 21:36:56 -0800
Subject: [PATCH 168/250] log warning if docker is stuck

---
 go.mod                     |  1 +
 go.sum                     |  1 +
 pkg/drivers/kic/kic.go     | 26 +++++++++++---------------
 pkg/drivers/kic/oci/oci.go | 10 ++++++++++
 4 files changed, 23 insertions(+), 15 deletions(-)

diff --git a/go.mod b/go.mod
index c31b89c1c9..09e880afb3 100644
--- a/go.mod
+++ b/go.mod
@@ -18,6 +18,7 @@ require (
 	github.com/elazarl/goproxy v0.0.0-20190421051319-9d40249d3c2f
 	github.com/elazarl/goproxy/ext v0.0.0-20190421051319-9d40249d3c2f // indirect
 	github.com/evanphx/json-patch v4.5.0+incompatible // indirect
+	github.com/go-acme/lego v2.5.0+incompatible
 	github.com/go-ole/go-ole v1.2.4 // indirect
 	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
diff --git a/go.sum b/go.sum
index 40b885f371..77a6bfb595 100644
--- a/go.sum
+++ b/go.sum
@@ -172,6 +172,7 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME
 github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
 github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
+github.com/go-acme/lego v2.5.0+incompatible h1:5fNN9yRQfv8ymH3DSsxla+4aYeQt2IgfZqHKVnK8f0s=
 github.com/go-acme/lego v2.5.0+incompatible/go.mod h1:yzMNe9CasVUhkquNvti5nAtPmG94USbYxYrZfTkIn0M=
 github.com/go-bindata/go-bindata v3.1.1+incompatible/go.mod h1:xK8Dsgwmeed+BBsSy2XTopBn/8uK2HWuGSnA11C3Joo=
 github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA=
diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go
index c461d2c3d6..cfc6b33f33 100644
--- a/pkg/drivers/kic/kic.go
+++ b/pkg/drivers/kic/kic.go
@@ -24,6 +24,7 @@ import (
 	"strings"
 
 	"github.com/docker/machine/libmachine/drivers"
+	"github.com/docker/machine/libmachine/log"
 	"github.com/docker/machine/libmachine/ssh"
 	"github.com/docker/machine/libmachine/state"
 	"github.com/golang/glog"
@@ -183,7 +184,7 @@ func (d *Driver) GetState() (state.State, error) {
 	out, err := cmd.CombinedOutput()
 	o := strings.Trim(string(out), "\n")
 	if err != nil {
-		return state.Error, errors.Wrapf(err, "error stop node %s", d.MachineName)
+		return state.Error, errors.Wrapf(err, "get container %s status", d.MachineName)
 	}
 	switch o {
 	case "running":
@@ -212,12 +213,17 @@ func (d *Driver) Kill() error {
 
 // Remove will delete the Kic Node Container
 func (d *Driver) Remove() error {
-	if _, err := d.nodeID(d.MachineName); err != nil {
-		return errors.Wrapf(err, "not found node %s", d.MachineName)
+	if _, err := oci.ContainerID(d.OCIBinary, d.MachineName); err != nil {
+		log.Warnf("could not find the container %s to remove it.", d.MachineName)
 	}
 	cmd := exec.Command(d.NodeConfig.OCIBinary, "rm", "-f", "-v", d.MachineName)
-	if err := cmd.Run(); err != nil {
-		return errors.Wrapf(err, "error removing node %s", d.MachineName)
+	o, err := cmd.CombinedOutput()
+	out := strings.Trim(string(o), "\n")
+	if err != nil {
+		if strings.Contains(out, "is already in progress") {
+			log.Warnf("Docker engine is stuck. please restart docker daemon on your computer.", d.MachineName)
+		}
+		return errors.Wrapf(err, "removing container %s, output %s", d.MachineName, out)
 	}
 	return nil
 }
@@ -286,13 +292,3 @@ func (d *Driver) Stop() error {
 func (d *Driver) RunSSHCommandFromDriver() error {
 	return fmt.Errorf("driver does not support RunSSHCommandFromDriver commands")
 }
-
-// looks up for a container node by name, will return error if not found.
-func (d *Driver) nodeID(nameOrID string) (string, error) {
-	cmd := exec.Command(d.NodeConfig.OCIBinary, "inspect", "-f", "{{.Id}}", nameOrID)
-	id, err := cmd.CombinedOutput()
-	if err != nil {
-		id = []byte{}
-	}
-	return string(id), err
-}
diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go
index 29805c1e9c..1ea1f53c3c 100644
--- a/pkg/drivers/kic/oci/oci.go
+++ b/pkg/drivers/kic/oci/oci.go
@@ -171,6 +171,16 @@ func ContainerIPs(ociBinary string, name string) (string, string, error) {
 
 }
 
+// ContainerID returns id of a container name
+func ContainerID(ociBinary string, nameOrID string) (string, error) {
+	cmd := exec.Command(ociBinary, "inspect", "-f", "{{.Id}}", nameOrID)
+	id, err := cmd.CombinedOutput()
+	if err != nil {
+		id = []byte{}
+	}
+	return string(id), err
+}
+
 // ListOwnedContainers lists all the containres that kic driver created on user's machine using a label
 func ListOwnedContainers(ociBinary string) ([]string, error) {
 	return listContainersByLabel(ociBinary, ClusterLabelKey)

From b46cdeca654b4ca5f0225adef3235e9d152d37f0 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Thu, 30 Jan 2020 21:39:41 -0800
Subject: [PATCH 169/250] gomod tidy

---
 go.mod | 1 -
 go.sum | 1 -
 2 files changed, 2 deletions(-)

diff --git a/go.mod b/go.mod
index 09e880afb3..c31b89c1c9 100644
--- a/go.mod
+++ b/go.mod
@@ -18,7 +18,6 @@ require (
 	github.com/elazarl/goproxy v0.0.0-20190421051319-9d40249d3c2f
 	github.com/elazarl/goproxy/ext v0.0.0-20190421051319-9d40249d3c2f // indirect
 	github.com/evanphx/json-patch v4.5.0+incompatible // indirect
-	github.com/go-acme/lego v2.5.0+incompatible
 	github.com/go-ole/go-ole v1.2.4 // indirect
 	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
diff --git a/go.sum b/go.sum
index 77a6bfb595..40b885f371 100644
--- a/go.sum
+++ b/go.sum
@@ -172,7 +172,6 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME
 github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
 github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
-github.com/go-acme/lego v2.5.0+incompatible h1:5fNN9yRQfv8ymH3DSsxla+4aYeQt2IgfZqHKVnK8f0s=
 github.com/go-acme/lego v2.5.0+incompatible/go.mod h1:yzMNe9CasVUhkquNvti5nAtPmG94USbYxYrZfTkIn0M=
 github.com/go-bindata/go-bindata v3.1.1+incompatible/go.mod h1:xK8Dsgwmeed+BBsSy2XTopBn/8uK2HWuGSnA11C3Joo=
 github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA=

From 6cd3b41bc86476f175ae9cbe96c8a3ba4db1af11 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Fri, 31 Jan 2020 11:43:18 -0800
Subject: [PATCH 170/250] Fix spacing in vm driver selection log

---
 cmd/minikube/cmd/start.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 4ccd39cf2f..3d1f1dfa07 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -596,14 +596,14 @@ func selectDriver(existing *config.MachineConfig) string {
 		if pick.Priority == registry.Experimental {
 			exp = "experimental"
 		}
-		out.T(out.Sparkle, `Selecting {{.experimental}} '{{.driver}}' driver from existing profile (alternates: {{.alternates}})`, out.V{"experimental": exp, "driver": existing.VMDriver, "alternates": alts})
+		out.T(out.Sparkle, `Selecting {{.experimental}}'{{.driver}}' driver from existing profile (alternates: {{.alternates}})`, out.V{"experimental": exp, "driver": existing.VMDriver, "alternates": alts})
 		return pick.Name
 	}
 
 	if len(options) > 1 {
-		out.T(out.Sparkle, `Automatically selected the {{.experimental}} '{{.driver}}' driver (alternates: {{.alternates}})`, out.V{"experimental": exp, "driver": pick.Name, "alternates": alts})
+		out.T(out.Sparkle, `Automatically selected the {{.experimental}}'{{.driver}}' driver (alternates: {{.alternates}})`, out.V{"experimental": exp, "driver": pick.Name, "alternates": alts})
 	} else {
-		out.T(out.Sparkle, `Automatically selected the {{.experimental}} '{{.driver}}' driver`, out.V{"experimental": exp, "driver": pick.Name})
+		out.T(out.Sparkle, `Automatically selected the {{.experimental}}'{{.driver}}' driver`, out.V{"experimental": exp, "driver": pick.Name})
 	}
 
 	if pick.Name == "" {

From 78ed12265d3eca947e1280afe6dd46faedb53345 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Fri, 31 Jan 2020 12:19:25 -0800
Subject: [PATCH 171/250] Add testing for Start function, make addon enablement
 testable

---
 cmd/minikube/cmd/config/addons_list.go |   4 +-
 cmd/minikube/cmd/config/open.go        |   7 +-
 cmd/minikube/cmd/dashboard.go          |   7 +-
 cmd/minikube/cmd/service.go            |   5 +-
 pkg/addons/addons.go                   |  39 ++++---
 pkg/addons/addons_test.go              | 144 ++++++++++++++-----------
 pkg/minikube/assets/addons.go          |   9 +-
 pkg/minikube/cluster/cluster.go        |  14 +--
 8 files changed, 131 insertions(+), 98 deletions(-)

diff --git a/cmd/minikube/cmd/config/addons_list.go b/cmd/minikube/cmd/config/addons_list.go
index c73c914c96..8737fe2224 100644
--- a/cmd/minikube/cmd/config/addons_list.go
+++ b/cmd/minikube/cmd/config/addons_list.go
@@ -102,7 +102,7 @@ var printAddonsList = func() {
 
 	for _, addonName := range addonNames {
 		addonBundle := assets.Addons[addonName]
-		addonStatus, err := addonBundle.IsEnabled()
+		addonStatus, err := addonBundle.IsEnabled(pName)
 		if err != nil {
 			out.WarningT("Unable to get addon status for {{.name}}: {{.error}}", out.V{"name": addonName, "error": err})
 		}
@@ -134,7 +134,7 @@ var printAddonsJSON = func() {
 	for _, addonName := range addonNames {
 		addonBundle := assets.Addons[addonName]
 
-		addonStatus, err := addonBundle.IsEnabled()
+		addonStatus, err := addonBundle.IsEnabled(pName)
 		if err != nil {
 			glog.Errorf("Unable to get addon status for %s: %v", addonName, err)
 			continue
diff --git a/cmd/minikube/cmd/config/open.go b/cmd/minikube/cmd/config/open.go
index 458950b917..1e53de657f 100644
--- a/cmd/minikube/cmd/config/open.go
+++ b/cmd/minikube/cmd/config/open.go
@@ -24,12 +24,14 @@ import (
 	"github.com/pkg/browser"
 
 	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
 	"k8s.io/minikube/pkg/minikube/assets"
 	"k8s.io/minikube/pkg/minikube/cluster"
 	"k8s.io/minikube/pkg/minikube/exit"
 	"k8s.io/minikube/pkg/minikube/machine"
 	"k8s.io/minikube/pkg/minikube/out"
 	"k8s.io/minikube/pkg/minikube/service"
+	pkg_config "k8s.io/minikube/pkg/minikube/config"
 )
 
 var (
@@ -66,7 +68,8 @@ var addonsOpenCmd = &cobra.Command{
 		}
 		defer api.Close()
 
-		if !cluster.IsMinikubeRunning(api) {
+		profileName := viper.GetString(pkg_config.MachineProfile)
+		if !cluster.IsHostRunning(api, profileName) {
 			os.Exit(1)
 		}
 		addon, ok := assets.Addons[addonName] // validate addon input
@@ -75,7 +78,7 @@ var addonsOpenCmd = &cobra.Command{
 To see the list of available addons run:
 minikube addons list`, out.V{"name": addonName})
 		}
-		ok, err = addon.IsEnabled()
+		ok, err = addon.IsEnabled(profileName)
 		if err != nil {
 			exit.WithError("IsEnabled failed", err)
 		}
diff --git a/cmd/minikube/cmd/dashboard.go b/cmd/minikube/cmd/dashboard.go
index 8b2e8e2017..4c465e265e 100644
--- a/cmd/minikube/cmd/dashboard.go
+++ b/cmd/minikube/cmd/dashboard.go
@@ -36,7 +36,6 @@ import (
 	pkgaddons "k8s.io/minikube/pkg/addons"
 	"k8s.io/minikube/pkg/minikube/assets"
 	"k8s.io/minikube/pkg/minikube/cluster"
-	"k8s.io/minikube/pkg/minikube/config"
 	pkg_config "k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/exit"
 	"k8s.io/minikube/pkg/minikube/machine"
@@ -103,18 +102,18 @@ var dashboardCmd = &cobra.Command{
 			exit.WithCodeT(exit.NoInput, "kubectl not found in PATH, but is required for the dashboard. Installation guide: https://kubernetes.io/docs/tasks/tools/install-kubectl/")
 		}
 
-		if !cluster.IsMinikubeRunning(api) {
+		if !cluster.IsHostRunning(api, profileName) {
 			os.Exit(1)
 		}
 
 		// Check dashboard status before enabling it
 		dashboardAddon := assets.Addons["dashboard"]
-		dashboardStatus, _ := dashboardAddon.IsEnabled()
+		dashboardStatus, _ := dashboardAddon.IsEnabled(profileName)
 		if !dashboardStatus {
 			// Send status messages to stderr for folks re-using this output.
 			out.ErrT(out.Enabling, "Enabling dashboard ...")
 			// Enable the dashboard add-on
-			err = pkgaddons.Set("dashboard", "true", viper.GetString(config.MachineProfile))
+			err = pkgaddons.Set("dashboard", "true", profileName)
 			if err != nil {
 				exit.WithError("Unable to enable dashboard", err)
 			}
diff --git a/cmd/minikube/cmd/service.go b/cmd/minikube/cmd/service.go
index 8d844deb06..bcb11759d0 100644
--- a/cmd/minikube/cmd/service.go
+++ b/cmd/minikube/cmd/service.go
@@ -25,8 +25,10 @@ import (
 	"github.com/golang/glog"
 	"github.com/pkg/browser"
 	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
 
 	"k8s.io/minikube/pkg/minikube/cluster"
+	pkg_config "k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/exit"
 	"k8s.io/minikube/pkg/minikube/machine"
 	"k8s.io/minikube/pkg/minikube/out"
@@ -71,7 +73,8 @@ var serviceCmd = &cobra.Command{
 		}
 		defer api.Close()
 
-		if !cluster.IsMinikubeRunning(api) {
+		profileName := viper.GetString(pkg_config.MachineProfile)
+		if !cluster.IsHostRunning(api, profileName) {
 			os.Exit(1)
 		}
 
diff --git a/pkg/addons/addons.go b/pkg/addons/addons.go
index 2070da2ca2..29a04fa51b 100644
--- a/pkg/addons/addons.go
+++ b/pkg/addons/addons.go
@@ -44,7 +44,6 @@ const defaultStorageClassProvisioner = "standard"
 // Set sets a value
 func Set(name, value, profile string) error {
 	glog.Infof("Setting %s=%s in profile %q", name, value, profile)
-
 	a, valid := isAddonValid(name)
 	if !valid {
 		return errors.Errorf("%s is not a valid addon", name)
@@ -70,6 +69,7 @@ func Set(name, value, profile string) error {
 		return errors.Wrap(err, "running callbacks")
 	}
 
+	glog.Infof("Writing new config for %q ...", profile)
 	// Write the value
 	return config.Write(profile, c)
 }
@@ -112,7 +112,7 @@ func enableOrDisableAddon(name, val, profile string) error {
 	addon := assets.Addons[name]
 
 	// check addon status before enabling/disabling it
-	alreadySet, err := isAddonAlreadySet(addon, enable)
+	alreadySet, err := isAddonAlreadySet(addon, enable, profile)
 	if err != nil {
 		out.ErrT(out.Conflict, "{{.error}}", out.V{"error": err})
 		return err
@@ -139,21 +139,15 @@ func enableOrDisableAddon(name, val, profile string) error {
 	}
 	defer api.Close()
 
-	//if minikube is not running, we return and simply update the value in the addon
-	//config and rewrite the file
-	if !cluster.IsMinikubeRunning(api) {
-		glog.Warningf("minikube is not running, writing to disk and doing nothing")
-		return nil
-	}
-
 	cfg, err := config.Load(profile)
 	if err != nil && !os.IsNotExist(err) {
 		exit.WithCodeT(exit.Data, "Unable to load config: {{.error}}", out.V{"error": err})
 	}
 
-	host, err := cluster.CheckIfHostExistsAndLoad(api, cfg.Name)
-	if err != nil {
-		return errors.Wrap(err, "getting host")
+	host, err := cluster.CheckIfHostExistsAndLoad(api, profile)
+	if err != nil || !cluster.IsHostRunning(api, profile) {
+		glog.Warningf("%q is not running, writing to %s=%v disk and skipping the rest (err=%v)", profile, addon.Name(), enable, err)
+		return nil
 	}
 
 	cmd, err := machine.CommandRunner(host)
@@ -165,11 +159,10 @@ func enableOrDisableAddon(name, val, profile string) error {
 	return enableOrDisableAddonInternal(addon, cmd, data, enable, profile)
 }
 
-func isAddonAlreadySet(addon *assets.Addon, enable bool) (bool, error) {
-	addonStatus, err := addon.IsEnabled()
-
+func isAddonAlreadySet(addon *assets.Addon, enable bool, profile string) (bool, error) {
+	addonStatus, err := addon.IsEnabled(profile)
 	if err != nil {
-		return false, errors.Wrap(err, "addon is enabled")
+		return false, errors.Wrap(err, "is enabled")
 	}
 
 	if addonStatus && enable {
@@ -227,6 +220,7 @@ func enableOrDisableAddonInternal(addon *assets.Addon, cmd command.Runner, data
 
 // enableOrDisableStorageClasses enables or disables storage classes
 func enableOrDisableStorageClasses(name, val, profile string) error {
+	glog.Infof("enableOrDisableStorageClasses %s=%v on %q", name, val, profile)
 	enable, err := strconv.ParseBool(val)
 	if err != nil {
 		return errors.Wrap(err, "Error parsing boolean")
@@ -241,6 +235,17 @@ func enableOrDisableStorageClasses(name, val, profile string) error {
 		return errors.Wrapf(err, "Error getting storagev1 interface %v ", err)
 	}
 
+	api, err := machine.NewAPIClient()
+	if err != nil {
+		return errors.Wrap(err, "machine client")
+	}
+	defer api.Close()
+
+	if !cluster.IsHostRunning(api, profile) {
+		glog.Warningf("%q is not running, writing to %s=%v disk and skipping enablement", profile, name, val)
+		return enableOrDisableAddon(name, val, profile)
+	}
+
 	if enable {
 		// Only StorageClass for 'name' should be marked as default
 		err = storageclass.SetDefaultStorageClass(storagev1, class)
@@ -269,7 +274,7 @@ func Start(profile string, additional []string) {
 
 	// Apply addons that are enabled by default
 	for name, a := range assets.Addons {
-		enabled, err := a.IsEnabled()
+		enabled, err := a.IsEnabled(profile)
 		if err != nil {
 			glog.Errorf("is-enabled failed for %q: %v", a.Name(), err)
 			continue
diff --git a/pkg/addons/addons_test.go b/pkg/addons/addons_test.go
index 2e7a6bed07..e0337705c1 100644
--- a/pkg/addons/addons_test.go
+++ b/pkg/addons/addons_test.go
@@ -17,93 +17,115 @@ limitations under the License.
 package addons
 
 import (
-	"fmt"
+	"io/ioutil"
 	"os"
+	"path/filepath"
 	"testing"
 
-	"gotest.tools/assert"
 	"k8s.io/minikube/pkg/minikube/assets"
 	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/localpath"
 )
 
+func createTestProfile(t *testing.T) string {
+	t.Helper()
+	td, err := ioutil.TempDir("", "profile")
+	if err != nil {
+		t.Fatalf("tempdir: %v", err)
+	}
+
+	err = os.Setenv(localpath.MinikubeHome, td)
+	if err != nil {
+		t.Errorf("error setting up test environment. could not set %s", localpath.MinikubeHome)
+	}
+
+	// Not necessary, but it is a handy random alphanumeric
+	name := filepath.Base(td)
+	if err := os.MkdirAll(config.ProfileFolderPath(name), 0777); err != nil {
+		t.Fatalf("error creating temporary directory")
+	}
+	if err := config.DefaultLoader.WriteConfigToFile(name, &config.MachineConfig{}); err != nil {
+		t.Fatalf("error creating temporary profile config: %v", err)
+	}
+	return name
+}
+
 func TestIsAddonAlreadySet(t *testing.T) {
-	testCases := []struct {
-		addonName string
-	}{
-		{
-			addonName: "ingress",
-		},
-
-		{
-			addonName: "registry",
-		},
+	profile := createTestProfile(t)
+	if err := Set("registry", "true", profile); err != nil {
+		t.Errorf("unable to set registry true: %v", err)
+	}
+	enabled, err := assets.Addons["registry"].IsEnabled(profile)
+	if err != nil {
+		t.Errorf("registry: %v", err)
+	}
+	if !enabled {
+		t.Errorf("expected registry to be enabled")
 	}
 
-	for _, test := range testCases {
-		addon := assets.Addons[test.addonName]
-		addonStatus, _ := addon.IsEnabled()
-
-		alreadySet, err := isAddonAlreadySet(addon, addonStatus)
-		if !alreadySet {
-			if addonStatus {
-				t.Errorf("Did not get expected status, \n\n expected %+v already enabled", test.addonName)
-			} else {
-				t.Errorf("Did not get expected status, \n\n expected %+v already disabled", test.addonName)
-			}
-		}
-		if err != nil {
-			t.Errorf("Got unexpected error: %+v", err)
-		}
+	enabled, err = assets.Addons["ingress"].IsEnabled(profile)
+	if err != nil {
+		t.Errorf("ingress: %v", err)
 	}
+	if enabled {
+		t.Errorf("expected ingress to not be enabled")
+	}
+
 }
 
 func TestDisableUnknownAddon(t *testing.T) {
-	tmpProfile := "temp-minikube-profile"
-	if err := Set("InvalidAddon", "false", tmpProfile); err == nil {
+	profile := createTestProfile(t)
+	if err := Set("InvalidAddon", "false", profile); err == nil {
 		t.Fatalf("Disable did not return error for unknown addon")
 	}
 }
 
 func TestEnableUnknownAddon(t *testing.T) {
-	tmpProfile := "temp-minikube-profile"
-	if err := Set("InvalidAddon", "true", tmpProfile); err == nil {
+	profile := createTestProfile(t)
+	if err := Set("InvalidAddon", "true", profile); err == nil {
 		t.Fatalf("Enable did not return error for unknown addon")
 	}
 }
 
 func TestEnableAndDisableAddon(t *testing.T) {
-	tests := []struct {
-		name   string
-		enable bool
-	}{
-		{
-			name:   "test enable",
-			enable: true,
-		}, {
-			name:   "test disable",
-			enable: false,
-		},
+	profile := createTestProfile(t)
+
+	// enable
+	if err := Set("dashboard", "true", profile); err != nil {
+		t.Errorf("Disable returned unexpected error: " + err.Error())
 	}
 
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			tmpProfile := "temp-minikube-profile"
-			if err := os.MkdirAll(config.ProfileFolderPath(tmpProfile), 0777); err != nil {
-				t.Fatalf("error creating temporary directory")
-			}
-			defer os.RemoveAll(config.ProfileFolderPath(tmpProfile))
+	c, err := config.DefaultLoader.LoadConfigFromFile(profile)
+	if err != nil {
+		t.Errorf("unable to load profile: %v", err)
+	}
+	if c.Addons["dashboard"] != true {
+		t.Errorf("expected dashboard to be enabled")
+	}
 
-			if err := config.DefaultLoader.WriteConfigToFile(tmpProfile, &config.MachineConfig{}); err != nil {
-				t.Fatalf("error creating temporary profile config: %v", err)
-			}
-			if err := Set("dashboard", fmt.Sprintf("%t", test.enable), tmpProfile); err != nil {
-				t.Fatalf("Disable returned unexpected error: " + err.Error())
-			}
-			c, err := config.DefaultLoader.LoadConfigFromFile(tmpProfile)
-			if err != nil {
-				t.Fatalf("error loading config: %v", err)
-			}
-			assert.Equal(t, c.Addons["dashboard"], test.enable)
-		})
+	// disable
+	if err := Set("dashboard", "false", profile); err != nil {
+		t.Errorf("Disable returned unexpected error: " + err.Error())
+	}
+
+	c, err = config.DefaultLoader.LoadConfigFromFile(profile)
+	if err != nil {
+		t.Errorf("unable to load profile: %v", err)
+	}
+	if c.Addons["dashboard"] != false {
+		t.Errorf("expected dashboard to be enabled")
+	}
+}
+
+func TestStart(t *testing.T) {
+	profile := createTestProfile(t)
+	Start(profile, []string{"dashboard"})
+
+	enabled, err := assets.Addons["dashboard"].IsEnabled(profile)
+	if err != nil {
+		t.Errorf("dashboard: %v", err)
+	}
+	if !enabled {
+		t.Errorf("expected dashboard to be enabled")
 	}
 }
diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go
index 14959a28ca..3bdcb758aa 100644
--- a/pkg/minikube/assets/addons.go
+++ b/pkg/minikube/assets/addons.go
@@ -25,7 +25,6 @@ import (
 
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
-	"github.com/spf13/viper"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/localpath"
@@ -55,16 +54,16 @@ func (a *Addon) Name() string {
 	return a.addonName
 }
 
-// IsEnabled checks if an Addon is enabled for the current profile
-func (a *Addon) IsEnabled() (bool, error) {
-	c, err := config.Load(viper.GetString(config.MachineProfile))
+// IsEnabled checks if an Addon is enabled for the given profile
+func (a *Addon) IsEnabled(profile string) (bool, error) {
+	c, err := config.Load(profile)
 	if err != nil {
 		return false, errors.Wrap(err, "load")
 	}
 
 	// Is this addon explicitly listed in their configuration?
 	status, ok := c.Addons[a.Name()]
-	glog.Infof("IsEnabled %q = %v (listed in config=%v)", a.Name(), status, ok)
+	glog.V(1).Infof("IsEnabled %q = %v (listed in config=%v)", a.Name(), status, ok)
 	if ok {
 		return status, nil
 	}
diff --git a/pkg/minikube/cluster/cluster.go b/pkg/minikube/cluster/cluster.go
index af3f6cc184..16f934cec1 100644
--- a/pkg/minikube/cluster/cluster.go
+++ b/pkg/minikube/cluster/cluster.go
@@ -627,17 +627,18 @@ func getIPForInterface(name string) (net.IP, error) {
 
 // CheckIfHostExistsAndLoad checks if a host exists, and loads it if it does
 func CheckIfHostExistsAndLoad(api libmachine.API, machineName string) (*host.Host, error) {
+	glog.Infof("Checking if %q exists ...", machineName)
 	exists, err := api.Exists(machineName)
 	if err != nil {
 		return nil, errors.Wrapf(err, "Error checking that machine exists: %s", machineName)
 	}
 	if !exists {
-		return nil, errors.Errorf("Machine does not exist for api.Exists(%s)", machineName)
+		return nil, errors.Errorf("machine %q does not exist", machineName)
 	}
 
 	host, err := api.Load(machineName)
 	if err != nil {
-		return nil, errors.Wrapf(err, "Error loading store for: %s", machineName)
+		return nil, errors.Wrapf(err, "loading machine %q", machineName)
 	}
 	return host, nil
 }
@@ -666,14 +667,15 @@ func CreateSSHShell(api libmachine.API, args []string) error {
 	return client.Shell(args...)
 }
 
-// IsMinikubeRunning checks that minikube has a status available and that
-// the status is `Running`
-func IsMinikubeRunning(api libmachine.API) bool {
-	s, err := GetHostStatus(api, viper.GetString(config.MachineProfile))
+// IsHostRunning asserts that this profile's primary host is in state "Running"
+func IsHostRunning(api libmachine.API, name string) bool {
+	s, err := GetHostStatus(api, name)
 	if err != nil {
+		glog.Warningf("host status for %q returned error: %v", name, err)
 		return false
 	}
 	if s != state.Running.String() {
+		glog.Warningf("%q host status: %s", name, s)
 		return false
 	}
 	return true

From 26cbae2d056302b9be6525ee7c27ad850b378efb Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Fri, 31 Jan 2020 12:36:28 -0800
Subject: [PATCH 172/250] Improve error message when profile does not exist

---
 pkg/minikube/config/config.go | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/pkg/minikube/config/config.go b/pkg/minikube/config/config.go
index 7abd2a8d81..6b32bf6028 100644
--- a/pkg/minikube/config/config.go
+++ b/pkg/minikube/config/config.go
@@ -18,8 +18,8 @@ package config
 
 import (
 	"encoding/json"
-	"errors"
 	"fmt"
+	"github.com/pkg/errors"
 	"io"
 	"io/ioutil"
 	"os"
@@ -148,17 +148,20 @@ func (c *simpleConfigLoader) LoadConfigFromFile(profileName string, miniHome ...
 	// Move to profile package
 	path := profileFilePath(profileName, miniHome...)
 
-	if _, err := os.Stat(path); os.IsNotExist(err) {
-		return nil, err
+	if _, err := os.Stat(path); err != nil {
+		if os.IsNotExist(err) {
+			return nil, fmt.Errorf("cluster %q does not exist", profileName)
+		}
+		return nil, errors.Wrap(err, "stat")
 	}
 
 	data, err := ioutil.ReadFile(path)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "read")
 	}
 
 	if err := json.Unmarshal(data, &cc); err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "unmarshal")
 	}
 	return &cc, nil
 }

From f6740b37be7d0af8ee8f89473b37780678b76f1d Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Fri, 31 Jan 2020 12:37:23 -0800
Subject: [PATCH 173/250] Run gofmt/goimports

---
 pkg/minikube/bootstrapper/bsutil/kubeadm_test.go |  6 +++---
 pkg/minikube/bootstrapper/bsutil/kubelet_test.go | 10 +++++-----
 pkg/minikube/config/config.go                    |  3 ++-
 3 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/pkg/minikube/bootstrapper/bsutil/kubeadm_test.go b/pkg/minikube/bootstrapper/bsutil/kubeadm_test.go
index a5aef997bd..4a94b9dc1e 100644
--- a/pkg/minikube/bootstrapper/bsutil/kubeadm_test.go
+++ b/pkg/minikube/bootstrapper/bsutil/kubeadm_test.go
@@ -120,7 +120,7 @@ func TestGenerateKubeadmYAMLDNS(t *testing.T) {
 			t.Run(tname, func(t *testing.T) {
 				cfg := tc.cfg
 				cfg.Nodes = []config.Node{
-					config.Node{
+					{
 						IP:           "1.1.1.1",
 						Name:         "mk",
 						ControlPlane: true,
@@ -179,7 +179,7 @@ func TestGenerateKubeadmYAML(t *testing.T) {
 		{"options", "docker", false, config.MachineConfig{KubernetesConfig: config.KubernetesConfig{ExtraOptions: extraOpts}}},
 		{"crio-options-gates", "crio", false, config.MachineConfig{KubernetesConfig: config.KubernetesConfig{ExtraOptions: extraOpts, FeatureGates: "a=b"}}},
 		{"unknown-component", "docker", true, config.MachineConfig{KubernetesConfig: config.KubernetesConfig{ExtraOptions: config.ExtraOptionSlice{config.ExtraOption{Component: "not-a-real-component", Key: "killswitch", Value: "true"}}}}},
-		{"containerd-api-port", "containerd", false, config.MachineConfig{Nodes: []config.Node{config.Node{Port: 12345}}}},
+		{"containerd-api-port", "containerd", false, config.MachineConfig{Nodes: []config.Node{{Port: 12345}}}},
 		{"containerd-pod-network-cidr", "containerd", false, config.MachineConfig{KubernetesConfig: config.KubernetesConfig{ExtraOptions: extraOptsPodCidr}}},
 		{"image-repository", "docker", false, config.MachineConfig{KubernetesConfig: config.KubernetesConfig{ImageRepository: "test/repo"}}},
 	}
@@ -199,7 +199,7 @@ func TestGenerateKubeadmYAML(t *testing.T) {
 					cfg.Nodes[0].ControlPlane = true
 				} else {
 					cfg.Nodes = []config.Node{
-						config.Node{
+						{
 							IP:           "1.1.1.1",
 							Name:         "mk",
 							ControlPlane: true,
diff --git a/pkg/minikube/bootstrapper/bsutil/kubelet_test.go b/pkg/minikube/bootstrapper/bsutil/kubelet_test.go
index 8b49bc653b..b50a4b89ef 100644
--- a/pkg/minikube/bootstrapper/bsutil/kubelet_test.go
+++ b/pkg/minikube/bootstrapper/bsutil/kubelet_test.go
@@ -42,7 +42,7 @@ func TestGenerateKubeletConfig(t *testing.T) {
 					ContainerRuntime:  "docker",
 				},
 				Nodes: []config.Node{
-					config.Node{
+					{
 						IP:           "192.168.1.100",
 						Name:         "minikube",
 						ControlPlane: true,
@@ -67,7 +67,7 @@ ExecStart=/var/lib/minikube/binaries/v1.11.10/kubelet --allow-privileged=true --
 					ContainerRuntime:  "cri-o",
 				},
 				Nodes: []config.Node{
-					config.Node{
+					{
 						IP:           "192.168.1.100",
 						Name:         "minikube",
 						ControlPlane: true,
@@ -92,7 +92,7 @@ ExecStart=/var/lib/minikube/binaries/v1.17.2/kubelet --authorization-mode=Webhoo
 					ContainerRuntime:  "containerd",
 				},
 				Nodes: []config.Node{
-					config.Node{
+					{
 						IP:           "192.168.1.100",
 						Name:         "minikube",
 						ControlPlane: true,
@@ -124,7 +124,7 @@ ExecStart=/var/lib/minikube/binaries/v1.17.2/kubelet --authorization-mode=Webhoo
 					},
 				},
 				Nodes: []config.Node{
-					config.Node{
+					{
 						IP:           "192.168.1.100",
 						Name:         "minikube",
 						ControlPlane: true,
@@ -150,7 +150,7 @@ ExecStart=/var/lib/minikube/binaries/v1.17.2/kubelet --authorization-mode=Webhoo
 					ImageRepository:   "docker-proxy-image.io/google_containers",
 				},
 				Nodes: []config.Node{
-					config.Node{
+					{
 						IP:           "192.168.1.100",
 						Name:         "minikube",
 						ControlPlane: true,
diff --git a/pkg/minikube/config/config.go b/pkg/minikube/config/config.go
index 6b32bf6028..134c969721 100644
--- a/pkg/minikube/config/config.go
+++ b/pkg/minikube/config/config.go
@@ -19,11 +19,12 @@ package config
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/pkg/errors"
 	"io"
 	"io/ioutil"
 	"os"
 
+	"github.com/pkg/errors"
+
 	"k8s.io/minikube/pkg/minikube/localpath"
 )
 

From c8f567998677f6ab6d7a7c0efb4ab05213e133e1 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Fri, 31 Jan 2020 12:38:06 -0800
Subject: [PATCH 174/250] Run gofmt/goimports on cmd/

---
 cmd/minikube/cmd/config/open.go | 2 +-
 cmd/minikube/cmd/start.go       | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/cmd/minikube/cmd/config/open.go b/cmd/minikube/cmd/config/open.go
index 1e53de657f..a65250810d 100644
--- a/cmd/minikube/cmd/config/open.go
+++ b/cmd/minikube/cmd/config/open.go
@@ -27,11 +27,11 @@ import (
 	"github.com/spf13/viper"
 	"k8s.io/minikube/pkg/minikube/assets"
 	"k8s.io/minikube/pkg/minikube/cluster"
+	pkg_config "k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/exit"
 	"k8s.io/minikube/pkg/minikube/machine"
 	"k8s.io/minikube/pkg/minikube/out"
 	"k8s.io/minikube/pkg/minikube/service"
-	pkg_config "k8s.io/minikube/pkg/minikube/config"
 )
 
 var (
diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 12d403df05..c7246f4394 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -46,6 +46,7 @@ import (
 	"github.com/spf13/viper"
 	"golang.org/x/sync/errgroup"
 	cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config"
+	"k8s.io/minikube/pkg/addons"
 	"k8s.io/minikube/pkg/minikube/bootstrapper"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
@@ -56,7 +57,6 @@ import (
 	"k8s.io/minikube/pkg/minikube/cruntime"
 	"k8s.io/minikube/pkg/minikube/driver"
 	"k8s.io/minikube/pkg/minikube/exit"
-	"k8s.io/minikube/pkg/addons"
 	"k8s.io/minikube/pkg/minikube/kubeconfig"
 	"k8s.io/minikube/pkg/minikube/localpath"
 	"k8s.io/minikube/pkg/minikube/logs"
@@ -171,7 +171,7 @@ func initMinikubeFlags() {
 	startCmd.Flags().String(containerRuntime, "docker", "The container runtime to be used (docker, crio, containerd).")
 	startCmd.Flags().Bool(createMount, false, "This will start the mount daemon and automatically mount files into minikube.")
 	startCmd.Flags().String(mountString, constants.DefaultMountDir+":/minikube-host", "The argument to pass the minikube mount command on start.")
-	startCmd.Flags().StringArrayVar(&addonList,"addons", nil, "Enable addons. see `minikube addons list` for a list of valid addon names.")
+	startCmd.Flags().StringArrayVar(&addonList, "addons", nil, "Enable addons. see `minikube addons list` for a list of valid addon names.")
 	startCmd.Flags().String(criSocket, "", "The cri socket path to be used.")
 	startCmd.Flags().String(networkPlugin, "", "The name of the network plugin.")
 	startCmd.Flags().Bool(enableDefaultCNI, false, "Enable the default CNI plugin (/etc/cni/net.d/k8s.conf). Used in conjunction with \"--network-plugin=cni\".")

From 02362d8d9c8f55d3e79be1f067ec65026ddfa863 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Fri, 31 Jan 2020 12:55:56 -0800
Subject: [PATCH 175/250] Loosen error message matching for unit test

---
 go.mod                                        |  1 -
 pkg/minikube/tunnel/cluster_inspector_test.go | 10 +++++++---
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/go.mod b/go.mod
index c31b89c1c9..346a954e35 100644
--- a/go.mod
+++ b/go.mod
@@ -72,7 +72,6 @@ require (
 	golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456
 	golang.org/x/text v0.3.2
 	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
-	gotest.tools v2.2.0+incompatible
 	k8s.io/api v0.17.2
 	k8s.io/apimachinery v0.17.2
 	k8s.io/client-go v11.0.0+incompatible
diff --git a/pkg/minikube/tunnel/cluster_inspector_test.go b/pkg/minikube/tunnel/cluster_inspector_test.go
index 4c8332facd..c3be2e2483 100644
--- a/pkg/minikube/tunnel/cluster_inspector_test.go
+++ b/pkg/minikube/tunnel/cluster_inspector_test.go
@@ -40,10 +40,14 @@ func TestAPIError(t *testing.T) {
 		machineAPI, configLoader, machineName,
 	}
 
-	s, r, err := inspector.getStateAndRoute()
+	_, _, err := inspector.getStateAndRoute()
+	if err == nil {
+		t.Errorf("expected error, got nil")
+	}
 
-	if err == nil || !strings.Contains(err.Error(), "Machine does not exist") {
-		t.Errorf("cluster inspector should propagate errors from API, getStateAndRoute() returned \"%v, %v\", %v", s, r, err)
+	// Make sure we properly propagate errors upward
+	if !strings.Contains(err.Error(), "exist") {
+		t.Errorf("getStateAndRoute error=%q, expected *exist*", err)
 	}
 }
 

From c518f3c55d56832c0428505d8d11c77a538389d6 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Fri, 31 Jan 2020 12:58:15 -0800
Subject: [PATCH 176/250] trying to correctly extract preloaded images

---
 pkg/provision/buildroot.go | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/pkg/provision/buildroot.go b/pkg/provision/buildroot.go
index 41798fe3ae..1b81235676 100644
--- a/pkg/provision/buildroot.go
+++ b/pkg/provision/buildroot.go
@@ -19,6 +19,7 @@ package provision
 import (
 	"bytes"
 	"fmt"
+	"os"
 	"os/exec"
 	"path"
 	"path/filepath"
@@ -186,9 +187,29 @@ WantedBy=multi-user.target
 		return nil, err
 	}
 
-	if err := p.Service("docker", serviceaction.Restart); err != nil {
+	log.Info("Moving folder...")
+	if _, err = p.SSHCommand("sudo mv /var/lib/all_of_docker/docker.tar /docker.tar"); err != nil {
 		return nil, err
 	}
+
+	log.Info("Extracting docker images...")
+	if _, err = p.SSHCommand("sudo tar xvf /docker.tar"); err != nil {
+		return nil, err
+	}
+
+	if _, err = p.SSHCommand("sudo systemctl stop docker"); err != nil {
+		return nil, err
+	}
+
+	if _, err = p.SSHCommand("sudo systemctl start docker"); err != nil {
+		return nil, err
+	}
+
+	// if err := p.Service("docker", serviceaction.Start); err != nil {
+	// 	return nil, err
+	// }
+
+	os.Exit(1)
 	return dockerCfg, nil
 }
 

From d9860b1eb798862c76491839dd91a5d62fd37a03 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Fri, 31 Jan 2020 12:58:53 -0800
Subject: [PATCH 177/250] Update spacing

---
 cmd/minikube/cmd/start.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 3d1f1dfa07..01fb64cceb 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -594,7 +594,7 @@ func selectDriver(existing *config.MachineConfig) string {
 	if existing != nil {
 		pick, alts := driver.Choose(existing.VMDriver, options)
 		if pick.Priority == registry.Experimental {
-			exp = "experimental"
+			exp = "experimental "
 		}
 		out.T(out.Sparkle, `Selecting {{.experimental}}'{{.driver}}' driver from existing profile (alternates: {{.alternates}})`, out.V{"experimental": exp, "driver": existing.VMDriver, "alternates": alts})
 		return pick.Name

From e5b240cbbc253add2d34cfbd940a891f3cbbba20 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Fri, 31 Jan 2020 12:59:28 -0800
Subject: [PATCH 178/250] Fix buildroot

---
 pkg/provision/buildroot.go | 23 +----------------------
 1 file changed, 1 insertion(+), 22 deletions(-)

diff --git a/pkg/provision/buildroot.go b/pkg/provision/buildroot.go
index 1b81235676..41798fe3ae 100644
--- a/pkg/provision/buildroot.go
+++ b/pkg/provision/buildroot.go
@@ -19,7 +19,6 @@ package provision
 import (
 	"bytes"
 	"fmt"
-	"os"
 	"os/exec"
 	"path"
 	"path/filepath"
@@ -187,29 +186,9 @@ WantedBy=multi-user.target
 		return nil, err
 	}
 
-	log.Info("Moving folder...")
-	if _, err = p.SSHCommand("sudo mv /var/lib/all_of_docker/docker.tar /docker.tar"); err != nil {
+	if err := p.Service("docker", serviceaction.Restart); err != nil {
 		return nil, err
 	}
-
-	log.Info("Extracting docker images...")
-	if _, err = p.SSHCommand("sudo tar xvf /docker.tar"); err != nil {
-		return nil, err
-	}
-
-	if _, err = p.SSHCommand("sudo systemctl stop docker"); err != nil {
-		return nil, err
-	}
-
-	if _, err = p.SSHCommand("sudo systemctl start docker"); err != nil {
-		return nil, err
-	}
-
-	// if err := p.Service("docker", serviceaction.Start); err != nil {
-	// 	return nil, err
-	// }
-
-	os.Exit(1)
 	return dockerCfg, nil
 }
 

From a7ac0339fdf60feb7404d5eb0cbb7ed5a0334054 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Fri, 31 Jan 2020 13:24:30 -0800
Subject: [PATCH 179/250] Address eview feedback

---
 pkg/addons/addons.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkg/addons/addons.go b/pkg/addons/addons.go
index 29a04fa51b..29c58029fc 100644
--- a/pkg/addons/addons.go
+++ b/pkg/addons/addons.go
@@ -146,7 +146,7 @@ func enableOrDisableAddon(name, val, profile string) error {
 
 	host, err := cluster.CheckIfHostExistsAndLoad(api, profile)
 	if err != nil || !cluster.IsHostRunning(api, profile) {
-		glog.Warningf("%q is not running, writing to %s=%v disk and skipping the rest (err=%v)", profile, addon.Name(), enable, err)
+		glog.Warningf("%q is not running, writing %s=%v to disk and skipping enablement (err=%v)", profile, addon.Name(), enable, err)
 		return nil
 	}
 
@@ -242,7 +242,7 @@ func enableOrDisableStorageClasses(name, val, profile string) error {
 	defer api.Close()
 
 	if !cluster.IsHostRunning(api, profile) {
-		glog.Warningf("%q is not running, writing to %s=%v disk and skipping enablement", profile, name, val)
+		glog.Warningf("%q is not running, writing %s=%v to disk and skipping enablement", profile, name, val)
 		return enableOrDisableAddon(name, val, profile)
 	}
 

From c3aafaeeb4541b0cd5303afe211fe281dbf5e4f3 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Fri, 31 Jan 2020 14:19:08 -0800
Subject: [PATCH 180/250] Add IsNotExist() method to the config package, so
 that we can have meaningful error messages and make them comparable

---
 cmd/minikube/cmd/config/profile.go      |  3 ++-
 cmd/minikube/cmd/config/profile_list.go |  2 +-
 cmd/minikube/cmd/dashboard.go           |  2 +-
 cmd/minikube/cmd/delete.go              |  4 ++--
 cmd/minikube/cmd/kubectl.go             |  5 ++---
 cmd/minikube/cmd/pause.go               |  2 +-
 cmd/minikube/cmd/start.go               |  4 ++--
 cmd/minikube/cmd/unpause.go             |  2 +-
 pkg/addons/addons.go                    |  3 +--
 pkg/addons/kubectl.go                   |  3 +--
 pkg/minikube/config/config.go           | 19 ++++++++++++++++++-
 11 files changed, 32 insertions(+), 17 deletions(-)

diff --git a/cmd/minikube/cmd/config/profile.go b/cmd/minikube/cmd/config/profile.go
index 03eb016426..67203ebdbf 100644
--- a/cmd/minikube/cmd/config/profile.go
+++ b/cmd/minikube/cmd/config/profile.go
@@ -22,6 +22,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	pkgConfig "k8s.io/minikube/pkg/minikube/config"
+	pkg_config "k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/exit"
 	"k8s.io/minikube/pkg/minikube/kubeconfig"
@@ -78,7 +79,7 @@ var ProfileCmd = &cobra.Command{
 		}
 		cc, err := pkgConfig.Load(profile)
 		// might err when loading older version of cfg file that doesn't have KeepContext field
-		if err != nil && !os.IsNotExist(err) {
+		if err != nil && !pkg_config.IsNotExist(err) {
 			out.ErrT(out.Sad, `Error loading profile config: {{.error}}`, out.V{"error": err})
 		}
 		if err == nil {
diff --git a/cmd/minikube/cmd/config/profile_list.go b/cmd/minikube/cmd/config/profile_list.go
index 8df06f2d2d..d3b90721a8 100644
--- a/cmd/minikube/cmd/config/profile_list.go
+++ b/cmd/minikube/cmd/config/profile_list.go
@@ -141,7 +141,7 @@ var printProfilesJSON = func() {
 
 	var body = map[string]interface{}{}
 
-	if err == nil || os.IsNotExist(err) {
+	if err == nil || config.IsNotExist(err) {
 		body["valid"] = valid
 		body["invalid"] = invalid
 		jsonString, _ := json.Marshal(body)
diff --git a/cmd/minikube/cmd/dashboard.go b/cmd/minikube/cmd/dashboard.go
index 4c465e265e..f247b365d7 100644
--- a/cmd/minikube/cmd/dashboard.go
+++ b/cmd/minikube/cmd/dashboard.go
@@ -60,7 +60,7 @@ var dashboardCmd = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 		profileName := viper.GetString(pkg_config.MachineProfile)
 		cc, err := pkg_config.Load(profileName)
-		if err != nil && !os.IsNotExist(err) {
+		if err != nil && !pkg_config.IsNotExist(err) {
 			exit.WithError("Error loading profile config", err)
 		}
 
diff --git a/cmd/minikube/cmd/delete.go b/cmd/minikube/cmd/delete.go
index 5efe7e2ee3..73af19bd66 100644
--- a/cmd/minikube/cmd/delete.go
+++ b/cmd/minikube/cmd/delete.go
@@ -188,7 +188,7 @@ func deleteProfile(profile *pkg_config.Profile) error {
 	}
 	defer api.Close()
 	cc, err := pkg_config.Load(profile.Name)
-	if err != nil && !os.IsNotExist(err) {
+	if err != nil && !pkg_config.IsNotExist(err) {
 		delErr := profileDeletionErr(profile.Name, fmt.Sprintf("error loading profile config: %v", err))
 		return DeletionError{Err: delErr, Errtype: MissingProfile}
 	}
@@ -223,7 +223,7 @@ func deleteProfile(profile *pkg_config.Profile) error {
 	deleteProfileDirectory(profile.Name)
 
 	if err := pkg_config.DeleteProfile(profile.Name); err != nil {
-		if os.IsNotExist(err) {
+		if pkg_config.IsNotExist(err) {
 			delErr := profileDeletionErr(profile.Name, fmt.Sprintf("\"%s\" profile does not exist", profile.Name))
 			return DeletionError{Err: delErr, Errtype: MissingProfile}
 		}
diff --git a/cmd/minikube/cmd/kubectl.go b/cmd/minikube/cmd/kubectl.go
index 706a9703e2..0a5eaea659 100644
--- a/cmd/minikube/cmd/kubectl.go
+++ b/cmd/minikube/cmd/kubectl.go
@@ -27,7 +27,6 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"k8s.io/minikube/pkg/minikube/config"
-	pkg_config "k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/exit"
 	"k8s.io/minikube/pkg/minikube/machine"
@@ -51,8 +50,8 @@ minikube kubectl -- get pods --namespace kube-system`,
 		}
 		defer api.Close()
 
-		cc, err := pkg_config.Load(viper.GetString(config.MachineProfile))
-		if err != nil && !os.IsNotExist(err) {
+		cc, err := config.Load(viper.GetString(config.MachineProfile))
+		if err != nil && !config.IsNotExist(err) {
 			out.ErrLn("Error loading profile config: %v", err)
 		}
 
diff --git a/cmd/minikube/cmd/pause.go b/cmd/minikube/cmd/pause.go
index 493c2c0990..1e7c68a01d 100644
--- a/cmd/minikube/cmd/pause.go
+++ b/cmd/minikube/cmd/pause.go
@@ -53,7 +53,7 @@ func runPause(cmd *cobra.Command, args []string) {
 	defer api.Close()
 	cc, err := config.Load(cname)
 
-	if err != nil && !os.IsNotExist(err) {
+	if err != nil && !config.IsNotExist(err) {
 		exit.WithError("Error loading profile config", err)
 	}
 
diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index c7246f4394..df4916aaf0 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -295,7 +295,7 @@ func runStart(cmd *cobra.Command, args []string) {
 	}
 
 	existing, err := config.Load(viper.GetString(config.MachineProfile))
-	if err != nil && !os.IsNotExist(err) {
+	if err != nil && !config.IsNotExist(err) {
 		exit.WithCodeT(exit.Data, "Unable to load config: {{.error}}", out.V{"error": err})
 	}
 
@@ -751,7 +751,7 @@ func validateUser(drvName string) {
 		os.Exit(exit.Permissions)
 	}
 	_, err = config.Load(viper.GetString(config.MachineProfile))
-	if err == nil || !os.IsNotExist(err) {
+	if err == nil || !config.IsNotExist(err) {
 		out.T(out.Tip, "Tip: To remove this root owned cluster, run: sudo {{.cmd}} delete", out.V{"cmd": minikubeCmd()})
 	}
 	if !useForce {
diff --git a/cmd/minikube/cmd/unpause.go b/cmd/minikube/cmd/unpause.go
index f2d86f8d9e..4a3356edff 100644
--- a/cmd/minikube/cmd/unpause.go
+++ b/cmd/minikube/cmd/unpause.go
@@ -45,7 +45,7 @@ var unpauseCmd = &cobra.Command{
 		defer api.Close()
 		cc, err := config.Load(cname)
 
-		if err != nil && !os.IsNotExist(err) {
+		if err != nil && !config.IsNotExist(err) {
 			exit.WithError("Error loading profile config", err)
 		}
 
diff --git a/pkg/addons/addons.go b/pkg/addons/addons.go
index 29c58029fc..06cf51d7dc 100644
--- a/pkg/addons/addons.go
+++ b/pkg/addons/addons.go
@@ -18,7 +18,6 @@ package addons
 
 import (
 	"fmt"
-	"os"
 	"path"
 	"strconv"
 	"strings"
@@ -140,7 +139,7 @@ func enableOrDisableAddon(name, val, profile string) error {
 	defer api.Close()
 
 	cfg, err := config.Load(profile)
-	if err != nil && !os.IsNotExist(err) {
+	if err != nil && !config.IsNotExist(err) {
 		exit.WithCodeT(exit.Data, "Unable to load config: {{.error}}", out.V{"error": err})
 	}
 
diff --git a/pkg/addons/kubectl.go b/pkg/addons/kubectl.go
index 5cc96383f8..1e3fbe2a97 100644
--- a/pkg/addons/kubectl.go
+++ b/pkg/addons/kubectl.go
@@ -17,7 +17,6 @@ limitations under the License.
 package addons
 
 import (
-	"os"
 	"os/exec"
 	"path"
 
@@ -53,7 +52,7 @@ func kubectlCommand(profile string, files []string, enable bool) (*exec.Cmd, err
 
 func kubernetesVersion(profile string) (string, error) {
 	cc, err := config.Load(profile)
-	if err != nil && !os.IsNotExist(err) {
+	if err != nil && !config.IsNotExist(err) {
 		return "", err
 	}
 	version := constants.DefaultKubernetesVersion
diff --git a/pkg/minikube/config/config.go b/pkg/minikube/config/config.go
index 134c969721..a5d16cc0c1 100644
--- a/pkg/minikube/config/config.go
+++ b/pkg/minikube/config/config.go
@@ -54,6 +54,23 @@ var (
 	ErrKeyNotFound = errors.New("specified key could not be found in config")
 )
 
+type ErrNotExist struct {
+	s string
+}
+
+func (e *ErrNotExist) Error() string {
+	return e.s
+}
+
+// IsNotExist returns whether the error means a nonexistent configuration
+func IsNotExist(err error) bool {
+	switch err.(type) {
+	case *ErrNotExist:
+		return true
+	}
+	return false
+}
+
 // MinikubeConfig represents minikube config
 type MinikubeConfig map[string]interface{}
 
@@ -151,7 +168,7 @@ func (c *simpleConfigLoader) LoadConfigFromFile(profileName string, miniHome ...
 
 	if _, err := os.Stat(path); err != nil {
 		if os.IsNotExist(err) {
-			return nil, fmt.Errorf("cluster %q does not exist", profileName)
+			return nil, &ErrNotExist{fmt.Sprintf("cluster %q does not exist", profileName)}
 		}
 		return nil, errors.Wrap(err, "stat")
 	}

From 5c7585a2c85b052ac3eaed22194bb1b4a01b3564 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Fri, 31 Jan 2020 14:26:04 -0800
Subject: [PATCH 181/250] Lint fix

---
 pkg/minikube/config/config.go | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/pkg/minikube/config/config.go b/pkg/minikube/config/config.go
index a5d16cc0c1..14a6aa5d33 100644
--- a/pkg/minikube/config/config.go
+++ b/pkg/minikube/config/config.go
@@ -64,8 +64,7 @@ func (e *ErrNotExist) Error() string {
 
 // IsNotExist returns whether the error means a nonexistent configuration
 func IsNotExist(err error) bool {
-	switch err.(type) {
-	case *ErrNotExist:
+	if _, ok := err.(*ErrNotExist); ok {
 		return true
 	}
 	return false

From 3ebbe4f1fc63758a93e8f5d036aea456ab774d9e Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Fri, 31 Jan 2020 14:32:54 -0800
Subject: [PATCH 182/250] Allow 'profiles list' to work even if a profile has
 no control plane

---
 cmd/minikube/cmd/config/profile_list.go | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/cmd/minikube/cmd/config/profile_list.go b/cmd/minikube/cmd/config/profile_list.go
index 8df06f2d2d..9121529f03 100644
--- a/cmd/minikube/cmd/config/profile_list.go
+++ b/cmd/minikube/cmd/config/profile_list.go
@@ -78,11 +78,12 @@ var printProfilesTable = func() {
 	for _, p := range validProfiles {
 		p.Status, err = cluster.GetHostStatus(api, p.Name)
 		if err != nil {
-			glog.Infof("error getting host status for %v", err)
+			glog.Infof("error getting host status for %s: %v", p.Name, err)
 		}
 		cp, err := config.PrimaryControlPlane(*p.Config)
 		if err != nil {
-			exit.WithError("profile has no control plane", err)
+			glog.Errorf("%q has no control plane: %v", p.Name, err)
+			// Print the data we know about anyways
 		}
 		validData = append(validData, []string{p.Name, p.Config.VMDriver, cp.IP, strconv.Itoa(cp.Port), p.Config.KubernetesConfig.KubernetesVersion, p.Status})
 	}
@@ -119,7 +120,7 @@ var printProfilesJSON = func() {
 	for _, v := range validProfiles {
 		status, err := cluster.GetHostStatus(api, v.Name)
 		if err != nil {
-			glog.Infof("error getting host status for  %v", err)
+			glog.Infof("error getting host status for %s: %v", v.Name, err)
 		}
 		v.Status = status
 	}

From 7d65ca31de9e9a1064e718a82a3f755a8c4493c6 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Fri, 31 Jan 2020 14:35:03 -0800
Subject: [PATCH 183/250] Promote Infofs to errors/warnings

---
 cmd/minikube/cmd/config/profile_list.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/cmd/minikube/cmd/config/profile_list.go b/cmd/minikube/cmd/config/profile_list.go
index 9121529f03..27d02bb18e 100644
--- a/cmd/minikube/cmd/config/profile_list.go
+++ b/cmd/minikube/cmd/config/profile_list.go
@@ -71,14 +71,14 @@ var printProfilesTable = func() {
 	}
 	api, err := machine.NewAPIClient()
 	if err != nil {
-		glog.Infof("failed to get machine api client %v", err)
+		glog.Errorf("failed to get machine api client %v", err)
 	}
 	defer api.Close()
 
 	for _, p := range validProfiles {
 		p.Status, err = cluster.GetHostStatus(api, p.Name)
 		if err != nil {
-			glog.Infof("error getting host status for %s: %v", p.Name, err)
+			glog.Warningf("error getting host status for %s: %v", p.Name, err)
 		}
 		cp, err := config.PrimaryControlPlane(*p.Config)
 		if err != nil {
@@ -112,7 +112,7 @@ var printProfilesTable = func() {
 var printProfilesJSON = func() {
 	api, err := machine.NewAPIClient()
 	if err != nil {
-		glog.Infof("failed to get machine api client %v", err)
+		glog.Errorf("failed to get machine api client %v", err)
 	}
 	defer api.Close()
 
@@ -120,7 +120,7 @@ var printProfilesJSON = func() {
 	for _, v := range validProfiles {
 		status, err := cluster.GetHostStatus(api, v.Name)
 		if err != nil {
-			glog.Infof("error getting host status for %s: %v", v.Name, err)
+			glog.Warningf("error getting host status for %s: %v", v.Name, err)
 		}
 		v.Status = status
 	}

From 9b5044bfb2e3ff721bde0bb90de9317f6ab27907 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Fri, 31 Jan 2020 14:53:04 -0800
Subject: [PATCH 184/250] Skip trySSH entirely if --force is set

---
 cmd/minikube/cmd/start.go | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 7143674c67..deecadb027 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -1083,6 +1083,10 @@ func validateNetwork(h *host.Host, r command.Runner) string {
 }
 
 func trySSH(h *host.Host, ip string) {
+	if viper.GetBool(force) {
+		return
+	}
+
 	sshAddr := net.JoinHostPort(ip, "22")
 
 	dial := func() (err error) {
@@ -1097,8 +1101,7 @@ func trySSH(h *host.Host, ip string) {
 	}
 
 	if err := retry.Expo(dial, time.Second, 13*time.Second); err != nil {
-		if !viper.GetBool(force) {
-			exit.WithCodeT(exit.IO, `minikube is unable to connect to the VM: {{.error}}
+		exit.WithCodeT(exit.IO, `minikube is unable to connect to the VM: {{.error}}
 
 	This is likely due to one of two reasons:
 

From 7ebd01e113bc3a39193d9b3f214b2eb41d188014 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Fri, 31 Jan 2020 14:55:24 -0800
Subject: [PATCH 185/250] Remove extra }

---
 cmd/minikube/cmd/start.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 4d9b8b9520..b353a6d878 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -1155,7 +1155,6 @@ func trySSH(h *host.Host, ip string) {
 	- Use an alternative --vm-driver
 	- Use --force to override this connectivity check
 	`, out.V{"error": err, "hypervisor": h.Driver.DriverName(), "ip": ip})
-		}
 	}
 }
 

From b3149a103133241ed342900679d44d1fa6da0efd Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Fri, 31 Jan 2020 14:32:54 -0800
Subject: [PATCH 186/250] Allow 'profiles list' to work even if a profile has
 no control plane

---
 cmd/minikube/cmd/config/profile_list.go | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/cmd/minikube/cmd/config/profile_list.go b/cmd/minikube/cmd/config/profile_list.go
index 6671f3fd34..acb1280d32 100644
--- a/cmd/minikube/cmd/config/profile_list.go
+++ b/cmd/minikube/cmd/config/profile_list.go
@@ -78,11 +78,12 @@ var printProfilesTable = func() {
 	for _, p := range validProfiles {
 		p.Status, err = cluster.GetHostStatus(api, p.Name)
 		if err != nil {
-			glog.Infof("error getting host status for %v", err)
+			glog.Infof("error getting host status for %s: %v", p.Name, err)
 		}
 		cp, err := config.PrimaryControlPlane(*p.Config)
 		if err != nil {
-			exit.WithError("profile has no control plane", err)
+			glog.Errorf("%q has no control plane: %v", p.Name, err)
+			// Print the data we know about anyways
 		}
 		validData = append(validData, []string{p.Name, p.Config.VMDriver, p.Config.KubernetesConfig.ContainerRuntime, cp.IP, strconv.Itoa(cp.Port), p.Config.KubernetesConfig.KubernetesVersion, p.Status})
 	}
@@ -119,7 +120,7 @@ var printProfilesJSON = func() {
 	for _, v := range validProfiles {
 		status, err := cluster.GetHostStatus(api, v.Name)
 		if err != nil {
-			glog.Infof("error getting host status for  %v", err)
+			glog.Infof("error getting host status for %s: %v", v.Name, err)
 		}
 		v.Status = status
 	}

From 40f2f7c28d8f6f25289aec5285b2f76296cd9fe4 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Fri, 31 Jan 2020 14:35:03 -0800
Subject: [PATCH 187/250] Promote Infofs to errors/warnings

---
 cmd/minikube/cmd/config/profile_list.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/cmd/minikube/cmd/config/profile_list.go b/cmd/minikube/cmd/config/profile_list.go
index acb1280d32..50b075a24f 100644
--- a/cmd/minikube/cmd/config/profile_list.go
+++ b/cmd/minikube/cmd/config/profile_list.go
@@ -71,14 +71,14 @@ var printProfilesTable = func() {
 	}
 	api, err := machine.NewAPIClient()
 	if err != nil {
-		glog.Infof("failed to get machine api client %v", err)
+		glog.Errorf("failed to get machine api client %v", err)
 	}
 	defer api.Close()
 
 	for _, p := range validProfiles {
 		p.Status, err = cluster.GetHostStatus(api, p.Name)
 		if err != nil {
-			glog.Infof("error getting host status for %s: %v", p.Name, err)
+			glog.Warningf("error getting host status for %s: %v", p.Name, err)
 		}
 		cp, err := config.PrimaryControlPlane(*p.Config)
 		if err != nil {
@@ -112,7 +112,7 @@ var printProfilesTable = func() {
 var printProfilesJSON = func() {
 	api, err := machine.NewAPIClient()
 	if err != nil {
-		glog.Infof("failed to get machine api client %v", err)
+		glog.Errorf("failed to get machine api client %v", err)
 	}
 	defer api.Close()
 
@@ -120,7 +120,7 @@ var printProfilesJSON = func() {
 	for _, v := range validProfiles {
 		status, err := cluster.GetHostStatus(api, v.Name)
 		if err != nil {
-			glog.Infof("error getting host status for %s: %v", v.Name, err)
+			glog.Warningf("error getting host status for %s: %v", v.Name, err)
 		}
 		v.Status = status
 	}

From dd4ad10b83ccab45017fc56909649882ad4f20eb Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Fri, 31 Jan 2020 15:27:08 -0800
Subject: [PATCH 188/250] Improve handling of paused none clusters.

---
 cmd/minikube/cmd/delete.go                    | 29 ++++++-
 cmd/minikube/cmd/status.go                    | 23 ++++--
 pkg/drivers/none/none.go                      | 23 ++----
 .../bootstrapper/bsutil/kverify/kverify.go    | 82 +++++++++++++++++--
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go  | 38 +--------
 5 files changed, 128 insertions(+), 67 deletions(-)

diff --git a/cmd/minikube/cmd/delete.go b/cmd/minikube/cmd/delete.go
index 5efe7e2ee3..832aa2af2d 100644
--- a/cmd/minikube/cmd/delete.go
+++ b/cmd/minikube/cmd/delete.go
@@ -35,6 +35,7 @@ import (
 	"k8s.io/minikube/pkg/minikube/cluster"
 	pkg_config "k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
+	"k8s.io/minikube/pkg/minikube/cruntime"
 	"k8s.io/minikube/pkg/minikube/driver"
 	"k8s.io/minikube/pkg/minikube/exit"
 	"k8s.io/minikube/pkg/minikube/kubeconfig"
@@ -194,7 +195,7 @@ func deleteProfile(profile *pkg_config.Profile) error {
 	}
 
 	if err == nil && driver.BareMetal(cc.VMDriver) {
-		if err := uninstallKubernetes(api, cc.KubernetesConfig, viper.GetString(cmdcfg.Bootstrapper)); err != nil {
+		if err := uninstallKubernetes(api, profile.Name, cc.KubernetesConfig, viper.GetString(cmdcfg.Bootstrapper)); err != nil {
 			deletionError, ok := err.(DeletionError)
 			if ok {
 				delErr := profileDeletionErr(profile.Name, fmt.Sprintf("%v", err))
@@ -276,12 +277,34 @@ func profileDeletionErr(profileName string, additionalInfo string) error {
 	return fmt.Errorf("error deleting profile \"%s\": %s", profileName, additionalInfo)
 }
 
-func uninstallKubernetes(api libmachine.API, kc pkg_config.KubernetesConfig, bsName string) error {
+func uninstallKubernetes(api libmachine.API, profile string, kc pkg_config.KubernetesConfig, bsName string) error {
 	out.T(out.Resetting, "Uninstalling Kubernetes {{.kubernetes_version}} using {{.bootstrapper_name}} ...", out.V{"kubernetes_version": kc.KubernetesVersion, "bootstrapper_name": bsName})
 	clusterBootstrapper, err := getClusterBootstrapper(api, bsName)
 	if err != nil {
 		return DeletionError{Err: fmt.Errorf("unable to get bootstrapper: %v", err), Errtype: Fatal}
-	} else if err = clusterBootstrapper.DeleteCluster(kc); err != nil {
+	}
+
+	host, err := cluster.CheckIfHostExistsAndLoad(api, profile)
+	if err != nil {
+		exit.WithError("Error getting host", err)
+	}
+	r, err := machine.CommandRunner(host)
+	if err != nil {
+		exit.WithError("Failed to get command runner", err)
+	}
+
+	cr, err := cruntime.New(cruntime.Config{Type: kc.ContainerRuntime, Runner: r})
+	if err != nil {
+		exit.WithError("Failed runtime", err)
+	}
+
+	// Unpause the cluster if necessary to avoid hung kubeadm
+	_, err = cluster.Unpause(cr, r, nil)
+	if err != nil {
+		glog.Errorf("unpause failed: %v", err)
+	}
+
+	if err = clusterBootstrapper.DeleteCluster(kc); err != nil {
 		return DeletionError{Err: fmt.Errorf("failed to delete cluster: %v", err), Errtype: Fatal}
 	}
 	return nil
diff --git a/cmd/minikube/cmd/status.go b/cmd/minikube/cmd/status.go
index ad050cf199..9af9a12a39 100644
--- a/cmd/minikube/cmd/status.go
+++ b/cmd/minikube/cmd/status.go
@@ -30,7 +30,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config"
+	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/kverify"
 	"k8s.io/minikube/pkg/minikube/cluster"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
@@ -142,24 +142,31 @@ func status(api libmachine.API, name string) (*Status, error) {
 		return st, errors.Wrap(err, "host")
 	}
 
-	// Nonexistent it is!
 	if hs == state.None.String() {
 		return st, nil
 	}
+
 	st.Host = hs
-	if st.Host != state.Running.String() {
+	if st.Host != state.Running.String() && st.Host != state.Paused.String() {
 		return st, nil
 	}
 
-	bs, err := getClusterBootstrapper(api, viper.GetString(cmdcfg.Bootstrapper))
+	host, err := cluster.CheckIfHostExistsAndLoad(api, name)
 	if err != nil {
-		return st, errors.Wrap(err, "bootstrapper")
+		return st, err
 	}
 
-	st.Kubelet, err = bs.GetKubeletStatus()
+	cr, err := machine.CommandRunner(host)
+	if err != nil {
+		return st, err
+	}
+
+	stk, err := kverify.KubeletStatus(cr)
 	if err != nil {
 		glog.Warningf("kubelet err: %v", err)
 		st.Kubelet = state.Error.String()
+	} else {
+		st.Kubelet = stk.String()
 	}
 
 	ip, err := cluster.GetHostDriverIP(api, name)
@@ -175,10 +182,12 @@ func status(api libmachine.API, name string) (*Status, error) {
 		port = constants.APIServerPort
 	}
 
-	st.APIServer, err = bs.GetAPIServerStatus(ip, port)
+	sta, err := kverify.APIServerStatus(cr, ip, port)
 	if err != nil {
 		glog.Errorln("Error apiserver status:", err)
 		st.APIServer = state.Error.String()
+	} else {
+		st.APIServer = sta.String()
 	}
 
 	st.Kubeconfig = Misconfigured
diff --git a/pkg/drivers/none/none.go b/pkg/drivers/none/none.go
index 7c431e8267..b5bfe49336 100644
--- a/pkg/drivers/none/none.go
+++ b/pkg/drivers/none/none.go
@@ -26,7 +26,7 @@ import (
 	"github.com/docker/machine/libmachine/state"
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
-	"k8s.io/apimachinery/pkg/util/net"
+	knet "k8s.io/apimachinery/pkg/util/net"
 	pkgdrivers "k8s.io/minikube/pkg/drivers"
 	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/minikube/cruntime"
@@ -94,7 +94,7 @@ func (d *Driver) DriverName() string {
 
 // GetIP returns an IP or hostname that this host is available at
 func (d *Driver) GetIP() (string, error) {
-	ip, err := net.ChooseHostInterface()
+	ip, err := knet.ChooseHostInterface()
 	if err != nil {
 		return "", err
 	}
@@ -123,10 +123,13 @@ func (d *Driver) GetURL() (string, error) {
 
 // GetState returns the state that the host is in (running, stopped, etc)
 func (d *Driver) GetState() (state.State, error) {
-	if err := checkKubelet(d.exec); err != nil {
-		glog.Infof("kubelet not running: %v", err)
-		return state.Stopped, nil
+	_, err := d.GetIP()
+	if err != nil {
+		return state.Error, err
 	}
+
+	// If we got this far, it's safe to call the host as running like a VM would.
+	// Rely on other checks to determine if kubelet and apiserver are healthy.
 	return state.Running, nil
 }
 
@@ -251,13 +254,3 @@ func restartKubelet(cr command.Runner) error {
 	}
 	return nil
 }
-
-// checkKubelet returns an error if the kubelet is not running.
-func checkKubelet(cr command.Runner) error {
-	glog.Infof("checking for running kubelet ...")
-	c := exec.Command("systemctl", "is-active", "--quiet", "service", "kubelet")
-	if _, err := cr.RunCmd(c); err != nil {
-		return errors.Wrap(err, "check kubelet")
-	}
-	return nil
-}
diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
index 00374eb23b..0a056adcf4 100644
--- a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
+++ b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
@@ -23,6 +23,8 @@ import (
 	"net"
 	"net/http"
 	"os/exec"
+	"path"
+	"strings"
 	"time"
 
 	"github.com/docker/machine/libmachine/state"
@@ -101,12 +103,12 @@ func APIServerIsRunning(start time.Time, ip string, port int, timeout time.Durat
 			return false, fmt.Errorf("cluster wait timed out during healthz check")
 		}
 
-		status, err := APIServerStatus(net.ParseIP(ip), port)
+		status, err := apiServerHealthz(net.ParseIP(ip), port)
 		if err != nil {
 			glog.Warningf("status: %v", err)
 			return false, nil
 		}
-		if status != "Running" {
+		if status != state.Running {
 			return false, nil
 		}
 		return true, nil
@@ -119,9 +121,53 @@ func APIServerIsRunning(start time.Time, ip string, port int, timeout time.Durat
 	return nil
 }
 
-// APIServerStatus hits the /healthz endpoint and returns libmachine style state.State
-func APIServerStatus(ip net.IP, apiserverPort int) (string, error) {
-	url := fmt.Sprintf("https://%s/healthz", net.JoinHostPort(ip.String(), fmt.Sprint(apiserverPort)))
+// APIServerStatus returns apiserver status in libmachine style state.State
+func APIServerStatus(cr command.Runner, ip net.IP, port int) (state.State, error) {
+	glog.Infof("Checking apiserver status ...")
+	// sudo, in case hidepid is set
+	rr, err := cr.RunCmd(exec.Command("sudo", "pgrep", "kube-apiserver"))
+	if err != nil {
+		return state.Stopped, nil
+	}
+	pids := strings.Split(strings.TrimSpace(rr.Stdout.String()), "\n")
+	pid := pids[len(pids)-1]
+	if len(pids) != 1 {
+		glog.Errorf("found %d apiserver pids: %v - choosing %s", len(pids), pids, pid)
+	}
+
+	// Get the freezer cgroup entry for this pid
+	rr, err = cr.RunCmd(exec.Command("sudo", "egrep", "^[0-9]+:freezer:", path.Join("/proc", pid, "cgroup")))
+	if err != nil {
+		glog.Warningf("unable to find freezer cgroup: %v", err)
+		return apiServerHealthz(ip, port)
+
+	}
+	freezer := strings.TrimSpace(rr.Stdout.String())
+	glog.Infof("apiserver freezer: %q", freezer)
+	fparts := strings.Split(freezer, ":")
+	if len(fparts) != 3 {
+		glog.Warningf("unable to parse freezer - found %d parts: %s", len(fparts), freezer)
+		return apiServerHealthz(ip, port)
+	}
+
+	rr, err = cr.RunCmd(exec.Command("sudo", "cat", path.Join("/sys/fs/cgroup/freezer", fparts[2], "freezer.state")))
+	if err != nil {
+		glog.Errorf("unable to get freezer state: %s", rr.Stderr.String())
+		return apiServerHealthz(ip, port)
+	}
+
+	fs := strings.TrimSpace(rr.Stdout.String())
+	glog.Infof("freezer state: %q", fs)
+	if fs == "FREEZING" || fs == "FROZEN" {
+		return state.Paused, nil
+	}
+	return apiServerHealthz(ip, port)
+}
+
+// apiServerHealthz hits the /healthz endpoint and returns libmachine style state.State
+func apiServerHealthz(ip net.IP, port int) (state.State, error) {
+	url := fmt.Sprintf("https://%s/healthz", net.JoinHostPort(ip.String(), fmt.Sprint(port)))
+	glog.Infof("Checking apiserver healthz at %s ...", url)
 	// To avoid: x509: certificate signed by unknown authority
 	tr := &http.Transport{
 		Proxy:           nil, // To avoid connectiv issue if http(s)_proxy is set.
@@ -131,11 +177,31 @@ func APIServerStatus(ip net.IP, apiserverPort int) (string, error) {
 	resp, err := client.Get(url)
 	// Connection refused, usually.
 	if err != nil {
-		return state.Stopped.String(), nil
+		return state.Stopped, nil
 	}
 	if resp.StatusCode != http.StatusOK {
 		glog.Warningf("%s response: %v %+v", url, err, resp)
-		return state.Error.String(), nil
+		return state.Error, nil
 	}
-	return state.Running.String(), nil
+	return state.Running, nil
+}
+
+func KubeletStatus(cr command.Runner) (state.State, error) {
+	glog.Infof("Checking kubelet status ...")
+	rr, err := cr.RunCmd(exec.Command("sudo", "systemctl", "is-active", "kubelet"))
+	if err != nil {
+		// Do not return now, as we still have parsing to do!
+		glog.Warningf("%s returned error: %v", rr.Command(), err)
+	}
+	s := strings.TrimSpace(rr.Stdout.String())
+	glog.Infof("kubelet is-active: %s", s)
+	switch s {
+	case "active":
+		return state.Running, nil
+	case "inactive":
+		return state.Stopped, nil
+	case "activating":
+		return state.Starting, nil
+	}
+	return state.Error, nil
 }
diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index 73aea4464e..e2f5074947 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -99,42 +99,12 @@ func (k *Bootstrapper) GetKubeletStatus() (string, error) {
 }
 
 // GetAPIServerStatus returns the api-server status
-func (k *Bootstrapper) GetAPIServerStatus(ip net.IP, apiserverPort int) (string, error) {
-	// sudo, in case hidepid is set
-	rr, err := k.c.RunCmd(exec.Command("sudo", "pgrep", "kube-apiserver"))
+func (k *Bootstrapper) GetAPIServerStatus(ip net.IP, port int) (string, error) {
+	s, err := kverify.APIServerStatus(k.c, ip, port)
 	if err != nil {
-		return state.Stopped.String(), nil
+		return state.Error.String(), err
 	}
-	pid := strings.TrimSpace(rr.Stdout.String())
-
-	// Get the freezer cgroup entry for this pid
-	rr, err = k.c.RunCmd(exec.Command("sudo", "egrep", "^[0-9]+:freezer:", path.Join("/proc", pid, "cgroup")))
-	if err != nil {
-		glog.Warningf("unable to find freezer cgroup: %v", err)
-		return kverify.APIServerStatus(ip, apiserverPort)
-
-	}
-	freezer := strings.TrimSpace(rr.Stdout.String())
-	glog.Infof("apiserver freezer: %q", freezer)
-	fparts := strings.Split(freezer, ":")
-	if len(fparts) != 3 {
-		glog.Warningf("unable to parse freezer - found %d parts: %s", len(fparts), freezer)
-		return kverify.APIServerStatus(ip, apiserverPort)
-	}
-
-	rr, err = k.c.RunCmd(exec.Command("sudo", "cat", path.Join("/sys/fs/cgroup/freezer", fparts[2], "freezer.state")))
-	if err != nil {
-		glog.Errorf("unable to get freezer state: %s", rr.Stderr.String())
-		return kverify.APIServerStatus(ip, apiserverPort)
-	}
-
-	fs := strings.TrimSpace(rr.Stdout.String())
-	glog.Infof("freezer state: %q", fs)
-	if fs == "FREEZING" || fs == "FROZEN" {
-		return state.Paused.String(), nil
-	}
-
-	return kverify.APIServerStatus(ip, apiserverPort)
+	return s.String(), nil
 }
 
 // LogCommands returns a map of log type to a command which will display that log.

From fbac776c1d10d0b519c23a7b1a5382355b0e80aa Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Fri, 31 Jan 2020 16:30:14 -0800
Subject: [PATCH 189/250] Update Makefile and CHANGELOG

---
 CHANGELOG.md | 26 ++++++++++++++++++++++++++
 Makefile     |  2 +-
 2 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3c360769bb..60df7c6f93 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,31 @@
 # Release Notes
 
+## Version 1.7.0 - 2020-01-31
+
+* Add docker run-time for kic driver [#6436](https://github.com/kubernetes/minikube/pull/6436)
+* Add addon enablement to start [#6440](https://github.com/kubernetes/minikube/pull/6440)
+* Configure etcd and kube-proxy metrics to listen on minikube node IP [#6322](https://github.com/kubernetes/minikube/pull/6322)
+* add container runtime info to profile list [#6409](https://github.com/kubernetes/minikube/pull/6409)
+* Allow 'profiles list' to work even if a profile has no control plane [#6450](https://github.com/kubernetes/minikube/pull/6450)
+* status: Explicitly state that the cluster does not exist [#6438](https://github.com/kubernetes/minikube/pull/6438)
+* Do not use an arch suffix for the coredns name [#6243](https://github.com/kubernetes/minikube/pull/6243)
+* Allow building storage-provisioner for other arch [#6257](https://github.com/kubernetes/minikube/pull/6257)
+* Prevent registry-creds configure from failing when a secret does not exist.  [#6380](https://github.com/kubernetes/minikube/pull/6380)
+* improve checking modprob netfilter [#6427](https://github.com/kubernetes/minikube/pull/6427)
+* Update kvm2.md [#6267](https://github.com/kubernetes/minikube/pull/6267)
+
+Huge thank you for this release towards our contributors: 
+
+- Anders Björklund
+- Bjørn Harald Fotland
+- Chance Zibolski
+- Kim Bao Long
+- Medya Ghazizadeh
+- Priya Wadhwa
+- Sharif Elgamal
+- Thomas Strömberg
+- akshay
+
 ## Version 1.7.0-beta.1 - 2020-01-24
 
 * Add 'pause' command to freeze Kubernetes cluster [#5962](https://github.com/kubernetes/minikube/pull/5962)
diff --git a/Makefile b/Makefile
index e460ac2886..1d299ac758 100755
--- a/Makefile
+++ b/Makefile
@@ -15,7 +15,7 @@
 # Bump these on release - and please check ISO_VERSION for correctness.
 VERSION_MAJOR ?= 1
 VERSION_MINOR ?= 7
-VERSION_BUILD ?= 0-beta.1
+VERSION_BUILD ?= 0
 RAW_VERSION=$(VERSION_MAJOR).$(VERSION_MINOR).${VERSION_BUILD}
 VERSION ?= v$(RAW_VERSION)
 

From 5be02875b5732681ed5bec1c7a002432bae79a92 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Fri, 31 Jan 2020 16:39:48 -0800
Subject: [PATCH 190/250] remove non user facing PR

---
 CHANGELOG.md | 1 -
 1 file changed, 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 60df7c6f93..225931ba9f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,7 +12,6 @@
 * Allow building storage-provisioner for other arch [#6257](https://github.com/kubernetes/minikube/pull/6257)
 * Prevent registry-creds configure from failing when a secret does not exist.  [#6380](https://github.com/kubernetes/minikube/pull/6380)
 * improve checking modprob netfilter [#6427](https://github.com/kubernetes/minikube/pull/6427)
-* Update kvm2.md [#6267](https://github.com/kubernetes/minikube/pull/6267)
 
 Huge thank you for this release towards our contributors: 
 

From ae34f00928979cebb55d6610c962a3dc45c28590 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Fri, 31 Jan 2020 17:00:11 -0800
Subject: [PATCH 191/250] Fix 'Failed to delete cluster: load: exit status 1'
 message if cluster does not exist

---
 pkg/minikube/cluster/cluster.go | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/pkg/minikube/cluster/cluster.go b/pkg/minikube/cluster/cluster.go
index 7d6e1d2852..e256fc4c1e 100644
--- a/pkg/minikube/cluster/cluster.go
+++ b/pkg/minikube/cluster/cluster.go
@@ -308,21 +308,21 @@ func StopHost(api libmachine.API) error {
 	return nil
 }
 
+// deleteOrphanedKIC attempts to delete an orphaned docker instance
+func deleteOrphanedKIC(name string) {
+	cmd := exec.Command(oci.Docker, "rm", "-f", "-v", name)
+	err := cmd.Run()
+	if err == nil {
+		glog.Infof("Found stale kic container and successfully cleaned it up!")
+	}
+}
+
 // DeleteHost deletes the host VM.
 func DeleteHost(api libmachine.API, machineName string) error {
 	host, err := api.Load(machineName)
 	if err != nil && host == nil {
-		// before we give up on deleting the host
-		//  we try to kill the possible orphan kic driver
-		// this case will happen if the user deleted both profile and machine folder
-		// inside minikube home
-		cmd := exec.Command(oci.Docker, "rm", "-f", "-v", machineName)
-		err := cmd.Run()
-		if err == nil {
-			glog.Infof("Found stale kic container and successfully cleaned it up.")
-			return nil
-		}
-		return errors.Wrap(err, "load")
+		deleteOrphanedKIC(machineName)
+		// keep going even if minikube  does not know about the host
 	}
 
 	// Get the status of the host. Ensure that it exists before proceeding ahead.
@@ -333,7 +333,7 @@ func DeleteHost(api libmachine.API, machineName string) error {
 	}
 
 	if status == state.None.String() {
-		return mcnerror.ErrHostDoesNotExist{Name: host.Name}
+		return mcnerror.ErrHostDoesNotExist{Name: machineName}
 	}
 
 	// This is slow if SSH is not responding, but HyperV hangs otherwise, See issue #2914

From eee5788ab365e97b763332af74e73689d17db742 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Fri, 31 Jan 2020 17:08:27 -0800
Subject: [PATCH 192/250] Make deletion output more terse

---
 cmd/minikube/cmd/delete.go     | 11 ++++-------
 pkg/minikube/out/style.go      |  2 +-
 pkg/minikube/out/style_enum.go |  2 +-
 3 files changed, 6 insertions(+), 9 deletions(-)

diff --git a/cmd/minikube/cmd/delete.go b/cmd/minikube/cmd/delete.go
index 73af19bd66..f5b6d42afa 100644
--- a/cmd/minikube/cmd/delete.go
+++ b/cmd/minikube/cmd/delete.go
@@ -131,14 +131,12 @@ func runDelete(cmd *cobra.Command, args []string) {
 		profileName := viper.GetString(pkg_config.MachineProfile)
 		profile, err := pkg_config.LoadProfile(profileName)
 		if err != nil {
-			out.ErrT(out.Meh, `"{{.name}}" profile does not exist`, out.V{"name": profileName})
+			out.ErrT(out.Meh, `"{{.name}}" profile does not exist, trying anyways.`, out.V{"name": profileName})
 		}
 
 		errs := DeleteProfiles([]*pkg_config.Profile{profile})
 		if len(errs) > 0 {
 			HandleDeletionErrors(errs)
-		} else {
-			out.T(out.DeletingHost, "Successfully deleted profile \"{{.name}}\"", out.V{"name": profileName})
 		}
 	}
 
@@ -153,7 +151,7 @@ func purgeMinikubeDirectory() {
 	if err := os.RemoveAll(localpath.MiniPath()); err != nil {
 		exit.WithError("unable to delete minikube config folder", err)
 	}
-	out.T(out.Crushed, "Successfully purged minikube directory located at - [{{.minikubeDirectory}}]", out.V{"minikubeDirectory": localpath.MiniPath()})
+	out.T(out.Deleted, "Successfully purged minikube directory located at - [{{.minikubeDirectory}}]", out.V{"minikubeDirectory": localpath.MiniPath()})
 }
 
 // DeleteProfiles deletes one or more profiles
@@ -212,7 +210,7 @@ func deleteProfile(profile *pkg_config.Profile) error {
 	if err = cluster.DeleteHost(api, profile.Name); err != nil {
 		switch errors.Cause(err).(type) {
 		case mcnerror.ErrHostDoesNotExist:
-			out.T(out.Meh, `"{{.name}}" cluster does not exist. Proceeding ahead with cleanup.`, out.V{"name": profile.Name})
+			glog.Infof("%s cluster does not exist. Proceeding ahead with cleanup.", profile.Name)
 		default:
 			out.T(out.FailureType, "Failed to delete cluster: {{.error}}", out.V{"error": err})
 			out.T(out.Notice, `You may need to manually remove the "{{.name}}" VM from your hypervisor`, out.V{"name": profile.Name})
@@ -231,11 +229,10 @@ func deleteProfile(profile *pkg_config.Profile) error {
 		return DeletionError{Err: delErr, Errtype: Fatal}
 	}
 
-	out.T(out.Crushed, `The "{{.name}}" cluster has been deleted.`, out.V{"name": profile.Name})
-
 	if err := deleteContext(profile.Name); err != nil {
 		return err
 	}
+	out.T(out.Deleted, `Removed all traces of the "{{.name}}" cluster.`, out.V{"name": profile.Name})
 	return nil
 }
 
diff --git a/pkg/minikube/out/style.go b/pkg/minikube/out/style.go
index 56f6bf8b35..00de29b68c 100644
--- a/pkg/minikube/out/style.go
+++ b/pkg/minikube/out/style.go
@@ -72,7 +72,7 @@ var styles = map[StyleEnum]style{
 	Option:        {Prefix: "    ▪ ", LowPrefix: lowIndent}, // Indented bullet
 	Command:       {Prefix: "    ▪ ", LowPrefix: lowIndent}, // Indented bullet
 	LogEntry:      {Prefix: "    "},                         // Indent
-	Crushed:       {Prefix: "💔  "},
+	Deleted:       {Prefix: "💀  "},
 	URL:           {Prefix: "👉  ", LowPrefix: lowIndent},
 	Documentation: {Prefix: "📘  "},
 	Issues:        {Prefix: "⁉️   "},
diff --git a/pkg/minikube/out/style_enum.go b/pkg/minikube/out/style_enum.go
index 150b19fa37..5b1571a042 100644
--- a/pkg/minikube/out/style_enum.go
+++ b/pkg/minikube/out/style_enum.go
@@ -45,7 +45,7 @@ const (
 	Option
 	Command
 	LogEntry
-	Crushed
+	Deleted
 	URL
 	Documentation
 	Issues

From 25401ceed45237b206b3e6878f89b5ca45770cc6 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Fri, 31 Jan 2020 17:14:57 -0800
Subject: [PATCH 193/250] The none driver does not stop. Improve test error
 messages

---
 test/integration/start_stop_delete_test.go | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/test/integration/start_stop_delete_test.go b/test/integration/start_stop_delete_test.go
index f71c78e6b2..eb1ad883df 100644
--- a/test/integration/start_stop_delete_test.go
+++ b/test/integration/start_stop_delete_test.go
@@ -105,9 +105,12 @@ func TestStartStop(t *testing.T) {
 					t.Errorf("%s failed: %v", rr.Args, err)
 				}
 
-				got := Status(ctx, t, Target(), profile, "Host")
-				if got != state.Stopped.String() {
-					t.Errorf("status = %q; want = %q", got, state.Stopped)
+				// The none driver never really stops
+				if !NoneDriver() {
+					got := Status(ctx, t, Target(), profile, "Host")
+					if got != state.Stopped.String() {
+						t.Errorf("post-stop host status = %q; want = %q", got, state.Stopped)
+					}
 				}
 
 				rr, err = Run(t, exec.CommandContext(ctx, Target(), startArgs...))
@@ -124,7 +127,7 @@ func TestStartStop(t *testing.T) {
 
 				got = Status(ctx, t, Target(), profile, "Host")
 				if got != state.Running.String() {
-					t.Errorf("host status = %q; want = %q", got, state.Running)
+					t.Errorf("post-start host status = %q; want = %q", got, state.Running)
 				}
 
 				if !NoneDriver() {
@@ -227,12 +230,12 @@ func testPause(ctx context.Context, t *testing.T, profile string) {
 
 	got := Status(ctx, t, Target(), profile, "APIServer")
 	if got != state.Paused.String() {
-		t.Errorf("apiserver status = %q; want = %q", got, state.Paused)
+		t.Errorf("post-pause apiserver status = %q; want = %q", got, state.Paused)
 	}
 
 	got = Status(ctx, t, Target(), profile, "Kubelet")
 	if got != state.Stopped.String() {
-		t.Errorf("kubelet status = %q; want = %q", got, state.Stopped)
+		t.Errorf("post-pause kubelet status = %q; want = %q", got, state.Stopped)
 	}
 
 	rr, err = Run(t, exec.CommandContext(ctx, Target(), "unpause", "-p", profile, "--alsologtostderr", "-v=1"))
@@ -242,12 +245,12 @@ func testPause(ctx context.Context, t *testing.T, profile string) {
 
 	got = Status(ctx, t, Target(), profile, "APIServer")
 	if got != state.Running.String() {
-		t.Errorf("apiserver status = %q; want = %q", got, state.Running)
+		t.Errorf("post-unpause apiserver status = %q; want = %q", got, state.Running)
 	}
 
 	got = Status(ctx, t, Target(), profile, "Kubelet")
 	if got != state.Running.String() {
-		t.Errorf("kubelet status = %q; want = %q", got, state.Running)
+		t.Errorf("post-unpause kubelet status = %q; want = %q", got, state.Running)
 	}
 
 }

From 921299b3d5949b2d41599b659a0fa5171a23174c Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Fri, 31 Jan 2020 17:28:17 -0800
Subject: [PATCH 194/250] Fix unset got variable

---
 test/integration/start_stop_delete_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/integration/start_stop_delete_test.go b/test/integration/start_stop_delete_test.go
index eb1ad883df..d9248155e5 100644
--- a/test/integration/start_stop_delete_test.go
+++ b/test/integration/start_stop_delete_test.go
@@ -125,7 +125,7 @@ func TestStartStop(t *testing.T) {
 					t.Fatalf("wait: %v", err)
 				}
 
-				got = Status(ctx, t, Target(), profile, "Host")
+				got := Status(ctx, t, Target(), profile, "Host")
 				if got != state.Running.String() {
 					t.Errorf("post-start host status = %q; want = %q", got, state.Running)
 				}

From 6760929a72fae6893c907f145bda3f9bae6f50c7 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Fri, 31 Jan 2020 18:36:38 -0800
Subject: [PATCH 195/250] Add logging of non-nil errors

---
 cmd/minikube/cmd/stop.go | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/cmd/minikube/cmd/stop.go b/cmd/minikube/cmd/stop.go
index f132fff277..f9f90b7917 100644
--- a/cmd/minikube/cmd/stop.go
+++ b/cmd/minikube/cmd/stop.go
@@ -20,6 +20,7 @@ import (
 	"time"
 
 	"github.com/docker/machine/libmachine/mcnerror"
+	"github.com/golang/glog"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -54,6 +55,11 @@ func runStop(cmd *cobra.Command, args []string) {
 	nonexistent := false
 	stop := func() (err error) {
 		err = cluster.StopHost(api)
+		if err == nil {
+			return nil
+		}
+		glog.Warningf("stop host returned error: %v", err)
+
 		switch err := errors.Cause(err).(type) {
 		case mcnerror.ErrHostDoesNotExist:
 			out.T(out.Meh, `"{{.profile_name}}" VM does not exist, nothing to stop`, out.V{"profile_name": profile})

From 7f0e83508a0371b13825aea7737831731d5ab71c Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Fri, 31 Jan 2020 18:44:00 -0800
Subject: [PATCH 196/250] Add more logging

---
 pkg/drivers/none/none.go        | 5 +++--
 pkg/minikube/cluster/cluster.go | 7 +++++++
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/pkg/drivers/none/none.go b/pkg/drivers/none/none.go
index b5bfe49336..13257044c7 100644
--- a/pkg/drivers/none/none.go
+++ b/pkg/drivers/none/none.go
@@ -200,7 +200,7 @@ func (d *Driver) Start() error {
 // Stop a host gracefully, including any containers that we are managing.
 func (d *Driver) Stop() error {
 	if err := stopKubelet(d.exec); err != nil {
-		return err
+		return errors.Wrap(err, "stop kubelet")
 	}
 	containers, err := d.runtime.ListContainers(cruntime.ListOptions{})
 	if err != nil {
@@ -208,9 +208,10 @@ func (d *Driver) Stop() error {
 	}
 	if len(containers) > 0 {
 		if err := d.runtime.StopContainers(containers); err != nil {
-			return errors.Wrap(err, "stop")
+			return errors.Wrap(err, "stop containers")
 		}
 	}
+	glog.Infof("none driver is stopped!")
 	return nil
 }
 
diff --git a/pkg/minikube/cluster/cluster.go b/pkg/minikube/cluster/cluster.go
index 7d6e1d2852..6b78b88cef 100644
--- a/pkg/minikube/cluster/cluster.go
+++ b/pkg/minikube/cluster/cluster.go
@@ -284,6 +284,12 @@ func trySSHPowerOff(h *host.Host) error {
 
 // StopHost stops the host VM, saving state to disk.
 func StopHost(api libmachine.API) error {
+	glog.Infof("Stopping host ...")
+	start := time.Now()
+	defer func() {
+		glog.Infof("Stopped host within %s", time.Since(start))
+	}()
+
 	machineName := viper.GetString(config.MachineProfile)
 	host, err := api.Load(machineName)
 	if err != nil {
@@ -299,6 +305,7 @@ func StopHost(api libmachine.API) error {
 	}
 
 	if err := host.Stop(); err != nil {
+		glog.Infof("host.Stop failed: %v", err)
 		alreadyInStateError, ok := err.(mcnerror.ErrHostAlreadyInState)
 		if ok && alreadyInStateError.State == state.Stopped {
 			return nil

From cee9448fa07e2e9230f7bf52716dd5db623a2779 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Fri, 31 Jan 2020 18:54:27 -0800
Subject: [PATCH 197/250] Make none.GetState more intelligent to fix stop
 issues

---
 pkg/drivers/none/none.go | 25 +++++++++++++++++++++----
 1 file changed, 21 insertions(+), 4 deletions(-)

diff --git a/pkg/drivers/none/none.go b/pkg/drivers/none/none.go
index 13257044c7..15a2170c84 100644
--- a/pkg/drivers/none/none.go
+++ b/pkg/drivers/none/none.go
@@ -18,6 +18,7 @@ package none
 
 import (
 	"fmt"
+	"net"
 	"os/exec"
 	"strings"
 	"time"
@@ -28,8 +29,11 @@ import (
 	"github.com/pkg/errors"
 	knet "k8s.io/apimachinery/pkg/util/net"
 	pkgdrivers "k8s.io/minikube/pkg/drivers"
+	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/kverify"
 	"k8s.io/minikube/pkg/minikube/command"
+	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/cruntime"
+	"k8s.io/minikube/pkg/minikube/kubeconfig"
 	"k8s.io/minikube/pkg/minikube/vmpath"
 	"k8s.io/minikube/pkg/util/retry"
 )
@@ -123,14 +127,27 @@ func (d *Driver) GetURL() (string, error) {
 
 // GetState returns the state that the host is in (running, stopped, etc)
 func (d *Driver) GetState() (state.State, error) {
-	_, err := d.GetIP()
+	ip, err := d.GetIP()
 	if err != nil {
 		return state.Error, err
 	}
 
-	// If we got this far, it's safe to call the host as running like a VM would.
-	// Rely on other checks to determine if kubelet and apiserver are healthy.
-	return state.Running, nil
+	port, err := kubeconfig.Port(d.BaseDriver.MachineName)
+	if err != nil {
+		glog.Warningf("unable to get port: %v", err)
+		port = constants.APIServerPort
+	}
+
+	// Confusing logic, as libmachine.Stop will loop until the state == Stopped
+	ast, err := kverify.APIServerStatus(d.exec, net.ParseIP(ip), port)
+	if err != nil {
+		return ast, err
+	}
+	if ast == state.Paused || ast == state.Running {
+		return ast, nil
+	}
+
+	return kverify.KubeletStatus(d.exec)
 }
 
 // Kill stops a host forcefully, including any containers that we are managing.

From 1176aa468f111af171457bc02739b79220529bff Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Fri, 31 Jan 2020 19:00:59 -0800
Subject: [PATCH 198/250] More status logging

---
 cmd/minikube/cmd/status.go | 7 +++++++
 pkg/drivers/none/none.go   | 1 +
 2 files changed, 8 insertions(+)

diff --git a/cmd/minikube/cmd/status.go b/cmd/minikube/cmd/status.go
index 9af9a12a39..700f691d99 100644
--- a/cmd/minikube/cmd/status.go
+++ b/cmd/minikube/cmd/status.go
@@ -138,6 +138,7 @@ func status(api libmachine.API, name string) (*Status, error) {
 	}
 
 	hs, err := cluster.GetHostStatus(api, name)
+	glog.Infof("%s host status = %q (err=%v)", name, hs, err)
 	if err != nil {
 		return st, errors.Wrap(err, "host")
 	}
@@ -162,6 +163,8 @@ func status(api libmachine.API, name string) (*Status, error) {
 	}
 
 	stk, err := kverify.KubeletStatus(cr)
+	glog.Infof("%s kubelet status = %s (err=%v)", stk, err)
+
 	if err != nil {
 		glog.Warningf("kubelet err: %v", err)
 		st.Kubelet = state.Error.String()
@@ -183,6 +186,8 @@ func status(api libmachine.API, name string) (*Status, error) {
 	}
 
 	sta, err := kverify.APIServerStatus(cr, ip, port)
+	glog.Infof("%s apiserver status = %s (err=%v)", stk, err)
+
 	if err != nil {
 		glog.Errorln("Error apiserver status:", err)
 		st.APIServer = state.Error.String()
@@ -192,6 +197,8 @@ func status(api libmachine.API, name string) (*Status, error) {
 
 	st.Kubeconfig = Misconfigured
 	ks, err := kubeconfig.IsClusterInConfig(ip, name)
+	glog.Infof("%s kubeconfig status = %s (err=%v)", ks, err)
+
 	if err != nil {
 		glog.Errorln("Error kubeconfig status:", err)
 	}
diff --git a/pkg/drivers/none/none.go b/pkg/drivers/none/none.go
index 15a2170c84..c9d8b1009d 100644
--- a/pkg/drivers/none/none.go
+++ b/pkg/drivers/none/none.go
@@ -127,6 +127,7 @@ func (d *Driver) GetURL() (string, error) {
 
 // GetState returns the state that the host is in (running, stopped, etc)
 func (d *Driver) GetState() (state.State, error) {
+	glog.Infof("GetState called")
 	ip, err := d.GetIP()
 	if err != nil {
 		return state.Error, err

From edbe92fb0159140294f3633f8912658297ae106c Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Fri, 31 Jan 2020 19:15:58 -0800
Subject: [PATCH 199/250] Re-arrange status calls to extract maximum
 information without being confusing

---
 cmd/minikube/cmd/status.go | 74 ++++++++++++++++++++------------------
 1 file changed, 39 insertions(+), 35 deletions(-)

diff --git a/cmd/minikube/cmd/status.go b/cmd/minikube/cmd/status.go
index 700f691d99..12dcfe644e 100644
--- a/cmd/minikube/cmd/status.go
+++ b/cmd/minikube/cmd/status.go
@@ -143,35 +143,13 @@ func status(api libmachine.API, name string) (*Status, error) {
 		return st, errors.Wrap(err, "host")
 	}
 
+	// We have no record of this host. Return nonexistent struct
 	if hs == state.None.String() {
 		return st, nil
 	}
-
 	st.Host = hs
-	if st.Host != state.Running.String() && st.Host != state.Paused.String() {
-		return st, nil
-	}
-
-	host, err := cluster.CheckIfHostExistsAndLoad(api, name)
-	if err != nil {
-		return st, err
-	}
-
-	cr, err := machine.CommandRunner(host)
-	if err != nil {
-		return st, err
-	}
-
-	stk, err := kverify.KubeletStatus(cr)
-	glog.Infof("%s kubelet status = %s (err=%v)", stk, err)
-
-	if err != nil {
-		glog.Warningf("kubelet err: %v", err)
-		st.Kubelet = state.Error.String()
-	} else {
-		st.Kubelet = stk.String()
-	}
 
+	// We have enough to query kubeconfig now!
 	ip, err := cluster.GetHostDriverIP(api, name)
 	if err != nil {
 		glog.Errorln("Error host driver ip status:", err)
@@ -185,8 +163,44 @@ func status(api libmachine.API, name string) (*Status, error) {
 		port = constants.APIServerPort
 	}
 
+	st.Kubeconfig = Misconfigured
+	ok, err := kubeconfig.IsClusterInConfig(ip, name)
+	glog.Infof("%s is in kubeconfig at ip %s: %v (err=%v)", name, ip, ok, err)
+	if ok {
+		st.Kubeconfig = Configured
+	}
+
+	// If it's not running, quickly bail out rather than delivering confusing information.
+	if st.Host != state.Running.String() {
+		glog.Infof("host is not running, skipping remaining checks")
+		st.APIServer = st.Host
+		st.Kubelet = st.Host
+		return st, nil
+	}
+
+	// At this point, we start executing commands on the cluster for more details
+	host, err := cluster.CheckIfHostExistsAndLoad(api, name)
+	if err != nil {
+		return st, err
+	}
+
+	cr, err := machine.CommandRunner(host)
+	if err != nil {
+		return st, err
+	}
+
+	stk, err := kverify.KubeletStatus(cr)
+	glog.Infof("%s kubelet status = %s (err=%v)", name, stk, err)
+
+	if err != nil {
+		glog.Warningf("kubelet err: %v", err)
+		st.Kubelet = state.Error.String()
+	} else {
+		st.Kubelet = stk.String()
+	}
+
 	sta, err := kverify.APIServerStatus(cr, ip, port)
-	glog.Infof("%s apiserver status = %s (err=%v)", stk, err)
+	glog.Infof("%s apiserver status = %s (err=%v)", name, stk, err)
 
 	if err != nil {
 		glog.Errorln("Error apiserver status:", err)
@@ -195,16 +209,6 @@ func status(api libmachine.API, name string) (*Status, error) {
 		st.APIServer = sta.String()
 	}
 
-	st.Kubeconfig = Misconfigured
-	ks, err := kubeconfig.IsClusterInConfig(ip, name)
-	glog.Infof("%s kubeconfig status = %s (err=%v)", ks, err)
-
-	if err != nil {
-		glog.Errorln("Error kubeconfig status:", err)
-	}
-	if ks {
-		st.Kubeconfig = Configured
-	}
 	return st, nil
 }
 

From af2060a8587aedd5142963e8f127dcccb60b968e Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Fri, 31 Jan 2020 19:24:30 -0800
Subject: [PATCH 200/250] Move kubeconfig down as we cannot get an IP if host
 is down

---
 cmd/minikube/cmd/status.go | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/cmd/minikube/cmd/status.go b/cmd/minikube/cmd/status.go
index 12dcfe644e..b554647b32 100644
--- a/cmd/minikube/cmd/status.go
+++ b/cmd/minikube/cmd/status.go
@@ -149,7 +149,16 @@ func status(api libmachine.API, name string) (*Status, error) {
 	}
 	st.Host = hs
 
-	// We have enough to query kubeconfig now!
+	// If it's not running, quickly bail out rather than delivering conflicting messages
+	if st.Host != state.Running.String() {
+		glog.Infof("host is not running, skipping remaining checks")
+		st.APIServer = st.Host
+		st.Kubelet = st.Host
+		st.Kubeconfig = st.Host
+		return st, nil
+	}
+
+	// We have a fully operational host, now we can check for details
 	ip, err := cluster.GetHostDriverIP(api, name)
 	if err != nil {
 		glog.Errorln("Error host driver ip status:", err)
@@ -170,15 +179,6 @@ func status(api libmachine.API, name string) (*Status, error) {
 		st.Kubeconfig = Configured
 	}
 
-	// If it's not running, quickly bail out rather than delivering confusing information.
-	if st.Host != state.Running.String() {
-		glog.Infof("host is not running, skipping remaining checks")
-		st.APIServer = st.Host
-		st.Kubelet = st.Host
-		return st, nil
-	}
-
-	// At this point, we start executing commands on the cluster for more details
 	host, err := cluster.CheckIfHostExistsAndLoad(api, name)
 	if err != nil {
 		return st, err

From a540310c564a9edce27033e24b94b2f32ed541fc Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Fri, 31 Jan 2020 22:13:25 -0800
Subject: [PATCH 201/250] Update version to beta.2

---
 CHANGELOG.md | 2 +-
 Makefile     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 225931ba9f..957e2e1025 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,6 @@
 # Release Notes
 
-## Version 1.7.0 - 2020-01-31
+## Version 1.7.0-beta.2 - 2020-01-31
 
 * Add docker run-time for kic driver [#6436](https://github.com/kubernetes/minikube/pull/6436)
 * Add addon enablement to start [#6440](https://github.com/kubernetes/minikube/pull/6440)
diff --git a/Makefile b/Makefile
index 1d299ac758..4a210cf7d5 100755
--- a/Makefile
+++ b/Makefile
@@ -15,7 +15,7 @@
 # Bump these on release - and please check ISO_VERSION for correctness.
 VERSION_MAJOR ?= 1
 VERSION_MINOR ?= 7
-VERSION_BUILD ?= 0
+VERSION_BUILD ?= 0-beta.2
 RAW_VERSION=$(VERSION_MAJOR).$(VERSION_MINOR).${VERSION_BUILD}
 VERSION ?= v$(RAW_VERSION)
 

From f6d5350651b6333cf82766444509964992bfa918 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= <anders.f.bjorklund@gmail.com>
Date: Sun, 2 Feb 2020 13:37:47 +0100
Subject: [PATCH 202/250] Add debugging target to output package versions

---
 Makefile | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/Makefile b/Makefile
index 4a210cf7d5..6775e0c397 100755
--- a/Makefile
+++ b/Makefile
@@ -374,6 +374,9 @@ mdlint:
 out/docs/minikube.md: $(shell find "cmd") $(shell find "pkg/minikube/constants") pkg/minikube/assets/assets.go pkg/minikube/translate/translations.go
 	go run -ldflags="$(MINIKUBE_LDFLAGS)" -tags gendocs hack/help_text/gen_help_text.go
 
+deb_version:
+	@echo $(DEB_VERSION)
+
 out/minikube_$(DEB_VERSION).deb: out/minikube_$(DEB_VERSION)-0_amd64.deb
 	cp $< $@
 
@@ -387,6 +390,9 @@ out/minikube_$(DEB_VERSION)-0_%.deb: out/minikube-linux-%
 	fakeroot dpkg-deb --build out/minikube_$(DEB_VERSION) $@
 	rm -rf out/minikube_$(DEB_VERSION)
 
+rpm_version:
+	@echo $(RPM_VERSION)
+
 out/minikube-$(RPM_VERSION).rpm: out/minikube-$(RPM_VERSION)-0.x86_64.rpm
 	cp $< $@
 

From 174d1fb9e56f9034c5d6d71b05f4818df79553b0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= <anders.f.bjorklund@gmail.com>
Date: Sun, 2 Feb 2020 13:55:07 +0100
Subject: [PATCH 203/250] Add arch to docker-machine-driver-kvm2 packages

But only support building for amd64 at the moment
---
 Makefile                                      | 22 ++++++++++++++-----
 .../deb/kvm2_deb_template/DEBIAN/control      |  2 +-
 2 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/Makefile b/Makefile
index 6775e0c397..6a17726cf6 100755
--- a/Makefile
+++ b/Makefile
@@ -531,13 +531,17 @@ release-minikube: out/minikube checksum ## Minikube release
 	gsutil cp out/minikube-$(GOOS)-$(GOARCH) $(MINIKUBE_UPLOAD_LOCATION)/$(MINIKUBE_VERSION)/minikube-$(GOOS)-$(GOARCH)
 	gsutil cp out/minikube-$(GOOS)-$(GOARCH).sha256 $(MINIKUBE_UPLOAD_LOCATION)/$(MINIKUBE_VERSION)/minikube-$(GOOS)-$(GOARCH).sha256
 
-out/docker-machine-driver-kvm2:
+out/docker-machine-driver-kvm2: out/docker-machine-driver-kvm2-amd64
+	cp $< $@
+
+out/docker-machine-driver-kvm2-%:
 ifeq ($(MINIKUBE_BUILD_IN_DOCKER),y)
 	docker inspect -f '{{.Id}} {{.RepoTags}}' $(KVM_BUILD_IMAGE) || $(MAKE) kvm-image
 	$(call DOCKER,$(KVM_BUILD_IMAGE),/usr/bin/make $@ COMMIT=$(COMMIT))
 	# make extra sure that we are linking with the older version of libvirt (1.3.1)
 	test "`strings $@ | grep '^LIBVIRT_[0-9]' | sort | tail -n 1`" = "LIBVIRT_1.2.9"
 else
+	GOARCH=$* \
 	go build \
 		-installsuffix "static" \
 		-ldflags="$(KVM2_LDFLAGS)" \
@@ -547,21 +551,29 @@ else
 endif
 	chmod +X $@
 
-out/docker-machine-driver-kvm2_$(DEB_VERSION).deb: out/docker-machine-driver-kvm2
+out/docker-machine-driver-kvm2_$(DEB_VERSION).deb: out/docker-machine-driver-kvm2_$(DEB_VERSION)-0_amd64.deb
+	cp $< $@
+
+out/docker-machine-driver-kvm2_$(DEB_VERSION)-0_%.deb: out/docker-machine-driver-kvm2-%
 	cp -r installers/linux/deb/kvm2_deb_template out/docker-machine-driver-kvm2_$(DEB_VERSION)
 	chmod 0755 out/docker-machine-driver-kvm2_$(DEB_VERSION)/DEBIAN
 	sed -E -i 's/--VERSION--/'$(DEB_VERSION)'/g' out/docker-machine-driver-kvm2_$(DEB_VERSION)/DEBIAN/control
+	sed -E -i 's/--ARCH--/'$*'/g' out/docker-machine-driver-kvm2_$(DEB_VERSION)/DEBIAN/control
 	mkdir -p out/docker-machine-driver-kvm2_$(DEB_VERSION)/usr/bin
 	cp out/docker-machine-driver-kvm2 out/docker-machine-driver-kvm2_$(DEB_VERSION)/usr/bin/docker-machine-driver-kvm2
-	fakeroot dpkg-deb --build out/docker-machine-driver-kvm2_$(DEB_VERSION)
+	fakeroot dpkg-deb --build out/docker-machine-driver-kvm2_$(DEB_VERSION) $@
 	rm -rf out/docker-machine-driver-kvm2_$(DEB_VERSION)
 
-out/docker-machine-driver-kvm2-$(RPM_VERSION).rpm: out/docker-machine-driver-kvm2
+out/docker-machine-driver-kvm2-$(RPM_VERSION).rpm: out/docker-machine-driver-kvm2-$(RPM_VERSION)-0.x86_64.deb
+	cp $< $@
+
+out/docker-machine-driver-kvm2-$(RPM_VERSION)-0.%.rpm: out/docker-machine-driver-kvm2-%
 	cp -r installers/linux/rpm/kvm2_rpm_template out/docker-machine-driver-kvm2-$(RPM_VERSION)
 	sed -E -i 's/--VERSION--/'$(RPM_VERSION)'/g' out/docker-machine-driver-kvm2-$(RPM_VERSION)/docker-machine-driver-kvm2.spec
 	sed -E -i 's|--OUT--|'$(PWD)/out'|g' out/docker-machine-driver-kvm2-$(RPM_VERSION)/docker-machine-driver-kvm2.spec
-	rpmbuild -bb -D "_rpmdir $(PWD)/out" -D "_rpmfilename docker-machine-driver-kvm2-$(RPM_VERSION).rpm" \
+	rpmbuild -bb -D "_rpmdir $(PWD)/out" --target $* \
 		out/docker-machine-driver-kvm2-$(RPM_VERSION)/docker-machine-driver-kvm2.spec
+	@mv out/$*/docker-machine-driver-kvm2-$(RPM_VERSION)-0.$*.rpm out/ && rmdir out/$*
 	rm -rf out/docker-machine-driver-kvm2-$(RPM_VERSION)
 
 .PHONY: kvm-image
diff --git a/installers/linux/deb/kvm2_deb_template/DEBIAN/control b/installers/linux/deb/kvm2_deb_template/DEBIAN/control
index 49a6075b6a..14b1fbe0d4 100644
--- a/installers/linux/deb/kvm2_deb_template/DEBIAN/control
+++ b/installers/linux/deb/kvm2_deb_template/DEBIAN/control
@@ -2,7 +2,7 @@ Package: docker-machine-driver-kvm2
 Version: --VERSION--
 Section: base
 Priority: optional
-Architecture: amd64
+Architecture: --ARCH--
 Depends: libvirt0 (>= 1.3.1)
 Recommends: minikube
 Maintainer: Thomas Strömberg <t+minikube@stromberg.org>

From c6340c78e726f7fdf2b86fffec82b084075be99f Mon Sep 17 00:00:00 2001
From: Zhongcheng Lao <Zhongcheng.Lao@microsoft.com>
Date: Mon, 3 Feb 2020 19:29:45 +0800
Subject: [PATCH 204/250] Fixes merge issue

Signed-off-by: Zhongcheng Lao <Zhongcheng.Lao@microsoft.com>
---
 pkg/minikube/bootstrapper/bsutil/kubelet.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkg/minikube/bootstrapper/bsutil/kubelet.go b/pkg/minikube/bootstrapper/bsutil/kubelet.go
index 3208df1d7f..3f3ae294c4 100644
--- a/pkg/minikube/bootstrapper/bsutil/kubelet.go
+++ b/pkg/minikube/bootstrapper/bsutil/kubelet.go
@@ -60,8 +60,8 @@ func NewKubeletConfig(mc config.MachineConfig, r cruntime.Manager) ([]byte, erro
 	if _, ok := extraOpts["node-ip"]; !ok {
 		extraOpts["node-ip"] = cp.IP
 	}
-	if k8s.NodeName != "" {
-		extraOpts["hostname-override"] = k8s.NodeName
+	if len(mc.Nodes) > 0 && mc.Nodes[0].Name != "" {
+		extraOpts["hostname-override"] = mc.Nodes[0].Name
 	}
 
 	pauseImage := images.Pause(k8s.ImageRepository)

From fc85dd6d45ddf38a1068ed533a44c4df79e91b67 Mon Sep 17 00:00:00 2001
From: Zhongcheng Lao <Zhongcheng.Lao@microsoft.com>
Date: Mon, 3 Feb 2020 22:31:59 +0800
Subject: [PATCH 205/250] Handle node info in kubelet config

---
 pkg/minikube/bootstrapper/bsutil/kubeadm.go      | 1 +
 pkg/minikube/bootstrapper/bsutil/kubelet.go      | 6 +++---
 pkg/minikube/bootstrapper/bsutil/kubelet_test.go | 2 +-
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go     | 3 ++-
 4 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/pkg/minikube/bootstrapper/bsutil/kubeadm.go b/pkg/minikube/bootstrapper/bsutil/kubeadm.go
index 8158ec151a..8e40f3ab95 100644
--- a/pkg/minikube/bootstrapper/bsutil/kubeadm.go
+++ b/pkg/minikube/bootstrapper/bsutil/kubeadm.go
@@ -88,6 +88,7 @@ func GenerateKubeadmYAML(mc config.MachineConfig, r cruntime.Manager) ([]byte, e
 		KubernetesVersion: k8s.KubernetesVersion,
 		EtcdDataDir:       EtcdDataDir(),
 		ClusterName:       k8s.ClusterName,
+		NodeName:          cp.Name,
 		CRISocket:         r.SocketPath(),
 		ImageRepository:   k8s.ImageRepository,
 		ExtraArgs:         extraComponentConfig,
diff --git a/pkg/minikube/bootstrapper/bsutil/kubelet.go b/pkg/minikube/bootstrapper/bsutil/kubelet.go
index 3f3ae294c4..f080a7eba7 100644
--- a/pkg/minikube/bootstrapper/bsutil/kubelet.go
+++ b/pkg/minikube/bootstrapper/bsutil/kubelet.go
@@ -30,7 +30,7 @@ import (
 
 // NewKubeletConfig generates a new systemd unit containing a configured kubelet
 // based on the options present in the KubernetesConfig.
-func NewKubeletConfig(mc config.MachineConfig, r cruntime.Manager) ([]byte, error) {
+func NewKubeletConfig(mc config.MachineConfig, nc config.Node, r cruntime.Manager) ([]byte, error) {
 	k8s := mc.KubernetesConfig
 	version, err := ParseKubernetesVersion(k8s.KubernetesVersion)
 	if err != nil {
@@ -60,8 +60,8 @@ func NewKubeletConfig(mc config.MachineConfig, r cruntime.Manager) ([]byte, erro
 	if _, ok := extraOpts["node-ip"]; !ok {
 		extraOpts["node-ip"] = cp.IP
 	}
-	if len(mc.Nodes) > 0 && mc.Nodes[0].Name != "" {
-		extraOpts["hostname-override"] = mc.Nodes[0].Name
+	if nc.Name != "" {
+		extraOpts["hostname-override"] = nc.Name
 	}
 
 	pauseImage := images.Pause(k8s.ImageRepository)
diff --git a/pkg/minikube/bootstrapper/bsutil/kubelet_test.go b/pkg/minikube/bootstrapper/bsutil/kubelet_test.go
index b50a4b89ef..d4bf9af482 100644
--- a/pkg/minikube/bootstrapper/bsutil/kubelet_test.go
+++ b/pkg/minikube/bootstrapper/bsutil/kubelet_test.go
@@ -177,7 +177,7 @@ ExecStart=/var/lib/minikube/binaries/v1.17.2/kubelet --authorization-mode=Webhoo
 				t.Fatalf("runtime: %v", err)
 			}
 
-			got, err := NewKubeletConfig(tc.cfg, runtime)
+			got, err := NewKubeletConfig(tc.cfg, tc.cfg.Nodes[0], runtime)
 			if err != nil && !tc.shouldErr {
 				t.Errorf("got unexpected error generating config: %v", err)
 				return
diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index d3801ddff8..e6b9505c44 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -464,7 +464,8 @@ func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error {
 		return errors.Wrap(err, "generating kubeadm cfg")
 	}
 
-	kubeletCfg, err := bsutil.NewKubeletConfig(cfg, r)
+	// TODO: multiple nodes
+	kubeletCfg, err := bsutil.NewKubeletConfig(cfg, cfg.Nodes[0], r)
 	if err != nil {
 		return errors.Wrap(err, "generating kubelet config")
 	}

From 6cc2fc8e4987559e55252914a9dc0a6fe5c7e9a5 Mon Sep 17 00:00:00 2001
From: tstromberg <tstromberg@google.com>
Date: Mon, 3 Feb 2020 10:49:06 -0800
Subject: [PATCH 206/250] Call createRequiredDirectories when starting
 pre-existing VM's, add logging

---
 pkg/minikube/cluster/cluster.go | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/pkg/minikube/cluster/cluster.go b/pkg/minikube/cluster/cluster.go
index e256fc4c1e..e45dec23ca 100644
--- a/pkg/minikube/cluster/cluster.go
+++ b/pkg/minikube/cluster/cluster.go
@@ -191,6 +191,10 @@ func configureHost(h *host.Host, e *engine.Options) error {
 		glog.Infof("configureHost completed within %s", time.Since(start))
 	}()
 
+	if err := createRequiredDirectories(h); err != nil {
+		errors.Wrap(err, "required directories")
+	}
+
 	if len(e.Env) > 0 {
 		h.HostOptions.EngineOptions.Env = e.Env
 		glog.Infof("Detecting provisioner ...")
@@ -284,6 +288,12 @@ func trySSHPowerOff(h *host.Host) error {
 
 // StopHost stops the host VM, saving state to disk.
 func StopHost(api libmachine.API) error {
+	glog.Infof("Stopping host ...")
+	start := time.Now()
+	defer func() {
+		glog.Infof("Stopped host within %s", time.Since(start))
+	}()
+
 	machineName := viper.GetString(config.MachineProfile)
 	host, err := api.Load(machineName)
 	if err != nil {
@@ -299,6 +309,7 @@ func StopHost(api libmachine.API) error {
 	}
 
 	if err := host.Stop(); err != nil {
+		glog.Infof("host.Stop failed: %v", err)
 		alreadyInStateError, ok := err.(mcnerror.ErrHostAlreadyInState)
 		if ok && alreadyInStateError.State == state.Stopped {
 			return nil
@@ -527,7 +538,7 @@ func createHost(api libmachine.API, cfg config.MachineConfig) (*host.Host, error
 	}
 
 	if err := createRequiredDirectories(h); err != nil {
-		return h, errors.Wrap(err, "required directories")
+		errors.Wrap(err, "required directories")
 	}
 
 	if driver.BareMetal(cfg.VMDriver) {

From bcc186b908499e18f9aae3afdf720477a734438e Mon Sep 17 00:00:00 2001
From: tstromberg <tstromberg@google.com>
Date: Mon, 3 Feb 2020 10:49:40 -0800
Subject: [PATCH 207/250] Startup previously enabled addons

---
 cmd/minikube/cmd/start.go |  8 ++++++--
 pkg/addons/addons.go      | 37 +++++++++++++++++++++++--------------
 2 files changed, 29 insertions(+), 16 deletions(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 9143bb8639..9b58c1b0f5 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -367,8 +367,12 @@ func runStart(cmd *cobra.Command, args []string) {
 	bootstrapCluster(bs, cr, mRunner, mc, preExists, isUpgrade)
 	configureMounts()
 
-	// enable addons
-	addons.Start(viper.GetString(config.MachineProfile), addonList)
+	// enable addons, both old and new!
+	existingAddons := map[string]bool{}
+	if existing != nil && existing.Addons != nil {
+		existingAddons = existing.Addons
+	}
+	addons.Start(viper.GetString(config.MachineProfile), existingAddons, addonList)
 
 	if err = cacheAndLoadImagesInConfig(); err != nil {
 		out.T(out.FailureType, "Unable to load cached images from config file.")
diff --git a/pkg/addons/addons.go b/pkg/addons/addons.go
index 06cf51d7dc..4fffa7e7ff 100644
--- a/pkg/addons/addons.go
+++ b/pkg/addons/addons.go
@@ -19,6 +19,7 @@ package addons
 import (
 	"fmt"
 	"path"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
@@ -68,8 +69,7 @@ func Set(name, value, profile string) error {
 		return errors.Wrap(err, "running callbacks")
 	}
 
-	glog.Infof("Writing new config for %q ...", profile)
-	// Write the value
+	glog.Infof("Writing out %q config to set %s=%v...", profile, name, value)
 	return config.Write(profile, c)
 }
 
@@ -263,33 +263,42 @@ func enableOrDisableStorageClasses(name, val, profile string) error {
 }
 
 // Start enables the default addons for a profile, plus any additional
-func Start(profile string, additional []string) {
+func Start(profile string, toEnable map[string]bool, additional []string) {
 	start := time.Now()
-	glog.Infof("enableAddons")
+	glog.Infof("enableAddons start: toEnable=%v, additional=%s", toEnable, additional)
 	defer func() {
 		glog.Infof("enableAddons completed in %s", time.Since(start))
 	}()
-	toEnable := []string{}
 
-	// Apply addons that are enabled by default
+	// Get the default values of any addons not saved to our config
 	for name, a := range assets.Addons {
-		enabled, err := a.IsEnabled(profile)
+		defaultVal, err := a.IsEnabled(profile)
 		if err != nil {
 			glog.Errorf("is-enabled failed for %q: %v", a.Name(), err)
 			continue
 		}
-		if enabled {
-			toEnable = append(toEnable, name)
+
+		_, exists := toEnable[name]
+		if !exists {
+			toEnable[name] = defaultVal
 		}
 	}
 
-	toEnable = append(toEnable, additional...)
-	if len(toEnable) == 0 {
-		return
+	// Apply new addons
+	for _, name := range additional {
+		toEnable[name] = true
 	}
 
-	out.T(out.AddonEnable, "Enabling addons: {{.addons}}", out.V{"addons": strings.Join(toEnable, ", ")})
-	for _, a := range toEnable {
+	toEnableList := []string{}
+	for k, v := range toEnable {
+		if v {
+			toEnableList = append(toEnableList, k)
+		}
+	}
+	sort.Strings(toEnableList)
+
+	out.T(out.AddonEnable, "Enabling addons: {{.addons}}", out.V{"addons": strings.Join(toEnableList, ", ")})
+	for _, a := range toEnableList {
 		err := Set(a, "true", profile)
 		if err != nil {
 			// Intentionally non-fatal

From 4ed5b4838e738d46f817c8b9e18616f3991b8628 Mon Sep 17 00:00:00 2001
From: tstromberg <tstromberg@google.com>
Date: Mon, 3 Feb 2020 10:50:06 -0800
Subject: [PATCH 208/250] Add integration test to ensure that we addons enabled
 while the cluster was stopped come up successfully

---
 test/integration/start_stop_delete_test.go | 36 +++++++++++++++-------
 1 file changed, 25 insertions(+), 11 deletions(-)

diff --git a/test/integration/start_stop_delete_test.go b/test/integration/start_stop_delete_test.go
index f71c78e6b2..6c9e2d6b48 100644
--- a/test/integration/start_stop_delete_test.go
+++ b/test/integration/start_stop_delete_test.go
@@ -105,9 +105,18 @@ func TestStartStop(t *testing.T) {
 					t.Errorf("%s failed: %v", rr.Args, err)
 				}
 
-				got := Status(ctx, t, Target(), profile, "Host")
-				if got != state.Stopped.String() {
-					t.Errorf("status = %q; want = %q", got, state.Stopped)
+				// The none driver never really stops
+				if !NoneDriver() {
+					got := Status(ctx, t, Target(), profile, "Host")
+					if got != state.Stopped.String() {
+						t.Errorf("post-stop host status = %q; want = %q", got, state.Stopped)
+					}
+				}
+
+				// Enable an addon to assert it comes up afterwards
+				rr, err = Run(t, exec.CommandContext(ctx, Target(), "addons", "enable", "dashboard", "-p", profile))
+				if err != nil {
+					t.Errorf("%s failed: %v", rr.Args, err)
 				}
 
 				rr, err = Run(t, exec.CommandContext(ctx, Target(), startArgs...))
@@ -118,13 +127,18 @@ func TestStartStop(t *testing.T) {
 
 				if strings.Contains(tc.name, "cni") {
 					t.Logf("WARNING: cni mode requires additional setup before pods can schedule :(")
-				} else if _, err := PodWait(ctx, t, profile, "default", "integration-test=busybox", 4*time.Minute); err != nil {
-					t.Fatalf("wait: %v", err)
+				} else {
+					if _, err := PodWait(ctx, t, profile, "default", "integration-test=busybox", 4*time.Minute); err != nil {
+						t.Fatalf("post-stop-start pod wait: %v", err)
+					}
+					if _, err := PodWait(ctx, t, profile, "kubernetes-dashboard", "k8s-app=kubernetes-dashboard", 4*time.Minute); err != nil {
+						t.Fatalf("post-stop-start addon wait: %v", err)
+					}
 				}
 
-				got = Status(ctx, t, Target(), profile, "Host")
+				got := Status(ctx, t, Target(), profile, "Host")
 				if got != state.Running.String() {
-					t.Errorf("host status = %q; want = %q", got, state.Running)
+					t.Errorf("post-start host status = %q; want = %q", got, state.Running)
 				}
 
 				if !NoneDriver() {
@@ -227,12 +241,12 @@ func testPause(ctx context.Context, t *testing.T, profile string) {
 
 	got := Status(ctx, t, Target(), profile, "APIServer")
 	if got != state.Paused.String() {
-		t.Errorf("apiserver status = %q; want = %q", got, state.Paused)
+		t.Errorf("post-pause apiserver status = %q; want = %q", got, state.Paused)
 	}
 
 	got = Status(ctx, t, Target(), profile, "Kubelet")
 	if got != state.Stopped.String() {
-		t.Errorf("kubelet status = %q; want = %q", got, state.Stopped)
+		t.Errorf("post-pause kubelet status = %q; want = %q", got, state.Stopped)
 	}
 
 	rr, err = Run(t, exec.CommandContext(ctx, Target(), "unpause", "-p", profile, "--alsologtostderr", "-v=1"))
@@ -242,12 +256,12 @@ func testPause(ctx context.Context, t *testing.T, profile string) {
 
 	got = Status(ctx, t, Target(), profile, "APIServer")
 	if got != state.Running.String() {
-		t.Errorf("apiserver status = %q; want = %q", got, state.Running)
+		t.Errorf("post-unpause apiserver status = %q; want = %q", got, state.Running)
 	}
 
 	got = Status(ctx, t, Target(), profile, "Kubelet")
 	if got != state.Running.String() {
-		t.Errorf("kubelet status = %q; want = %q", got, state.Running)
+		t.Errorf("post-unpause kubelet status = %q; want = %q", got, state.Running)
 	}
 
 }

From 769dd93aa0b87d905ef78e70c59f983a23e24961 Mon Sep 17 00:00:00 2001
From: tstromberg <tstromberg@google.com>
Date: Mon, 3 Feb 2020 10:56:53 -0800
Subject: [PATCH 209/250] Oops, make sure we return the error code

---
 pkg/minikube/cluster/cluster.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/minikube/cluster/cluster.go b/pkg/minikube/cluster/cluster.go
index e45dec23ca..dba72f50ca 100644
--- a/pkg/minikube/cluster/cluster.go
+++ b/pkg/minikube/cluster/cluster.go
@@ -538,7 +538,7 @@ func createHost(api libmachine.API, cfg config.MachineConfig) (*host.Host, error
 	}
 
 	if err := createRequiredDirectories(h); err != nil {
-		errors.Wrap(err, "required directories")
+		return h, errors.Wrap(err, "required directories")
 	}
 
 	if driver.BareMetal(cfg.VMDriver) {

From 2eddd08ae924282ea99019105d1f9e0ffcc4d517 Mon Sep 17 00:00:00 2001
From: tstromberg <tstromberg@google.com>
Date: Mon, 3 Feb 2020 12:57:52 -0800
Subject: [PATCH 210/250] Fix test signature

---
 pkg/addons/addons_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/addons/addons_test.go b/pkg/addons/addons_test.go
index e0337705c1..14449917a2 100644
--- a/pkg/addons/addons_test.go
+++ b/pkg/addons/addons_test.go
@@ -119,7 +119,7 @@ func TestEnableAndDisableAddon(t *testing.T) {
 
 func TestStart(t *testing.T) {
 	profile := createTestProfile(t)
-	Start(profile, []string{"dashboard"})
+	Start(profile, map[string]bool{}, []string{"dashboard"})
 
 	enabled, err := assets.Addons["dashboard"].IsEnabled(profile)
 	if err != nil {

From 1f1cfa5b528d6d81e0f8b3e1e3ce31cd1e192285 Mon Sep 17 00:00:00 2001
From: tstromberg <tstromberg@google.com>
Date: Mon, 3 Feb 2020 12:58:16 -0800
Subject: [PATCH 211/250] Update golangci-lint from v1.21 to v1.23.2

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 6a17726cf6..93ddd3ac28 100755
--- a/Makefile
+++ b/Makefile
@@ -51,7 +51,7 @@ MINIKUBE_RELEASES_URL=https://github.com/kubernetes/minikube/releases/download
 
 KERNEL_VERSION ?= 4.19.88
 # latest from https://github.com/golangci/golangci-lint/releases
-GOLINT_VERSION ?= v1.21.0
+GOLINT_VERSION ?= v1.23.2
 # Limit number of default jobs, to avoid the CI builds running out of memory
 GOLINT_JOBS ?= 4
 # see https://github.com/golangci/golangci-lint#memory-usage-of-golangci-lint

From 39da4432c76c73c17eb89b292263a60d133bb08b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= <anders.f.bjorklund@gmail.com>
Date: Mon, 3 Feb 2020 22:15:02 +0100
Subject: [PATCH 212/250] Fix building packages for architectures

Missed a suffix, and the rpm architecture names
---
 Makefile                                                  | 8 +++++++-
 .../rpm/kvm2_rpm_template/docker-machine-driver-kvm2.spec | 2 +-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 6a17726cf6..9e29595a8c 100755
--- a/Makefile
+++ b/Makefile
@@ -534,6 +534,12 @@ release-minikube: out/minikube checksum ## Minikube release
 out/docker-machine-driver-kvm2: out/docker-machine-driver-kvm2-amd64
 	cp $< $@
 
+out/docker-machine-driver-kvm2-x86_64: out/docker-machine-driver-kvm2-amd64
+	cp $< $@
+
+out/docker-machine-driver-kvm2-aarch64: out/docker-machine-driver-kvm2-arm64
+	cp $< $@
+
 out/docker-machine-driver-kvm2-%:
 ifeq ($(MINIKUBE_BUILD_IN_DOCKER),y)
 	docker inspect -f '{{.Id}} {{.RepoTags}}' $(KVM_BUILD_IMAGE) || $(MAKE) kvm-image
@@ -560,7 +566,7 @@ out/docker-machine-driver-kvm2_$(DEB_VERSION)-0_%.deb: out/docker-machine-driver
 	sed -E -i 's/--VERSION--/'$(DEB_VERSION)'/g' out/docker-machine-driver-kvm2_$(DEB_VERSION)/DEBIAN/control
 	sed -E -i 's/--ARCH--/'$*'/g' out/docker-machine-driver-kvm2_$(DEB_VERSION)/DEBIAN/control
 	mkdir -p out/docker-machine-driver-kvm2_$(DEB_VERSION)/usr/bin
-	cp out/docker-machine-driver-kvm2 out/docker-machine-driver-kvm2_$(DEB_VERSION)/usr/bin/docker-machine-driver-kvm2
+	cp $< out/docker-machine-driver-kvm2_$(DEB_VERSION)/usr/bin/docker-machine-driver-kvm2
 	fakeroot dpkg-deb --build out/docker-machine-driver-kvm2_$(DEB_VERSION) $@
 	rm -rf out/docker-machine-driver-kvm2_$(DEB_VERSION)
 
diff --git a/installers/linux/rpm/kvm2_rpm_template/docker-machine-driver-kvm2.spec b/installers/linux/rpm/kvm2_rpm_template/docker-machine-driver-kvm2.spec
index 372e7e18cc..ec309ffb66 100644
--- a/installers/linux/rpm/kvm2_rpm_template/docker-machine-driver-kvm2.spec
+++ b/installers/linux/rpm/kvm2_rpm_template/docker-machine-driver-kvm2.spec
@@ -18,7 +18,7 @@ a consistent way to manage various VM providers.
 %prep
 mkdir -p %{name}-%{version}
 cd %{name}-%{version}
-cp --OUT--/docker-machine-driver-kvm2 .
+cp --OUT--/docker-machine-driver-kvm2-%{_arch} docker-machine-driver-kvm2
 
 %install
 cd %{name}-%{version}

From a8283d8a3cbb146d8aa0f4973ad8dc6c196480f0 Mon Sep 17 00:00:00 2001
From: tstromberg <tstromberg@google.com>
Date: Mon, 3 Feb 2020 13:48:30 -0800
Subject: [PATCH 213/250] Fix lint issue

---
 pkg/minikube/cluster/cluster.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/minikube/cluster/cluster.go b/pkg/minikube/cluster/cluster.go
index dba72f50ca..86ac6e615e 100644
--- a/pkg/minikube/cluster/cluster.go
+++ b/pkg/minikube/cluster/cluster.go
@@ -192,7 +192,7 @@ func configureHost(h *host.Host, e *engine.Options) error {
 	}()
 
 	if err := createRequiredDirectories(h); err != nil {
-		errors.Wrap(err, "required directories")
+		return errors.Wrap(err, "required directories")
 	}
 
 	if len(e.Env) > 0 {

From 4124fbdb69c8c55274dd55bb324a37943e1ca69c Mon Sep 17 00:00:00 2001
From: tstromberg <tstromberg@google.com>
Date: Mon, 3 Feb 2020 14:20:57 -0800
Subject: [PATCH 214/250] Consider a paused apiserver to be controlled by a
 running host

---
 pkg/drivers/none/none.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/pkg/drivers/none/none.go b/pkg/drivers/none/none.go
index c9d8b1009d..3fd00b76b9 100644
--- a/pkg/drivers/none/none.go
+++ b/pkg/drivers/none/none.go
@@ -144,8 +144,10 @@ func (d *Driver) GetState() (state.State, error) {
 	if err != nil {
 		return ast, err
 	}
+
+	// If the apiserver is up, we'll claim to be up.
 	if ast == state.Paused || ast == state.Running {
-		return ast, nil
+		return state.Running, nil
 	}
 
 	return kverify.KubeletStatus(d.exec)

From e5e47db1c734bb7529689b1e3496f5a32ebe9077 Mon Sep 17 00:00:00 2001
From: tstromberg <tstromberg@google.com>
Date: Mon, 3 Feb 2020 14:21:09 -0800
Subject: [PATCH 215/250] Choose the newest apiserver pid, and choose more
 exactly

---
 .../bootstrapper/bsutil/kverify/kverify.go    | 30 +++++++++++--------
 1 file changed, 18 insertions(+), 12 deletions(-)

diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
index 0a056adcf4..de8f797ddb 100644
--- a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
+++ b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
@@ -24,6 +24,7 @@ import (
 	"net/http"
 	"os/exec"
 	"path"
+	"strconv"
 	"strings"
 	"time"
 
@@ -39,13 +40,12 @@ import (
 // APIServerProcess waits for api server to be healthy returns error if it doesn't
 func APIServerProcess(runner command.Runner, start time.Time, timeout time.Duration) error {
 	glog.Infof("waiting for apiserver process to appear ...")
-	err := wait.PollImmediate(time.Second*1, timeout, func() (bool, error) {
+	err := wait.PollImmediate(time.Millisecond*500, timeout, func() (bool, error) {
 		if time.Since(start) > timeout {
 			return false, fmt.Errorf("cluster wait timed out during process check")
 		}
-		rr, ierr := runner.RunCmd(exec.Command("sudo", "pgrep", "kube-apiserver"))
-		if ierr != nil {
-			glog.Warningf("pgrep apiserver: %v cmd: %s", ierr, rr.Command())
+
+		if _, ierr := apiServerPID(runner); ierr != nil {
 			return false, nil
 		}
 		return true, nil
@@ -57,6 +57,16 @@ func APIServerProcess(runner command.Runner, start time.Time, timeout time.Durat
 	return nil
 }
 
+// apiServerPID returns our best guess to the apiserver pid
+func apiServerPID(cr command.Runner) (int, error) {
+	rr, err := cr.RunCmd(exec.Command("sudo", "pgrep", "-xnf", "kube-apiserver.*minikube.*"))
+	if err != nil {
+		return 0, err
+	}
+	s := strings.TrimSpace(rr.Stdout.String())
+	return strconv.Atoi(s)
+}
+
 // SystemPods verifies essential pods for running kurnetes is running
 func SystemPods(client *kubernetes.Clientset, start time.Time, timeout time.Duration) error {
 	glog.Info("waiting for kube-system pods to appear ...")
@@ -124,19 +134,15 @@ func APIServerIsRunning(start time.Time, ip string, port int, timeout time.Durat
 // APIServerStatus returns apiserver status in libmachine style state.State
 func APIServerStatus(cr command.Runner, ip net.IP, port int) (state.State, error) {
 	glog.Infof("Checking apiserver status ...")
-	// sudo, in case hidepid is set
-	rr, err := cr.RunCmd(exec.Command("sudo", "pgrep", "kube-apiserver"))
+
+	pid, err := apiServerPID(cr)
 	if err != nil {
+		glog.Warningf("unable to get apiserver pid: %v", err)
 		return state.Stopped, nil
 	}
-	pids := strings.Split(strings.TrimSpace(rr.Stdout.String()), "\n")
-	pid := pids[len(pids)-1]
-	if len(pids) != 1 {
-		glog.Errorf("found %d apiserver pids: %v - choosing %s", len(pids), pids, pid)
-	}
 
 	// Get the freezer cgroup entry for this pid
-	rr, err = cr.RunCmd(exec.Command("sudo", "egrep", "^[0-9]+:freezer:", path.Join("/proc", pid, "cgroup")))
+	rr, err := cr.RunCmd(exec.Command("sudo", "egrep", "^[0-9]+:freezer:", fmt.Sprintf("/proc/%d/cgroup", pid)))
 	if err != nil {
 		glog.Warningf("unable to find freezer cgroup: %v", err)
 		return apiServerHealthz(ip, port)

From 38a4a4ab8b1e31ba4e58cb75067fe0341db07353 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Mon, 3 Feb 2020 15:09:25 -0800
Subject: [PATCH 216/250] more inclusive nslookup

---
 cmd/minikube/cmd/start.go | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 9143bb8639..05370ba939 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -1144,9 +1144,13 @@ Suggested workarounds:
 
 func tryLookup(r command.Runner) {
 	// DNS check
-	if rr, err := r.RunCmd(exec.Command("nslookup", "kubernetes.io")); err != nil {
-		glog.Warningf("%s failed: %v", rr.Args, err)
-		out.WarningT("VM may be unable to resolve external DNS records")
+	if rr, err := r.RunCmd(exec.Command("nslookup", "kubernetes.io", "-type=ns")); err != nil {
+		glog.Infof("%s failed: %v which might be okay will retry nslookup with query type", rr.Args, err)
+		// will try with different command format for ISOs with diffrent busybox versions.
+		if rr, err = r.RunCmd(exec.Command("nslookup", "kubernetes.io")); err != nil {
+			glog.Warningf("nslookup failed: %v", rr.Args, err)
+			out.WarningT("VM may be unable to resolve external DNS records")
+		}
 	}
 }
 

From b4c1db6e1487af718b72ec618e011cd52a019578 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Mon, 3 Feb 2020 15:11:12 -0800
Subject: [PATCH 217/250] change wording VM to node

---
 cmd/minikube/cmd/start.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 05370ba939..c0277bcc85 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -1149,7 +1149,7 @@ func tryLookup(r command.Runner) {
 		// will try with different command format for ISOs with diffrent busybox versions.
 		if rr, err = r.RunCmd(exec.Command("nslookup", "kubernetes.io")); err != nil {
 			glog.Warningf("nslookup failed: %v", rr.Args, err)
-			out.WarningT("VM may be unable to resolve external DNS records")
+			out.WarningT("Node may be unable to resolve external DNS records")
 		}
 	}
 }

From 3dd5495d2f919d4d5d12b9b7b1a6b5736ce05dba Mon Sep 17 00:00:00 2001
From: Medya Ghazizadeh <medyagh@users.noreply.github.com>
Date: Mon, 3 Feb 2020 16:31:52 -0700
Subject: [PATCH 218/250] trim kic base image (#6478)

---
 Makefile                       |  4 ++--
 hack/images/kicbase.Dockerfile | 33 ++++++++++++++++++++-------------
 pkg/drivers/kic/types.go       |  5 +++--
 3 files changed, 25 insertions(+), 17 deletions(-)

diff --git a/Makefile b/Makefile
index 5d2fecad1a..564359501b 100755
--- a/Makefile
+++ b/Makefile
@@ -500,8 +500,8 @@ storage-provisioner-image: out/storage-provisioner-$(GOARCH) ## Build storage-pr
 
 .PHONY: kic-base-image
 kic-base-image: ## builds the base image used for kic.
-	docker rmi -f $(REGISTRY)/kicbase:v0.0.3-snapshot || true
-	docker build -f ./hack/images/kicbase.Dockerfile -t $(REGISTRY)/kicbase:v0.0.3-snapshot  --build-arg COMMIT_SHA=${VERSION}-$(COMMIT)  .
+	docker rmi -f $(REGISTRY)/kicbase:v0.0.4-snapshot || true
+	docker build -f ./hack/images/kicbase.Dockerfile -t $(REGISTRY)/kicbase:v0.0.4-snapshot  --build-arg COMMIT_SHA=${VERSION}-$(COMMIT)  .
 
 
 
diff --git a/hack/images/kicbase.Dockerfile b/hack/images/kicbase.Dockerfile
index aeafd04403..1636d288bc 100644
--- a/hack/images/kicbase.Dockerfile
+++ b/hack/images/kicbase.Dockerfile
@@ -1,7 +1,8 @@
 ARG COMMIT_SHA
-# for now using node image created by kind https://github.com/kubernetes-sigs/kind
-# could be changed to slim ubuntu with systemd. 
-FROM kindest/node:v1.16.2
+# using base image created by kind https://github.com/kubernetes-sigs/kind
+# which is an ubuntu 19.10 with an entry-point that helps running systemd
+# could be changed to any debian that can run systemd
+FROM kindest/base:v20200122-2dfe64b2
 USER root
 RUN apt-get update && apt-get install -y \
   sudo \
@@ -9,14 +10,28 @@ RUN apt-get update && apt-get install -y \
   openssh-server \
   docker.io \
   && apt-get clean -y 
-# based on https://github.com/rastasheep/ubuntu-sshd/blob/master/18.04/Dockerfile
+# disable containerd by default
+RUN systemctl disable containerd
+# enable docker which is default
+RUN systemctl enable docker
 # making SSH work for docker container 
+# based on https://github.com/rastasheep/ubuntu-sshd/blob/master/18.04/Dockerfile
 RUN mkdir /var/run/sshd
 RUN echo 'root:root' |chpasswd
 RUN sed -ri 's/^#?PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config
 RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config
 EXPOSE 22
-# Deleting all "kind" related stuff from the image.
+# for minikube ssh. to match VM using "docker" as username
+RUN adduser --ingroup docker --disabled-password --gecos '' docker 
+RUN adduser docker sudo
+RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+USER docker
+RUN mkdir /home/docker/.ssh
+# Deleting leftovers
+USER root
+# kind base-image entry-point expects a "kind" folder for product_name,product_uuid
+# https://github.com/kubernetes-sigs/kind/blob/master/images/base/files/usr/local/bin/entrypoint
+RUN mkdir -p /kind
 RUN rm -rf \
     /var/cache/debconf/* \
     /var/lib/apt/lists/* \
@@ -26,12 +41,4 @@ RUN rm -rf \
     /usr/share/doc/* \
     /usr/share/man/* \
     /usr/share/local/* \
-    /kind/bin/kubeadm /kind/bin/kubelet /kind/systemd /kind/images /kind/manifests
 RUN echo "kic! Build: ${COMMIT_SHA} Time :$(date)" > "/kic.txt"
-# for minikube ssh. to match VM using docker username
-RUN adduser --ingroup docker --disabled-password --gecos '' docker 
-RUN adduser docker sudo
-RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
-USER docker
-RUN mkdir /home/docker/.ssh
-USER root
diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go
index 32f2a254ba..62826954ca 100644
--- a/pkg/drivers/kic/types.go
+++ b/pkg/drivers/kic/types.go
@@ -25,8 +25,9 @@ const (
 	DefaultPodCIDR = "10.244.0.0/16"
 	// DefaultBindIPV4 is The default IP the container will bind to.
 	DefaultBindIPV4 = "127.0.0.1"
-	// BaseImage is the base image is used to spin up kic containers created by kind.
-	BaseImage = "gcr.io/k8s-minikube/kicbase:v0.0.3@sha256:34db5e30f8830c0d5e49b62f3ea6b2844f805980592fe0084cbea799bfb12664" // OverlayImage is the cni plugin used for overlay image, created by kind.
+	// BaseImage is the base image is used to spin up kic containers. it uses same base-image as kind.
+	BaseImage = "gcr.io/k8s-minikube/kicbase:v0.0.4@sha256:01552b7ffc7cb0c49c6106796ed73f69647691d8fbae89fed8ee317dc7f1015a"
+	// OverlayImage is the cni plugin used for overlay image, created by kind.
 	// CNI plugin image used for kic drivers created by kind.
 	OverlayImage = "kindest/kindnetd:0.5.3"
 )

From 4a4e7b78594dd4583a0ece8b62699dca3930d04f Mon Sep 17 00:00:00 2001
From: Karol Zadora-Przylecki <karolz@microsoft.com>
Date: Mon, 13 Jan 2020 14:07:22 -0800
Subject: [PATCH 219/250] Add Azure Container Registry support

---
 cmd/minikube/cmd/config/configure.go          | 30 +++++++++++++++++++
 .../registry-creds-rc.yaml.tmpl               | 17 ++++++++++-
 .../content/en/docs/Tasks/Registry/private.md |  6 ++--
 3 files changed, 50 insertions(+), 3 deletions(-)

diff --git a/cmd/minikube/cmd/config/configure.go b/cmd/minikube/cmd/config/configure.go
index b51d240063..412fa1204d 100644
--- a/cmd/minikube/cmd/config/configure.go
+++ b/cmd/minikube/cmd/config/configure.go
@@ -53,6 +53,9 @@ var addonsConfigureCmd = &cobra.Command{
 			dockerUser := "changeme"
 			dockerPass := "changeme"
 			gcrURL := "https://gcr.io"
+			acrURL := "changeme"
+			acrClientID := "changeme"
+			acrPassword := "changeme"
 
 			enableAWSECR := AskForYesNoConfirmation("\nDo you want to enable AWS Elastic Container Registry?", posResponses, negResponses)
 			if enableAWSECR {
@@ -90,6 +93,13 @@ var addonsConfigureCmd = &cobra.Command{
 				dockerPass = AskForPasswordValue("-- Enter docker registry password: ")
 			}
 
+			enableACR := AskForYesNoConfirmation("\nDo you want to enable Azure Container Registry?", posResponses, negResponses)
+			if enableACR {
+				acrURL = AskForStaticValue("-- Enter Azure Container Registry (ACR) URL: ")
+				acrClientID = AskForStaticValue("-- Enter client ID (service principal ID) to access ACR: ")
+				acrPassword = AskForPasswordValue("-- Enter service principal password to access Azure Container Registry: ")
+			}
+
 			// Create ECR Secret
 			err := service.CreateSecret(
 				"kube-system",
@@ -148,6 +158,26 @@ var addonsConfigureCmd = &cobra.Command{
 			if err != nil {
 				out.WarningT("ERROR creating `registry-creds-dpr` secret")
 			}
+
+			// Create Azure Container Registry Secret
+			err = service.CreateSecret(
+				"kube-system",
+				"registry-creds-acr",
+				map[string]string{
+					"ACR_URL":       acrURL,
+					"ACR_CLIENT_ID": acrClientID,
+					"ACR_PASSWORD":  acrPassword,
+				},
+				map[string]string{
+					"app":                           "registry-creds",
+					"cloud":                         "acr",
+					"kubernetes.io/minikube-addons": "registry-creds",
+				})
+
+			if err != nil {
+				out.WarningT("ERROR creating `registry-creds-acr` secret")
+			}
+
 		default:
 			out.FailureT("{{.name}} has no available configuration options", out.V{"name": addon})
 			return
diff --git a/deploy/addons/registry-creds/registry-creds-rc.yaml.tmpl b/deploy/addons/registry-creds/registry-creds-rc.yaml.tmpl
index 8d3c3f1fd4..4af7e7400a 100644
--- a/deploy/addons/registry-creds/registry-creds-rc.yaml.tmpl
+++ b/deploy/addons/registry-creds/registry-creds-rc.yaml.tmpl
@@ -18,7 +18,7 @@ spec:
         addonmanager.kubernetes.io/mode: Reconcile
     spec:
       containers:
-      - image: upmcenterprises/registry-creds:1.9
+      - image: upmcenterprises/registry-creds:1.10
         name: registry-creds
         imagePullPolicy: Always
         env:
@@ -77,6 +77,21 @@ spec:
               secretKeyRef:
                 name: registry-creds-gcr
                 key: gcrurl
+          - name: ACR_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: registry-creds-acr
+                key: ACR_PASSWORD
+          - name: ACR_URL
+            valueFrom:
+              secretKeyRef:
+                name: registry-creds-acr
+                key: ACR_URL
+          - name: ACR_CLIENT_ID
+            valueFrom:
+              secretKeyRef:
+                name: registry-creds-acr
+                key: ACR_CLIENT_ID
         volumeMounts:
         - name: gcr-creds
           mountPath: "/root/.config/gcloud"
diff --git a/site/content/en/docs/Tasks/Registry/private.md b/site/content/en/docs/Tasks/Registry/private.md
index b2a835e7e8..270ad665e4 100644
--- a/site/content/en/docs/Tasks/Registry/private.md
+++ b/site/content/en/docs/Tasks/Registry/private.md
@@ -2,13 +2,13 @@
 title: "Private"
 linkTitle: "Private"
 weight: 6
-date: 2019-08-01
+date: 2020-01-14
 description: >
   How to use a private registry within minikube
 ---
 
 
-**GCR/ECR/Docker**: minikube has an addon, `registry-creds` which maps credentials into minikube to support pulling from Google Container Registry (GCR), Amazon's EC2 Container Registry (ECR), and Private Docker registries.  You will need to run `minikube addons configure registry-creds` and `minikube addons enable registry-creds` to get up and running.  An example of this is below:
+**GCR/ECR/ACR/Docker**: minikube has an addon, `registry-creds` which maps credentials into minikube to support pulling from Google Container Registry (GCR), Amazon's EC2 Container Registry (ECR), Azure Container Registry (ACR), and Private Docker registries.  You will need to run `minikube addons configure registry-creds` and `minikube addons enable registry-creds` to get up and running.  An example of this is below:
 
 ```shell
 $ minikube addons configure registry-creds
@@ -18,6 +18,8 @@ Do you want to enable Google Container Registry? [y/n]: y
 -- Enter path to credentials (e.g. /home/user/.config/gcloud/application_default_credentials.json):/home/user/.config/gcloud/application_default_credentials.json
 
 Do you want to enable Docker Registry? [y/n]: n
+
+Do you want to enable Azure Container Registry? [y/n]: n
 registry-creds was successfully configured
 $ minikube addons enable registry-creds
 ```

From 16fd9beea368a6542557cedbce58e6f2e928b25d Mon Sep 17 00:00:00 2001
From: tstromberg <tstromberg@google.com>
Date: Mon, 3 Feb 2020 16:19:43 -0800
Subject: [PATCH 220/250] Split cluster package into multiple files

---
 pkg/minikube/cluster/cluster.go    | 691 -----------------------------
 pkg/minikube/cluster/delete.go     |  60 +++
 pkg/minikube/cluster/docker_env.go |  50 +++
 pkg/minikube/cluster/info.go       | 100 +++++
 pkg/minikube/cluster/ip.go         |  92 ++++
 pkg/minikube/cluster/iso.go        |  14 +
 pkg/minikube/cluster/restart.go    | 103 +++++
 pkg/minikube/cluster/ssh.go        |  33 ++
 pkg/minikube/cluster/start.go      | 255 +++++++++++
 pkg/minikube/cluster/status.go     |  63 +++
 pkg/minikube/cluster/stop.go       |  60 +++
 11 files changed, 830 insertions(+), 691 deletions(-)
 create mode 100644 pkg/minikube/cluster/delete.go
 create mode 100644 pkg/minikube/cluster/docker_env.go
 create mode 100644 pkg/minikube/cluster/info.go
 create mode 100644 pkg/minikube/cluster/ip.go
 create mode 100644 pkg/minikube/cluster/iso.go
 create mode 100644 pkg/minikube/cluster/restart.go
 create mode 100644 pkg/minikube/cluster/ssh.go
 create mode 100644 pkg/minikube/cluster/start.go
 create mode 100644 pkg/minikube/cluster/status.go
 create mode 100644 pkg/minikube/cluster/stop.go

diff --git a/pkg/minikube/cluster/cluster.go b/pkg/minikube/cluster/cluster.go
index e256fc4c1e..9f4a87cb7f 100644
--- a/pkg/minikube/cluster/cluster.go
+++ b/pkg/minikube/cluster/cluster.go
@@ -17,50 +17,11 @@ limitations under the License.
 package cluster
 
 import (
-	"encoding/json"
 	"flag"
-	"fmt"
-	"io/ioutil"
-	"math"
-	"net"
-	"os/exec"
-	"path"
-	"path/filepath"
-	"regexp"
-	"strconv"
-	"strings"
-	"time"
 
-	"github.com/docker/machine/libmachine"
-	"github.com/docker/machine/libmachine/drivers"
-	"github.com/docker/machine/libmachine/engine"
-	"github.com/docker/machine/libmachine/host"
-	"github.com/docker/machine/libmachine/mcnerror"
-	"github.com/docker/machine/libmachine/provision"
 	"github.com/docker/machine/libmachine/ssh"
-	"github.com/docker/machine/libmachine/state"
-	"github.com/golang/glog"
-	"github.com/juju/mutex"
-	"github.com/pkg/errors"
-	"github.com/shirou/gopsutil/cpu"
-	"github.com/shirou/gopsutil/disk"
-	"github.com/shirou/gopsutil/mem"
-	"github.com/spf13/viper"
 
-	"k8s.io/minikube/pkg/drivers/kic"
-	"k8s.io/minikube/pkg/drivers/kic/oci"
-	"k8s.io/minikube/pkg/minikube/command"
-	"k8s.io/minikube/pkg/minikube/config"
-	"k8s.io/minikube/pkg/minikube/constants"
-	"k8s.io/minikube/pkg/minikube/driver"
 	"k8s.io/minikube/pkg/minikube/exit"
-	"k8s.io/minikube/pkg/minikube/localpath"
-	"k8s.io/minikube/pkg/minikube/out"
-	"k8s.io/minikube/pkg/minikube/registry"
-	"k8s.io/minikube/pkg/minikube/sshutil"
-	"k8s.io/minikube/pkg/minikube/vmpath"
-	"k8s.io/minikube/pkg/util/lock"
-	"k8s.io/minikube/pkg/util/retry"
 )
 
 // hostRunner is a minimal host.Host based interface for running commands
@@ -68,23 +29,6 @@ type hostRunner interface {
 	RunSSHCommand(string) (string, error)
 }
 
-var (
-	// The maximum the guest VM clock is allowed to be ahead and behind. This value is intentionally
-	// large to allow for inaccurate methodology, but still small enough so that certificates are likely valid.
-	maxClockDesyncSeconds = 2.1
-
-	// requiredDirectories are directories to create on the host during setup
-	requiredDirectories = []string{
-		vmpath.GuestAddonsDir,
-		vmpath.GuestManifestsDir,
-		vmpath.GuestEphemeralDir,
-		vmpath.GuestPersistentDir,
-		vmpath.GuestCertsDir,
-		path.Join(vmpath.GuestPersistentDir, "images"),
-		path.Join(vmpath.GuestPersistentDir, "binaries"),
-	}
-)
-
 // This init function is used to set the logtostderr variable to false so that INFO level log info does not clutter the CLI
 // INFO lvl logging is displayed due to the kubernetes api calling flag.Set("logtostderr", "true") in its init()
 // see: https://github.com/kubernetes/kubernetes/blob/master/pkg/kubectl/util/logs/logs.go#L32-L34
@@ -96,638 +40,3 @@ func init() {
 	// Setting the default client to native gives much better performance.
 	ssh.SetDefaultClient(ssh.Native)
 }
-
-// CacheISO downloads and caches ISO.
-func CacheISO(cfg config.MachineConfig) error {
-	if driver.BareMetal(cfg.VMDriver) {
-		return nil
-	}
-	return cfg.Downloader.CacheMinikubeISOFromURL(cfg.MinikubeISO)
-}
-
-// StartHost starts a host VM.
-func StartHost(api libmachine.API, cfg config.MachineConfig) (*host.Host, error) {
-	// Prevent machine-driver boot races, as well as our own certificate race
-	releaser, err := acquireMachinesLock(cfg.Name)
-	if err != nil {
-		return nil, errors.Wrap(err, "boot lock")
-	}
-	start := time.Now()
-	defer func() {
-		glog.Infof("releasing machines lock for %q, held for %s", cfg.Name, time.Since(start))
-		releaser.Release()
-	}()
-
-	exists, err := api.Exists(cfg.Name)
-	if err != nil {
-		return nil, errors.Wrapf(err, "exists: %s", cfg.Name)
-	}
-	if !exists {
-		glog.Infoln("Machine does not exist... provisioning new machine")
-		glog.Infof("Provisioning machine with config: %+v", cfg)
-		return createHost(api, cfg)
-	}
-
-	glog.Infoln("Skipping create...Using existing machine configuration")
-
-	h, err := api.Load(cfg.Name)
-	if err != nil {
-		return nil, errors.Wrap(err, "Error loading existing host. Please try running [minikube delete], then run [minikube start] again.")
-	}
-
-	if exists && cfg.Name == constants.DefaultMachineName {
-		out.T(out.Tip, "Tip: Use 'minikube start -p <name>' to create a new cluster, or 'minikube delete' to delete this one.")
-	}
-
-	s, err := h.Driver.GetState()
-	glog.Infoln("Machine state: ", s)
-	if err != nil {
-		return nil, errors.Wrap(err, "Error getting state for host")
-	}
-
-	if s == state.Running {
-		out.T(out.Running, `Using the running {{.driver_name}} "{{.profile_name}}" VM ...`, out.V{"driver_name": cfg.VMDriver, "profile_name": cfg.Name})
-	} else {
-		out.T(out.Restarting, `Starting existing {{.driver_name}} VM for "{{.profile_name}}" ...`, out.V{"driver_name": cfg.VMDriver, "profile_name": cfg.Name})
-		if err := h.Driver.Start(); err != nil {
-			return nil, errors.Wrap(err, "start")
-		}
-		if err := api.Save(h); err != nil {
-			return nil, errors.Wrap(err, "save")
-		}
-	}
-
-	e := engineOptions(cfg)
-	glog.Infof("engine options: %+v", e)
-
-	out.T(out.Waiting, "Waiting for the host to be provisioned ...")
-	err = configureHost(h, e)
-	if err != nil {
-		return nil, err
-	}
-	return h, nil
-}
-
-// acquireMachinesLock protects against code that is not parallel-safe (libmachine, cert setup)
-func acquireMachinesLock(name string) (mutex.Releaser, error) {
-	spec := lock.PathMutexSpec(filepath.Join(localpath.MiniPath(), "machines"))
-	// NOTE: Provisioning generally completes within 60 seconds
-	spec.Timeout = 10 * time.Minute
-
-	glog.Infof("acquiring machines lock for %s: %+v", name, spec)
-	start := time.Now()
-	r, err := mutex.Acquire(spec)
-	if err == nil {
-		glog.Infof("acquired machines lock for %q in %s", name, time.Since(start))
-	}
-	return r, err
-}
-
-// configureHost handles any post-powerup configuration required
-func configureHost(h *host.Host, e *engine.Options) error {
-	start := time.Now()
-	glog.Infof("configureHost: %+v", h.Driver)
-	defer func() {
-		glog.Infof("configureHost completed within %s", time.Since(start))
-	}()
-
-	if len(e.Env) > 0 {
-		h.HostOptions.EngineOptions.Env = e.Env
-		glog.Infof("Detecting provisioner ...")
-		provisioner, err := provision.DetectProvisioner(h.Driver)
-		if err != nil {
-			return errors.Wrap(err, "detecting provisioner")
-		}
-		glog.Infof("Provisioning with %s: %+v", provisioner.String(), *h.HostOptions)
-		if err := provisioner.Provision(*h.HostOptions.SwarmOptions, *h.HostOptions.AuthOptions, *h.HostOptions.EngineOptions); err != nil {
-			return errors.Wrap(err, "provision")
-		}
-	}
-
-	if driver.BareMetal(h.Driver.DriverName()) {
-		glog.Infof("%s is a local driver, skipping auth/time setup", h.Driver.DriverName())
-		return nil
-	}
-	glog.Infof("Configuring auth for driver %s ...", h.Driver.DriverName())
-	if err := h.ConfigureAuth(); err != nil {
-		return &retry.RetriableError{Err: errors.Wrap(err, "Error configuring auth on host")}
-	}
-	return ensureSyncedGuestClock(h)
-}
-
-// ensureGuestClockSync ensures that the guest system clock is relatively in-sync
-func ensureSyncedGuestClock(h hostRunner) error {
-	d, err := guestClockDelta(h, time.Now())
-	if err != nil {
-		glog.Warningf("Unable to measure system clock delta: %v", err)
-		return nil
-	}
-	if math.Abs(d.Seconds()) < maxClockDesyncSeconds {
-		glog.Infof("guest clock delta is within tolerance: %s", d)
-		return nil
-	}
-	if err := adjustGuestClock(h, time.Now()); err != nil {
-		return errors.Wrap(err, "adjusting system clock")
-	}
-	return nil
-}
-
-// guestClockDelta returns the approximate difference between the host and guest system clock
-// NOTE: This does not currently take into account ssh latency.
-func guestClockDelta(h hostRunner, local time.Time) (time.Duration, error) {
-	out, err := h.RunSSHCommand("date +%s.%N")
-	if err != nil {
-		return 0, errors.Wrap(err, "get clock")
-	}
-	glog.Infof("guest clock: %s", out)
-	ns := strings.Split(strings.TrimSpace(out), ".")
-	secs, err := strconv.ParseInt(strings.TrimSpace(ns[0]), 10, 64)
-	if err != nil {
-		return 0, errors.Wrap(err, "atoi")
-	}
-	nsecs, err := strconv.ParseInt(strings.TrimSpace(ns[1]), 10, 64)
-	if err != nil {
-		return 0, errors.Wrap(err, "atoi")
-	}
-	// NOTE: In a synced state, remote is a few hundred ms ahead of local
-	remote := time.Unix(secs, nsecs)
-	d := remote.Sub(local)
-	glog.Infof("Guest: %s Remote: %s (delta=%s)", remote, local, d)
-	return d, nil
-}
-
-// adjustSystemClock adjusts the guest system clock to be nearer to the host system clock
-func adjustGuestClock(h hostRunner, t time.Time) error {
-	out, err := h.RunSSHCommand(fmt.Sprintf("sudo date -s @%d", t.Unix()))
-	glog.Infof("clock set: %s (err=%v)", out, err)
-	return err
-}
-
-// trySSHPowerOff runs the poweroff command on the guest VM to speed up deletion
-func trySSHPowerOff(h *host.Host) error {
-	s, err := h.Driver.GetState()
-	if err != nil {
-		glog.Warningf("unable to get state: %v", err)
-		return err
-	}
-	if s != state.Running {
-		glog.Infof("host is in state %s", s)
-		return nil
-	}
-
-	out.T(out.Shutdown, `Powering off "{{.profile_name}}" via SSH ...`, out.V{"profile_name": h.Name})
-	out, err := h.RunSSHCommand("sudo poweroff")
-	// poweroff always results in an error, since the host disconnects.
-	glog.Infof("poweroff result: out=%s, err=%v", out, err)
-	return nil
-}
-
-// StopHost stops the host VM, saving state to disk.
-func StopHost(api libmachine.API) error {
-	machineName := viper.GetString(config.MachineProfile)
-	host, err := api.Load(machineName)
-	if err != nil {
-		return errors.Wrapf(err, "load")
-	}
-
-	out.T(out.Stopping, `Stopping "{{.profile_name}}" in {{.driver_name}} ...`, out.V{"profile_name": machineName, "driver_name": host.DriverName})
-	if host.DriverName == driver.HyperV {
-		glog.Infof("As there are issues with stopping Hyper-V VMs using API, trying to shut down using SSH")
-		if err := trySSHPowerOff(host); err != nil {
-			return errors.Wrap(err, "ssh power off")
-		}
-	}
-
-	if err := host.Stop(); err != nil {
-		alreadyInStateError, ok := err.(mcnerror.ErrHostAlreadyInState)
-		if ok && alreadyInStateError.State == state.Stopped {
-			return nil
-		}
-		return &retry.RetriableError{Err: errors.Wrapf(err, "Stop: %s", machineName)}
-	}
-	return nil
-}
-
-// deleteOrphanedKIC attempts to delete an orphaned docker instance
-func deleteOrphanedKIC(name string) {
-	cmd := exec.Command(oci.Docker, "rm", "-f", "-v", name)
-	err := cmd.Run()
-	if err == nil {
-		glog.Infof("Found stale kic container and successfully cleaned it up!")
-	}
-}
-
-// DeleteHost deletes the host VM.
-func DeleteHost(api libmachine.API, machineName string) error {
-	host, err := api.Load(machineName)
-	if err != nil && host == nil {
-		deleteOrphanedKIC(machineName)
-		// keep going even if minikube  does not know about the host
-	}
-
-	// Get the status of the host. Ensure that it exists before proceeding ahead.
-	status, err := GetHostStatus(api, machineName)
-	if err != nil {
-		// Warn, but proceed
-		out.WarningT("Unable to get the status of the {{.name}} cluster.", out.V{"name": machineName})
-	}
-
-	if status == state.None.String() {
-		return mcnerror.ErrHostDoesNotExist{Name: machineName}
-	}
-
-	// This is slow if SSH is not responding, but HyperV hangs otherwise, See issue #2914
-	if host.Driver.DriverName() == driver.HyperV {
-		if err := trySSHPowerOff(host); err != nil {
-			glog.Infof("Unable to power off minikube because the host was not found.")
-		}
-		out.T(out.DeletingHost, "Successfully powered off Hyper-V. minikube driver -- {{.driver}}", out.V{"driver": host.Driver.DriverName()})
-	}
-
-	out.T(out.DeletingHost, `Deleting "{{.profile_name}}" in {{.driver_name}} ...`, out.V{"profile_name": machineName, "driver_name": host.DriverName})
-	if err := host.Driver.Remove(); err != nil {
-		return errors.Wrap(err, "host remove")
-	}
-	if err := api.Remove(machineName); err != nil {
-		return errors.Wrap(err, "api remove")
-	}
-	return nil
-}
-
-// GetHostStatus gets the status of the host VM.
-func GetHostStatus(api libmachine.API, machineName string) (string, error) {
-	exists, err := api.Exists(machineName)
-	if err != nil {
-		return "", errors.Wrapf(err, "%s exists", machineName)
-	}
-	if !exists {
-		return state.None.String(), nil
-	}
-
-	host, err := api.Load(machineName)
-	if err != nil {
-		return "", errors.Wrapf(err, "load")
-	}
-
-	s, err := host.Driver.GetState()
-	if err != nil {
-		return "", errors.Wrap(err, "state")
-	}
-	return s.String(), nil
-}
-
-// GetHostDriverIP gets the ip address of the current minikube cluster
-func GetHostDriverIP(api libmachine.API, machineName string) (net.IP, error) {
-	host, err := CheckIfHostExistsAndLoad(api, machineName)
-	if err != nil {
-		return nil, err
-	}
-
-	ipStr, err := host.Driver.GetIP()
-	if err != nil {
-		return nil, errors.Wrap(err, "getting IP")
-	}
-	if driver.IsKIC(host.DriverName) {
-		ipStr = kic.DefaultBindIPV4
-	}
-	ip := net.ParseIP(ipStr)
-	if ip == nil {
-		return nil, fmt.Errorf("parsing IP: %s", ipStr)
-	}
-	return ip, nil
-}
-
-func engineOptions(cfg config.MachineConfig) *engine.Options {
-	o := engine.Options{
-		Env:              cfg.DockerEnv,
-		InsecureRegistry: append([]string{constants.DefaultServiceCIDR}, cfg.InsecureRegistry...),
-		RegistryMirror:   cfg.RegistryMirror,
-		ArbitraryFlags:   cfg.DockerOpt,
-		InstallURL:       drivers.DefaultEngineInstallURL,
-	}
-	return &o
-}
-
-type hostInfo struct {
-	Memory   int
-	CPUs     int
-	DiskSize int
-}
-
-func megs(bytes uint64) int {
-	return int(bytes / 1024 / 1024)
-}
-
-func getHostInfo() (*hostInfo, error) {
-	i, err := cpu.Info()
-	if err != nil {
-		glog.Warningf("Unable to get CPU info: %v", err)
-		return nil, err
-	}
-	v, err := mem.VirtualMemory()
-	if err != nil {
-		glog.Warningf("Unable to get mem info: %v", err)
-		return nil, err
-	}
-	d, err := disk.Usage("/")
-	if err != nil {
-		glog.Warningf("Unable to get disk info: %v", err)
-		return nil, err
-	}
-
-	var info hostInfo
-	info.CPUs = len(i)
-	info.Memory = megs(v.Total)
-	info.DiskSize = megs(d.Total)
-	return &info, nil
-}
-
-// showLocalOsRelease shows systemd information about the current linux distribution, on the local host
-func showLocalOsRelease() {
-	osReleaseOut, err := ioutil.ReadFile("/etc/os-release")
-	if err != nil {
-		glog.Errorf("ReadFile: %v", err)
-		return
-	}
-
-	osReleaseInfo, err := provision.NewOsRelease(osReleaseOut)
-	if err != nil {
-		glog.Errorf("NewOsRelease: %v", err)
-		return
-	}
-
-	out.T(out.Provisioner, "OS release is {{.pretty_name}}", out.V{"pretty_name": osReleaseInfo.PrettyName})
-}
-
-// showRemoteOsRelease shows systemd information about the current linux distribution, on the remote VM
-func showRemoteOsRelease(driver drivers.Driver) {
-	provisioner, err := provision.DetectProvisioner(driver)
-	if err != nil {
-		glog.Errorf("DetectProvisioner: %v", err)
-		return
-	}
-
-	osReleaseInfo, err := provisioner.GetOsReleaseInfo()
-	if err != nil {
-		glog.Errorf("GetOsReleaseInfo: %v", err)
-		return
-	}
-
-	glog.Infof("Provisioned with %s", osReleaseInfo.PrettyName)
-}
-
-// showHostInfo shows host information
-func showHostInfo(cfg config.MachineConfig) {
-	if driver.BareMetal(cfg.VMDriver) {
-		info, err := getHostInfo()
-		if err == nil {
-			out.T(out.StartingNone, "Running on localhost (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...", out.V{"number_of_cpus": info.CPUs, "memory_size": info.Memory, "disk_size": info.DiskSize})
-		}
-	} else if driver.IsKIC(cfg.VMDriver) {
-		info, err := getHostInfo() // TODO medyagh: get docker-machine info for non linux
-		if err == nil {
-			out.T(out.StartingVM, "Creating Kubernetes in {{.driver_name}} container with (CPUs={{.number_of_cpus}}), Memory={{.memory_size}}MB ({{.host_memory_size}}MB available) ...", out.V{"driver_name": cfg.VMDriver, "number_of_cpus": cfg.CPUs, "number_of_host_cpus": info.CPUs, "memory_size": cfg.Memory, "host_memory_size": info.Memory})
-		}
-	} else {
-		out.T(out.StartingVM, "Creating {{.driver_name}} VM (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...", out.V{"driver_name": cfg.VMDriver, "number_of_cpus": cfg.CPUs, "memory_size": cfg.Memory, "disk_size": cfg.DiskSize})
-	}
-}
-
-func createHost(api libmachine.API, cfg config.MachineConfig) (*host.Host, error) {
-	if cfg.VMDriver == driver.VMwareFusion && viper.GetBool(config.ShowDriverDeprecationNotification) {
-		out.WarningT(`The vmwarefusion driver is deprecated and support for it will be removed in a future release.
-			Please consider switching to the new vmware unified driver, which is intended to replace the vmwarefusion driver.
-			See https://minikube.sigs.k8s.io/docs/reference/drivers/vmware/ for more information.
-			To disable this message, run [minikube config set ShowDriverDeprecationNotification false]`)
-	}
-	showHostInfo(cfg)
-	def := registry.Driver(cfg.VMDriver)
-	if def.Empty() {
-		return nil, fmt.Errorf("unsupported/missing driver: %s", cfg.VMDriver)
-	}
-	dd := def.Config(cfg)
-	data, err := json.Marshal(dd)
-	if err != nil {
-		return nil, errors.Wrap(err, "marshal")
-	}
-
-	h, err := api.NewHost(cfg.VMDriver, data)
-	if err != nil {
-		return nil, errors.Wrap(err, "new host")
-	}
-
-	h.HostOptions.AuthOptions.CertDir = localpath.MiniPath()
-	h.HostOptions.AuthOptions.StorePath = localpath.MiniPath()
-	h.HostOptions.EngineOptions = engineOptions(cfg)
-
-	if err := api.Create(h); err != nil {
-		// Wait for all the logs to reach the client
-		time.Sleep(2 * time.Second)
-		return nil, errors.Wrap(err, "create")
-	}
-
-	if err := createRequiredDirectories(h); err != nil {
-		return h, errors.Wrap(err, "required directories")
-	}
-
-	if driver.BareMetal(cfg.VMDriver) {
-		showLocalOsRelease()
-	} else if !driver.BareMetal(cfg.VMDriver) && !driver.IsKIC(cfg.VMDriver) {
-		showRemoteOsRelease(h.Driver)
-		// Ensure that even new VM's have proper time synchronization up front
-		// It's 2019, and I can't believe I am still dealing with time desync as a problem.
-		if err := ensureSyncedGuestClock(h); err != nil {
-			return h, err
-		}
-	} // TODO:medyagh add show-os release for kic
-
-	if err := api.Save(h); err != nil {
-		return nil, errors.Wrap(err, "save")
-	}
-	return h, nil
-}
-
-// GetHostDockerEnv gets the necessary docker env variables to allow the use of docker through minikube's vm
-func GetHostDockerEnv(api libmachine.API) (map[string]string, error) {
-	pName := viper.GetString(config.MachineProfile)
-	host, err := CheckIfHostExistsAndLoad(api, pName)
-	if err != nil {
-		return nil, errors.Wrap(err, "Error checking that api exists and loading it")
-	}
-
-	ip := kic.DefaultBindIPV4
-	if !driver.IsKIC(host.Driver.DriverName()) { // kic externally accessible ip is different that node ip
-		ip, err = host.Driver.GetIP()
-		if err != nil {
-			return nil, errors.Wrap(err, "Error getting ip from host")
-		}
-
-	}
-
-	tcpPrefix := "tcp://"
-	port := constants.DockerDaemonPort
-	if driver.IsKIC(host.Driver.DriverName()) { // for kic we need to find out what port docker allocated during creation
-		port, err = oci.HostPortBinding(host.Driver.DriverName(), pName, constants.DockerDaemonPort)
-		if err != nil {
-			return nil, errors.Wrapf(err, "get hostbind port for %d", constants.DockerDaemonPort)
-		}
-	}
-
-	envMap := map[string]string{
-		"DOCKER_TLS_VERIFY": "1",
-		"DOCKER_HOST":       tcpPrefix + net.JoinHostPort(ip, fmt.Sprint(port)),
-		"DOCKER_CERT_PATH":  localpath.MakeMiniPath("certs"),
-	}
-	return envMap, nil
-}
-
-// GetVMHostIP gets the ip address to be used for mapping host -> VM and VM -> host
-func GetVMHostIP(host *host.Host) (net.IP, error) {
-	switch host.DriverName {
-	case driver.KVM2:
-		return net.ParseIP("192.168.39.1"), nil
-	case driver.HyperV:
-		re := regexp.MustCompile(`"VSwitch": "(.*?)",`)
-		// TODO(aprindle) Change this to deserialize the driver instead
-		hypervVirtualSwitch := re.FindStringSubmatch(string(host.RawDriver))[1]
-		ip, err := getIPForInterface(fmt.Sprintf("vEthernet (%s)", hypervVirtualSwitch))
-		if err != nil {
-			return []byte{}, errors.Wrap(err, fmt.Sprintf("ip for interface (%s)", hypervVirtualSwitch))
-		}
-		return ip, nil
-	case driver.VirtualBox:
-		out, err := exec.Command(driver.VBoxManagePath(), "showvminfo", host.Name, "--machinereadable").Output()
-		if err != nil {
-			return []byte{}, errors.Wrap(err, "vboxmanage")
-		}
-		re := regexp.MustCompile(`hostonlyadapter2="(.*?)"`)
-		iface := re.FindStringSubmatch(string(out))[1]
-		ip, err := getIPForInterface(iface)
-		if err != nil {
-			return []byte{}, errors.Wrap(err, "Error getting VM/Host IP address")
-		}
-		return ip, nil
-	case driver.HyperKit:
-		return net.ParseIP("192.168.64.1"), nil
-	case driver.VMware:
-		vmIPString, err := host.Driver.GetIP()
-		if err != nil {
-			return []byte{}, errors.Wrap(err, "Error getting VM IP address")
-		}
-		vmIP := net.ParseIP(vmIPString).To4()
-		if vmIP == nil {
-			return []byte{}, errors.Wrap(err, "Error converting VM IP address to IPv4 address")
-		}
-		return net.IPv4(vmIP[0], vmIP[1], vmIP[2], byte(1)), nil
-	default:
-		return []byte{}, errors.New("Error, attempted to get host ip address for unsupported driver")
-	}
-}
-
-// Based on code from http://stackoverflow.com/questions/23529663/how-to-get-all-addresses-and-masks-from-local-interfaces-in-go
-func getIPForInterface(name string) (net.IP, error) {
-	i, _ := net.InterfaceByName(name)
-	addrs, _ := i.Addrs()
-	for _, a := range addrs {
-		if ipnet, ok := a.(*net.IPNet); ok {
-			if ip := ipnet.IP.To4(); ip != nil {
-				return ip, nil
-			}
-		}
-	}
-	return nil, errors.Errorf("Error finding IPV4 address for %s", name)
-}
-
-// CheckIfHostExistsAndLoad checks if a host exists, and loads it if it does
-func CheckIfHostExistsAndLoad(api libmachine.API, machineName string) (*host.Host, error) {
-	glog.Infof("Checking if %q exists ...", machineName)
-	exists, err := api.Exists(machineName)
-	if err != nil {
-		return nil, errors.Wrapf(err, "Error checking that machine exists: %s", machineName)
-	}
-	if !exists {
-		return nil, errors.Errorf("machine %q does not exist", machineName)
-	}
-
-	host, err := api.Load(machineName)
-	if err != nil {
-		return nil, errors.Wrapf(err, "loading machine %q", machineName)
-	}
-	return host, nil
-}
-
-// CreateSSHShell creates a new SSH shell / client
-func CreateSSHShell(api libmachine.API, args []string) error {
-	machineName := viper.GetString(config.MachineProfile)
-	host, err := CheckIfHostExistsAndLoad(api, machineName)
-	if err != nil {
-		return errors.Wrap(err, "host exists and load")
-	}
-
-	currentState, err := host.Driver.GetState()
-	if err != nil {
-		return errors.Wrap(err, "state")
-	}
-
-	if currentState != state.Running {
-		return errors.Errorf("%q is not running", machineName)
-	}
-
-	client, err := host.CreateSSHClient()
-	if err != nil {
-		return errors.Wrap(err, "Creating ssh client")
-	}
-	return client.Shell(args...)
-}
-
-// IsHostRunning asserts that this profile's primary host is in state "Running"
-func IsHostRunning(api libmachine.API, name string) bool {
-	s, err := GetHostStatus(api, name)
-	if err != nil {
-		glog.Warningf("host status for %q returned error: %v", name, err)
-		return false
-	}
-	if s != state.Running.String() {
-		glog.Warningf("%q host status: %s", name, s)
-		return false
-	}
-	return true
-}
-
-// createRequiredDirectories creates directories expected by minikube to exist
-func createRequiredDirectories(h *host.Host) error {
-	if h.DriverName == driver.Mock {
-		glog.Infof("skipping createRequiredDirectories")
-		return nil
-	}
-	glog.Infof("creating required directories: %v", requiredDirectories)
-	r, err := commandRunner(h)
-	if err != nil {
-		return errors.Wrap(err, "command runner")
-	}
-
-	args := append([]string{"mkdir", "-p"}, requiredDirectories...)
-	if _, err := r.RunCmd(exec.Command("sudo", args...)); err != nil {
-		return errors.Wrapf(err, "sudo mkdir (%s)", h.DriverName)
-	}
-	return nil
-}
-
-// commandRunner returns best available command runner for this host
-func commandRunner(h *host.Host) (command.Runner, error) {
-	if h.DriverName == driver.Mock {
-		glog.Errorf("commandRunner: returning unconfigured FakeCommandRunner, commands will fail!")
-		return &command.FakeCommandRunner{}, nil
-	}
-	if driver.BareMetal(h.Driver.DriverName()) {
-		return &command.ExecRunner{}, nil
-	}
-	if h.Driver.DriverName() == driver.Docker {
-		return command.NewKICRunner(h.Name, "docker"), nil
-	}
-	client, err := sshutil.NewSSHClient(h.Driver)
-	if err != nil {
-		return nil, errors.Wrap(err, "getting ssh client for bootstrapper")
-	}
-	return command.NewSSHRunner(client), nil
-}
diff --git a/pkg/minikube/cluster/delete.go b/pkg/minikube/cluster/delete.go
new file mode 100644
index 0000000000..ed11d7c3d5
--- /dev/null
+++ b/pkg/minikube/cluster/delete.go
@@ -0,0 +1,60 @@
+package cluster
+
+import (
+	"os/exec"
+
+	"github.com/docker/machine/libmachine"
+	"github.com/docker/machine/libmachine/mcnerror"
+	"github.com/docker/machine/libmachine/state"
+	"github.com/golang/glog"
+	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/drivers/kic/oci"
+	"k8s.io/minikube/pkg/minikube/driver"
+	"k8s.io/minikube/pkg/minikube/out"
+)
+
+// deleteOrphanedKIC attempts to delete an orphaned docker instance
+func deleteOrphanedKIC(name string) {
+	cmd := exec.Command(oci.Docker, "rm", "-f", "-v", name)
+	err := cmd.Run()
+	if err == nil {
+		glog.Infof("Found stale kic container and successfully cleaned it up!")
+	}
+}
+
+// DeleteHost deletes the host VM.
+func DeleteHost(api libmachine.API, machineName string) error {
+	host, err := api.Load(machineName)
+	if err != nil && host == nil {
+		deleteOrphanedKIC(machineName)
+		// keep going even if minikube  does not know about the host
+	}
+
+	// Get the status of the host. Ensure that it exists before proceeding ahead.
+	status, err := GetHostStatus(api, machineName)
+	if err != nil {
+		// Warn, but proceed
+		out.WarningT("Unable to get the status of the {{.name}} cluster.", out.V{"name": machineName})
+	}
+
+	if status == state.None.String() {
+		return mcnerror.ErrHostDoesNotExist{Name: machineName}
+	}
+
+	// This is slow if SSH is not responding, but HyperV hangs otherwise, See issue #2914
+	if host.Driver.DriverName() == driver.HyperV {
+		if err := trySSHPowerOff(host); err != nil {
+			glog.Infof("Unable to power off minikube because the host was not found.")
+		}
+		out.T(out.DeletingHost, "Successfully powered off Hyper-V. minikube driver -- {{.driver}}", out.V{"driver": host.Driver.DriverName()})
+	}
+
+	out.T(out.DeletingHost, `Deleting "{{.profile_name}}" in {{.driver_name}} ...`, out.V{"profile_name": machineName, "driver_name": host.DriverName})
+	if err := host.Driver.Remove(); err != nil {
+		return errors.Wrap(err, "host remove")
+	}
+	if err := api.Remove(machineName); err != nil {
+		return errors.Wrap(err, "api remove")
+	}
+	return nil
+}
diff --git a/pkg/minikube/cluster/docker_env.go b/pkg/minikube/cluster/docker_env.go
new file mode 100644
index 0000000000..23c901dcd2
--- /dev/null
+++ b/pkg/minikube/cluster/docker_env.go
@@ -0,0 +1,50 @@
+package cluster
+
+import (
+	"fmt"
+	"net"
+
+	"github.com/docker/machine/libmachine"
+	"github.com/pkg/errors"
+	"github.com/spf13/viper"
+	"k8s.io/minikube/pkg/drivers/kic"
+	"k8s.io/minikube/pkg/drivers/kic/oci"
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/constants"
+	"k8s.io/minikube/pkg/minikube/driver"
+	"k8s.io/minikube/pkg/minikube/localpath"
+)
+
+// GetHostDockerEnv gets the necessary docker env variables to allow the use of docker through minikube's vm
+func GetHostDockerEnv(api libmachine.API) (map[string]string, error) {
+	pName := viper.GetString(config.MachineProfile)
+	host, err := CheckIfHostExistsAndLoad(api, pName)
+	if err != nil {
+		return nil, errors.Wrap(err, "Error checking that api exists and loading it")
+	}
+
+	ip := kic.DefaultBindIPV4
+	if !driver.IsKIC(host.Driver.DriverName()) { // kic externally accessible ip is different that node ip
+		ip, err = host.Driver.GetIP()
+		if err != nil {
+			return nil, errors.Wrap(err, "Error getting ip from host")
+		}
+
+	}
+
+	tcpPrefix := "tcp://"
+	port := constants.DockerDaemonPort
+	if driver.IsKIC(host.Driver.DriverName()) { // for kic we need to find out what port docker allocated during creation
+		port, err = oci.HostPortBinding(host.Driver.DriverName(), pName, constants.DockerDaemonPort)
+		if err != nil {
+			return nil, errors.Wrapf(err, "get hostbind port for %d", constants.DockerDaemonPort)
+		}
+	}
+
+	envMap := map[string]string{
+		"DOCKER_TLS_VERIFY": "1",
+		"DOCKER_HOST":       tcpPrefix + net.JoinHostPort(ip, fmt.Sprint(port)),
+		"DOCKER_CERT_PATH":  localpath.MakeMiniPath("certs"),
+	}
+	return envMap, nil
+}
diff --git a/pkg/minikube/cluster/info.go b/pkg/minikube/cluster/info.go
new file mode 100644
index 0000000000..6065d938b7
--- /dev/null
+++ b/pkg/minikube/cluster/info.go
@@ -0,0 +1,100 @@
+package cluster
+
+import (
+	"io/ioutil"
+
+	"github.com/docker/machine/libmachine/drivers"
+	"github.com/docker/machine/libmachine/provision"
+	"github.com/golang/glog"
+	"github.com/shirou/gopsutil/cpu"
+	"github.com/shirou/gopsutil/disk"
+	"github.com/shirou/gopsutil/mem"
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/driver"
+	"k8s.io/minikube/pkg/minikube/out"
+)
+
+type hostInfo struct {
+	Memory   int
+	CPUs     int
+	DiskSize int
+}
+
+func megs(bytes uint64) int {
+	return int(bytes / 1024 / 1024)
+}
+
+func getHostInfo() (*hostInfo, error) {
+	i, err := cpu.Info()
+	if err != nil {
+		glog.Warningf("Unable to get CPU info: %v", err)
+		return nil, err
+	}
+	v, err := mem.VirtualMemory()
+	if err != nil {
+		glog.Warningf("Unable to get mem info: %v", err)
+		return nil, err
+	}
+	d, err := disk.Usage("/")
+	if err != nil {
+		glog.Warningf("Unable to get disk info: %v", err)
+		return nil, err
+	}
+
+	var info hostInfo
+	info.CPUs = len(i)
+	info.Memory = megs(v.Total)
+	info.DiskSize = megs(d.Total)
+	return &info, nil
+}
+
+// showLocalOsRelease shows systemd information about the current linux distribution, on the local host
+func showLocalOsRelease() {
+	osReleaseOut, err := ioutil.ReadFile("/etc/os-release")
+	if err != nil {
+		glog.Errorf("ReadFile: %v", err)
+		return
+	}
+
+	osReleaseInfo, err := provision.NewOsRelease(osReleaseOut)
+	if err != nil {
+		glog.Errorf("NewOsRelease: %v", err)
+		return
+	}
+
+	out.T(out.Provisioner, "OS release is {{.pretty_name}}", out.V{"pretty_name": osReleaseInfo.PrettyName})
+}
+
+// showRemoteOsRelease shows systemd information about the current linux distribution, on the remote VM
+func showRemoteOsRelease(driver drivers.Driver) {
+	provisioner, err := provision.DetectProvisioner(driver)
+	if err != nil {
+		glog.Errorf("DetectProvisioner: %v", err)
+		return
+	}
+
+	osReleaseInfo, err := provisioner.GetOsReleaseInfo()
+	if err != nil {
+		glog.Errorf("GetOsReleaseInfo: %v", err)
+		return
+	}
+
+	glog.Infof("Provisioned with %s", osReleaseInfo.PrettyName)
+}
+
+// showHostInfo shows host information
+func showHostInfo(cfg config.MachineConfig) {
+	if driver.BareMetal(cfg.VMDriver) {
+		info, err := getHostInfo()
+		if err == nil {
+			out.T(out.StartingNone, "Running on localhost (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...", out.V{"number_of_cpus": info.CPUs, "memory_size": info.Memory, "disk_size": info.DiskSize})
+		}
+	} else if driver.IsKIC(cfg.VMDriver) {
+		info, err := getHostInfo() // TODO medyagh: get docker-machine info for non linux
+		if err == nil {
+			out.T(out.StartingVM, "Creating Kubernetes in {{.driver_name}} container with (CPUs={{.number_of_cpus}}), Memory={{.memory_size}}MB ({{.host_memory_size}}MB available) ...", out.V{"driver_name": cfg.VMDriver, "number_of_cpus": cfg.CPUs, "number_of_host_cpus": info.CPUs, "memory_size": cfg.Memory, "host_memory_size": info.Memory})
+		}
+	} else {
+		out.T(out.StartingVM, "Creating {{.driver_name}} VM (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...", out.V{"driver_name": cfg.VMDriver, "number_of_cpus": cfg.CPUs, "memory_size": cfg.Memory, "disk_size": cfg.DiskSize})
+	}
+}
diff --git a/pkg/minikube/cluster/ip.go b/pkg/minikube/cluster/ip.go
new file mode 100644
index 0000000000..cafc22027b
--- /dev/null
+++ b/pkg/minikube/cluster/ip.go
@@ -0,0 +1,92 @@
+package cluster
+
+import (
+	"fmt"
+	"net"
+	"os/exec"
+	"regexp"
+
+	"github.com/docker/machine/libmachine"
+	"github.com/docker/machine/libmachine/host"
+	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/drivers/kic"
+	"k8s.io/minikube/pkg/minikube/driver"
+)
+
+// GetVMHostIP gets the ip address to be used for mapping host -> VM and VM -> host
+func GetVMHostIP(host *host.Host) (net.IP, error) {
+	switch host.DriverName {
+	case driver.KVM2:
+		return net.ParseIP("192.168.39.1"), nil
+	case driver.HyperV:
+		re := regexp.MustCompile(`"VSwitch": "(.*?)",`)
+		// TODO(aprindle) Change this to deserialize the driver instead
+		hypervVirtualSwitch := re.FindStringSubmatch(string(host.RawDriver))[1]
+		ip, err := getIPForInterface(fmt.Sprintf("vEthernet (%s)", hypervVirtualSwitch))
+		if err != nil {
+			return []byte{}, errors.Wrap(err, fmt.Sprintf("ip for interface (%s)", hypervVirtualSwitch))
+		}
+		return ip, nil
+	case driver.VirtualBox:
+		out, err := exec.Command(driver.VBoxManagePath(), "showvminfo", host.Name, "--machinereadable").Output()
+		if err != nil {
+			return []byte{}, errors.Wrap(err, "vboxmanage")
+		}
+		re := regexp.MustCompile(`hostonlyadapter2="(.*?)"`)
+		iface := re.FindStringSubmatch(string(out))[1]
+		ip, err := getIPForInterface(iface)
+		if err != nil {
+			return []byte{}, errors.Wrap(err, "Error getting VM/Host IP address")
+		}
+		return ip, nil
+	case driver.HyperKit:
+		return net.ParseIP("192.168.64.1"), nil
+	case driver.VMware:
+		vmIPString, err := host.Driver.GetIP()
+		if err != nil {
+			return []byte{}, errors.Wrap(err, "Error getting VM IP address")
+		}
+		vmIP := net.ParseIP(vmIPString).To4()
+		if vmIP == nil {
+			return []byte{}, errors.Wrap(err, "Error converting VM IP address to IPv4 address")
+		}
+		return net.IPv4(vmIP[0], vmIP[1], vmIP[2], byte(1)), nil
+	default:
+		return []byte{}, errors.New("Error, attempted to get host ip address for unsupported driver")
+	}
+}
+
+// GetHostDriverIP gets the ip address of the current minikube cluster
+func GetHostDriverIP(api libmachine.API, machineName string) (net.IP, error) {
+	host, err := CheckIfHostExistsAndLoad(api, machineName)
+	if err != nil {
+		return nil, err
+	}
+
+	ipStr, err := host.Driver.GetIP()
+	if err != nil {
+		return nil, errors.Wrap(err, "getting IP")
+	}
+	if driver.IsKIC(host.DriverName) {
+		ipStr = kic.DefaultBindIPV4
+	}
+	ip := net.ParseIP(ipStr)
+	if ip == nil {
+		return nil, fmt.Errorf("parsing IP: %s", ipStr)
+	}
+	return ip, nil
+}
+
+// Based on code from http://stackoverflow.com/questions/23529663/how-to-get-all-addresses-and-masks-from-local-interfaces-in-go
+func getIPForInterface(name string) (net.IP, error) {
+	i, _ := net.InterfaceByName(name)
+	addrs, _ := i.Addrs()
+	for _, a := range addrs {
+		if ipnet, ok := a.(*net.IPNet); ok {
+			if ip := ipnet.IP.To4(); ip != nil {
+				return ip, nil
+			}
+		}
+	}
+	return nil, errors.Errorf("Error finding IPV4 address for %s", name)
+}
diff --git a/pkg/minikube/cluster/iso.go b/pkg/minikube/cluster/iso.go
new file mode 100644
index 0000000000..9aa1810cd6
--- /dev/null
+++ b/pkg/minikube/cluster/iso.go
@@ -0,0 +1,14 @@
+package cluster
+
+import (
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/driver"
+)
+
+// CacheISO downloads and caches ISO.
+func CacheISO(cfg config.MachineConfig) error {
+	if driver.BareMetal(cfg.VMDriver) {
+		return nil
+	}
+	return cfg.Downloader.CacheMinikubeISOFromURL(cfg.MinikubeISO)
+}
diff --git a/pkg/minikube/cluster/restart.go b/pkg/minikube/cluster/restart.go
new file mode 100644
index 0000000000..724a20b847
--- /dev/null
+++ b/pkg/minikube/cluster/restart.go
@@ -0,0 +1,103 @@
+package cluster
+
+import (
+	"fmt"
+	"math"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/docker/machine/libmachine/engine"
+	"github.com/docker/machine/libmachine/host"
+	"github.com/docker/machine/libmachine/provision"
+	"github.com/golang/glog"
+	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/minikube/driver"
+	"k8s.io/minikube/pkg/util/retry"
+)
+
+var (
+	// The maximum the guest VM clock is allowed to be ahead and behind. This value is intentionally
+	// large to allow for inaccurate methodology, but still small enough so that certificates are likely valid.
+	maxClockDesyncSeconds = 2.1
+)
+
+// restartHost fixes up a previously configured VM
+func restartHost(h *host.Host, e *engine.Options) error {
+	start := time.Now()
+	glog.Infof("ressurectHost: %+v", h.Driver)
+	defer func() {
+		glog.Infof("ressurectHost completed within %s", time.Since(start))
+	}()
+
+	if len(e.Env) > 0 {
+		h.HostOptions.EngineOptions.Env = e.Env
+		glog.Infof("Detecting provisioner ...")
+		provisioner, err := provision.DetectProvisioner(h.Driver)
+		if err != nil {
+			return errors.Wrap(err, "detecting provisioner")
+		}
+		glog.Infof("Provisioning with %s: %+v", provisioner.String(), *h.HostOptions)
+		if err := provisioner.Provision(*h.HostOptions.SwarmOptions, *h.HostOptions.AuthOptions, *h.HostOptions.EngineOptions); err != nil {
+			return errors.Wrap(err, "provision")
+		}
+	}
+
+	if driver.BareMetal(h.Driver.DriverName()) {
+		glog.Infof("%s is a local driver, skipping auth/time setup", h.Driver.DriverName())
+		return nil
+	}
+	glog.Infof("Configuring auth for driver %s ...", h.Driver.DriverName())
+	if err := h.ConfigureAuth(); err != nil {
+		return &retry.RetriableError{Err: errors.Wrap(err, "Error configuring auth on host")}
+	}
+	return ensureSyncedGuestClock(h)
+}
+
+// ensureGuestClockSync ensures that the guest system clock is relatively in-sync
+func ensureSyncedGuestClock(h hostRunner) error {
+	d, err := guestClockDelta(h, time.Now())
+	if err != nil {
+		glog.Warningf("Unable to measure system clock delta: %v", err)
+		return nil
+	}
+	if math.Abs(d.Seconds()) < maxClockDesyncSeconds {
+		glog.Infof("guest clock delta is within tolerance: %s", d)
+		return nil
+	}
+	if err := adjustGuestClock(h, time.Now()); err != nil {
+		return errors.Wrap(err, "adjusting system clock")
+	}
+	return nil
+}
+
+// guestClockDelta returns the approximate difference between the host and guest system clock
+// NOTE: This does not currently take into account ssh latency.
+func guestClockDelta(h hostRunner, local time.Time) (time.Duration, error) {
+	out, err := h.RunSSHCommand("date +%s.%N")
+	if err != nil {
+		return 0, errors.Wrap(err, "get clock")
+	}
+	glog.Infof("guest clock: %s", out)
+	ns := strings.Split(strings.TrimSpace(out), ".")
+	secs, err := strconv.ParseInt(strings.TrimSpace(ns[0]), 10, 64)
+	if err != nil {
+		return 0, errors.Wrap(err, "atoi")
+	}
+	nsecs, err := strconv.ParseInt(strings.TrimSpace(ns[1]), 10, 64)
+	if err != nil {
+		return 0, errors.Wrap(err, "atoi")
+	}
+	// NOTE: In a synced state, remote is a few hundred ms ahead of local
+	remote := time.Unix(secs, nsecs)
+	d := remote.Sub(local)
+	glog.Infof("Guest: %s Remote: %s (delta=%s)", remote, local, d)
+	return d, nil
+}
+
+// adjustSystemClock adjusts the guest system clock to be nearer to the host system clock
+func adjustGuestClock(h hostRunner, t time.Time) error {
+	out, err := h.RunSSHCommand(fmt.Sprintf("sudo date -s @%d", t.Unix()))
+	glog.Infof("clock set: %s (err=%v)", out, err)
+	return err
+}
diff --git a/pkg/minikube/cluster/ssh.go b/pkg/minikube/cluster/ssh.go
new file mode 100644
index 0000000000..0fdc955053
--- /dev/null
+++ b/pkg/minikube/cluster/ssh.go
@@ -0,0 +1,33 @@
+package cluster
+
+import (
+	"github.com/docker/machine/libmachine"
+	"github.com/docker/machine/libmachine/state"
+	"github.com/pkg/errors"
+	"github.com/spf13/viper"
+	"k8s.io/minikube/pkg/minikube/config"
+)
+
+// CreateSSHShell creates a new SSH shell / client
+func CreateSSHShell(api libmachine.API, args []string) error {
+	machineName := viper.GetString(config.MachineProfile)
+	host, err := CheckIfHostExistsAndLoad(api, machineName)
+	if err != nil {
+		return errors.Wrap(err, "host exists and load")
+	}
+
+	currentState, err := host.Driver.GetState()
+	if err != nil {
+		return errors.Wrap(err, "state")
+	}
+
+	if currentState != state.Running {
+		return errors.Errorf("%q is not running", machineName)
+	}
+
+	client, err := host.CreateSSHClient()
+	if err != nil {
+		return errors.Wrap(err, "Creating ssh client")
+	}
+	return client.Shell(args...)
+}
diff --git a/pkg/minikube/cluster/start.go b/pkg/minikube/cluster/start.go
new file mode 100644
index 0000000000..2a71c8dc1a
--- /dev/null
+++ b/pkg/minikube/cluster/start.go
@@ -0,0 +1,255 @@
+package cluster
+
+import (
+	"encoding/json"
+	"fmt"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"time"
+
+	"github.com/docker/machine/libmachine"
+	"github.com/docker/machine/libmachine/drivers"
+	"github.com/docker/machine/libmachine/engine"
+	"github.com/docker/machine/libmachine/host"
+	"github.com/docker/machine/libmachine/state"
+	"github.com/golang/glog"
+	"github.com/juju/mutex"
+	"github.com/pkg/errors"
+	"github.com/spf13/viper"
+	"k8s.io/minikube/pkg/minikube/assets"
+	"k8s.io/minikube/pkg/minikube/command"
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/constants"
+	"k8s.io/minikube/pkg/minikube/driver"
+	"k8s.io/minikube/pkg/minikube/localpath"
+	"k8s.io/minikube/pkg/minikube/out"
+	"k8s.io/minikube/pkg/minikube/registry"
+	"k8s.io/minikube/pkg/minikube/sshutil"
+	"k8s.io/minikube/pkg/minikube/vmpath"
+	"k8s.io/minikube/pkg/util/lock"
+)
+
+var (
+	// requiredDirectories are directories to create on the host during setup
+	requiredDirectories = []string{
+		vmpath.GuestAddonsDir,
+		vmpath.GuestManifestsDir,
+		vmpath.GuestEphemeralDir,
+		vmpath.GuestPersistentDir,
+		vmpath.GuestCertsDir,
+		path.Join(vmpath.GuestPersistentDir, "images"),
+		path.Join(vmpath.GuestPersistentDir, "binaries"),
+	}
+)
+
+// StartHost starts a host VM.
+func StartHost(api libmachine.API, cfg config.MachineConfig) (*host.Host, error) {
+	// Prevent machine-driver boot races, as well as our own certificate race
+	releaser, err := acquireMachinesLock(cfg.Name)
+	if err != nil {
+		return nil, errors.Wrap(err, "boot lock")
+	}
+	start := time.Now()
+	defer func() {
+		glog.Infof("releasing machines lock for %q, held for %s", cfg.Name, time.Since(start))
+		releaser.Release()
+	}()
+
+	exists, err := api.Exists(cfg.Name)
+	if err != nil {
+		return nil, errors.Wrapf(err, "exists: %s", cfg.Name)
+	}
+	if !exists {
+		glog.Infoln("Machine does not exist... provisioning new machine")
+		glog.Infof("Provisioning machine with config: %+v", cfg)
+		return createHost(api, cfg)
+	}
+
+	glog.Infoln("Skipping create...Using existing machine configuration")
+
+	h, err := api.Load(cfg.Name)
+	if err != nil {
+		return nil, errors.Wrap(err, "Error loading existing host. Please try running [minikube delete], then run [minikube start] again.")
+	}
+
+	if exists && cfg.Name == constants.DefaultMachineName {
+		out.T(out.Tip, "Tip: Use 'minikube start -p <name>' to create a new cluster, or 'minikube delete' to delete this one.")
+	}
+
+	s, err := h.Driver.GetState()
+	glog.Infoln("Machine state: ", s)
+	if err != nil {
+		return nil, errors.Wrap(err, "Error getting state for host")
+	}
+
+	if s == state.Running {
+		out.T(out.Running, `Using the running {{.driver_name}} "{{.profile_name}}" VM ...`, out.V{"driver_name": cfg.VMDriver, "profile_name": cfg.Name})
+	} else {
+		out.T(out.Restarting, `Starting existing {{.driver_name}} VM for "{{.profile_name}}" ...`, out.V{"driver_name": cfg.VMDriver, "profile_name": cfg.Name})
+		if err := h.Driver.Start(); err != nil {
+			return nil, errors.Wrap(err, "start")
+		}
+		if err := api.Save(h); err != nil {
+			return nil, errors.Wrap(err, "save")
+		}
+	}
+
+	e := engineOptions(cfg)
+	glog.Infof("engine options: %+v", e)
+
+	out.T(out.Waiting, "Waiting for the host to be provisioned ...")
+	err = ressurectHost(h, e)
+	if err != nil {
+		return nil, err
+	}
+	return h, nil
+}
+
+func engineOptions(cfg config.MachineConfig) *engine.Options {
+	o := engine.Options{
+		Env:              cfg.DockerEnv,
+		InsecureRegistry: append([]string{constants.DefaultServiceCIDR}, cfg.InsecureRegistry...),
+		RegistryMirror:   cfg.RegistryMirror,
+		ArbitraryFlags:   cfg.DockerOpt,
+		InstallURL:       drivers.DefaultEngineInstallURL,
+	}
+	return &o
+}
+
+func createHost(api libmachine.API, cfg config.MachineConfig) (*host.Host, error) {
+	if cfg.VMDriver == driver.VMwareFusion && viper.GetBool(config.ShowDriverDeprecationNotification) {
+		out.WarningT(`The vmwarefusion driver is deprecated and support for it will be removed in a future release.
+			Please consider switching to the new vmware unified driver, which is intended to replace the vmwarefusion driver.
+			See https://minikube.sigs.k8s.io/docs/reference/drivers/vmware/ for more information.
+			To disable this message, run [minikube config set ShowDriverDeprecationNotification false]`)
+	}
+	showHostInfo(cfg)
+	def := registry.Driver(cfg.VMDriver)
+	if def.Empty() {
+		return nil, fmt.Errorf("unsupported/missing driver: %s", cfg.VMDriver)
+	}
+	dd := def.Config(cfg)
+	data, err := json.Marshal(dd)
+	if err != nil {
+		return nil, errors.Wrap(err, "marshal")
+	}
+
+	h, err := api.NewHost(cfg.VMDriver, data)
+	if err != nil {
+		return nil, errors.Wrap(err, "new host")
+	}
+
+	h.HostOptions.AuthOptions.CertDir = localpath.MiniPath()
+	h.HostOptions.AuthOptions.StorePath = localpath.MiniPath()
+	h.HostOptions.EngineOptions = engineOptions(cfg)
+
+	if err := api.Create(h); err != nil {
+		// Wait for all the logs to reach the client
+		time.Sleep(2 * time.Second)
+		return nil, errors.Wrap(err, "create")
+	}
+
+	if err := createRequiredDirectories(h); err != nil {
+		return h, errors.Wrap(err, "required directories")
+	}
+
+	for _, addon := range addon.Assets {
+		var f assets.CopyableFile
+		var err error
+		if addon.IsTemplate() {
+			f, err = addon.Evaluate(data)
+			if err != nil {
+				return errors.Wrapf(err, "evaluate bundled addon %s asset", addon.GetAssetName())
+			}
+
+		} else {
+			f = addon
+		}
+		fPath := path.Join(f.GetTargetDir(), f.GetTargetName())
+
+		if enable {
+			glog.Infof("installing %s", fPath)
+			if err := cmd.Copy(f); err != nil {
+				return err
+			}
+		} else {
+			glog.Infof("Removing %+v", fPath)
+			defer func() {
+				if err := cmd.Remove(f); err != nil {
+					glog.Warningf("error removing %s; addon should still be disabled as expected", fPath)
+				}
+			}()
+		}
+		files = append(files, fPath)
+	}
+
+	if driver.BareMetal(cfg.VMDriver) {
+		showLocalOsRelease()
+	} else if !driver.BareMetal(cfg.VMDriver) && !driver.IsKIC(cfg.VMDriver) {
+		showRemoteOsRelease(h.Driver)
+		// Ensure that even new VM's have proper time synchronization up front
+		// It's 2019, and I can't believe I am still dealing with time desync as a problem.
+		if err := ensureSyncedGuestClock(h); err != nil {
+			return h, err
+		}
+	} // TODO:medyagh add show-os release for kic
+
+	if err := api.Save(h); err != nil {
+		return nil, errors.Wrap(err, "save")
+	}
+	return h, nil
+}
+
+// createRequiredDirectories creates directories expected by minikube to exist
+func createRequiredDirectories(h *host.Host) error {
+	if h.DriverName == driver.Mock {
+		glog.Infof("skipping createRequiredDirectories")
+		return nil
+	}
+	glog.Infof("creating required directories: %v", requiredDirectories)
+	r, err := commandRunner(h)
+	if err != nil {
+		return errors.Wrap(err, "command runner")
+	}
+
+	args := append([]string{"mkdir", "-p"}, requiredDirectories...)
+	if _, err := r.RunCmd(exec.Command("sudo", args...)); err != nil {
+		return errors.Wrapf(err, "sudo mkdir (%s)", h.DriverName)
+	}
+	return nil
+}
+
+// commandRunner returns best available command runner for this host
+func commandRunner(h *host.Host) (command.Runner, error) {
+	if h.DriverName == driver.Mock {
+		glog.Errorf("commandRunner: returning unconfigured FakeCommandRunner, commands will fail!")
+		return &command.FakeCommandRunner{}, nil
+	}
+	if driver.BareMetal(h.Driver.DriverName()) {
+		return &command.ExecRunner{}, nil
+	}
+	if h.Driver.DriverName() == driver.Docker {
+		return command.NewKICRunner(h.Name, "docker"), nil
+	}
+	client, err := sshutil.NewSSHClient(h.Driver)
+	if err != nil {
+		return nil, errors.Wrap(err, "getting ssh client for bootstrapper")
+	}
+	return command.NewSSHRunner(client), nil
+}
+
+// acquireMachinesLock protects against code that is not parallel-safe (libmachine, cert setup)
+func acquireMachinesLock(name string) (mutex.Releaser, error) {
+	spec := lock.PathMutexSpec(filepath.Join(localpath.MiniPath(), "machines"))
+	// NOTE: Provisioning generally completes within 60 seconds
+	spec.Timeout = 15 * time.Minute
+
+	glog.Infof("acquiring machines lock for %s: %+v", name, spec)
+	start := time.Now()
+	r, err := mutex.Acquire(spec)
+	if err == nil {
+		glog.Infof("acquired machines lock for %q in %s", name, time.Since(start))
+	}
+	return r, err
+}
diff --git a/pkg/minikube/cluster/status.go b/pkg/minikube/cluster/status.go
new file mode 100644
index 0000000000..8b880e778d
--- /dev/null
+++ b/pkg/minikube/cluster/status.go
@@ -0,0 +1,63 @@
+package cluster
+
+import (
+	"github.com/docker/machine/libmachine"
+	"github.com/docker/machine/libmachine/host"
+	"github.com/docker/machine/libmachine/state"
+	"github.com/golang/glog"
+	"github.com/pkg/errors"
+)
+
+// GetHostStatus gets the status of the host VM.
+func GetHostStatus(api libmachine.API, machineName string) (string, error) {
+	exists, err := api.Exists(machineName)
+	if err != nil {
+		return "", errors.Wrapf(err, "%s exists", machineName)
+	}
+	if !exists {
+		return state.None.String(), nil
+	}
+
+	host, err := api.Load(machineName)
+	if err != nil {
+		return "", errors.Wrapf(err, "load")
+	}
+
+	s, err := host.Driver.GetState()
+	if err != nil {
+		return "", errors.Wrap(err, "state")
+	}
+	return s.String(), nil
+}
+
+// IsHostRunning asserts that this profile's primary host is in state "Running"
+func IsHostRunning(api libmachine.API, name string) bool {
+	s, err := GetHostStatus(api, name)
+	if err != nil {
+		glog.Warningf("host status for %q returned error: %v", name, err)
+		return false
+	}
+	if s != state.Running.String() {
+		glog.Warningf("%q host status: %s", name, s)
+		return false
+	}
+	return true
+}
+
+// CheckIfHostExistsAndLoad checks if a host exists, and loads it if it does
+func CheckIfHostExistsAndLoad(api libmachine.API, machineName string) (*host.Host, error) {
+	glog.Infof("Checking if %q exists ...", machineName)
+	exists, err := api.Exists(machineName)
+	if err != nil {
+		return nil, errors.Wrapf(err, "Error checking that machine exists: %s", machineName)
+	}
+	if !exists {
+		return nil, errors.Errorf("machine %q does not exist", machineName)
+	}
+
+	host, err := api.Load(machineName)
+	if err != nil {
+		return nil, errors.Wrapf(err, "loading machine %q", machineName)
+	}
+	return host, nil
+}
diff --git a/pkg/minikube/cluster/stop.go b/pkg/minikube/cluster/stop.go
new file mode 100644
index 0000000000..f0cf7623c6
--- /dev/null
+++ b/pkg/minikube/cluster/stop.go
@@ -0,0 +1,60 @@
+package cluster
+
+import (
+	"github.com/docker/machine/libmachine"
+	"github.com/docker/machine/libmachine/host"
+	"github.com/docker/machine/libmachine/mcnerror"
+	"github.com/docker/machine/libmachine/state"
+	"github.com/golang/glog"
+	"github.com/pkg/errors"
+	"github.com/spf13/viper"
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/driver"
+	"k8s.io/minikube/pkg/minikube/out"
+	"k8s.io/minikube/pkg/util/retry"
+)
+
+// StopHost stops the host VM, saving state to disk.
+func StopHost(api libmachine.API) error {
+	machineName := viper.GetString(config.MachineProfile)
+	host, err := api.Load(machineName)
+	if err != nil {
+		return errors.Wrapf(err, "load")
+	}
+
+	out.T(out.Stopping, `Stopping "{{.profile_name}}" in {{.driver_name}} ...`, out.V{"profile_name": machineName, "driver_name": host.DriverName})
+	if host.DriverName == driver.HyperV {
+		glog.Infof("As there are issues with stopping Hyper-V VMs using API, trying to shut down using SSH")
+		if err := trySSHPowerOff(host); err != nil {
+			return errors.Wrap(err, "ssh power off")
+		}
+	}
+
+	if err := host.Stop(); err != nil {
+		alreadyInStateError, ok := err.(mcnerror.ErrHostAlreadyInState)
+		if ok && alreadyInStateError.State == state.Stopped {
+			return nil
+		}
+		return &retry.RetriableError{Err: errors.Wrapf(err, "Stop: %s", machineName)}
+	}
+	return nil
+}
+
+// trySSHPowerOff runs the poweroff command on the guest VM to speed up deletion
+func trySSHPowerOff(h *host.Host) error {
+	s, err := h.Driver.GetState()
+	if err != nil {
+		glog.Warningf("unable to get state: %v", err)
+		return err
+	}
+	if s != state.Running {
+		glog.Infof("host is in state %s", s)
+		return nil
+	}
+
+	out.T(out.Shutdown, `Powering off "{{.profile_name}}" via SSH ...`, out.V{"profile_name": h.Name})
+	out, err := h.RunSSHCommand("sudo poweroff")
+	// poweroff always results in an error, since the host disconnects.
+	glog.Infof("poweroff result: out=%s, err=%v", out, err)
+	return nil
+}

From 286396344c0df9177b3f0e5285d6ab18832994db Mon Sep 17 00:00:00 2001
From: tstromberg <tstromberg@google.com>
Date: Mon, 3 Feb 2020 16:24:36 -0800
Subject: [PATCH 221/250] Name restart/ressurect functions fix

---
 pkg/minikube/cluster/{restart.go => fix.go} | 8 ++++----
 pkg/minikube/cluster/start.go               | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)
 rename pkg/minikube/cluster/{restart.go => fix.go} (93%)

diff --git a/pkg/minikube/cluster/restart.go b/pkg/minikube/cluster/fix.go
similarity index 93%
rename from pkg/minikube/cluster/restart.go
rename to pkg/minikube/cluster/fix.go
index 724a20b847..cf2c21db0a 100644
--- a/pkg/minikube/cluster/restart.go
+++ b/pkg/minikube/cluster/fix.go
@@ -22,12 +22,12 @@ var (
 	maxClockDesyncSeconds = 2.1
 )
 
-// restartHost fixes up a previously configured VM
-func restartHost(h *host.Host, e *engine.Options) error {
+// fixHost fixes up a previously configured VM so that it is ready to run Kubernetes
+func fixHost(h *host.Host, e *engine.Options) error {
 	start := time.Now()
-	glog.Infof("ressurectHost: %+v", h.Driver)
+	glog.Infof("fixHost: %+v", h.Driver)
 	defer func() {
-		glog.Infof("ressurectHost completed within %s", time.Since(start))
+		glog.Infof("fixHost completed within %s", time.Since(start))
 	}()
 
 	if len(e.Env) > 0 {
diff --git a/pkg/minikube/cluster/start.go b/pkg/minikube/cluster/start.go
index 2a71c8dc1a..d6822b45ba 100644
--- a/pkg/minikube/cluster/start.go
+++ b/pkg/minikube/cluster/start.go
@@ -99,7 +99,7 @@ func StartHost(api libmachine.API, cfg config.MachineConfig) (*host.Host, error)
 	glog.Infof("engine options: %+v", e)
 
 	out.T(out.Waiting, "Waiting for the host to be provisioned ...")
-	err = ressurectHost(h, e)
+	err = fixHost(h, e)
 	if err != nil {
 		return nil, err
 	}

From 718ab997d5f74023c0a3475788516125a6d89623 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Mon, 3 Feb 2020 17:11:35 -0800
Subject: [PATCH 222/250] spell

---
 cmd/minikube/cmd/start.go | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index c0277bcc85..f95157aa25 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -1145,15 +1145,14 @@ Suggested workarounds:
 func tryLookup(r command.Runner) {
 	// DNS check
 	if rr, err := r.RunCmd(exec.Command("nslookup", "kubernetes.io", "-type=ns")); err != nil {
-		glog.Infof("%s failed: %v which might be okay will retry nslookup with query type", rr.Args, err)
-		// will try with different command format for ISOs with diffrent busybox versions.
-		if rr, err = r.RunCmd(exec.Command("nslookup", "kubernetes.io")); err != nil {
-			glog.Warningf("nslookup failed: %v", rr.Args, err)
+		glog.Infof("%s failed: %v which might be okay will retry nslookup without query type", rr.Args, err)
+		// will try with without query type for ISOs with different busybox versions.
+		if _, err = r.RunCmd(exec.Command("nslookup", "kubernetes.io")); err != nil {
+			glog.Warningf("nslookup failed: %v", err)
 			out.WarningT("Node may be unable to resolve external DNS records")
 		}
 	}
 }
-
 func tryRegistry(r command.Runner) {
 	// Try an HTTPS connection to the image repository
 	proxy := os.Getenv("HTTPS_PROXY")

From a2f2df9f81a28d610a3444e93c2c851118836bff Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Mon, 3 Feb 2020 20:21:41 -0800
Subject: [PATCH 223/250] add options to docker service for kic

---
 hack/images/docker.service     | 48 ++++++++++++++++++++++++++++++++++
 hack/images/kicbase.Dockerfile |  2 ++
 pkg/drivers/kic/types.go       |  3 ++-
 3 files changed, 52 insertions(+), 1 deletion(-)
 create mode 100644 hack/images/docker.service

diff --git a/hack/images/docker.service b/hack/images/docker.service
new file mode 100644
index 0000000000..275f4f9b26
--- /dev/null
+++ b/hack/images/docker.service
@@ -0,0 +1,48 @@
+[Unit]
+Description=Docker Application Container Engine
+Documentation=https://docs.docker.com
+BindsTo=containerd.service
+After=network-online.target firewalld.service containerd.service
+Wants=network-online.target
+Requires=docker.socket
+
+[Service]
+Type=notify
+# the default is not to use systemd for cgroups because the delegate issues still
+# exists and systemd currently does not support the cgroup feature set required
+# for containers run by docker
+# ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock
+ExecStart=/usr/bin/dockerd -H tcp://0.0.0.0:2376 -H unix:///var/run/docker.sock --default-ulimit=nofile=1048576:1048576  --label provider=docker --insecure-registry 10.96.0.0/12 
+ExecReload=/bin/kill -s HUP $MAINPID
+TimeoutSec=0
+RestartSec=2
+Restart=always
+
+# Note that StartLimit* options were moved from "Service" to "Unit" in systemd 229.
+# Both the old, and new location are accepted by systemd 229 and up, so using the old location
+# to make them work for either version of systemd.
+StartLimitBurst=3
+
+# Note that StartLimitInterval was renamed to StartLimitIntervalSec in systemd 230.
+# Both the old, and new name are accepted by systemd 230 and up, so using the old name to make
+# this option work for either version of systemd.
+StartLimitInterval=60s
+
+# Having non-zero Limit*s causes performance problems due to accounting overhead
+# in the kernel. We recommend using cgroups to do container-local accounting.
+LimitNOFILE=infinity
+LimitNPROC=infinity
+LimitCORE=infinity
+
+# Comment TasksMax if your systemd version does not support it.
+# Only systemd 226 and above support this option.
+TasksMax=infinity
+
+# set delegate yes so that systemd does not reset the cgroups of docker containers
+Delegate=yes
+
+# kill only the docker process, not all processes in the cgroup
+KillMode=process
+
+[Install]
+WantedBy=multi-user.target
diff --git a/hack/images/kicbase.Dockerfile b/hack/images/kicbase.Dockerfile
index 1636d288bc..542b2a6f7b 100644
--- a/hack/images/kicbase.Dockerfile
+++ b/hack/images/kicbase.Dockerfile
@@ -10,6 +10,8 @@ RUN apt-get update && apt-get install -y \
   openssh-server \
   docker.io \
   && apt-get clean -y 
+# we will use provision package in future https://github.com/kubernetes/minikube/issues/6485
+ADD hack/images/docker.service /lib/systemd/system/docker.service
 # disable containerd by default
 RUN systemctl disable containerd
 # enable docker which is default
diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go
index 62826954ca..9f2e2e90b5 100644
--- a/pkg/drivers/kic/types.go
+++ b/pkg/drivers/kic/types.go
@@ -26,7 +26,8 @@ const (
 	// DefaultBindIPV4 is The default IP the container will bind to.
 	DefaultBindIPV4 = "127.0.0.1"
 	// BaseImage is the base image is used to spin up kic containers. it uses same base-image as kind.
-	BaseImage = "gcr.io/k8s-minikube/kicbase:v0.0.4@sha256:01552b7ffc7cb0c49c6106796ed73f69647691d8fbae89fed8ee317dc7f1015a"
+	// BaseImage = "gcr.io/k8s-minikube/kicbase:v0.0.4@sha256:01552b7ffc7cb0c49c6106796ed73f69647691d8fbae89fed8ee317dc7f1015a"
+	BaseImage = "gcr.io/k8s-minikube/kicbase:v0.0.4-snapshot"
 	// OverlayImage is the cni plugin used for overlay image, created by kind.
 	// CNI plugin image used for kic drivers created by kind.
 	OverlayImage = "kindest/kindnetd:0.5.3"

From 92eeb3c10b9c87ab5c558a60ed0a0ce16f3db08b Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Mon, 3 Feb 2020 22:29:27 -0800
Subject: [PATCH 224/250] refactor provision

---
 pkg/minikube/machine/client.go |  11 +-
 pkg/provision/buildroot.go     | 207 +--------------------------
 pkg/provision/provision.go     | 252 +++++++++++++++++++++++++++++++++
 pkg/provision/ubuntu.go        | 210 +++++++++++++++++++++++++++
 4 files changed, 472 insertions(+), 208 deletions(-)
 create mode 100644 pkg/provision/provision.go
 create mode 100644 pkg/provision/ubuntu.go

diff --git a/pkg/minikube/machine/client.go b/pkg/minikube/machine/client.go
index 9d8b0e7236..b1e1112e81 100644
--- a/pkg/minikube/machine/client.go
+++ b/pkg/minikube/machine/client.go
@@ -36,6 +36,7 @@ import (
 	"github.com/docker/machine/libmachine/host"
 	"github.com/docker/machine/libmachine/mcnutils"
 	"github.com/docker/machine/libmachine/persist"
+	lib_provision "github.com/docker/machine/libmachine/provision"
 	"github.com/docker/machine/libmachine/ssh"
 	"github.com/docker/machine/libmachine/state"
 	"github.com/docker/machine/libmachine/swarm"
@@ -207,10 +208,16 @@ func (api *LocalClient) Create(h *host.Host) error {
 		{
 			"provisioning",
 			func() error {
-				if driver.BareMetal(h.Driver.DriverName()) || driver.IsKIC(h.Driver.DriverName()) {
+				if driver.BareMetal(h.Driver.DriverName()) {
 					return nil
 				}
-				pv := provision.NewBuildrootProvisioner(h.Driver)
+				var pv lib_provision.Provisioner
+				if driver.IsKIC(h.Driver.DriverName()) {
+					pv = provision.NewUbuntuProvisioner(h.Driver)
+					fmt.Println("choosing ubuntu provisioner")
+				} else {
+					pv = provision.NewBuildrootProvisioner(h.Driver)
+				}
 				return pv.Provision(*h.HostOptions.SwarmOptions, *h.HostOptions.AuthOptions, *h.HostOptions.EngineOptions)
 			},
 		},
diff --git a/pkg/provision/buildroot.go b/pkg/provision/buildroot.go
index 41798fe3ae..e1701baf18 100644
--- a/pkg/provision/buildroot.go
+++ b/pkg/provision/buildroot.go
@@ -19,29 +19,18 @@ package provision
 import (
 	"bytes"
 	"fmt"
-	"os/exec"
 	"path"
-	"path/filepath"
-	"strings"
 	"text/template"
 	"time"
 
 	"github.com/docker/machine/libmachine/auth"
-	"github.com/docker/machine/libmachine/cert"
 	"github.com/docker/machine/libmachine/drivers"
 	"github.com/docker/machine/libmachine/engine"
 	"github.com/docker/machine/libmachine/log"
-	"github.com/docker/machine/libmachine/mcnutils"
 	"github.com/docker/machine/libmachine/provision"
 	"github.com/docker/machine/libmachine/provision/pkgaction"
 	"github.com/docker/machine/libmachine/provision/serviceaction"
 	"github.com/docker/machine/libmachine/swarm"
-	"github.com/pkg/errors"
-	"k8s.io/minikube/pkg/minikube/assets"
-	"k8s.io/minikube/pkg/minikube/command"
-	"k8s.io/minikube/pkg/minikube/config"
-	"k8s.io/minikube/pkg/minikube/sshutil"
-	"k8s.io/minikube/pkg/util"
 	"k8s.io/minikube/pkg/util/retry"
 )
 
@@ -50,15 +39,6 @@ type BuildrootProvisioner struct {
 	provision.SystemdProvisioner
 }
 
-// for escaping systemd template specifiers (e.g. '%i'), which are not supported by minikube
-var systemdSpecifierEscaper = strings.NewReplacer("%", "%%")
-
-func init() {
-	provision.Register("Buildroot", &provision.RegisteredProvisioner{
-		New: NewBuildrootProvisioner,
-	})
-}
-
 // NewBuildrootProvisioner creates a new BuildrootProvisioner
 func NewBuildrootProvisioner(d drivers.Driver) provision.Provisioner {
 	return &BuildrootProvisioner{
@@ -75,17 +55,6 @@ func (p *BuildrootProvisioner) CompatibleWithHost() bool {
 	return p.OsReleaseInfo.ID == "buildroot"
 }
 
-// escapeSystemdDirectives escapes special characters in the input variables used to create the
-// systemd unit file, which would otherwise be interpreted as systemd directives. An example
-// are template specifiers (e.g. '%i') which are predefined variables that get evaluated dynamically
-// (see systemd man pages for more info). This is not supported by minikube, thus needs to be escaped.
-func escapeSystemdDirectives(engineConfigContext *provision.EngineConfigContext) {
-	// escape '%' in Environment option so that it does not evaluate into a template specifier
-	engineConfigContext.EngineOptions.Env = util.ReplaceChars(engineConfigContext.EngineOptions.Env, systemdSpecifierEscaper)
-	// input might contain whitespaces, wrap it in quotes
-	engineConfigContext.EngineOptions.Env = util.ConcatStrings(engineConfigContext.EngineOptions.Env, "\"", "\"")
-}
-
 // GenerateDockerOptions generates the *provision.DockerOptions for this provisioner
 func (p *BuildrootProvisioner) GenerateDockerOptions(dockerPort int) (*provision.DockerOptions, error) {
 	var engineCfg bytes.Buffer
@@ -192,14 +161,6 @@ WantedBy=multi-user.target
 	return dockerCfg, nil
 }
 
-func rootFileSystemType(p *BuildrootProvisioner) (string, error) {
-	fs, err := p.SSHCommand("df --output=fstype / | tail -n 1")
-	if err != nil {
-		return "", err
-	}
-	return strings.TrimSpace(fs), nil
-}
-
 // Package installs a package
 func (p *BuildrootProvisioner) Package(name string, action pkgaction.PackageAction) error {
 	return nil
@@ -235,176 +196,10 @@ func (p *BuildrootProvisioner) Provision(swarmOptions swarm.Options, authOptions
 	}
 
 	log.Debugf("setting minikube options for container-runtime")
-	if err := setContainerRuntimeOptions(p); err != nil {
+	if err := setContainerRuntimeOptions(p.Driver.GetMachineName(), p); err != nil {
 		log.Debugf("Error setting container-runtime options during provisioning %v", err)
 		return err
 	}
 
 	return nil
 }
-
-func setRemoteAuthOptions(p provision.Provisioner) auth.Options {
-	dockerDir := p.GetDockerOptionsDir()
-	authOptions := p.GetAuthOptions()
-
-	// due to windows clients, we cannot use filepath.Join as the paths
-	// will be mucked on the linux hosts
-	authOptions.CaCertRemotePath = path.Join(dockerDir, "ca.pem")
-	authOptions.ServerCertRemotePath = path.Join(dockerDir, "server.pem")
-	authOptions.ServerKeyRemotePath = path.Join(dockerDir, "server-key.pem")
-
-	return authOptions
-}
-
-func setContainerRuntimeOptions(p *BuildrootProvisioner) error {
-	c, err := config.Load(p.Driver.GetMachineName())
-	if err != nil {
-		return errors.Wrap(err, "getting cluster config")
-	}
-
-	switch c.KubernetesConfig.ContainerRuntime {
-	case "crio", "cri-o":
-		return p.setCrioOptions()
-	case "containerd":
-		return nil
-	default:
-		_, err := p.GenerateDockerOptions(engine.DefaultPort)
-		return err
-	}
-}
-
-func (p *BuildrootProvisioner) setCrioOptions() error {
-	// pass through --insecure-registry
-	var (
-		crioOptsTmpl = `
-CRIO_MINIKUBE_OPTIONS='{{ range .EngineOptions.InsecureRegistry }}--insecure-registry {{.}} {{ end }}'
-`
-		crioOptsPath = "/etc/sysconfig/crio.minikube"
-	)
-	t, err := template.New("crioOpts").Parse(crioOptsTmpl)
-	if err != nil {
-		return err
-	}
-	var crioOptsBuf bytes.Buffer
-	if err := t.Execute(&crioOptsBuf, p); err != nil {
-		return err
-	}
-
-	if _, err = p.SSHCommand(fmt.Sprintf("sudo mkdir -p %s && printf %%s \"%s\" | sudo tee %s", path.Dir(crioOptsPath), crioOptsBuf.String(), crioOptsPath)); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func configureAuth(p *BuildrootProvisioner) error {
-	log.Infof("configureAuth start")
-	start := time.Now()
-	defer func() {
-		log.Infof("configureAuth took %s", time.Since(start))
-	}()
-
-	driver := p.GetDriver()
-	machineName := driver.GetMachineName()
-	authOptions := p.GetAuthOptions()
-	org := mcnutils.GetUsername() + "." + machineName
-	bits := 2048
-
-	ip, err := driver.GetIP()
-	if err != nil {
-		return errors.Wrap(err, "error getting ip during provisioning")
-	}
-
-	if err := copyHostCerts(authOptions); err != nil {
-		return err
-	}
-
-	// The Host IP is always added to the certificate's SANs list
-	hosts := append(authOptions.ServerCertSANs, ip, "localhost")
-	log.Debugf("generating server cert: %s ca-key=%s private-key=%s org=%s san=%s",
-		authOptions.ServerCertPath,
-		authOptions.CaCertPath,
-		authOptions.CaPrivateKeyPath,
-		org,
-		hosts,
-	)
-
-	err = cert.GenerateCert(&cert.Options{
-		Hosts:     hosts,
-		CertFile:  authOptions.ServerCertPath,
-		KeyFile:   authOptions.ServerKeyPath,
-		CAFile:    authOptions.CaCertPath,
-		CAKeyFile: authOptions.CaPrivateKeyPath,
-		Org:       org,
-		Bits:      bits,
-	})
-
-	if err != nil {
-		return fmt.Errorf("error generating server cert: %v", err)
-	}
-
-	return copyRemoteCerts(authOptions, driver)
-}
-
-func copyHostCerts(authOptions auth.Options) error {
-	log.Infof("copyHostCerts")
-	execRunner := &command.ExecRunner{}
-	hostCerts := map[string]string{
-		authOptions.CaCertPath:     path.Join(authOptions.StorePath, "ca.pem"),
-		authOptions.ClientCertPath: path.Join(authOptions.StorePath, "cert.pem"),
-		authOptions.ClientKeyPath:  path.Join(authOptions.StorePath, "key.pem"),
-	}
-
-	if _, err := execRunner.RunCmd(exec.Command("mkdir", "-p", authOptions.StorePath)); err != nil {
-		return err
-	}
-	for src, dst := range hostCerts {
-		f, err := assets.NewFileAsset(src, path.Dir(dst), filepath.Base(dst), "0777")
-		if err != nil {
-			return errors.Wrapf(err, "open cert file: %s", src)
-		}
-		if err := execRunner.Copy(f); err != nil {
-			return errors.Wrapf(err, "transferring file: %+v", f)
-		}
-	}
-
-	return nil
-}
-
-func copyRemoteCerts(authOptions auth.Options, driver drivers.Driver) error {
-	log.Infof("copyRemoteCerts")
-
-	remoteCerts := map[string]string{
-		authOptions.CaCertPath:     authOptions.CaCertRemotePath,
-		authOptions.ServerCertPath: authOptions.ServerCertRemotePath,
-		authOptions.ServerKeyPath:  authOptions.ServerKeyRemotePath,
-	}
-
-	sshClient, err := sshutil.NewSSHClient(driver)
-	if err != nil {
-		return errors.Wrap(err, "provisioning: error getting ssh client")
-	}
-	sshRunner := command.NewSSHRunner(sshClient)
-
-	dirs := []string{}
-	for _, dst := range remoteCerts {
-		dirs = append(dirs, path.Dir(dst))
-	}
-
-	args := append([]string{"mkdir", "-p"}, dirs...)
-	if _, err = sshRunner.RunCmd(exec.Command("sudo", args...)); err != nil {
-		return err
-	}
-
-	for src, dst := range remoteCerts {
-		f, err := assets.NewFileAsset(src, path.Dir(dst), filepath.Base(dst), "0640")
-		if err != nil {
-			return errors.Wrapf(err, "error copying %s to %s", src, dst)
-		}
-		if err := sshRunner.Copy(f); err != nil {
-			return errors.Wrapf(err, "transferring file to machine %v", f)
-		}
-	}
-
-	return nil
-}
diff --git a/pkg/provision/provision.go b/pkg/provision/provision.go
new file mode 100644
index 0000000000..0ef9f69664
--- /dev/null
+++ b/pkg/provision/provision.go
@@ -0,0 +1,252 @@
+/*
+Copyright 2019 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package provision
+
+import (
+	"bytes"
+	"fmt"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"strings"
+	"text/template"
+	"time"
+
+	"github.com/docker/machine/libmachine/auth"
+	"github.com/docker/machine/libmachine/cert"
+	"github.com/docker/machine/libmachine/drivers"
+	"github.com/docker/machine/libmachine/engine"
+	"github.com/docker/machine/libmachine/log"
+	"github.com/docker/machine/libmachine/mcnutils"
+	"github.com/docker/machine/libmachine/provision"
+	"github.com/docker/machine/libmachine/swarm"
+	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/minikube/assets"
+	"k8s.io/minikube/pkg/minikube/command"
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/sshutil"
+	"k8s.io/minikube/pkg/util"
+)
+
+// generic interface for minikube provisioner
+type miniProvisioner interface {
+	String() string
+	CompatibleWithHost() bool
+	GenerateDockerOptions(int) (*provision.DockerOptions, error)
+	Provision(swarmOptions swarm.Options, authOptions auth.Options, engineOptions engine.Options) error
+	GetDriver() drivers.Driver
+	GetAuthOptions() auth.Options
+	SSHCommand(string) (string, error)
+}
+
+// for escaping systemd template specifiers (e.g. '%i'), which are not supported by minikube
+var systemdSpecifierEscaper = strings.NewReplacer("%", "%%")
+
+func init() {
+	provision.Register("Buildroot", &provision.RegisteredProvisioner{
+		New: NewBuildrootProvisioner,
+	})
+	provision.Register("Ubuntu", &provision.RegisteredProvisioner{
+		New: NewUbuntuProvisioner,
+	})
+
+}
+
+func configureAuth(p miniProvisioner) error {
+	log.Infof("configureAuth start")
+	start := time.Now()
+	defer func() {
+		log.Infof("configureAuth took %s", time.Since(start))
+	}()
+
+	driver := p.GetDriver()
+	machineName := driver.GetMachineName()
+	authOptions := p.GetAuthOptions()
+	org := mcnutils.GetUsername() + "." + machineName
+	bits := 2048
+
+	ip, err := driver.GetIP()
+	if err != nil {
+		return errors.Wrap(err, "error getting ip during provisioning")
+	}
+
+	if err := copyHostCerts(authOptions); err != nil {
+		return err
+	}
+
+	// The Host IP is always added to the certificate's SANs list
+	hosts := append(authOptions.ServerCertSANs, ip, "localhost", "127.0.0.1")
+	log.Debugf("generating server cert: %s ca-key=%s private-key=%s org=%s san=%s",
+		authOptions.ServerCertPath,
+		authOptions.CaCertPath,
+		authOptions.CaPrivateKeyPath,
+		org,
+		hosts,
+	)
+
+	err = cert.GenerateCert(&cert.Options{
+		Hosts:     hosts,
+		CertFile:  authOptions.ServerCertPath,
+		KeyFile:   authOptions.ServerKeyPath,
+		CAFile:    authOptions.CaCertPath,
+		CAKeyFile: authOptions.CaPrivateKeyPath,
+		Org:       org,
+		Bits:      bits,
+	})
+
+	if err != nil {
+		return fmt.Errorf("error generating server cert: %v", err)
+	}
+
+	return copyRemoteCerts(authOptions, driver)
+}
+
+func copyHostCerts(authOptions auth.Options) error {
+	log.Infof("copyHostCerts")
+	execRunner := &command.ExecRunner{}
+	hostCerts := map[string]string{
+		authOptions.CaCertPath:     path.Join(authOptions.StorePath, "ca.pem"),
+		authOptions.ClientCertPath: path.Join(authOptions.StorePath, "cert.pem"),
+		authOptions.ClientKeyPath:  path.Join(authOptions.StorePath, "key.pem"),
+	}
+
+	if _, err := execRunner.RunCmd(exec.Command("mkdir", "-p", authOptions.StorePath)); err != nil {
+		return err
+	}
+	for src, dst := range hostCerts {
+		f, err := assets.NewFileAsset(src, path.Dir(dst), filepath.Base(dst), "0777")
+		if err != nil {
+			return errors.Wrapf(err, "open cert file: %s", src)
+		}
+		if err := execRunner.Copy(f); err != nil {
+			return errors.Wrapf(err, "transferring file: %+v", f)
+		}
+	}
+
+	return nil
+}
+
+func copyRemoteCerts(authOptions auth.Options, driver drivers.Driver) error {
+	log.Infof("copyRemoteCerts")
+
+	remoteCerts := map[string]string{
+		authOptions.CaCertPath:     authOptions.CaCertRemotePath,
+		authOptions.ServerCertPath: authOptions.ServerCertRemotePath,
+		authOptions.ServerKeyPath:  authOptions.ServerKeyRemotePath,
+	}
+
+	sshClient, err := sshutil.NewSSHClient(driver)
+	if err != nil {
+		return errors.Wrap(err, "provisioning: error getting ssh client")
+	}
+	sshRunner := command.NewSSHRunner(sshClient)
+
+	dirs := []string{}
+	for _, dst := range remoteCerts {
+		dirs = append(dirs, path.Dir(dst))
+	}
+
+	args := append([]string{"mkdir", "-p"}, dirs...)
+	if _, err = sshRunner.RunCmd(exec.Command("sudo", args...)); err != nil {
+		return err
+	}
+
+	for src, dst := range remoteCerts {
+		f, err := assets.NewFileAsset(src, path.Dir(dst), filepath.Base(dst), "0640")
+		if err != nil {
+			return errors.Wrapf(err, "error copying %s to %s", src, dst)
+		}
+		if err := sshRunner.Copy(f); err != nil {
+			return errors.Wrapf(err, "transferring file to machine %v", f)
+		}
+	}
+
+	return nil
+}
+
+func setRemoteAuthOptions(p provision.Provisioner) auth.Options {
+	dockerDir := p.GetDockerOptionsDir()
+	authOptions := p.GetAuthOptions()
+
+	// due to windows clients, we cannot use filepath.Join as the paths
+	// will be mucked on the linux hosts
+	authOptions.CaCertRemotePath = path.Join(dockerDir, "ca.pem")
+	authOptions.ServerCertRemotePath = path.Join(dockerDir, "server.pem")
+	authOptions.ServerKeyRemotePath = path.Join(dockerDir, "server-key.pem")
+
+	return authOptions
+}
+
+func setContainerRuntimeOptions(name string, p miniProvisioner) error {
+	c, err := config.Load(name)
+	if err != nil {
+		return errors.Wrap(err, "getting cluster config")
+	}
+
+	switch c.KubernetesConfig.ContainerRuntime {
+	case "crio", "cri-o":
+		return setCrioOptions(p)
+	case "containerd":
+		return nil
+	default:
+		_, err := p.GenerateDockerOptions(engine.DefaultPort)
+		return err
+	}
+}
+
+func setCrioOptions(p provision.SSHCommander) error {
+	// pass through --insecure-registry
+	var (
+		crioOptsTmpl = `
+CRIO_MINIKUBE_OPTIONS='{{ range .EngineOptions.InsecureRegistry }}--insecure-registry {{.}} {{ end }}'
+`
+		crioOptsPath = "/etc/sysconfig/crio.minikube"
+	)
+	t, err := template.New("crioOpts").Parse(crioOptsTmpl)
+	if err != nil {
+		return err
+	}
+	var crioOptsBuf bytes.Buffer
+	if err := t.Execute(&crioOptsBuf, p); err != nil {
+		return err
+	}
+
+	if _, err = p.SSHCommand(fmt.Sprintf("sudo mkdir -p %s && printf %%s \"%s\" | sudo tee %s", path.Dir(crioOptsPath), crioOptsBuf.String(), crioOptsPath)); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func rootFileSystemType(p provision.SSHCommander) (string, error) {
+	fs, err := p.SSHCommand("df --output=fstype / | tail -n 1")
+	if err != nil {
+		return "", err
+	}
+	return strings.TrimSpace(fs), nil
+}
+
+// escapeSystemdDirectives escapes special characters in the input variables used to create the
+// systemd unit file, which would otherwise be interpreted as systemd directives. An example
+// are template specifiers (e.g. '%i') which are predefined variables that get evaluated dynamically
+// (see systemd man pages for more info). This is not supported by minikube, thus needs to be escaped.
+func escapeSystemdDirectives(engineConfigContext *provision.EngineConfigContext) {
+	// escape '%' in Environment option so that it does not evaluate into a template specifier
+	engineConfigContext.EngineOptions.Env = util.ReplaceChars(engineConfigContext.EngineOptions.Env, systemdSpecifierEscaper)
+	// input might contain whitespaces, wrap it in quotes
+	engineConfigContext.EngineOptions.Env = util.ConcatStrings(engineConfigContext.EngineOptions.Env, "\"", "\"")
+}
diff --git a/pkg/provision/ubuntu.go b/pkg/provision/ubuntu.go
new file mode 100644
index 0000000000..575dc53535
--- /dev/null
+++ b/pkg/provision/ubuntu.go
@@ -0,0 +1,210 @@
+/*
+Copyright 2019 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package provision
+
+import (
+	"bytes"
+	"fmt"
+	"path"
+	"text/template"
+	"time"
+
+	"github.com/docker/machine/libmachine/auth"
+	"github.com/docker/machine/libmachine/drivers"
+	"github.com/docker/machine/libmachine/engine"
+	"github.com/docker/machine/libmachine/log"
+	"github.com/docker/machine/libmachine/provision"
+	"github.com/docker/machine/libmachine/provision/pkgaction"
+	"github.com/docker/machine/libmachine/provision/serviceaction"
+	"github.com/docker/machine/libmachine/swarm"
+	"k8s.io/minikube/pkg/util/retry"
+)
+
+// UbuntuProvisioner provisions the ubuntu
+type UbuntuProvisioner struct {
+	BuildrootProvisioner
+}
+
+// NewUbuntuProvisioner creates a new UbuntuProvisioner
+func NewUbuntuProvisioner(d drivers.Driver) provision.Provisioner {
+	return &UbuntuProvisioner{
+		BuildrootProvisioner{
+			provision.NewSystemdProvisioner("ubuntu", d),
+		},
+	}
+}
+
+func (p *UbuntuProvisioner) String() string {
+	return "ubuntu"
+}
+
+// CompatibleWithHost checks if provisioner is compatible with host
+func (p *UbuntuProvisioner) CompatibleWithHost() bool {
+	return p.OsReleaseInfo.ID == "ubuntu"
+}
+
+// GenerateDockerOptions generates the *provision.DockerOptions for this provisioner
+func (p *UbuntuProvisioner) GenerateDockerOptions(dockerPort int) (*provision.DockerOptions, error) {
+	var engineCfg bytes.Buffer
+
+	drvLabel := fmt.Sprintf("provider=%s", p.Driver.DriverName())
+	p.EngineOptions.Labels = append(p.EngineOptions.Labels, drvLabel)
+
+	noPivot := true
+	// Using pivot_root is not supported on fstype rootfs
+	if fstype, err := rootFileSystemType(p); err == nil {
+		log.Debugf("root file system type: %s", fstype)
+		noPivot = fstype == "rootfs"
+	}
+
+	engineConfigTmpl := `[Unit]
+Description=Docker Application Container Engine
+Documentation=https://docs.docker.com
+BindsTo=containerd.service
+After=network-online.target firewalld.service containerd.service
+Wants=network-online.target
+Requires=docker.socket
+
+[Service]
+Type=notify
+`
+	if noPivot {
+		log.Warn("Using fundamentally insecure --no-pivot option")
+		engineConfigTmpl += `
+# DOCKER_RAMDISK disables pivot_root in Docker, using MS_MOVE instead.
+Environment=DOCKER_RAMDISK=yes
+`
+	}
+	engineConfigTmpl += `
+{{range .EngineOptions.Env}}Environment={{.}}
+{{end}}
+
+# This file is a systemd drop-in unit that inherits from the base dockerd configuration.
+# The base configuration already specifies an 'ExecStart=...' command. The first directive
+# here is to clear out that command inherited from the base configuration. Without this,
+# the command from the base configuration and the command specified here are treated as
+# a sequence of commands, which is not the desired behavior, nor is it valid -- systemd
+# will catch this invalid input and refuse to start the service with an error like:
+#  Service has more than one ExecStart= setting, which is only allowed for Type=oneshot services.
+
+# NOTE: default-ulimit=nofile is set to an arbitrary number for consistency with other
+# container runtimes. If left unlimited, it may result in OOM issues with MySQL.
+ExecStart=
+ExecStart=/usr/bin/dockerd -H tcp://0.0.0.0:{{.DockerPort}} -H unix:///var/run/docker.sock --default-ulimit=nofile=1048576:1048576 --tlsverify --tlscacert {{.AuthOptions.CaCertRemotePath}} --tlscert {{.AuthOptions.ServerCertRemotePath}} --tlskey {{.AuthOptions.ServerKeyRemotePath}} {{ range .EngineOptions.Labels }}--label {{.}} {{ end }}{{ range .EngineOptions.InsecureRegistry }}--insecure-registry {{.}} {{ end }}{{ range .EngineOptions.RegistryMirror }}--registry-mirror {{.}} {{ end }}{{ range .EngineOptions.ArbitraryFlags }}--{{.}} {{ end }}
+ExecReload=/bin/kill -s HUP $MAINPID
+
+# Having non-zero Limit*s causes performance problems due to accounting overhead
+# in the kernel. We recommend using cgroups to do container-local accounting.
+LimitNOFILE=infinity
+LimitNPROC=infinity
+LimitCORE=infinity
+
+# Uncomment TasksMax if your systemd version supports it.
+# Only systemd 226 and above support this version.
+TasksMax=infinity
+TimeoutStartSec=0
+
+# set delegate yes so that systemd does not reset the cgroups of docker containers
+Delegate=yes
+
+# kill only the docker process, not all processes in the cgroup
+KillMode=process
+
+[Install]
+WantedBy=multi-user.target
+`
+	t, err := template.New("engineConfig").Parse(engineConfigTmpl)
+	if err != nil {
+		return nil, err
+	}
+
+	engineConfigContext := provision.EngineConfigContext{
+		DockerPort:    dockerPort,
+		AuthOptions:   p.AuthOptions,
+		EngineOptions: p.EngineOptions,
+	}
+
+	escapeSystemdDirectives(&engineConfigContext)
+
+	if err := t.Execute(&engineCfg, engineConfigContext); err != nil {
+		return nil, err
+	}
+
+	dockerCfg := &provision.DockerOptions{
+		EngineOptions:     engineCfg.String(),
+		EngineOptionsPath: "/lib/systemd/system/docker.service",
+	}
+
+	log.Info("Setting Docker configuration on the remote daemon...")
+
+	if _, err = p.SSHCommand(fmt.Sprintf("sudo mkdir -p %s && printf %%s \"%s\" | sudo tee %s", path.Dir(dockerCfg.EngineOptionsPath), dockerCfg.EngineOptions, dockerCfg.EngineOptionsPath)); err != nil {
+		return nil, err
+	}
+
+	if err := p.Service("docker", serviceaction.Enable); err != nil {
+		return nil, err
+	}
+
+	if err := p.Service("docker", serviceaction.Restart); err != nil {
+		return nil, err
+	}
+	return dockerCfg, nil
+}
+
+// Package installs a package
+func (p *UbuntuProvisioner) Package(name string, action pkgaction.PackageAction) error {
+	return nil
+}
+
+// Provision does the provisioning
+func (p *UbuntuProvisioner) Provision(swarmOptions swarm.Options, authOptions auth.Options, engineOptions engine.Options) error {
+	fmt.Println("************ INSISDE ******* PRVOVISION ******************")
+	p.SwarmOptions = swarmOptions
+	p.AuthOptions = authOptions
+	p.EngineOptions = engineOptions
+
+	log.Infof("provisioning hostname %q", p.Driver.GetMachineName())
+	if err := p.SetHostname(p.Driver.GetMachineName()); err != nil {
+		return err
+	}
+
+	p.AuthOptions = setRemoteAuthOptions(p)
+	log.Debugf("set auth options %+v", p.AuthOptions)
+
+	log.Debugf("setting up certificates")
+	configAuth := func() error {
+		if err := configureAuth(p); err != nil {
+			log.Warnf("configureAuth failed: %v", err)
+			return &retry.RetriableError{Err: err}
+		}
+		return nil
+	}
+
+	err := retry.Expo(configAuth, time.Second, 2*time.Minute)
+	if err != nil {
+		log.Debugf("Error configuring auth during provisioning %v", err)
+		return err
+	}
+
+	log.Debugf("setting minikube options for container-runtime")
+	if err := setContainerRuntimeOptions(p.Driver.GetMachineName(), p); err != nil {
+		log.Debugf("Error setting container-runtime options during provisioning %v", err)
+		return err
+	}
+
+	return nil
+}

From e231e7ae96e18bb3ad36bcc9cce688777e516241 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Mon, 3 Feb 2020 22:32:08 -0800
Subject: [PATCH 225/250] rever dockefile changes

---
 hack/images/docker.service     | 48 ----------------------------------
 hack/images/kicbase.Dockerfile |  2 --
 pkg/drivers/kic/types.go       |  3 +--
 3 files changed, 1 insertion(+), 52 deletions(-)
 delete mode 100644 hack/images/docker.service

diff --git a/hack/images/docker.service b/hack/images/docker.service
deleted file mode 100644
index 275f4f9b26..0000000000
--- a/hack/images/docker.service
+++ /dev/null
@@ -1,48 +0,0 @@
-[Unit]
-Description=Docker Application Container Engine
-Documentation=https://docs.docker.com
-BindsTo=containerd.service
-After=network-online.target firewalld.service containerd.service
-Wants=network-online.target
-Requires=docker.socket
-
-[Service]
-Type=notify
-# the default is not to use systemd for cgroups because the delegate issues still
-# exists and systemd currently does not support the cgroup feature set required
-# for containers run by docker
-# ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock
-ExecStart=/usr/bin/dockerd -H tcp://0.0.0.0:2376 -H unix:///var/run/docker.sock --default-ulimit=nofile=1048576:1048576  --label provider=docker --insecure-registry 10.96.0.0/12 
-ExecReload=/bin/kill -s HUP $MAINPID
-TimeoutSec=0
-RestartSec=2
-Restart=always
-
-# Note that StartLimit* options were moved from "Service" to "Unit" in systemd 229.
-# Both the old, and new location are accepted by systemd 229 and up, so using the old location
-# to make them work for either version of systemd.
-StartLimitBurst=3
-
-# Note that StartLimitInterval was renamed to StartLimitIntervalSec in systemd 230.
-# Both the old, and new name are accepted by systemd 230 and up, so using the old name to make
-# this option work for either version of systemd.
-StartLimitInterval=60s
-
-# Having non-zero Limit*s causes performance problems due to accounting overhead
-# in the kernel. We recommend using cgroups to do container-local accounting.
-LimitNOFILE=infinity
-LimitNPROC=infinity
-LimitCORE=infinity
-
-# Comment TasksMax if your systemd version does not support it.
-# Only systemd 226 and above support this option.
-TasksMax=infinity
-
-# set delegate yes so that systemd does not reset the cgroups of docker containers
-Delegate=yes
-
-# kill only the docker process, not all processes in the cgroup
-KillMode=process
-
-[Install]
-WantedBy=multi-user.target
diff --git a/hack/images/kicbase.Dockerfile b/hack/images/kicbase.Dockerfile
index 542b2a6f7b..1636d288bc 100644
--- a/hack/images/kicbase.Dockerfile
+++ b/hack/images/kicbase.Dockerfile
@@ -10,8 +10,6 @@ RUN apt-get update && apt-get install -y \
   openssh-server \
   docker.io \
   && apt-get clean -y 
-# we will use provision package in future https://github.com/kubernetes/minikube/issues/6485
-ADD hack/images/docker.service /lib/systemd/system/docker.service
 # disable containerd by default
 RUN systemctl disable containerd
 # enable docker which is default
diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go
index 9f2e2e90b5..62826954ca 100644
--- a/pkg/drivers/kic/types.go
+++ b/pkg/drivers/kic/types.go
@@ -26,8 +26,7 @@ const (
 	// DefaultBindIPV4 is The default IP the container will bind to.
 	DefaultBindIPV4 = "127.0.0.1"
 	// BaseImage is the base image is used to spin up kic containers. it uses same base-image as kind.
-	// BaseImage = "gcr.io/k8s-minikube/kicbase:v0.0.4@sha256:01552b7ffc7cb0c49c6106796ed73f69647691d8fbae89fed8ee317dc7f1015a"
-	BaseImage = "gcr.io/k8s-minikube/kicbase:v0.0.4-snapshot"
+	BaseImage = "gcr.io/k8s-minikube/kicbase:v0.0.4@sha256:01552b7ffc7cb0c49c6106796ed73f69647691d8fbae89fed8ee317dc7f1015a"
 	// OverlayImage is the cni plugin used for overlay image, created by kind.
 	// CNI plugin image used for kic drivers created by kind.
 	OverlayImage = "kindest/kindnetd:0.5.3"

From 98dbe067580aea9112bab68ecb600b57e32c73da Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Mon, 3 Feb 2020 22:39:12 -0800
Subject: [PATCH 226/250] remove debugging messages

---
 pkg/minikube/machine/client.go | 1 -
 pkg/provision/ubuntu.go        | 1 -
 2 files changed, 2 deletions(-)

diff --git a/pkg/minikube/machine/client.go b/pkg/minikube/machine/client.go
index b1e1112e81..a07cda9d22 100644
--- a/pkg/minikube/machine/client.go
+++ b/pkg/minikube/machine/client.go
@@ -214,7 +214,6 @@ func (api *LocalClient) Create(h *host.Host) error {
 				var pv lib_provision.Provisioner
 				if driver.IsKIC(h.Driver.DriverName()) {
 					pv = provision.NewUbuntuProvisioner(h.Driver)
-					fmt.Println("choosing ubuntu provisioner")
 				} else {
 					pv = provision.NewBuildrootProvisioner(h.Driver)
 				}
diff --git a/pkg/provision/ubuntu.go b/pkg/provision/ubuntu.go
index 575dc53535..8ff0c778ee 100644
--- a/pkg/provision/ubuntu.go
+++ b/pkg/provision/ubuntu.go
@@ -172,7 +172,6 @@ func (p *UbuntuProvisioner) Package(name string, action pkgaction.PackageAction)
 
 // Provision does the provisioning
 func (p *UbuntuProvisioner) Provision(swarmOptions swarm.Options, authOptions auth.Options, engineOptions engine.Options) error {
-	fmt.Println("************ INSISDE ******* PRVOVISION ******************")
 	p.SwarmOptions = swarmOptions
 	p.AuthOptions = authOptions
 	p.EngineOptions = engineOptions

From 6c95a28c10b96af51e5427d6fb9c8bdb07194b43 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Mon, 3 Feb 2020 23:25:55 -0800
Subject: [PATCH 227/250] remove default crictl setting

---
 Makefile                       | 4 ++--
 hack/images/kicbase.Dockerfile | 1 +
 pkg/drivers/kic/types.go       | 2 +-
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index 564359501b..6c4f73b826 100755
--- a/Makefile
+++ b/Makefile
@@ -500,8 +500,8 @@ storage-provisioner-image: out/storage-provisioner-$(GOARCH) ## Build storage-pr
 
 .PHONY: kic-base-image
 kic-base-image: ## builds the base image used for kic.
-	docker rmi -f $(REGISTRY)/kicbase:v0.0.4-snapshot || true
-	docker build -f ./hack/images/kicbase.Dockerfile -t $(REGISTRY)/kicbase:v0.0.4-snapshot  --build-arg COMMIT_SHA=${VERSION}-$(COMMIT)  .
+	docker rmi -f $(REGISTRY)/kicbase:v0.0.5-snapshot || true
+	docker build -f ./hack/images/kicbase.Dockerfile -t $(REGISTRY)/kicbase:v0.0.5-snapshot  --build-arg COMMIT_SHA=${VERSION}-$(COMMIT)  .
 
 
 
diff --git a/hack/images/kicbase.Dockerfile b/hack/images/kicbase.Dockerfile
index 1636d288bc..836f359a66 100644
--- a/hack/images/kicbase.Dockerfile
+++ b/hack/images/kicbase.Dockerfile
@@ -12,6 +12,7 @@ RUN apt-get update && apt-get install -y \
   && apt-get clean -y 
 # disable containerd by default
 RUN systemctl disable containerd
+RUN rm /etc/crictl.yaml
 # enable docker which is default
 RUN systemctl enable docker
 # making SSH work for docker container 
diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go
index 62826954ca..317ee956f0 100644
--- a/pkg/drivers/kic/types.go
+++ b/pkg/drivers/kic/types.go
@@ -26,7 +26,7 @@ const (
 	// DefaultBindIPV4 is The default IP the container will bind to.
 	DefaultBindIPV4 = "127.0.0.1"
 	// BaseImage is the base image is used to spin up kic containers. it uses same base-image as kind.
-	BaseImage = "gcr.io/k8s-minikube/kicbase:v0.0.4@sha256:01552b7ffc7cb0c49c6106796ed73f69647691d8fbae89fed8ee317dc7f1015a"
+	BaseImage = "gcr.io/k8s-minikube/kicbase:v0.0.5@sha256:3ddd8461dfb5c3e452ccc44d87750b87a574ec23fc425da67dccc1f0c57d428a"
 	// OverlayImage is the cni plugin used for overlay image, created by kind.
 	// CNI plugin image used for kic drivers created by kind.
 	OverlayImage = "kindest/kindnetd:0.5.3"

From 42ec2c8d6c4199c16fabcbb4b5eebaea0258698b Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Tue, 4 Feb 2020 07:45:09 -0800
Subject: [PATCH 228/250] Minor refactor fixes

---
 pkg/minikube/cluster/delete.go     | 16 ++++++
 pkg/minikube/cluster/docker_env.go | 16 ++++++
 pkg/minikube/cluster/fix.go        | 92 ++++++++++++++++++------------
 pkg/minikube/cluster/info.go       | 16 ++++++
 pkg/minikube/cluster/ip.go         | 16 ++++++
 pkg/minikube/cluster/iso.go        | 16 ++++++
 pkg/minikube/cluster/ssh.go        | 16 ++++++
 pkg/minikube/cluster/start.go      | 16 ++++++
 pkg/minikube/cluster/status.go     | 16 ++++++
 pkg/minikube/cluster/stop.go       | 16 ++++++
 10 files changed, 198 insertions(+), 38 deletions(-)

diff --git a/pkg/minikube/cluster/delete.go b/pkg/minikube/cluster/delete.go
index ed11d7c3d5..f11f0e4c25 100644
--- a/pkg/minikube/cluster/delete.go
+++ b/pkg/minikube/cluster/delete.go
@@ -1,3 +1,19 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package cluster
 
 import (
diff --git a/pkg/minikube/cluster/docker_env.go b/pkg/minikube/cluster/docker_env.go
index 23c901dcd2..6c349a4128 100644
--- a/pkg/minikube/cluster/docker_env.go
+++ b/pkg/minikube/cluster/docker_env.go
@@ -1,3 +1,19 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package cluster
 
 import (
diff --git a/pkg/minikube/cluster/fix.go b/pkg/minikube/cluster/fix.go
index dd23799493..1a5e9c6cea 100644
--- a/pkg/minikube/cluster/fix.go
+++ b/pkg/minikube/cluster/fix.go
@@ -1,3 +1,19 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package cluster
 
 import (
@@ -8,6 +24,7 @@ import (
 	"time"
 
 	"github.com/docker/machine/libmachine"
+	"github.com/docker/machine/libmachine/host"
 	"github.com/docker/machine/libmachine/provision"
 	"github.com/docker/machine/libmachine/state"
 	"github.com/golang/glog"
@@ -17,72 +34,71 @@ import (
 	"k8s.io/minikube/pkg/util/retry"
 )
 
-var (
+// hostRunner is a minimal host.Host based interface for running commands
+type hostRunner interface {
+	RunSSHCommand(string) (string, error)
+}
+
+const (
 	// The maximum the guest VM clock is allowed to be ahead and behind. This value is intentionally
 	// large to allow for inaccurate methodology, but still small enough so that certificates are likely valid.
 	maxClockDesyncSeconds = 2.1
 )
 
 // fixHost fixes up a previously configured VM so that it is ready to run Kubernetes
-func fixHost(api libmachine.API, mc config.MachineConfig) error {
-	h, err := api.Load(cfg.Name)
-	if err != nil {
-		return nil, errors.Wrap(err, "Error loading existing host. Please try running [minikube delete], then run [minikube start] again.")
-	}
-
-	s, err := h.Driver.GetState()
-	if err != nil {
-		return nil, errors.Wrap(err, "Error getting state for host")
-	}
-
-	if s == state.Running {
-		out.T(out.Running, `Using the running {{.driver_name}} "{{.profile_name}}" VM ...`, out.V{"driver_name": cfg.VMDriver, "profile_name": cfg.Name})
-	} else {
-		out.T(out.Restarting, `Starting existing {{.driver_name}} VM for "{{.profile_name}}" ...`, out.V{"driver_name": cfg.VMDriver, "profile_name": cfg.Name})
-		if err := h.Driver.Start(); err != nil {
-			return nil, errors.Wrap(err, "start")
-		}
-		if err := api.Save(h); err != nil {
-			return nil, errors.Wrap(err, "save")
-		}
-	}
-
-	e := engineOptions(cfg)
-	out.T(out.Waiting, "Waiting for the host to be provisioned ...")
-	err = fixHost(h, e, cfg)
-	if err != nil {
-		return nil, err
-	}
-	return h, nil
+func fixHost(api libmachine.API, mc config.MachineConfig) (*host.Host, error) {
+	out.T(out.Waiting, "Reconfiguring existing host ...")
 
 	start := time.Now()
-	glog.Infof("fixHost: %+v", h.Driver)
+	glog.Infof("fixHost starting: %s", mc.Name)
 	defer func() {
 		glog.Infof("fixHost completed within %s", time.Since(start))
 	}()
 
+	h, err := api.Load(mc.Name)
+	if err != nil {
+		return h, errors.Wrap(err, "Error loading existing host. Please try running [minikube delete], then run [minikube start] again.")
+	}
+
+	s, err := h.Driver.GetState()
+	if err != nil {
+		return h, errors.Wrap(err, "Error getting state for host")
+	}
+
+	if s == state.Running {
+		out.T(out.Running, `Using the running {{.driver_name}} "{{.profile_name}}" VM ...`, out.V{"driver_name": mc.VMDriver, "profile_name": mc.Name})
+	} else {
+		out.T(out.Restarting, `Starting existing {{.driver_name}} VM for "{{.profile_name}}" ...`, out.V{"driver_name": mc.VMDriver, "profile_name": mc.Name})
+		if err := h.Driver.Start(); err != nil {
+			return h, errors.Wrap(err, "driver start")
+		}
+		if err := api.Save(h); err != nil {
+			return h, errors.Wrap(err, "save")
+		}
+	}
+
+	e := engineOptions(mc)
 	if len(e.Env) > 0 {
 		h.HostOptions.EngineOptions.Env = e.Env
 		glog.Infof("Detecting provisioner ...")
 		provisioner, err := provision.DetectProvisioner(h.Driver)
 		if err != nil {
-			return errors.Wrap(err, "detecting provisioner")
+			return h, errors.Wrap(err, "detecting provisioner")
 		}
-		glog.Infof("Provisioning with %s: %+v", provisioner.String(), *h.HostOptions)
 		if err := provisioner.Provision(*h.HostOptions.SwarmOptions, *h.HostOptions.AuthOptions, *h.HostOptions.EngineOptions); err != nil {
-			return errors.Wrap(err, "provision")
+			return h, errors.Wrap(err, "provision")
 		}
 	}
 
 	if err := postStartSetup(h, mc); err != nil {
-		return errors.Wrap(err, "post-start")
+		return h, errors.Wrap(err, "post-start")
 	}
 
 	glog.Infof("Configuring auth for driver %s ...", h.Driver.DriverName())
 	if err := h.ConfigureAuth(); err != nil {
-		return &retry.RetriableError{Err: errors.Wrap(err, "Error configuring auth on host")}
+		return h, &retry.RetriableError{Err: errors.Wrap(err, "Error configuring auth on host")}
 	}
-	return ensureSyncedGuestClock(h)
+	return h, ensureSyncedGuestClock(h)
 }
 
 // ensureGuestClockSync ensures that the guest system clock is relatively in-sync
diff --git a/pkg/minikube/cluster/info.go b/pkg/minikube/cluster/info.go
index 6065d938b7..49c133502c 100644
--- a/pkg/minikube/cluster/info.go
+++ b/pkg/minikube/cluster/info.go
@@ -1,3 +1,19 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package cluster
 
 import (
diff --git a/pkg/minikube/cluster/ip.go b/pkg/minikube/cluster/ip.go
index cafc22027b..8ea0cb5b10 100644
--- a/pkg/minikube/cluster/ip.go
+++ b/pkg/minikube/cluster/ip.go
@@ -1,3 +1,19 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package cluster
 
 import (
diff --git a/pkg/minikube/cluster/iso.go b/pkg/minikube/cluster/iso.go
index 9aa1810cd6..2f78587a21 100644
--- a/pkg/minikube/cluster/iso.go
+++ b/pkg/minikube/cluster/iso.go
@@ -1,3 +1,19 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package cluster
 
 import (
diff --git a/pkg/minikube/cluster/ssh.go b/pkg/minikube/cluster/ssh.go
index 0fdc955053..0f3c28c841 100644
--- a/pkg/minikube/cluster/ssh.go
+++ b/pkg/minikube/cluster/ssh.go
@@ -1,3 +1,19 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package cluster
 
 import (
diff --git a/pkg/minikube/cluster/start.go b/pkg/minikube/cluster/start.go
index 848cf5eb89..4895ff9484 100644
--- a/pkg/minikube/cluster/start.go
+++ b/pkg/minikube/cluster/start.go
@@ -1,3 +1,19 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package cluster
 
 import (
diff --git a/pkg/minikube/cluster/status.go b/pkg/minikube/cluster/status.go
index 8b880e778d..889389be47 100644
--- a/pkg/minikube/cluster/status.go
+++ b/pkg/minikube/cluster/status.go
@@ -1,3 +1,19 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package cluster
 
 import (
diff --git a/pkg/minikube/cluster/stop.go b/pkg/minikube/cluster/stop.go
index f0cf7623c6..ba4e717697 100644
--- a/pkg/minikube/cluster/stop.go
+++ b/pkg/minikube/cluster/stop.go
@@ -1,3 +1,19 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package cluster
 
 import (

From d35c825a053bc42fd161cb4381d53964248842d5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Str=C3=B6mberg?= <tstromberg@google.com>
Date: Tue, 4 Feb 2020 10:25:33 -0800
Subject: [PATCH 229/250] Split cluster package into multiple files (#6484)

* Split cluster package into multiple files

* Name restart/ressurect functions fix

* Minor refactor fixes
---
 pkg/minikube/cluster/cluster.go    | 707 -----------------------------
 pkg/minikube/cluster/delete.go     |  76 ++++
 pkg/minikube/cluster/docker_env.go |  66 +++
 pkg/minikube/cluster/fix.go        | 150 ++++++
 pkg/minikube/cluster/info.go       | 116 +++++
 pkg/minikube/cluster/ip.go         | 108 +++++
 pkg/minikube/cluster/iso.go        |  30 ++
 pkg/minikube/cluster/ssh.go        |  49 ++
 pkg/minikube/cluster/start.go      | 202 +++++++++
 pkg/minikube/cluster/status.go     |  79 ++++
 pkg/minikube/cluster/stop.go       |  76 ++++
 11 files changed, 952 insertions(+), 707 deletions(-)
 create mode 100644 pkg/minikube/cluster/delete.go
 create mode 100644 pkg/minikube/cluster/docker_env.go
 create mode 100644 pkg/minikube/cluster/fix.go
 create mode 100644 pkg/minikube/cluster/info.go
 create mode 100644 pkg/minikube/cluster/ip.go
 create mode 100644 pkg/minikube/cluster/iso.go
 create mode 100644 pkg/minikube/cluster/ssh.go
 create mode 100644 pkg/minikube/cluster/start.go
 create mode 100644 pkg/minikube/cluster/status.go
 create mode 100644 pkg/minikube/cluster/stop.go

diff --git a/pkg/minikube/cluster/cluster.go b/pkg/minikube/cluster/cluster.go
index 86ac6e615e..ee7ca7f055 100644
--- a/pkg/minikube/cluster/cluster.go
+++ b/pkg/minikube/cluster/cluster.go
@@ -17,72 +17,11 @@ limitations under the License.
 package cluster
 
 import (
-	"encoding/json"
 	"flag"
-	"fmt"
-	"io/ioutil"
-	"math"
-	"net"
-	"os/exec"
-	"path"
-	"path/filepath"
-	"regexp"
-	"strconv"
-	"strings"
-	"time"
 
-	"github.com/docker/machine/libmachine"
-	"github.com/docker/machine/libmachine/drivers"
-	"github.com/docker/machine/libmachine/engine"
-	"github.com/docker/machine/libmachine/host"
-	"github.com/docker/machine/libmachine/mcnerror"
-	"github.com/docker/machine/libmachine/provision"
 	"github.com/docker/machine/libmachine/ssh"
-	"github.com/docker/machine/libmachine/state"
-	"github.com/golang/glog"
-	"github.com/juju/mutex"
-	"github.com/pkg/errors"
-	"github.com/shirou/gopsutil/cpu"
-	"github.com/shirou/gopsutil/disk"
-	"github.com/shirou/gopsutil/mem"
-	"github.com/spf13/viper"
 
-	"k8s.io/minikube/pkg/drivers/kic"
-	"k8s.io/minikube/pkg/drivers/kic/oci"
-	"k8s.io/minikube/pkg/minikube/command"
-	"k8s.io/minikube/pkg/minikube/config"
-	"k8s.io/minikube/pkg/minikube/constants"
-	"k8s.io/minikube/pkg/minikube/driver"
 	"k8s.io/minikube/pkg/minikube/exit"
-	"k8s.io/minikube/pkg/minikube/localpath"
-	"k8s.io/minikube/pkg/minikube/out"
-	"k8s.io/minikube/pkg/minikube/registry"
-	"k8s.io/minikube/pkg/minikube/sshutil"
-	"k8s.io/minikube/pkg/minikube/vmpath"
-	"k8s.io/minikube/pkg/util/lock"
-	"k8s.io/minikube/pkg/util/retry"
-)
-
-// hostRunner is a minimal host.Host based interface for running commands
-type hostRunner interface {
-	RunSSHCommand(string) (string, error)
-}
-
-var (
-	// The maximum the guest VM clock is allowed to be ahead and behind. This value is intentionally
-	// large to allow for inaccurate methodology, but still small enough so that certificates are likely valid.
-	maxClockDesyncSeconds = 2.1
-
-	// requiredDirectories are directories to create on the host during setup
-	requiredDirectories = []string{
-		vmpath.GuestAddonsDir,
-		vmpath.GuestManifestsDir,
-		vmpath.GuestEphemeralDir,
-		vmpath.GuestPersistentDir,
-		vmpath.GuestCertsDir,
-		path.Join(vmpath.GuestPersistentDir, "images"),
-		path.Join(vmpath.GuestPersistentDir, "binaries"),
-	}
 )
 
 // This init function is used to set the logtostderr variable to false so that INFO level log info does not clutter the CLI
@@ -96,649 +35,3 @@ func init() {
 	// Setting the default client to native gives much better performance.
 	ssh.SetDefaultClient(ssh.Native)
 }
-
-// CacheISO downloads and caches ISO.
-func CacheISO(cfg config.MachineConfig) error {
-	if driver.BareMetal(cfg.VMDriver) {
-		return nil
-	}
-	return cfg.Downloader.CacheMinikubeISOFromURL(cfg.MinikubeISO)
-}
-
-// StartHost starts a host VM.
-func StartHost(api libmachine.API, cfg config.MachineConfig) (*host.Host, error) {
-	// Prevent machine-driver boot races, as well as our own certificate race
-	releaser, err := acquireMachinesLock(cfg.Name)
-	if err != nil {
-		return nil, errors.Wrap(err, "boot lock")
-	}
-	start := time.Now()
-	defer func() {
-		glog.Infof("releasing machines lock for %q, held for %s", cfg.Name, time.Since(start))
-		releaser.Release()
-	}()
-
-	exists, err := api.Exists(cfg.Name)
-	if err != nil {
-		return nil, errors.Wrapf(err, "exists: %s", cfg.Name)
-	}
-	if !exists {
-		glog.Infoln("Machine does not exist... provisioning new machine")
-		glog.Infof("Provisioning machine with config: %+v", cfg)
-		return createHost(api, cfg)
-	}
-
-	glog.Infoln("Skipping create...Using existing machine configuration")
-
-	h, err := api.Load(cfg.Name)
-	if err != nil {
-		return nil, errors.Wrap(err, "Error loading existing host. Please try running [minikube delete], then run [minikube start] again.")
-	}
-
-	if exists && cfg.Name == constants.DefaultMachineName {
-		out.T(out.Tip, "Tip: Use 'minikube start -p <name>' to create a new cluster, or 'minikube delete' to delete this one.")
-	}
-
-	s, err := h.Driver.GetState()
-	glog.Infoln("Machine state: ", s)
-	if err != nil {
-		return nil, errors.Wrap(err, "Error getting state for host")
-	}
-
-	if s == state.Running {
-		out.T(out.Running, `Using the running {{.driver_name}} "{{.profile_name}}" VM ...`, out.V{"driver_name": cfg.VMDriver, "profile_name": cfg.Name})
-	} else {
-		out.T(out.Restarting, `Starting existing {{.driver_name}} VM for "{{.profile_name}}" ...`, out.V{"driver_name": cfg.VMDriver, "profile_name": cfg.Name})
-		if err := h.Driver.Start(); err != nil {
-			return nil, errors.Wrap(err, "start")
-		}
-		if err := api.Save(h); err != nil {
-			return nil, errors.Wrap(err, "save")
-		}
-	}
-
-	e := engineOptions(cfg)
-	glog.Infof("engine options: %+v", e)
-
-	out.T(out.Waiting, "Waiting for the host to be provisioned ...")
-	err = configureHost(h, e)
-	if err != nil {
-		return nil, err
-	}
-	return h, nil
-}
-
-// acquireMachinesLock protects against code that is not parallel-safe (libmachine, cert setup)
-func acquireMachinesLock(name string) (mutex.Releaser, error) {
-	spec := lock.PathMutexSpec(filepath.Join(localpath.MiniPath(), "machines"))
-	// NOTE: Provisioning generally completes within 60 seconds
-	spec.Timeout = 10 * time.Minute
-
-	glog.Infof("acquiring machines lock for %s: %+v", name, spec)
-	start := time.Now()
-	r, err := mutex.Acquire(spec)
-	if err == nil {
-		glog.Infof("acquired machines lock for %q in %s", name, time.Since(start))
-	}
-	return r, err
-}
-
-// configureHost handles any post-powerup configuration required
-func configureHost(h *host.Host, e *engine.Options) error {
-	start := time.Now()
-	glog.Infof("configureHost: %+v", h.Driver)
-	defer func() {
-		glog.Infof("configureHost completed within %s", time.Since(start))
-	}()
-
-	if err := createRequiredDirectories(h); err != nil {
-		return errors.Wrap(err, "required directories")
-	}
-
-	if len(e.Env) > 0 {
-		h.HostOptions.EngineOptions.Env = e.Env
-		glog.Infof("Detecting provisioner ...")
-		provisioner, err := provision.DetectProvisioner(h.Driver)
-		if err != nil {
-			return errors.Wrap(err, "detecting provisioner")
-		}
-		glog.Infof("Provisioning with %s: %+v", provisioner.String(), *h.HostOptions)
-		if err := provisioner.Provision(*h.HostOptions.SwarmOptions, *h.HostOptions.AuthOptions, *h.HostOptions.EngineOptions); err != nil {
-			return errors.Wrap(err, "provision")
-		}
-	}
-
-	if driver.BareMetal(h.Driver.DriverName()) {
-		glog.Infof("%s is a local driver, skipping auth/time setup", h.Driver.DriverName())
-		return nil
-	}
-	glog.Infof("Configuring auth for driver %s ...", h.Driver.DriverName())
-	if err := h.ConfigureAuth(); err != nil {
-		return &retry.RetriableError{Err: errors.Wrap(err, "Error configuring auth on host")}
-	}
-	return ensureSyncedGuestClock(h)
-}
-
-// ensureGuestClockSync ensures that the guest system clock is relatively in-sync
-func ensureSyncedGuestClock(h hostRunner) error {
-	d, err := guestClockDelta(h, time.Now())
-	if err != nil {
-		glog.Warningf("Unable to measure system clock delta: %v", err)
-		return nil
-	}
-	if math.Abs(d.Seconds()) < maxClockDesyncSeconds {
-		glog.Infof("guest clock delta is within tolerance: %s", d)
-		return nil
-	}
-	if err := adjustGuestClock(h, time.Now()); err != nil {
-		return errors.Wrap(err, "adjusting system clock")
-	}
-	return nil
-}
-
-// guestClockDelta returns the approximate difference between the host and guest system clock
-// NOTE: This does not currently take into account ssh latency.
-func guestClockDelta(h hostRunner, local time.Time) (time.Duration, error) {
-	out, err := h.RunSSHCommand("date +%s.%N")
-	if err != nil {
-		return 0, errors.Wrap(err, "get clock")
-	}
-	glog.Infof("guest clock: %s", out)
-	ns := strings.Split(strings.TrimSpace(out), ".")
-	secs, err := strconv.ParseInt(strings.TrimSpace(ns[0]), 10, 64)
-	if err != nil {
-		return 0, errors.Wrap(err, "atoi")
-	}
-	nsecs, err := strconv.ParseInt(strings.TrimSpace(ns[1]), 10, 64)
-	if err != nil {
-		return 0, errors.Wrap(err, "atoi")
-	}
-	// NOTE: In a synced state, remote is a few hundred ms ahead of local
-	remote := time.Unix(secs, nsecs)
-	d := remote.Sub(local)
-	glog.Infof("Guest: %s Remote: %s (delta=%s)", remote, local, d)
-	return d, nil
-}
-
-// adjustSystemClock adjusts the guest system clock to be nearer to the host system clock
-func adjustGuestClock(h hostRunner, t time.Time) error {
-	out, err := h.RunSSHCommand(fmt.Sprintf("sudo date -s @%d", t.Unix()))
-	glog.Infof("clock set: %s (err=%v)", out, err)
-	return err
-}
-
-// trySSHPowerOff runs the poweroff command on the guest VM to speed up deletion
-func trySSHPowerOff(h *host.Host) error {
-	s, err := h.Driver.GetState()
-	if err != nil {
-		glog.Warningf("unable to get state: %v", err)
-		return err
-	}
-	if s != state.Running {
-		glog.Infof("host is in state %s", s)
-		return nil
-	}
-
-	out.T(out.Shutdown, `Powering off "{{.profile_name}}" via SSH ...`, out.V{"profile_name": h.Name})
-	out, err := h.RunSSHCommand("sudo poweroff")
-	// poweroff always results in an error, since the host disconnects.
-	glog.Infof("poweroff result: out=%s, err=%v", out, err)
-	return nil
-}
-
-// StopHost stops the host VM, saving state to disk.
-func StopHost(api libmachine.API) error {
-	glog.Infof("Stopping host ...")
-	start := time.Now()
-	defer func() {
-		glog.Infof("Stopped host within %s", time.Since(start))
-	}()
-
-	machineName := viper.GetString(config.MachineProfile)
-	host, err := api.Load(machineName)
-	if err != nil {
-		return errors.Wrapf(err, "load")
-	}
-
-	out.T(out.Stopping, `Stopping "{{.profile_name}}" in {{.driver_name}} ...`, out.V{"profile_name": machineName, "driver_name": host.DriverName})
-	if host.DriverName == driver.HyperV {
-		glog.Infof("As there are issues with stopping Hyper-V VMs using API, trying to shut down using SSH")
-		if err := trySSHPowerOff(host); err != nil {
-			return errors.Wrap(err, "ssh power off")
-		}
-	}
-
-	if err := host.Stop(); err != nil {
-		glog.Infof("host.Stop failed: %v", err)
-		alreadyInStateError, ok := err.(mcnerror.ErrHostAlreadyInState)
-		if ok && alreadyInStateError.State == state.Stopped {
-			return nil
-		}
-		return &retry.RetriableError{Err: errors.Wrapf(err, "Stop: %s", machineName)}
-	}
-	return nil
-}
-
-// deleteOrphanedKIC attempts to delete an orphaned docker instance
-func deleteOrphanedKIC(name string) {
-	cmd := exec.Command(oci.Docker, "rm", "-f", "-v", name)
-	err := cmd.Run()
-	if err == nil {
-		glog.Infof("Found stale kic container and successfully cleaned it up!")
-	}
-}
-
-// DeleteHost deletes the host VM.
-func DeleteHost(api libmachine.API, machineName string) error {
-	host, err := api.Load(machineName)
-	if err != nil && host == nil {
-		deleteOrphanedKIC(machineName)
-		// keep going even if minikube  does not know about the host
-	}
-
-	// Get the status of the host. Ensure that it exists before proceeding ahead.
-	status, err := GetHostStatus(api, machineName)
-	if err != nil {
-		// Warn, but proceed
-		out.WarningT("Unable to get the status of the {{.name}} cluster.", out.V{"name": machineName})
-	}
-
-	if status == state.None.String() {
-		return mcnerror.ErrHostDoesNotExist{Name: machineName}
-	}
-
-	// This is slow if SSH is not responding, but HyperV hangs otherwise, See issue #2914
-	if host.Driver.DriverName() == driver.HyperV {
-		if err := trySSHPowerOff(host); err != nil {
-			glog.Infof("Unable to power off minikube because the host was not found.")
-		}
-		out.T(out.DeletingHost, "Successfully powered off Hyper-V. minikube driver -- {{.driver}}", out.V{"driver": host.Driver.DriverName()})
-	}
-
-	out.T(out.DeletingHost, `Deleting "{{.profile_name}}" in {{.driver_name}} ...`, out.V{"profile_name": machineName, "driver_name": host.DriverName})
-	if err := host.Driver.Remove(); err != nil {
-		return errors.Wrap(err, "host remove")
-	}
-	if err := api.Remove(machineName); err != nil {
-		return errors.Wrap(err, "api remove")
-	}
-	return nil
-}
-
-// GetHostStatus gets the status of the host VM.
-func GetHostStatus(api libmachine.API, machineName string) (string, error) {
-	exists, err := api.Exists(machineName)
-	if err != nil {
-		return "", errors.Wrapf(err, "%s exists", machineName)
-	}
-	if !exists {
-		return state.None.String(), nil
-	}
-
-	host, err := api.Load(machineName)
-	if err != nil {
-		return "", errors.Wrapf(err, "load")
-	}
-
-	s, err := host.Driver.GetState()
-	if err != nil {
-		return "", errors.Wrap(err, "state")
-	}
-	return s.String(), nil
-}
-
-// GetHostDriverIP gets the ip address of the current minikube cluster
-func GetHostDriverIP(api libmachine.API, machineName string) (net.IP, error) {
-	host, err := CheckIfHostExistsAndLoad(api, machineName)
-	if err != nil {
-		return nil, err
-	}
-
-	ipStr, err := host.Driver.GetIP()
-	if err != nil {
-		return nil, errors.Wrap(err, "getting IP")
-	}
-	if driver.IsKIC(host.DriverName) {
-		ipStr = kic.DefaultBindIPV4
-	}
-	ip := net.ParseIP(ipStr)
-	if ip == nil {
-		return nil, fmt.Errorf("parsing IP: %s", ipStr)
-	}
-	return ip, nil
-}
-
-func engineOptions(cfg config.MachineConfig) *engine.Options {
-	o := engine.Options{
-		Env:              cfg.DockerEnv,
-		InsecureRegistry: append([]string{constants.DefaultServiceCIDR}, cfg.InsecureRegistry...),
-		RegistryMirror:   cfg.RegistryMirror,
-		ArbitraryFlags:   cfg.DockerOpt,
-		InstallURL:       drivers.DefaultEngineInstallURL,
-	}
-	return &o
-}
-
-type hostInfo struct {
-	Memory   int
-	CPUs     int
-	DiskSize int
-}
-
-func megs(bytes uint64) int {
-	return int(bytes / 1024 / 1024)
-}
-
-func getHostInfo() (*hostInfo, error) {
-	i, err := cpu.Info()
-	if err != nil {
-		glog.Warningf("Unable to get CPU info: %v", err)
-		return nil, err
-	}
-	v, err := mem.VirtualMemory()
-	if err != nil {
-		glog.Warningf("Unable to get mem info: %v", err)
-		return nil, err
-	}
-	d, err := disk.Usage("/")
-	if err != nil {
-		glog.Warningf("Unable to get disk info: %v", err)
-		return nil, err
-	}
-
-	var info hostInfo
-	info.CPUs = len(i)
-	info.Memory = megs(v.Total)
-	info.DiskSize = megs(d.Total)
-	return &info, nil
-}
-
-// showLocalOsRelease shows systemd information about the current linux distribution, on the local host
-func showLocalOsRelease() {
-	osReleaseOut, err := ioutil.ReadFile("/etc/os-release")
-	if err != nil {
-		glog.Errorf("ReadFile: %v", err)
-		return
-	}
-
-	osReleaseInfo, err := provision.NewOsRelease(osReleaseOut)
-	if err != nil {
-		glog.Errorf("NewOsRelease: %v", err)
-		return
-	}
-
-	out.T(out.Provisioner, "OS release is {{.pretty_name}}", out.V{"pretty_name": osReleaseInfo.PrettyName})
-}
-
-// showRemoteOsRelease shows systemd information about the current linux distribution, on the remote VM
-func showRemoteOsRelease(driver drivers.Driver) {
-	provisioner, err := provision.DetectProvisioner(driver)
-	if err != nil {
-		glog.Errorf("DetectProvisioner: %v", err)
-		return
-	}
-
-	osReleaseInfo, err := provisioner.GetOsReleaseInfo()
-	if err != nil {
-		glog.Errorf("GetOsReleaseInfo: %v", err)
-		return
-	}
-
-	glog.Infof("Provisioned with %s", osReleaseInfo.PrettyName)
-}
-
-// showHostInfo shows host information
-func showHostInfo(cfg config.MachineConfig) {
-	if driver.BareMetal(cfg.VMDriver) {
-		info, err := getHostInfo()
-		if err == nil {
-			out.T(out.StartingNone, "Running on localhost (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...", out.V{"number_of_cpus": info.CPUs, "memory_size": info.Memory, "disk_size": info.DiskSize})
-		}
-	} else if driver.IsKIC(cfg.VMDriver) {
-		info, err := getHostInfo() // TODO medyagh: get docker-machine info for non linux
-		if err == nil {
-			out.T(out.StartingVM, "Creating Kubernetes in {{.driver_name}} container with (CPUs={{.number_of_cpus}}), Memory={{.memory_size}}MB ({{.host_memory_size}}MB available) ...", out.V{"driver_name": cfg.VMDriver, "number_of_cpus": cfg.CPUs, "number_of_host_cpus": info.CPUs, "memory_size": cfg.Memory, "host_memory_size": info.Memory})
-		}
-	} else {
-		out.T(out.StartingVM, "Creating {{.driver_name}} VM (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...", out.V{"driver_name": cfg.VMDriver, "number_of_cpus": cfg.CPUs, "memory_size": cfg.Memory, "disk_size": cfg.DiskSize})
-	}
-}
-
-func createHost(api libmachine.API, cfg config.MachineConfig) (*host.Host, error) {
-	if cfg.VMDriver == driver.VMwareFusion && viper.GetBool(config.ShowDriverDeprecationNotification) {
-		out.WarningT(`The vmwarefusion driver is deprecated and support for it will be removed in a future release.
-			Please consider switching to the new vmware unified driver, which is intended to replace the vmwarefusion driver.
-			See https://minikube.sigs.k8s.io/docs/reference/drivers/vmware/ for more information.
-			To disable this message, run [minikube config set ShowDriverDeprecationNotification false]`)
-	}
-	showHostInfo(cfg)
-	def := registry.Driver(cfg.VMDriver)
-	if def.Empty() {
-		return nil, fmt.Errorf("unsupported/missing driver: %s", cfg.VMDriver)
-	}
-	dd := def.Config(cfg)
-	data, err := json.Marshal(dd)
-	if err != nil {
-		return nil, errors.Wrap(err, "marshal")
-	}
-
-	h, err := api.NewHost(cfg.VMDriver, data)
-	if err != nil {
-		return nil, errors.Wrap(err, "new host")
-	}
-
-	h.HostOptions.AuthOptions.CertDir = localpath.MiniPath()
-	h.HostOptions.AuthOptions.StorePath = localpath.MiniPath()
-	h.HostOptions.EngineOptions = engineOptions(cfg)
-
-	if err := api.Create(h); err != nil {
-		// Wait for all the logs to reach the client
-		time.Sleep(2 * time.Second)
-		return nil, errors.Wrap(err, "create")
-	}
-
-	if err := createRequiredDirectories(h); err != nil {
-		return h, errors.Wrap(err, "required directories")
-	}
-
-	if driver.BareMetal(cfg.VMDriver) {
-		showLocalOsRelease()
-	} else if !driver.BareMetal(cfg.VMDriver) && !driver.IsKIC(cfg.VMDriver) {
-		showRemoteOsRelease(h.Driver)
-		// Ensure that even new VM's have proper time synchronization up front
-		// It's 2019, and I can't believe I am still dealing with time desync as a problem.
-		if err := ensureSyncedGuestClock(h); err != nil {
-			return h, err
-		}
-	} // TODO:medyagh add show-os release for kic
-
-	if err := api.Save(h); err != nil {
-		return nil, errors.Wrap(err, "save")
-	}
-	return h, nil
-}
-
-// GetHostDockerEnv gets the necessary docker env variables to allow the use of docker through minikube's vm
-func GetHostDockerEnv(api libmachine.API) (map[string]string, error) {
-	pName := viper.GetString(config.MachineProfile)
-	host, err := CheckIfHostExistsAndLoad(api, pName)
-	if err != nil {
-		return nil, errors.Wrap(err, "Error checking that api exists and loading it")
-	}
-
-	ip := kic.DefaultBindIPV4
-	if !driver.IsKIC(host.Driver.DriverName()) { // kic externally accessible ip is different that node ip
-		ip, err = host.Driver.GetIP()
-		if err != nil {
-			return nil, errors.Wrap(err, "Error getting ip from host")
-		}
-
-	}
-
-	tcpPrefix := "tcp://"
-	port := constants.DockerDaemonPort
-	if driver.IsKIC(host.Driver.DriverName()) { // for kic we need to find out what port docker allocated during creation
-		port, err = oci.HostPortBinding(host.Driver.DriverName(), pName, constants.DockerDaemonPort)
-		if err != nil {
-			return nil, errors.Wrapf(err, "get hostbind port for %d", constants.DockerDaemonPort)
-		}
-	}
-
-	envMap := map[string]string{
-		"DOCKER_TLS_VERIFY": "1",
-		"DOCKER_HOST":       tcpPrefix + net.JoinHostPort(ip, fmt.Sprint(port)),
-		"DOCKER_CERT_PATH":  localpath.MakeMiniPath("certs"),
-	}
-	return envMap, nil
-}
-
-// GetVMHostIP gets the ip address to be used for mapping host -> VM and VM -> host
-func GetVMHostIP(host *host.Host) (net.IP, error) {
-	switch host.DriverName {
-	case driver.KVM2:
-		return net.ParseIP("192.168.39.1"), nil
-	case driver.HyperV:
-		re := regexp.MustCompile(`"VSwitch": "(.*?)",`)
-		// TODO(aprindle) Change this to deserialize the driver instead
-		hypervVirtualSwitch := re.FindStringSubmatch(string(host.RawDriver))[1]
-		ip, err := getIPForInterface(fmt.Sprintf("vEthernet (%s)", hypervVirtualSwitch))
-		if err != nil {
-			return []byte{}, errors.Wrap(err, fmt.Sprintf("ip for interface (%s)", hypervVirtualSwitch))
-		}
-		return ip, nil
-	case driver.VirtualBox:
-		out, err := exec.Command(driver.VBoxManagePath(), "showvminfo", host.Name, "--machinereadable").Output()
-		if err != nil {
-			return []byte{}, errors.Wrap(err, "vboxmanage")
-		}
-		re := regexp.MustCompile(`hostonlyadapter2="(.*?)"`)
-		iface := re.FindStringSubmatch(string(out))[1]
-		ip, err := getIPForInterface(iface)
-		if err != nil {
-			return []byte{}, errors.Wrap(err, "Error getting VM/Host IP address")
-		}
-		return ip, nil
-	case driver.HyperKit:
-		return net.ParseIP("192.168.64.1"), nil
-	case driver.VMware:
-		vmIPString, err := host.Driver.GetIP()
-		if err != nil {
-			return []byte{}, errors.Wrap(err, "Error getting VM IP address")
-		}
-		vmIP := net.ParseIP(vmIPString).To4()
-		if vmIP == nil {
-			return []byte{}, errors.Wrap(err, "Error converting VM IP address to IPv4 address")
-		}
-		return net.IPv4(vmIP[0], vmIP[1], vmIP[2], byte(1)), nil
-	default:
-		return []byte{}, errors.New("Error, attempted to get host ip address for unsupported driver")
-	}
-}
-
-// Based on code from http://stackoverflow.com/questions/23529663/how-to-get-all-addresses-and-masks-from-local-interfaces-in-go
-func getIPForInterface(name string) (net.IP, error) {
-	i, _ := net.InterfaceByName(name)
-	addrs, _ := i.Addrs()
-	for _, a := range addrs {
-		if ipnet, ok := a.(*net.IPNet); ok {
-			if ip := ipnet.IP.To4(); ip != nil {
-				return ip, nil
-			}
-		}
-	}
-	return nil, errors.Errorf("Error finding IPV4 address for %s", name)
-}
-
-// CheckIfHostExistsAndLoad checks if a host exists, and loads it if it does
-func CheckIfHostExistsAndLoad(api libmachine.API, machineName string) (*host.Host, error) {
-	glog.Infof("Checking if %q exists ...", machineName)
-	exists, err := api.Exists(machineName)
-	if err != nil {
-		return nil, errors.Wrapf(err, "Error checking that machine exists: %s", machineName)
-	}
-	if !exists {
-		return nil, errors.Errorf("machine %q does not exist", machineName)
-	}
-
-	host, err := api.Load(machineName)
-	if err != nil {
-		return nil, errors.Wrapf(err, "loading machine %q", machineName)
-	}
-	return host, nil
-}
-
-// CreateSSHShell creates a new SSH shell / client
-func CreateSSHShell(api libmachine.API, args []string) error {
-	machineName := viper.GetString(config.MachineProfile)
-	host, err := CheckIfHostExistsAndLoad(api, machineName)
-	if err != nil {
-		return errors.Wrap(err, "host exists and load")
-	}
-
-	currentState, err := host.Driver.GetState()
-	if err != nil {
-		return errors.Wrap(err, "state")
-	}
-
-	if currentState != state.Running {
-		return errors.Errorf("%q is not running", machineName)
-	}
-
-	client, err := host.CreateSSHClient()
-	if err != nil {
-		return errors.Wrap(err, "Creating ssh client")
-	}
-	return client.Shell(args...)
-}
-
-// IsHostRunning asserts that this profile's primary host is in state "Running"
-func IsHostRunning(api libmachine.API, name string) bool {
-	s, err := GetHostStatus(api, name)
-	if err != nil {
-		glog.Warningf("host status for %q returned error: %v", name, err)
-		return false
-	}
-	if s != state.Running.String() {
-		glog.Warningf("%q host status: %s", name, s)
-		return false
-	}
-	return true
-}
-
-// createRequiredDirectories creates directories expected by minikube to exist
-func createRequiredDirectories(h *host.Host) error {
-	if h.DriverName == driver.Mock {
-		glog.Infof("skipping createRequiredDirectories")
-		return nil
-	}
-	glog.Infof("creating required directories: %v", requiredDirectories)
-	r, err := commandRunner(h)
-	if err != nil {
-		return errors.Wrap(err, "command runner")
-	}
-
-	args := append([]string{"mkdir", "-p"}, requiredDirectories...)
-	if _, err := r.RunCmd(exec.Command("sudo", args...)); err != nil {
-		return errors.Wrapf(err, "sudo mkdir (%s)", h.DriverName)
-	}
-	return nil
-}
-
-// commandRunner returns best available command runner for this host
-func commandRunner(h *host.Host) (command.Runner, error) {
-	if h.DriverName == driver.Mock {
-		glog.Errorf("commandRunner: returning unconfigured FakeCommandRunner, commands will fail!")
-		return &command.FakeCommandRunner{}, nil
-	}
-	if driver.BareMetal(h.Driver.DriverName()) {
-		return &command.ExecRunner{}, nil
-	}
-	if h.Driver.DriverName() == driver.Docker {
-		return command.NewKICRunner(h.Name, "docker"), nil
-	}
-	client, err := sshutil.NewSSHClient(h.Driver)
-	if err != nil {
-		return nil, errors.Wrap(err, "getting ssh client for bootstrapper")
-	}
-	return command.NewSSHRunner(client), nil
-}
diff --git a/pkg/minikube/cluster/delete.go b/pkg/minikube/cluster/delete.go
new file mode 100644
index 0000000000..f11f0e4c25
--- /dev/null
+++ b/pkg/minikube/cluster/delete.go
@@ -0,0 +1,76 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cluster
+
+import (
+	"os/exec"
+
+	"github.com/docker/machine/libmachine"
+	"github.com/docker/machine/libmachine/mcnerror"
+	"github.com/docker/machine/libmachine/state"
+	"github.com/golang/glog"
+	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/drivers/kic/oci"
+	"k8s.io/minikube/pkg/minikube/driver"
+	"k8s.io/minikube/pkg/minikube/out"
+)
+
+// deleteOrphanedKIC attempts to delete an orphaned docker instance
+func deleteOrphanedKIC(name string) {
+	cmd := exec.Command(oci.Docker, "rm", "-f", "-v", name)
+	err := cmd.Run()
+	if err == nil {
+		glog.Infof("Found stale kic container and successfully cleaned it up!")
+	}
+}
+
+// DeleteHost deletes the host VM.
+func DeleteHost(api libmachine.API, machineName string) error {
+	host, err := api.Load(machineName)
+	if err != nil && host == nil {
+		deleteOrphanedKIC(machineName)
+		// keep going even if minikube  does not know about the host
+	}
+
+	// Get the status of the host. Ensure that it exists before proceeding ahead.
+	status, err := GetHostStatus(api, machineName)
+	if err != nil {
+		// Warn, but proceed
+		out.WarningT("Unable to get the status of the {{.name}} cluster.", out.V{"name": machineName})
+	}
+
+	if status == state.None.String() {
+		return mcnerror.ErrHostDoesNotExist{Name: machineName}
+	}
+
+	// This is slow if SSH is not responding, but HyperV hangs otherwise, See issue #2914
+	if host.Driver.DriverName() == driver.HyperV {
+		if err := trySSHPowerOff(host); err != nil {
+			glog.Infof("Unable to power off minikube because the host was not found.")
+		}
+		out.T(out.DeletingHost, "Successfully powered off Hyper-V. minikube driver -- {{.driver}}", out.V{"driver": host.Driver.DriverName()})
+	}
+
+	out.T(out.DeletingHost, `Deleting "{{.profile_name}}" in {{.driver_name}} ...`, out.V{"profile_name": machineName, "driver_name": host.DriverName})
+	if err := host.Driver.Remove(); err != nil {
+		return errors.Wrap(err, "host remove")
+	}
+	if err := api.Remove(machineName); err != nil {
+		return errors.Wrap(err, "api remove")
+	}
+	return nil
+}
diff --git a/pkg/minikube/cluster/docker_env.go b/pkg/minikube/cluster/docker_env.go
new file mode 100644
index 0000000000..6c349a4128
--- /dev/null
+++ b/pkg/minikube/cluster/docker_env.go
@@ -0,0 +1,66 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cluster
+
+import (
+	"fmt"
+	"net"
+
+	"github.com/docker/machine/libmachine"
+	"github.com/pkg/errors"
+	"github.com/spf13/viper"
+	"k8s.io/minikube/pkg/drivers/kic"
+	"k8s.io/minikube/pkg/drivers/kic/oci"
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/constants"
+	"k8s.io/minikube/pkg/minikube/driver"
+	"k8s.io/minikube/pkg/minikube/localpath"
+)
+
+// GetHostDockerEnv gets the necessary docker env variables to allow the use of docker through minikube's vm
+func GetHostDockerEnv(api libmachine.API) (map[string]string, error) {
+	pName := viper.GetString(config.MachineProfile)
+	host, err := CheckIfHostExistsAndLoad(api, pName)
+	if err != nil {
+		return nil, errors.Wrap(err, "Error checking that api exists and loading it")
+	}
+
+	ip := kic.DefaultBindIPV4
+	if !driver.IsKIC(host.Driver.DriverName()) { // kic externally accessible ip is different that node ip
+		ip, err = host.Driver.GetIP()
+		if err != nil {
+			return nil, errors.Wrap(err, "Error getting ip from host")
+		}
+
+	}
+
+	tcpPrefix := "tcp://"
+	port := constants.DockerDaemonPort
+	if driver.IsKIC(host.Driver.DriverName()) { // for kic we need to find out what port docker allocated during creation
+		port, err = oci.HostPortBinding(host.Driver.DriverName(), pName, constants.DockerDaemonPort)
+		if err != nil {
+			return nil, errors.Wrapf(err, "get hostbind port for %d", constants.DockerDaemonPort)
+		}
+	}
+
+	envMap := map[string]string{
+		"DOCKER_TLS_VERIFY": "1",
+		"DOCKER_HOST":       tcpPrefix + net.JoinHostPort(ip, fmt.Sprint(port)),
+		"DOCKER_CERT_PATH":  localpath.MakeMiniPath("certs"),
+	}
+	return envMap, nil
+}
diff --git a/pkg/minikube/cluster/fix.go b/pkg/minikube/cluster/fix.go
new file mode 100644
index 0000000000..1a5e9c6cea
--- /dev/null
+++ b/pkg/minikube/cluster/fix.go
@@ -0,0 +1,150 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cluster
+
+import (
+	"fmt"
+	"math"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/docker/machine/libmachine"
+	"github.com/docker/machine/libmachine/host"
+	"github.com/docker/machine/libmachine/provision"
+	"github.com/docker/machine/libmachine/state"
+	"github.com/golang/glog"
+	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/out"
+	"k8s.io/minikube/pkg/util/retry"
+)
+
+// hostRunner is a minimal host.Host based interface for running commands
+type hostRunner interface {
+	RunSSHCommand(string) (string, error)
+}
+
+const (
+	// The maximum the guest VM clock is allowed to be ahead and behind. This value is intentionally
+	// large to allow for inaccurate methodology, but still small enough so that certificates are likely valid.
+	maxClockDesyncSeconds = 2.1
+)
+
+// fixHost fixes up a previously configured VM so that it is ready to run Kubernetes
+func fixHost(api libmachine.API, mc config.MachineConfig) (*host.Host, error) {
+	out.T(out.Waiting, "Reconfiguring existing host ...")
+
+	start := time.Now()
+	glog.Infof("fixHost starting: %s", mc.Name)
+	defer func() {
+		glog.Infof("fixHost completed within %s", time.Since(start))
+	}()
+
+	h, err := api.Load(mc.Name)
+	if err != nil {
+		return h, errors.Wrap(err, "Error loading existing host. Please try running [minikube delete], then run [minikube start] again.")
+	}
+
+	s, err := h.Driver.GetState()
+	if err != nil {
+		return h, errors.Wrap(err, "Error getting state for host")
+	}
+
+	if s == state.Running {
+		out.T(out.Running, `Using the running {{.driver_name}} "{{.profile_name}}" VM ...`, out.V{"driver_name": mc.VMDriver, "profile_name": mc.Name})
+	} else {
+		out.T(out.Restarting, `Starting existing {{.driver_name}} VM for "{{.profile_name}}" ...`, out.V{"driver_name": mc.VMDriver, "profile_name": mc.Name})
+		if err := h.Driver.Start(); err != nil {
+			return h, errors.Wrap(err, "driver start")
+		}
+		if err := api.Save(h); err != nil {
+			return h, errors.Wrap(err, "save")
+		}
+	}
+
+	e := engineOptions(mc)
+	if len(e.Env) > 0 {
+		h.HostOptions.EngineOptions.Env = e.Env
+		glog.Infof("Detecting provisioner ...")
+		provisioner, err := provision.DetectProvisioner(h.Driver)
+		if err != nil {
+			return h, errors.Wrap(err, "detecting provisioner")
+		}
+		if err := provisioner.Provision(*h.HostOptions.SwarmOptions, *h.HostOptions.AuthOptions, *h.HostOptions.EngineOptions); err != nil {
+			return h, errors.Wrap(err, "provision")
+		}
+	}
+
+	if err := postStartSetup(h, mc); err != nil {
+		return h, errors.Wrap(err, "post-start")
+	}
+
+	glog.Infof("Configuring auth for driver %s ...", h.Driver.DriverName())
+	if err := h.ConfigureAuth(); err != nil {
+		return h, &retry.RetriableError{Err: errors.Wrap(err, "Error configuring auth on host")}
+	}
+	return h, ensureSyncedGuestClock(h)
+}
+
+// ensureGuestClockSync ensures that the guest system clock is relatively in-sync
+func ensureSyncedGuestClock(h hostRunner) error {
+	d, err := guestClockDelta(h, time.Now())
+	if err != nil {
+		glog.Warningf("Unable to measure system clock delta: %v", err)
+		return nil
+	}
+	if math.Abs(d.Seconds()) < maxClockDesyncSeconds {
+		glog.Infof("guest clock delta is within tolerance: %s", d)
+		return nil
+	}
+	if err := adjustGuestClock(h, time.Now()); err != nil {
+		return errors.Wrap(err, "adjusting system clock")
+	}
+	return nil
+}
+
+// guestClockDelta returns the approximate difference between the host and guest system clock
+// NOTE: This does not currently take into account ssh latency.
+func guestClockDelta(h hostRunner, local time.Time) (time.Duration, error) {
+	out, err := h.RunSSHCommand("date +%s.%N")
+	if err != nil {
+		return 0, errors.Wrap(err, "get clock")
+	}
+	glog.Infof("guest clock: %s", out)
+	ns := strings.Split(strings.TrimSpace(out), ".")
+	secs, err := strconv.ParseInt(strings.TrimSpace(ns[0]), 10, 64)
+	if err != nil {
+		return 0, errors.Wrap(err, "atoi")
+	}
+	nsecs, err := strconv.ParseInt(strings.TrimSpace(ns[1]), 10, 64)
+	if err != nil {
+		return 0, errors.Wrap(err, "atoi")
+	}
+	// NOTE: In a synced state, remote is a few hundred ms ahead of local
+	remote := time.Unix(secs, nsecs)
+	d := remote.Sub(local)
+	glog.Infof("Guest: %s Remote: %s (delta=%s)", remote, local, d)
+	return d, nil
+}
+
+// adjustSystemClock adjusts the guest system clock to be nearer to the host system clock
+func adjustGuestClock(h hostRunner, t time.Time) error {
+	out, err := h.RunSSHCommand(fmt.Sprintf("sudo date -s @%d", t.Unix()))
+	glog.Infof("clock set: %s (err=%v)", out, err)
+	return err
+}
diff --git a/pkg/minikube/cluster/info.go b/pkg/minikube/cluster/info.go
new file mode 100644
index 0000000000..49c133502c
--- /dev/null
+++ b/pkg/minikube/cluster/info.go
@@ -0,0 +1,116 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cluster
+
+import (
+	"io/ioutil"
+
+	"github.com/docker/machine/libmachine/drivers"
+	"github.com/docker/machine/libmachine/provision"
+	"github.com/golang/glog"
+	"github.com/shirou/gopsutil/cpu"
+	"github.com/shirou/gopsutil/disk"
+	"github.com/shirou/gopsutil/mem"
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/driver"
+	"k8s.io/minikube/pkg/minikube/out"
+)
+
+type hostInfo struct {
+	Memory   int
+	CPUs     int
+	DiskSize int
+}
+
+func megs(bytes uint64) int {
+	return int(bytes / 1024 / 1024)
+}
+
+func getHostInfo() (*hostInfo, error) {
+	i, err := cpu.Info()
+	if err != nil {
+		glog.Warningf("Unable to get CPU info: %v", err)
+		return nil, err
+	}
+	v, err := mem.VirtualMemory()
+	if err != nil {
+		glog.Warningf("Unable to get mem info: %v", err)
+		return nil, err
+	}
+	d, err := disk.Usage("/")
+	if err != nil {
+		glog.Warningf("Unable to get disk info: %v", err)
+		return nil, err
+	}
+
+	var info hostInfo
+	info.CPUs = len(i)
+	info.Memory = megs(v.Total)
+	info.DiskSize = megs(d.Total)
+	return &info, nil
+}
+
+// showLocalOsRelease shows systemd information about the current linux distribution, on the local host
+func showLocalOsRelease() {
+	osReleaseOut, err := ioutil.ReadFile("/etc/os-release")
+	if err != nil {
+		glog.Errorf("ReadFile: %v", err)
+		return
+	}
+
+	osReleaseInfo, err := provision.NewOsRelease(osReleaseOut)
+	if err != nil {
+		glog.Errorf("NewOsRelease: %v", err)
+		return
+	}
+
+	out.T(out.Provisioner, "OS release is {{.pretty_name}}", out.V{"pretty_name": osReleaseInfo.PrettyName})
+}
+
+// showRemoteOsRelease shows systemd information about the current linux distribution, on the remote VM
+func showRemoteOsRelease(driver drivers.Driver) {
+	provisioner, err := provision.DetectProvisioner(driver)
+	if err != nil {
+		glog.Errorf("DetectProvisioner: %v", err)
+		return
+	}
+
+	osReleaseInfo, err := provisioner.GetOsReleaseInfo()
+	if err != nil {
+		glog.Errorf("GetOsReleaseInfo: %v", err)
+		return
+	}
+
+	glog.Infof("Provisioned with %s", osReleaseInfo.PrettyName)
+}
+
+// showHostInfo shows host information
+func showHostInfo(cfg config.MachineConfig) {
+	if driver.BareMetal(cfg.VMDriver) {
+		info, err := getHostInfo()
+		if err == nil {
+			out.T(out.StartingNone, "Running on localhost (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...", out.V{"number_of_cpus": info.CPUs, "memory_size": info.Memory, "disk_size": info.DiskSize})
+		}
+	} else if driver.IsKIC(cfg.VMDriver) {
+		info, err := getHostInfo() // TODO medyagh: get docker-machine info for non linux
+		if err == nil {
+			out.T(out.StartingVM, "Creating Kubernetes in {{.driver_name}} container with (CPUs={{.number_of_cpus}}), Memory={{.memory_size}}MB ({{.host_memory_size}}MB available) ...", out.V{"driver_name": cfg.VMDriver, "number_of_cpus": cfg.CPUs, "number_of_host_cpus": info.CPUs, "memory_size": cfg.Memory, "host_memory_size": info.Memory})
+		}
+	} else {
+		out.T(out.StartingVM, "Creating {{.driver_name}} VM (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...", out.V{"driver_name": cfg.VMDriver, "number_of_cpus": cfg.CPUs, "memory_size": cfg.Memory, "disk_size": cfg.DiskSize})
+	}
+}
diff --git a/pkg/minikube/cluster/ip.go b/pkg/minikube/cluster/ip.go
new file mode 100644
index 0000000000..8ea0cb5b10
--- /dev/null
+++ b/pkg/minikube/cluster/ip.go
@@ -0,0 +1,108 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cluster
+
+import (
+	"fmt"
+	"net"
+	"os/exec"
+	"regexp"
+
+	"github.com/docker/machine/libmachine"
+	"github.com/docker/machine/libmachine/host"
+	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/drivers/kic"
+	"k8s.io/minikube/pkg/minikube/driver"
+)
+
+// GetVMHostIP gets the ip address to be used for mapping host -> VM and VM -> host
+func GetVMHostIP(host *host.Host) (net.IP, error) {
+	switch host.DriverName {
+	case driver.KVM2:
+		return net.ParseIP("192.168.39.1"), nil
+	case driver.HyperV:
+		re := regexp.MustCompile(`"VSwitch": "(.*?)",`)
+		// TODO(aprindle) Change this to deserialize the driver instead
+		hypervVirtualSwitch := re.FindStringSubmatch(string(host.RawDriver))[1]
+		ip, err := getIPForInterface(fmt.Sprintf("vEthernet (%s)", hypervVirtualSwitch))
+		if err != nil {
+			return []byte{}, errors.Wrap(err, fmt.Sprintf("ip for interface (%s)", hypervVirtualSwitch))
+		}
+		return ip, nil
+	case driver.VirtualBox:
+		out, err := exec.Command(driver.VBoxManagePath(), "showvminfo", host.Name, "--machinereadable").Output()
+		if err != nil {
+			return []byte{}, errors.Wrap(err, "vboxmanage")
+		}
+		re := regexp.MustCompile(`hostonlyadapter2="(.*?)"`)
+		iface := re.FindStringSubmatch(string(out))[1]
+		ip, err := getIPForInterface(iface)
+		if err != nil {
+			return []byte{}, errors.Wrap(err, "Error getting VM/Host IP address")
+		}
+		return ip, nil
+	case driver.HyperKit:
+		return net.ParseIP("192.168.64.1"), nil
+	case driver.VMware:
+		vmIPString, err := host.Driver.GetIP()
+		if err != nil {
+			return []byte{}, errors.Wrap(err, "Error getting VM IP address")
+		}
+		vmIP := net.ParseIP(vmIPString).To4()
+		if vmIP == nil {
+			return []byte{}, errors.Wrap(err, "Error converting VM IP address to IPv4 address")
+		}
+		return net.IPv4(vmIP[0], vmIP[1], vmIP[2], byte(1)), nil
+	default:
+		return []byte{}, errors.New("Error, attempted to get host ip address for unsupported driver")
+	}
+}
+
+// GetHostDriverIP gets the ip address of the current minikube cluster
+func GetHostDriverIP(api libmachine.API, machineName string) (net.IP, error) {
+	host, err := CheckIfHostExistsAndLoad(api, machineName)
+	if err != nil {
+		return nil, err
+	}
+
+	ipStr, err := host.Driver.GetIP()
+	if err != nil {
+		return nil, errors.Wrap(err, "getting IP")
+	}
+	if driver.IsKIC(host.DriverName) {
+		ipStr = kic.DefaultBindIPV4
+	}
+	ip := net.ParseIP(ipStr)
+	if ip == nil {
+		return nil, fmt.Errorf("parsing IP: %s", ipStr)
+	}
+	return ip, nil
+}
+
+// Based on code from http://stackoverflow.com/questions/23529663/how-to-get-all-addresses-and-masks-from-local-interfaces-in-go
+func getIPForInterface(name string) (net.IP, error) {
+	i, _ := net.InterfaceByName(name)
+	addrs, _ := i.Addrs()
+	for _, a := range addrs {
+		if ipnet, ok := a.(*net.IPNet); ok {
+			if ip := ipnet.IP.To4(); ip != nil {
+				return ip, nil
+			}
+		}
+	}
+	return nil, errors.Errorf("Error finding IPV4 address for %s", name)
+}
diff --git a/pkg/minikube/cluster/iso.go b/pkg/minikube/cluster/iso.go
new file mode 100644
index 0000000000..2f78587a21
--- /dev/null
+++ b/pkg/minikube/cluster/iso.go
@@ -0,0 +1,30 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cluster
+
+import (
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/driver"
+)
+
+// CacheISO downloads and caches ISO.
+func CacheISO(cfg config.MachineConfig) error {
+	if driver.BareMetal(cfg.VMDriver) {
+		return nil
+	}
+	return cfg.Downloader.CacheMinikubeISOFromURL(cfg.MinikubeISO)
+}
diff --git a/pkg/minikube/cluster/ssh.go b/pkg/minikube/cluster/ssh.go
new file mode 100644
index 0000000000..0f3c28c841
--- /dev/null
+++ b/pkg/minikube/cluster/ssh.go
@@ -0,0 +1,49 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cluster
+
+import (
+	"github.com/docker/machine/libmachine"
+	"github.com/docker/machine/libmachine/state"
+	"github.com/pkg/errors"
+	"github.com/spf13/viper"
+	"k8s.io/minikube/pkg/minikube/config"
+)
+
+// CreateSSHShell creates a new SSH shell / client
+func CreateSSHShell(api libmachine.API, args []string) error {
+	machineName := viper.GetString(config.MachineProfile)
+	host, err := CheckIfHostExistsAndLoad(api, machineName)
+	if err != nil {
+		return errors.Wrap(err, "host exists and load")
+	}
+
+	currentState, err := host.Driver.GetState()
+	if err != nil {
+		return errors.Wrap(err, "state")
+	}
+
+	if currentState != state.Running {
+		return errors.Errorf("%q is not running", machineName)
+	}
+
+	client, err := host.CreateSSHClient()
+	if err != nil {
+		return errors.Wrap(err, "Creating ssh client")
+	}
+	return client.Shell(args...)
+}
diff --git a/pkg/minikube/cluster/start.go b/pkg/minikube/cluster/start.go
new file mode 100644
index 0000000000..4895ff9484
--- /dev/null
+++ b/pkg/minikube/cluster/start.go
@@ -0,0 +1,202 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cluster
+
+import (
+	"encoding/json"
+	"fmt"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"time"
+
+	"github.com/docker/machine/libmachine"
+	"github.com/docker/machine/libmachine/drivers"
+	"github.com/docker/machine/libmachine/engine"
+	"github.com/docker/machine/libmachine/host"
+	"github.com/golang/glog"
+	"github.com/juju/mutex"
+	"github.com/pkg/errors"
+	"github.com/spf13/viper"
+	"k8s.io/minikube/pkg/minikube/command"
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/constants"
+	"k8s.io/minikube/pkg/minikube/driver"
+	"k8s.io/minikube/pkg/minikube/localpath"
+	"k8s.io/minikube/pkg/minikube/out"
+	"k8s.io/minikube/pkg/minikube/registry"
+	"k8s.io/minikube/pkg/minikube/sshutil"
+	"k8s.io/minikube/pkg/minikube/vmpath"
+	"k8s.io/minikube/pkg/util/lock"
+)
+
+var (
+	// requiredDirectories are directories to create on the host during setup
+	requiredDirectories = []string{
+		vmpath.GuestAddonsDir,
+		vmpath.GuestManifestsDir,
+		vmpath.GuestEphemeralDir,
+		vmpath.GuestPersistentDir,
+		vmpath.GuestCertsDir,
+		path.Join(vmpath.GuestPersistentDir, "images"),
+		path.Join(vmpath.GuestPersistentDir, "binaries"),
+	}
+)
+
+// StartHost starts a host VM.
+func StartHost(api libmachine.API, cfg config.MachineConfig) (*host.Host, error) {
+	// Prevent machine-driver boot races, as well as our own certificate race
+	releaser, err := acquireMachinesLock(cfg.Name)
+	if err != nil {
+		return nil, errors.Wrap(err, "boot lock")
+	}
+	start := time.Now()
+	defer func() {
+		glog.Infof("releasing machines lock for %q, held for %s", cfg.Name, time.Since(start))
+		releaser.Release()
+	}()
+
+	exists, err := api.Exists(cfg.Name)
+	if err != nil {
+		return nil, errors.Wrapf(err, "exists: %s", cfg.Name)
+	}
+	if !exists {
+		glog.Infof("Provisioning new machine with config: %+v", cfg)
+		return createHost(api, cfg)
+	}
+	glog.Infoln("Skipping create...Using existing machine configuration")
+	return fixHost(api, cfg)
+}
+
+func engineOptions(cfg config.MachineConfig) *engine.Options {
+	o := engine.Options{
+		Env:              cfg.DockerEnv,
+		InsecureRegistry: append([]string{constants.DefaultServiceCIDR}, cfg.InsecureRegistry...),
+		RegistryMirror:   cfg.RegistryMirror,
+		ArbitraryFlags:   cfg.DockerOpt,
+		InstallURL:       drivers.DefaultEngineInstallURL,
+	}
+	return &o
+}
+
+func createHost(api libmachine.API, cfg config.MachineConfig) (*host.Host, error) {
+	if cfg.VMDriver == driver.VMwareFusion && viper.GetBool(config.ShowDriverDeprecationNotification) {
+		out.WarningT(`The vmwarefusion driver is deprecated and support for it will be removed in a future release.
+			Please consider switching to the new vmware unified driver, which is intended to replace the vmwarefusion driver.
+			See https://minikube.sigs.k8s.io/docs/reference/drivers/vmware/ for more information.
+			To disable this message, run [minikube config set ShowDriverDeprecationNotification false]`)
+	}
+	showHostInfo(cfg)
+	def := registry.Driver(cfg.VMDriver)
+	if def.Empty() {
+		return nil, fmt.Errorf("unsupported/missing driver: %s", cfg.VMDriver)
+	}
+	dd := def.Config(cfg)
+	data, err := json.Marshal(dd)
+	if err != nil {
+		return nil, errors.Wrap(err, "marshal")
+	}
+
+	h, err := api.NewHost(cfg.VMDriver, data)
+	if err != nil {
+		return nil, errors.Wrap(err, "new host")
+	}
+
+	h.HostOptions.AuthOptions.CertDir = localpath.MiniPath()
+	h.HostOptions.AuthOptions.StorePath = localpath.MiniPath()
+	h.HostOptions.EngineOptions = engineOptions(cfg)
+
+	if err := api.Create(h); err != nil {
+		// Wait for all the logs to reach the client
+		time.Sleep(2 * time.Second)
+		return nil, errors.Wrap(err, "create")
+	}
+
+	if err := postStartSetup(h, cfg); err != nil {
+		return h, errors.Wrap(err, "post-start")
+	}
+
+	if err := api.Save(h); err != nil {
+		return nil, errors.Wrap(err, "save")
+	}
+	return h, nil
+}
+
+// postStart are functions shared between startHost and fixHost
+func postStartSetup(h *host.Host, mc config.MachineConfig) error {
+	if h.DriverName == driver.Mock {
+		glog.Infof("mock driver: skipping postStart")
+		return nil
+	}
+
+	glog.Infof("creating required directories: %v", requiredDirectories)
+	r, err := commandRunner(h)
+	if err != nil {
+		return errors.Wrap(err, "command runner")
+	}
+
+	args := append([]string{"mkdir", "-p"}, requiredDirectories...)
+	if _, err := r.RunCmd(exec.Command("sudo", args...)); err != nil {
+		return errors.Wrapf(err, "sudo mkdir (%s)", h.DriverName)
+	}
+
+	if driver.BareMetal(mc.VMDriver) {
+		showLocalOsRelease()
+	}
+
+	if !driver.BareMetal(mc.VMDriver) && !driver.IsKIC(mc.VMDriver) {
+		showRemoteOsRelease(h.Driver)
+		if err := ensureSyncedGuestClock(h); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// commandRunner returns best available command runner for this host
+func commandRunner(h *host.Host) (command.Runner, error) {
+	if h.DriverName == driver.Mock {
+		glog.Errorf("commandRunner: returning unconfigured FakeCommandRunner, commands will fail!")
+		return &command.FakeCommandRunner{}, nil
+	}
+	if driver.BareMetal(h.Driver.DriverName()) {
+		return &command.ExecRunner{}, nil
+	}
+	if h.Driver.DriverName() == driver.Docker {
+		return command.NewKICRunner(h.Name, "docker"), nil
+	}
+	client, err := sshutil.NewSSHClient(h.Driver)
+	if err != nil {
+		return nil, errors.Wrap(err, "getting ssh client for bootstrapper")
+	}
+	return command.NewSSHRunner(client), nil
+}
+
+// acquireMachinesLock protects against code that is not parallel-safe (libmachine, cert setup)
+func acquireMachinesLock(name string) (mutex.Releaser, error) {
+	spec := lock.PathMutexSpec(filepath.Join(localpath.MiniPath(), "machines"))
+	// NOTE: Provisioning generally completes within 60 seconds
+	spec.Timeout = 15 * time.Minute
+
+	glog.Infof("acquiring machines lock for %s: %+v", name, spec)
+	start := time.Now()
+	r, err := mutex.Acquire(spec)
+	if err == nil {
+		glog.Infof("acquired machines lock for %q in %s", name, time.Since(start))
+	}
+	return r, err
+}
diff --git a/pkg/minikube/cluster/status.go b/pkg/minikube/cluster/status.go
new file mode 100644
index 0000000000..889389be47
--- /dev/null
+++ b/pkg/minikube/cluster/status.go
@@ -0,0 +1,79 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cluster
+
+import (
+	"github.com/docker/machine/libmachine"
+	"github.com/docker/machine/libmachine/host"
+	"github.com/docker/machine/libmachine/state"
+	"github.com/golang/glog"
+	"github.com/pkg/errors"
+)
+
+// GetHostStatus gets the status of the host VM.
+func GetHostStatus(api libmachine.API, machineName string) (string, error) {
+	exists, err := api.Exists(machineName)
+	if err != nil {
+		return "", errors.Wrapf(err, "%s exists", machineName)
+	}
+	if !exists {
+		return state.None.String(), nil
+	}
+
+	host, err := api.Load(machineName)
+	if err != nil {
+		return "", errors.Wrapf(err, "load")
+	}
+
+	s, err := host.Driver.GetState()
+	if err != nil {
+		return "", errors.Wrap(err, "state")
+	}
+	return s.String(), nil
+}
+
+// IsHostRunning asserts that this profile's primary host is in state "Running"
+func IsHostRunning(api libmachine.API, name string) bool {
+	s, err := GetHostStatus(api, name)
+	if err != nil {
+		glog.Warningf("host status for %q returned error: %v", name, err)
+		return false
+	}
+	if s != state.Running.String() {
+		glog.Warningf("%q host status: %s", name, s)
+		return false
+	}
+	return true
+}
+
+// CheckIfHostExistsAndLoad checks if a host exists, and loads it if it does
+func CheckIfHostExistsAndLoad(api libmachine.API, machineName string) (*host.Host, error) {
+	glog.Infof("Checking if %q exists ...", machineName)
+	exists, err := api.Exists(machineName)
+	if err != nil {
+		return nil, errors.Wrapf(err, "Error checking that machine exists: %s", machineName)
+	}
+	if !exists {
+		return nil, errors.Errorf("machine %q does not exist", machineName)
+	}
+
+	host, err := api.Load(machineName)
+	if err != nil {
+		return nil, errors.Wrapf(err, "loading machine %q", machineName)
+	}
+	return host, nil
+}
diff --git a/pkg/minikube/cluster/stop.go b/pkg/minikube/cluster/stop.go
new file mode 100644
index 0000000000..ba4e717697
--- /dev/null
+++ b/pkg/minikube/cluster/stop.go
@@ -0,0 +1,76 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cluster
+
+import (
+	"github.com/docker/machine/libmachine"
+	"github.com/docker/machine/libmachine/host"
+	"github.com/docker/machine/libmachine/mcnerror"
+	"github.com/docker/machine/libmachine/state"
+	"github.com/golang/glog"
+	"github.com/pkg/errors"
+	"github.com/spf13/viper"
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/driver"
+	"k8s.io/minikube/pkg/minikube/out"
+	"k8s.io/minikube/pkg/util/retry"
+)
+
+// StopHost stops the host VM, saving state to disk.
+func StopHost(api libmachine.API) error {
+	machineName := viper.GetString(config.MachineProfile)
+	host, err := api.Load(machineName)
+	if err != nil {
+		return errors.Wrapf(err, "load")
+	}
+
+	out.T(out.Stopping, `Stopping "{{.profile_name}}" in {{.driver_name}} ...`, out.V{"profile_name": machineName, "driver_name": host.DriverName})
+	if host.DriverName == driver.HyperV {
+		glog.Infof("As there are issues with stopping Hyper-V VMs using API, trying to shut down using SSH")
+		if err := trySSHPowerOff(host); err != nil {
+			return errors.Wrap(err, "ssh power off")
+		}
+	}
+
+	if err := host.Stop(); err != nil {
+		alreadyInStateError, ok := err.(mcnerror.ErrHostAlreadyInState)
+		if ok && alreadyInStateError.State == state.Stopped {
+			return nil
+		}
+		return &retry.RetriableError{Err: errors.Wrapf(err, "Stop: %s", machineName)}
+	}
+	return nil
+}
+
+// trySSHPowerOff runs the poweroff command on the guest VM to speed up deletion
+func trySSHPowerOff(h *host.Host) error {
+	s, err := h.Driver.GetState()
+	if err != nil {
+		glog.Warningf("unable to get state: %v", err)
+		return err
+	}
+	if s != state.Running {
+		glog.Infof("host is in state %s", s)
+		return nil
+	}
+
+	out.T(out.Shutdown, `Powering off "{{.profile_name}}" via SSH ...`, out.V{"profile_name": h.Name})
+	out, err := h.RunSSHCommand("sudo poweroff")
+	// poweroff always results in an error, since the host disconnects.
+	glog.Infof("poweroff result: out=%s, err=%v", out, err)
+	return nil
+}

From dd6e5de82c2ec9bf0b84c0b59cdfa13cfeb2aeb4 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Tue, 4 Feb 2020 11:28:03 -0800
Subject: [PATCH 230/250] Update translation files with new strings

---
 translations/de.json    | 149 +++++++++++++++++++++++++++----------
 translations/es.json    | 149 +++++++++++++++++++++++++++----------
 translations/fr.json    | 147 +++++++++++++++++++++++++++----------
 translations/ja.json    |  43 +++++++----
 translations/pl.json    | 159 ++++++++++++++++++++++++++++------------
 translations/zh-CN.json |  37 +++++++++-
 6 files changed, 505 insertions(+), 179 deletions(-)

diff --git a/translations/de.json b/translations/de.json
index f56676d146..e268a479cf 100644
--- a/translations/de.json
+++ b/translations/de.json
@@ -1,15 +1,17 @@
 {
-	"\"{{.minikube_addon}}\" was successfully disabled": "",
-	"\"{{.name}}\" cluster does not exist. Proceeding ahead with cleanup.": "",
+	"\"The '{{.minikube_addon}}' addon is disabled": "",
 	"\"{{.name}}\" profile does not exist": "",
+	"\"{{.name}}\" profile does not exist, trying anyways.": "",
 	"\"{{.profile_name}}\" VM does not exist, nothing to stop": "",
 	"\"{{.profile_name}}\" host does not exist, unable to show an IP": "",
 	"\"{{.profile_name}}\" stopped.": "",
 	"'none' driver does not support 'minikube docker-env' command": "",
 	"'none' driver does not support 'minikube mount' command": "",
 	"'none' driver does not support 'minikube ssh' command": "",
+	"'{{.driver}}' driver reported an issue: {{.error}}": "",
+	"- {{.profile}}": "",
 	"A VPN or firewall is interfering with HTTP access to the minikube VM. Alternatively, try a different VM driver: https://minikube.sigs.k8s.io/docs/start/": "",
-	"A firewall is blocking Docker within the minikube VM from reaching the internet. You may need to configure it to use a proxy.": "",
+	"A firewall is blocking Docker the minikube VM from reaching the internet. You may need to configure it to use a proxy.": "",
 	"A firewall is interfering with minikube's ability to make outgoing HTTPS requests. You may need to change the value of the HTTPS_PROXY environment variable.": "",
 	"A firewall is likely blocking minikube from reaching the internet. You may need to configure minikube to use a proxy.": "",
 	"A set of apiserver IP Addresses which are used in the generated certificate for kubernetes.  This can be used if you want to make the apiserver available from outside the machine": "",
@@ -32,21 +34,33 @@
 	"Amount of RAM allocated to the minikube VM (format: \u003cnumber\u003e[\u003cunit\u003e], where unit = b, k, m or g).": "",
 	"Amount of time to wait for a service in seconds": "",
 	"Amount of time to wait for service in seconds": "",
+	"Another hypervisor, such as VirtualBox, is conflicting with KVM. Please stop the other hypervisor, or use --vm-driver to switch to it.": "",
+	"Automatically selected the {{.experimental}}'{{.driver}}' driver": "",
+	"Automatically selected the {{.experimental}}'{{.driver}}' driver (alternates: {{.alternates}})": "",
 	"Available Commands": "",
 	"Basic Commands:": "",
+	"Block until the apiserver is servicing API requests": "",
 	"Cannot find directory {{.path}} for mount": "",
+	"Cannot use both --output and --format options": "",
+	"Check output of 'journalctl -xeu kubelet', try passing --extra-config=kubelet.cgroup-driver=systemd to minikube start": "",
+	"Check that SELinux is disabled, and that the provided apiserver flags are valid": "",
 	"Check that minikube is running and that you have specified the correct namespace (-n flag) if required.": "",
+	"Check that the provided apiserver flags are valid": "",
 	"Check that your --kubernetes-version has a leading 'v'. For example: 'v1.1.14'": "",
-	"Check that your apiserver flags are valid, or run 'minikube delete'": "",
 	"Check your firewall rules for interference, and run 'virt-host-validate' to check for KVM configuration issues. If you are running minikube within a VM, consider using --vm-driver=none": "",
 	"Configuration and Management Commands:": "",
+	"Configure a default route on this Linux host, or use another --vm-driver that does not require it": "",
 	"Configure an external network switch following the official documentation, then add `--hyperv-virtual-switch=\u003cswitch-name\u003e` to `minikube start`": "",
 	"Configures the addon w/ADDON_NAME within minikube (example: minikube addons configure registry-creds). For a list of available addons use: minikube addons list": "",
 	"Configuring local host environment ...": "",
 	"Confirm that you have a working internet connection and that your VM has not run out of resources by using: 'minikube logs'": "",
 	"Confirm that you have supplied the correct value to --hyperv-virtual-switch using the 'Get-VMSwitch' command": "",
+	"Could not get profile flag": "",
+	"Could not process error from failed deletion": "",
+	"Could not process errors from failed deletion": "",
 	"Country code of the image mirror to be used. Leave empty to use the global one. For Chinese mainland users, set it to cn.": "Ländercode des zu verwendenden Image Mirror. Lassen Sie dieses Feld leer, um den globalen zu verwenden. Nutzer vom chinesischen Festland stellen cn ein.",
 	"Created a new profile : {{.profile_name}}": "",
+	"Creating Kubernetes in {{.driver_name}} container with (CPUs={{.number_of_cpus}}), Memory={{.memory_size}}MB ({{.host_memory_size}}MB available) ...": "",
 	"Creating a new profile failed": "",
 	"Creating mount {{.name}} ...": "Bereitstellung {{.name}} wird erstellt...",
 	"Creating {{.driver_name}} VM (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...": "",
@@ -82,26 +96,29 @@
 	"ERROR creating `registry-creds-ecr` secret: {{.error}}": "",
 	"ERROR creating `registry-creds-gcr` secret: {{.error}}": "",
 	"Either systemctl is not installed, or Docker is broken. Run 'sudo systemctl start docker' and 'journalctl -u docker'": "",
+	"Enable addons. see `minikube addons list` for a list of valid addon names.": "",
 	"Enable experimental NVIDIA GPU support in minikube": "Experimentellen NVIDIA GPU-Support in minikube aktivieren",
 	"Enable host resolver for NAT DNS requests (virtualbox driver only)": "Host Resolver für NAT DNS-Anfragen aktivieren (nur Virtualbox-Treiber)",
+	"Enable istio needs {{.minMem}} MB of memory and {{.minCpus}} CPUs.": "",
 	"Enable proxy for NAT DNS requests (virtualbox driver only)": "Proxy für NAT-DNS-Anforderungen aktivieren (nur Virtualbox-Treiber)",
 	"Enable the default CNI plugin (/etc/cni/net.d/k8s.conf). Used in conjunction with \\\"--network-plugin=cni\\": "Standard-CNI-Plugin-in (/etc/cni/net.d/k8s.conf) aktivieren. Wird in Verbindung mit \"--network-plugin = cni\" verwendet",
 	"Enable the default CNI plugin (/etc/cni/net.d/k8s.conf). Used in conjunction with \\\"--network-plugin=cni\\\".": "",
 	"Enables the addon w/ADDON_NAME within minikube (example: minikube addons enable dashboard). For a list of available addons use: minikube addons list": "",
+	"Enabling '{{.name}}' returned an error: {{.error}}": "",
+	"Enabling addons: {{.addons}}": "",
 	"Enabling dashboard ...": "",
+	"Ensure that CRI-O is installed and healthy: Run 'sudo systemctl start crio' and 'journalctl -u crio'. Alternatively, use --container-runtime=docker": "",
+	"Ensure that Docker is installed and healthy: Run 'sudo systemctl start docker' and 'journalctl -u docker'. Alternatively, select another value for --vm-driver": "",
+	"Ensure that the user listed in /etc/libvirt/qemu.conf has access to your home directory": "",
+	"Ensure that your value for HTTPS_PROXY points to an HTTPS proxy rather than an HTTP proxy": "",
 	"Environment variables to pass to the Docker daemon. (format: key=value)": "Umgebungsvariablen, die an den Docker-Daemon übergeben werden. (Format: Schlüssel = Wert)",
 	"Error checking driver version: {{.error}}": "Fehler beim Prüfen der Treiberversion: {{.error}}",
-	"Error creating list template": "",
 	"Error creating minikube directory": "",
-	"Error creating status template": "",
 	"Error creating view template": "",
-	"Error executing list template": "",
-	"Error executing status template": "",
 	"Error executing template": "",
 	"Error executing view template": "",
 	"Error finding port for mount": "",
 	"Error getting IP": "",
-	"Error getting bootstrapper": "",
 	"Error getting client": "",
 	"Error getting client: {{.error}}": "",
 	"Error getting cluster": "",
@@ -110,7 +127,7 @@
 	"Error getting host": "",
 	"Error getting host status": "",
 	"Error getting machine logs": "",
-	"Error getting machine status": "",
+	"Error getting profiles to delete": "",
 	"Error getting service status": "",
 	"Error getting service with namespace: {{.namespace}} and labels {{.labelName}}:{{.addonName}}: {{.error}}": "",
 	"Error getting the host IP address to use from within the VM": "",
@@ -124,7 +141,6 @@
 	"Error parsing minikube version: {{.error}}": "Fehler beim Parsen der minikube-Version: {{.error}}",
 	"Error parsing vmDriver version: {{.error}}": "Fehler beim Parsen der vmDriver-Version: {{.error}}",
 	"Error reading {{.path}}: {{.error}}": "",
-	"Error restarting cluster": "",
 	"Error setting shell variables": "",
 	"Error starting cluster": "",
 	"Error starting mount": "",
@@ -136,12 +152,12 @@
 	"Error: [{{.id}}] {{.error}}": "",
 	"Examples": "",
 	"Exiting": "Wird beendet",
-	"Exiting due to driver incompatibility": "",
+	"Exiting.": "",
 	"Failed runtime": "",
 	"Failed to cache ISO": "",
 	"Failed to cache and load images": "",
 	"Failed to cache binaries": "",
-	"Failed to cache images": "",
+	"Failed to cache images to tar": "",
 	"Failed to change permissions for {{.minikube_dir_path}}: {{.error}}": "Fehler beim Ändern der Berechtigungen für {{.minikube_dir_path}}: {{.error}}",
 	"Failed to check if machine exists": "",
 	"Failed to check main repository and mirrors for images for images": "",
@@ -160,7 +176,7 @@
 	"Failed to get service URL: {{.error}}": "",
 	"Failed to kill mount process: {{.error}}": "Fehler beim Beenden des Bereitstellungsprozesses: {{.error}}",
 	"Failed to list cached images": "",
-	"Failed to remove profile": "",
+	"Failed to reload cached images": "",
 	"Failed to save config": "",
 	"Failed to set NO_PROXY Env. Please use `export NO_PROXY=$NO_PROXY,{{.ip}}": "NO_PROXY Env konnte nicht festgelegt werden. Benutzen Sie `export NO_PROXY = $ NO_PROXY, {{. Ip}}",
 	"Failed to set NO_PROXY Env. Please use `export NO_PROXY=$NO_PROXY,{{.ip}}`.": "",
@@ -185,14 +201,19 @@
 	"Gets the status of a local kubernetes cluster": "",
 	"Gets the status of a local kubernetes cluster.\n\tExit status contains the status of minikube's VM, cluster and kubernetes encoded on it's bits in this order from right to left.\n\tEg: 7 meaning: 1 (for minikube NOK) + 2 (for cluster NOK) + 4 (for kubernetes NOK)": "",
 	"Gets the value of PROPERTY_NAME from the minikube config file": "",
+	"Getting machine config failed": "",
 	"Global Flags": "",
-	"Go template format string for the addon list output.  The format for Go templates can be found here: https://golang.org/pkg/text/template/\nFor the list of accessible variables for the template, see the struct values here: https://godoc.org/k8s.io/minikube/cmd/minikube/cmd/config#AddonListTemplate": "",
 	"Go template format string for the cache list output.  The format for Go templates can be found here: https://golang.org/pkg/text/template/\nFor the list of accessible variables for the template, see the struct values here: https://godoc.org/k8s.io/minikube/cmd/minikube/cmd#CacheListTemplate": "",
 	"Go template format string for the config view output.  The format for Go templates can be found here: https://golang.org/pkg/text/template/\nFor the list of accessible variables for the template, see the struct values here: https://godoc.org/k8s.io/minikube/cmd/minikube/cmd/config#ConfigViewTemplate": "",
 	"Go template format string for the status output.  The format for Go templates can be found here: https://golang.org/pkg/text/template/\nFor the list accessible variables for the template, see the struct values here: https://godoc.org/k8s.io/minikube/cmd/minikube/cmd#Status": "",
 	"Group ID:     {{.groupID}}": "",
 	"Have you set up libvirt correctly?": "",
 	"Hide the hypervisor signature from the guest in minikube (kvm2 driver only)": "Hypervisor-Signatur vor dem Gast in minikube verbergen (nur kvm2-Treiber)",
+	"Hyperkit is broken. Upgrade to the latest hyperkit version and/or Docker for Desktop. Alternatively, you may choose an alternate --vm-driver": "",
+	"Hyperkit networking is broken. Upgrade to the latest hyperkit version and/or Docker for Desktop. Alternatively, you may choose an alternate --vm-driver": "",
+	"If set, automatically updates drivers to the latest version. Defaults to true.": "",
+	"If set, pause all namespaces": "",
+	"If set, unpause all namespaces": "",
 	"If the above advice does not help, please let us know:": "",
 	"If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --vm-driver=none.": "Wenn true, speichern Sie Docker-Images für den aktuellen Bootstrapper zwischen und laden Sie sie auf den Computer. Immer falsch mit --vm-driver = none.",
 	"If true, only download and cache files for later use - don't install or start anything.": "Wenn true, laden Sie nur Dateien für die spätere Verwendung herunter und speichern Sie sie – installieren oder starten Sie nichts.",
@@ -206,6 +227,7 @@
 	"Invalid size passed in argument: {{.error}}": "",
 	"IsEnabled failed": "",
 	"Kill the mount process spawned by minikube start": "",
+	"Kubernetes {{.new}} is now available. If you would like to upgrade, specify: --kubernetes-version={{.new}}": "",
 	"Kubernetes {{.version}} is not supported by this release of minikube": "",
 	"Launching Kubernetes ...": "Kubernetes wird gestartet...",
 	"Launching proxy ...": "",
@@ -230,9 +252,13 @@
 	"Mounting host path {{.sourcePath}} into VM as {{.destinationPath}} ...": "",
 	"Mounts the specified directory into minikube": "",
 	"Mounts the specified directory into minikube.": "",
+	"Multiple errors deleting profiles": "",
+	"Multiple minikube profiles were found -": "",
+	"NIC Type used for host only network. One of Am79C970A, Am79C973, 82540EM, 82543GC, 82545EM, or virtio (virtualbox driver only)": "",
 	"NOTE: This process must stay alive for the mount to be accessible ...": "",
 	"Networking and Connectivity Commands:": "",
 	"No minikube profile was found. You can create one using `minikube start`.": "",
+	"Node may be unable to resolve external DNS records": "",
 	"None of the known repositories in your location are accessible. Using {{.image_repository_name}} as fallback.": "Keines der bekannten Repositories an Ihrem Standort ist zugänglich. {{.image_repository_name}} wird als Fallback verwendet.",
 	"None of the known repositories is accessible. Consider specifying an alternative image repository with --image-repository flag": "Keines der bekannten Repositories ist zugänglich. Erwägen Sie, ein alternatives Image-Repository mit der Kennzeichnung --image-repository anzugeben",
 	"Not passing {{.name}}={{.value}} to docker env.": "",
@@ -243,11 +269,15 @@
 	"Open the addons URL with https instead of http": "",
 	"Open the service URL with https instead of http": "",
 	"Opening kubernetes service  {{.namespace_name}}/{{.service_name}} in default browser...": "",
+	"Opening service {{.namespace_name}}/{{.service_name}} in default browser...": "",
 	"Opening {{.url}} in your default browser...": "",
 	"Opens the addon w/ADDON_NAME within minikube (example: minikube addons open dashboard). For a list of available addons use: minikube addons list": "",
 	"Options:      {{.options}}": "",
 	"Outputs minikube shell completion for the given shell (bash or zsh)": "",
 	"Outputs minikube shell completion for the given shell (bash or zsh)\n\n\tThis depends on the bash-completion binary.  Example installation instructions:\n\tOS X:\n\t\t$ brew install bash-completion\n\t\t$ source $(brew --prefix)/etc/bash_completion\n\t\t$ minikube completion bash \u003e ~/.minikube-completion  # for bash users\n\t\t$ minikube completion zsh \u003e ~/.minikube-completion  # for zsh users\n\t\t$ source ~/.minikube-completion\n\tUbuntu:\n\t\t$ apt-get install bash-completion\n\t\t$ source /etc/bash-completion\n\t\t$ source \u003c(minikube completion bash) # for bash users\n\t\t$ source \u003c(minikube completion zsh) # for zsh users\n\n\tAdditionally, you may want to output the completion to a file and source in your .bashrc\n\n\tNote for zsh users: [1] zsh completions are only supported in versions of zsh \u003e= 5.2": "",
+	"Pause": "",
+	"Paused kubelet and {{.count}} containers": "",
+	"Paused kubelet and {{.count}} containers in: {{.namespaces}}": "",
 	"Permissions:  {{.octalMode}} ({{.writtenMode}})": "",
 	"Please enter a value:": "",
 	"Please install the minikube hyperkit VM driver, or select an alternative --vm-driver": "",
@@ -264,22 +294,24 @@
 	"Problems detected in {{.entry}}:": "",
 	"Problems detected in {{.name}}:": "",
 	"Profile gets or sets the current minikube profile": "",
+	"Profile name \"{{.profilename}}\" is minikube keyword. To delete profile use command minikube delete -p \u003cprofile name\u003e": "",
 	"Provide VM UUID to restore MAC address (hyperkit driver only)": "Geben Sie die VM-UUID an, um die MAC-Adresse wiederherzustellen (nur Hyperkit-Treiber)",
 	"Pulling images ...": "",
-	"Reboot to complete VirtualBox installation, and verify that VirtualBox is not blocked by your system": "",
+	"Reboot to complete VirtualBox installation, verify that VirtualBox is not blocked by your system, and/or use another hypervisor": "",
 	"Rebuild libvirt with virt-network support": "",
 	"Received {{.name}} signal": "",
+	"Reconfiguring existing host ...": "",
 	"Registry mirrors to pass to the Docker daemon": "Registry-Mirror, die an den Docker-Daemon übergeben werden",
 	"Reinstall VirtualBox and reboot. Alternatively, try the kvm2 driver: https://minikube.sigs.k8s.io/docs/reference/drivers/kvm2/": "",
 	"Reinstall VirtualBox and verify that it is not blocked: System Preferences -\u003e Security \u0026 Privacy -\u003e General -\u003e Some system software was blocked from loading": "",
 	"Related issues:": "",
 	"Relaunching Kubernetes using {{.bootstrapper}} ...": "Kubernetes mit {{.bootstrapper}} neu starten...",
+	"Removed all traces of the \"{{.name}}\" cluster.": "",
 	"Removing {{.directory}} ...": "{{.directory}} wird entfernt...",
-	"Requested CPU count {{.requested_cpus}} is less than the minimum allowed of {{.minimum_cpus}}": "",
+	"Requested cpu count {{.requested_cpus}} is less than the minimum allowed of {{.minimum_cpus}}": "",
 	"Requested disk size {{.requested_size}} is less than minimum of {{.minimum_size}}": "Die angeforderte Festplattengröße {{.requested_size}} liegt unter dem Mindestwert von {{.minimum_size}}.",
 	"Requested memory allocation ({{.memory}}MB) is less than the default memory allocation of {{.default_memorysize}}MB. Beware that minikube might not work correctly or crash unexpectedly.": "Die angeforderte Speicherzuordnung ({{.memory}} MB) ist geringer als die Standardspeicherzuordnung von {{.default_memorysize}} MB. Beachten Sie, dass minikube möglicherweise nicht richtig funktioniert oder unerwartet abstürzt.",
 	"Requested memory allocation {{.requested_size}} is less than the minimum allowed of {{.minimum_size}}": "Die angeforderte Speicherzuweisung {{.requested_size}} liegt unter dem zulässigen Mindestwert von {{.minimum_size}}.",
-	"Retriable failure: {{.error}}": "",
 	"Retrieve the ssh identity key path of the specified cluster": "",
 	"Retrieve the ssh identity key path of the specified cluster.": "",
 	"Retrieves the IP address of the running cluster": "",
@@ -287,22 +319,27 @@
 	"Retrieves the IP address of the running cluster, checks it\n\t\t\twith IP in kubeconfig, and corrects kubeconfig if incorrect.": "",
 	"Returns the value of PROPERTY_NAME from the minikube config file.  Can be overwritten at runtime by flags or environmental variables.": "",
 	"Run 'kubectl describe pod coredns -n kube-system' and check for a firewall or DNS conflict": "",
-	"Run 'minikube delete' to delete the stale VM": "",
+	"Run 'minikube delete' to delete the stale VM, or and ensure that minikube is running as the same user you are issuing this command with": "",
 	"Run kubectl": "",
 	"Run minikube from the C: drive.": "",
 	"Run the kubernetes client, download it if necessary. Remember -- after kubectl!\n\nExamples:\nminikube kubectl -- --help\nminikube kubectl -- get pods --namespace kube-system": "",
 	"Run the minikube command as an Administrator": "",
+	"Run: 'chmod 600 $HOME/.kube/config'": "",
 	"Running on localhost (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...": "",
+	"Selecting {{.experimental}}'{{.driver}}' driver from existing profile (alternates: {{.alternates}})": "",
+	"Selecting {{.experimental}}'{{.driver}}' driver from user configuration (alternates: {{.alternates}})": "",
 	"Set failed": "",
+	"Set flag to delete all profiles": "",
+	"Set this flag to delete the '.minikube' folder from your user directory.": "",
 	"Sets an individual value in a minikube config file": "",
 	"Sets the PROPERTY_NAME config value to PROPERTY_VALUE\n\tThese values can be overwritten by flags or environment variables at runtime.": "",
 	"Sets up docker env variables; similar to '$(docker-machine env)'": "",
 	"Sets up docker env variables; similar to '$(docker-machine env)'.": "",
 	"Setting profile failed": "",
+	"Show a list of global command-line options (applies to all commands).": "",
 	"Show only log entries which point to known problems": "",
 	"Show only the most recent journal entries, and continuously print new entries as they are appended to the journal.": "",
 	"Skipped switching kubectl context for {{.profile_name}} because --keep-context was set.": "",
-	"Sorry that minikube crashed. If this was unexpected, we would love to hear from you:": "",
 	"Sorry, Kubernetes {{.version}} is not supported by this release of minikube": "",
 	"Sorry, completion support is not yet implemented for {{.name}}": "",
 	"Sorry, the kubeadm.{{.parameter_name}} parameter is currently not supported by --extra-config": "Leider wird der Parameter kubeadm.{{.parameter_name}} momentan von --extra-config nicht unterstützt.",
@@ -319,16 +356,23 @@
 	"Stopping \"{{.profile_name}}\" in {{.driver_name}} ...": "",
 	"Stops a local kubernetes cluster running in Virtualbox. This command stops the VM\nitself, leaving all files intact. The cluster can be started again with the \"start\" command.": "",
 	"Stops a running local kubernetes cluster": "",
+	"Successfully deleted all profiles": "",
 	"Successfully mounted {{.sourcePath}} to {{.destinationPath}}": "",
+	"Successfully powered off Hyper-V. minikube driver -- {{.driver}}": "",
+	"Successfully purged minikube directory located at - [{{.minikubeDirectory}}]": "",
 	"Suggestion: {{.advice}}": "",
+	"Suggestion: {{.fix}}": "",
 	"Target directory {{.path}} must be an absolute path": "",
 	"The \"{{.driver_name}}\" driver requires root privileges. Please run minikube using 'sudo minikube --vm-driver={{.driver_name}}": "Der Treiber \"{{.driver_name}}\" benötigt Root-Rechte. Führen Sie minikube aus mit 'sudo minikube --vm-driver = {{. Driver_name}}.",
 	"The \"{{.driver_name}}\" driver requires root privileges. Please run minikube using 'sudo minikube --vm-driver={{.driver_name}}'.": "",
 	"The \"{{.driver_name}}\" driver should not be used with root privileges.": "",
 	"The \"{{.name}}\" cluster has been deleted.": "Der Cluster \"{{.name}}\" wurde gelöscht.",
 	"The \"{{.name}}\" cluster has been deleted.__1": "Der Cluster \"{{.name}}\" wurde gelöscht.",
+	"The 'none' driver does not respect the --cpus flag": "",
+	"The 'none' driver does not respect the --memory flag": "",
 	"The 'none' driver does not support multiple profiles: https://minikube.sigs.k8s.io/docs/reference/drivers/none/": "",
 	"The 'none' driver provides limited isolation and may reduce system security and reliability.": "Der Treiber \"Keine\" bietet eine eingeschränkte Isolation und beeinträchtigt möglicherweise Sicherheit und Zuverlässigkeit des Systems.",
+	"The '{{.addonName}}' addon is enabled": "",
 	"The '{{.driver}}' driver requires elevated permissions. The following commands will be executed:\\n\\n{{ .example }}\\n": "",
 	"The CIDR to be used for service cluster IPs.": "Die CIDR, die für Service-Cluster-IPs verwendet werden soll.",
 	"The CIDR to be used for the minikube VM (virtualbox driver only)": "Die CIDR, die für die minikube-VM verwendet werden soll (nur Virtualbox-Treiber)",
@@ -351,14 +395,18 @@
 	"The docker host is currently not running": "",
 	"The docker service is currently not active": "",
 	"The driver '{{.driver}}' is not supported on {{.os}}": "Der Treiber '{{.driver}}' wird auf {{.os}} nicht unterstützt",
+	"The driver {{.experimental}} '{{.driver}}' is not supported on {{.os}}": "",
 	"The existing \"{{.profile_name}}\" VM that was created using the \"{{.old_driver}}\" driver, and is incompatible with the \"{{.driver}}\" driver.": "",
 	"The hyperv virtual switch name. Defaults to first found. (hyperv driver only)": "Der Name des virtuellen Hyperv-Switch. Standardmäßig zuerst gefunden. (nur Hyperv-Treiber)",
+	"The hypervisor does not appear to be configured properly. Run 'minikube start --alsologtostderr -v=1' and inspect the error code": "",
 	"The initial time interval for each check that wait performs in seconds": "",
 	"The kubernetes version that the minikube VM will use (ex: v1.2.3)": "Die von der minikube-VM verwendete Kubernetes-Version (Beispiel: v1.2.3)",
+	"The machine-driver specified is failing to start. Try running 'docker-machine-driver-\u003ctype\u003e version'": "",
 	"The minikube VM is offline. Please run 'minikube start' to start it again.": "",
 	"The name of the network plugin": "Der Name des Netzwerk-Plugins",
 	"The name of the network plugin.": "",
 	"The number of bytes to use for 9p packet payload": "",
+	"The output format. One of 'json', 'table'": "",
 	"The path on the file system where the docs in markdown need to be saved": "",
 	"The service namespace": "",
 	"The services namespace": "",
@@ -367,7 +415,6 @@
 	"The value passed to --format is invalid: {{.error}}": "",
 	"The vmwarefusion driver is deprecated and support for it will be removed in a future release.\n\t\t\tPlease consider switching to the new vmware unified driver, which is intended to replace the vmwarefusion driver.\n\t\t\tSee https://minikube.sigs.k8s.io/docs/reference/drivers/vmware/ for more information.\n\t\t\tTo disable this message, run [minikube config set ShowDriverDeprecationNotification false]": "",
 	"The {{.driver_name}} driver should not be used with root privileges.": "Der Treiber {{.driver_name}} sollte nicht mit Root-Rechten verwendet werden.",
-	"There appears to be another hypervisor conflicting with KVM. Please stop the other hypervisor, or use --vm-driver to switch to it.": "",
 	"There's a new version for '{{.driver_executable}}'. Please consider upgrading. {{.documentation_url}}": "Es gibt eine neue Version für '{{.driver_executable}}'. Bitte erwägen Sie ein Upgrade. {{.documentation_url}}",
 	"These changes will take effect upon a minikube delete and then a minikube start": "",
 	"This addon does not have an endpoint defined for the 'addons open' command.\nYou can add one by annotating a service with the label {{.labelName}}:{{.addonName}}": "",
@@ -376,38 +423,45 @@
 	"This will start the mount daemon and automatically mount files into minikube": "Dadurch wird der Mount-Daemon gestartet und die Dateien werden automatisch in minikube geladen",
 	"This will start the mount daemon and automatically mount files into minikube.": "",
 	"Tip: To remove this root owned cluster, run: sudo {{.cmd}} delete": "Tipp: Um diesen Root-Cluster zu entfernen, führen Sie Folgendes aus: sudo {{.cmd}} delete",
-	"Tip: Use 'minikube start -p \u003cname\u003e' to create a new cluster, or 'minikube delete' to delete this one.": "",
 	"To connect to this cluster, use: kubectl --context={{.name}}": "Verwenden Sie zum Herstellen einer Verbindung zu diesem Cluster: kubectl --context = {{.name}}",
 	"To connect to this cluster, use: kubectl --context={{.name}}__1": "Verwenden Sie zum Herstellen einer Verbindung zu diesem Cluster: kubectl --context = {{.name}}",
 	"To connect to this cluster, use: kubectl --context={{.profile_name}}": "",
 	"To disable this notice, run: 'minikube config set WantUpdateNotification false'\\n": "",
-	"To proceed, either:\n      1) Delete the existing VM using: '{{.command}} delete'\n      or\n      2) Restart with the existing driver: '{{.command}} start --vm-driver={{.old_driver}}'": "",
+	"To proceed, either:\n\n    1) Delete the existing \"{{.profile_name}}\" cluster using: '{{.command}} delete'\n\n    * or *\n\n    2) Start the existing \"{{.profile_name}}\" cluster using: '{{.command}} start --vm-driver={{.old_driver}}'": "",
+	"To see addons list for other profiles use: `minikube addons -p name list`": "",
 	"To start minikube with HyperV Powershell must be in your PATH`": "",
 	"To use kubectl or minikube commands as your own user, you may need to relocate them. For example, to overwrite your own settings, run:": "Möglicherweise müssen Sie Kubectl- oder minikube-Befehle verschieben, um sie als eigenen Nutzer zu verwenden. Um beispielsweise Ihre eigenen Einstellungen zu überschreiben, führen Sie aus:",
 	"Troubleshooting Commands:": "",
+	"Trying to delete invalid profile {{.profile}}": "",
 	"Unable to bind flags": "",
+	"Unable to determine a default driver to use. Try specifying --vm-driver, or see https://minikube.sigs.k8s.io/docs/start/": "",
 	"Unable to enable dashboard": "",
 	"Unable to fetch latest version info": "",
 	"Unable to generate docs": "",
 	"Unable to generate the documentation. Please ensure that the path specified is a directory, exists \u0026 you have permission to write to it.": "",
 	"Unable to get VM IP address": "",
+	"Unable to get addon status for {{.name}}: {{.error}}": "",
 	"Unable to get bootstrapper: {{.error}}": "Bootstrapper kann nicht abgerufen werden: {{.error}}",
 	"Unable to get current user": "",
 	"Unable to get runtime": "",
-	"Unable to get the status of the cluster.": "",
+	"Unable to get the status of the {{.name}} cluster.": "",
 	"Unable to kill mount process: {{.error}}": "",
 	"Unable to load cached images from config file.": "Zwischengespeicherte Bilder können nicht aus der Konfigurationsdatei geladen werden.",
 	"Unable to load cached images: {{.error}}": "",
 	"Unable to load config: {{.error}}": "Konfig kann nicht geladen werden: {{.error}}",
 	"Unable to parse \"{{.kubernetes_version}}\": {{.error}}": "\"{{.Kubernetes_version}}\" kann nicht geparst werden: {{.error}}",
+	"Unable to parse default Kubernetes version from constants: {{.error}}": "",
 	"Unable to parse oldest Kubernetes version from constants: {{.error}}": "",
 	"Unable to pull images, which may be OK: {{.error}}": "Bilder können nicht abgerufen werden, was möglicherweise kein Problem darstellt: {{.error}}",
 	"Unable to remove machine directory: %v": "",
-	"Unable to start VM": "",
+	"Unable to start VM. Please investigate and run 'minikube delete' if possible": "",
 	"Unable to stop VM": "",
 	"Unable to update {{.driver}} driver: {{.error}}": "",
+	"Unable to verify SSH connectivity: {{.error}}. Will retry...": "",
 	"Uninstalling Kubernetes {{.kubernetes_version}} using {{.bootstrapper_name}} ...": "Kubernetes {{.kubernetes_version}} wird mit {{.bootstrapper_name}} deinstalliert...",
 	"Unmounting {{.path}} ...": "",
+	"Unpaused kubelet and {{.count}} containers": "",
+	"Unpaused kubelet and {{.count}} containers in: {{.namespaces}}": "",
 	"Unset the KUBECONFIG environment variable, or verify that it does not point to an empty or otherwise invalid path": "",
 	"Unset variables instead of setting them": "",
 	"Update server returned an empty list": "",
@@ -416,15 +470,20 @@
 	"Usage": "",
 	"Usage: minikube completion SHELL": "",
 	"Usage: minikube delete": "",
+	"Usage: minikube delete --all --purge": "",
 	"Use \"{{.CommandPath}} [command] --help\" for more information about a command.": "",
+	"Use 'kubect get po -A' to find the correct and namespace name": "",
+	"Use -A to specify all namespaces": "",
 	"Use VirtualBox to remove the conflicting VM and/or network interfaces": "",
 	"Use native Golang SSH client (default true). Set to 'false' to use the command line 'ssh' command when accessing the docker machine. Useful for the machine drivers when they will not start with 'Waiting for SSH'.": "",
 	"User ID:      {{.userID}}": "",
 	"Userspace file server is shutdown": "",
 	"Userspace file server:": "",
 	"Using image repository {{.name}}": "Verwenden des Image-Repositorys {{.name}}",
+	"Using the '{{.runtime}}' runtime with the 'none' driver is an untested configuration!": "",
 	"Using the running {{.driver_name}} \"{{.profile_name}}\" VM ...": "",
 	"VM driver is one of: %v": "VM-Treiber ist einer von: %v",
+	"VM is unable to access {{.repository}}, you may need to configure a proxy or set --image-repository": "",
 	"Verify that your HTTP_PROXY and HTTPS_PROXY environment variables are set correctly.": "",
 	"Verify the IP address of the running cluster in kubeconfig.": "",
 	"Verifying dashboard health ...": "",
@@ -434,12 +493,12 @@
 	"VirtualBox cannot create a network, probably because it conflicts with an existing network that minikube no longer knows about. Try running 'minikube delete'": "",
 	"VirtualBox is broken. Disable real-time anti-virus software, reboot, and reinstall VirtualBox if the problem continues.": "",
 	"VirtualBox is broken. Reinstall VirtualBox, reboot, and run 'minikube delete'.": "",
+	"VirtualBox is unable to find its network interface. Try upgrading to the latest release and rebooting.": "",
+	"Virtualization support is disabled on your computer. If you are running minikube within a VM, try '--vm-driver=none'. Otherwise, consult your systems BIOS manual for how to enable virtualization.": "",
 	"Wait failed": "",
 	"Wait failed: {{.error}}": "",
 	"Wait until Kubernetes core services are healthy before exiting": "Warten Sie vor dem Beenden, bis die Kerndienste von Kubernetes fehlerfrei arbeiten",
-	"Wait until Kubernetes core services are healthy before exiting.": "",
-	"Waiting for the host to be provisioned ...": "",
-	"Waiting for:": "",
+	"Waiting for cluster to come online ...": "",
 	"Where to root the NFS Shares, defaults to /nfsshares (hyperkit driver only)": "Als Root für die NFS-Freigaben wird standardmäßig /nfsshares verwendet (nur Hyperkit-Treiber)",
 	"You appear to be using a proxy, but your NO_PROXY environment does not include the minikube IP ({{.ip_address}}). Please see {{.documentation_url}} for more details": "Sie scheinen einen Proxy zu verwenden, aber Ihre NO_PROXY-Umgebung enthält keine minikube-IP ({{.ip_address}}). Weitere Informationen finden Sie unter {{.documentation_url}}",
 	"You can delete them using the following command(s):": "",
@@ -452,50 +511,62 @@
 	"Your minikube vm is not running, try minikube start.": "",
 	"addon '{{.name}}' is currently not enabled.\nTo enable this addon run:\nminikube addons enable {{.name}}": "",
 	"addon '{{.name}}' is not a valid addon packaged with minikube.\nTo see the list of available addons run:\nminikube addons list": "",
-	"addon list failed": "",
-	"addons modifies minikube addons files using subcommands like \"minikube addons enable heapster\"": "",
+	"addons modifies minikube addons files using subcommands like \"minikube addons enable dashboard\"": "",
 	"api load": "",
 	"bash completion failed": "",
-	"browser failed to open url: {{.error}}": "",
 	"call with cleanup=true to remove old tunnels": "",
 	"command runner": "",
 	"config modifies minikube config files using subcommands like \"minikube config set vm-driver kvm\"\nConfigurable fields:\\n\\n": "",
 	"config view failed": "",
 	"dashboard service is not running: {{.error}}": "",
 	"disable failed": "",
+	"dry-run mode. Validates configuration, but does not mutate system state": "",
+	"dry-run validation complete!": "",
 	"enable failed": "",
 	"error creating clientset": "",
 	"error creating machine client": "",
-	"error getting driver": "",
 	"error parsing the input ip address for mount": "",
 	"error starting tunnel": "",
 	"failed to open browser: {{.error}}": "",
 	"if true, will embed the certs in kubeconfig.": "",
+	"kubeadm detected a TCP port conflict with another process: probably another local Kubernetes installation. Run lsof -p\u003cport\u003e to find the process and kill it": "",
 	"kubectl and minikube configuration will be stored in {{.home_folder}}": "Konfiguration von Kubectl und minikube wird in {{.home_folder}} gespeichert",
 	"kubectl not found in PATH, but is required for the dashboard. Installation guide: https://kubernetes.io/docs/tasks/tools/install-kubectl/": "",
 	"kubectl proxy": "",
 	"logdir set failed": "",
 	"max time to wait per Kubernetes core services to be healthy.": "",
-	"minikube is not running, so the service cannot be accessed": "",
+	"minikube addons list --output OUTPUT. json, list": "",
+	"minikube is exiting due to an error. If the above message is not useful, open an issue:": "",
 	"minikube is unable to access the Google Container Registry. You may need to configure it to use a HTTP proxy.": "",
+	"minikube is unable to connect to the VM: {{.error}}\n\nThis is likely due to one of two reasons:\n\n- VPN or firewall interference\n- {{.hypervisor}} network configuration issue\n\nSuggested workarounds:\n\n- Disable your local VPN or firewall software\n- Configure your local VPN or firewall to allow access to {{.ip}}\n- Restart or reinstall {{.hypervisor}}\n- Use an alternative --vm-driver": "",
 	"minikube profile was successfully set to {{.profile_name}}": "",
+	"minikube status --output OUTPUT. json, text": "",
 	"minikube {{.version}} is available! Download it: {{.url}}": "",
+	"mkcmp is used to compare performance of two minikube binaries": "",
 	"mount argument \"{{.value}}\" must be in form: \u003csource directory\u003e:\u003ctarget directory\u003e": "",
 	"mount failed": "",
+	"namespaces to pause": "",
+	"namespaces to unpause": "",
 	"not enough arguments ({{.ArgCount}}).\\nusage: minikube config set PROPERTY_NAME PROPERTY_VALUE": "",
+	"pause containers": "",
 	"profile sets the current minikube profile, or gets the current profile if no arguments are provided.  This is used to run and manage multiple minikube instance.  You can return to the default minikube profile by running `minikube profile default`": "",
+	"reload cached images.": "",
+	"reloads images previously added using the 'cache add' subcommand": "",
 	"service {{.namespace_name}}/{{.service_name}} has no node port": "",
 	"stat failed": "",
+	"status json failure": "",
+	"status text failure": "",
 	"toom any arguments ({{.ArgCount}}).\\nusage: minikube config set PROPERTY_NAME PROPERTY_VALUE": "",
-	"tunnel creates a route to services deployed with type LoadBalancer and sets their Ingress to their ClusterIP": "",
+	"tunnel creates a route to services deployed with type LoadBalancer and sets their Ingress to their ClusterIP. for a detailed example see https://minikube.sigs.k8s.io/docs/tasks/loadbalancer": "",
 	"tunnel makes services of type LoadBalancer accessible on localhost": "",
 	"unable to bind flags": "",
+	"unable to delete minikube config folder": "",
 	"unable to set logtostderr": "",
+	"unpause Kubernetes": "",
 	"unset failed": "",
-	"unset minikube profile": "",
 	"unsets PROPERTY_NAME from the minikube config file.  Can be overwritten by flags or environmental variables": "",
 	"unsets an individual value in a minikube config file": "",
-	"unsupported driver: {{.name}}": "",
+	"unsupported or missing driver: {{.name}}": "",
 	"update config": "",
 	"usage: minikube addons configure ADDON_NAME": "",
 	"usage: minikube addons disable ADDON_NAME": "",
@@ -503,9 +574,12 @@
 	"usage: minikube addons list": "",
 	"usage: minikube addons open ADDON_NAME": "",
 	"usage: minikube config unset PROPERTY_NAME": "",
+	"usage: minikube delete": "",
+	"usage: minikube delete --all": "",
 	"usage: minikube profile [MINIKUBE_PROFILE_NAME]": "",
 	"zsh completion failed": "",
-	"{{.addonName}} was successfully enabled": "",
+	"{{.driver}} does not appear to be installed": "",
+	"{{.driver}} does not appear to be installed, but is specified by an existing profile. Please run 'minikube delete' or install {{.driver}}": "",
 	"{{.extra_option_component_name}}.{{.key}}={{.value}}": "",
 	"{{.machine}} IP has been updated to point at {{.ip}}": "",
 	"{{.machine}} IP was already correctly configured for {{.ip}}": "",
@@ -513,6 +587,7 @@
 	"{{.name}} has no available configuration options": "",
 	"{{.name}} was successfully configured": "",
 	"{{.name}}\" profile does not exist": "Profil \"{{.name}}\" existiert nicht",
+	"{{.path}} is version {{.client_version}}, and is incompatible with Kubernetes {{.cluster_version}}. You will need to update {{.path}} or use 'minikube kubectl' to connect with this cluster": "",
 	"{{.prefix}}minikube {{.version}} on {{.platform}}": "{{.prefix}}minikube {{.version}} auf {{.platform}}",
 	"{{.type}} is not yet a supported filesystem. We will try anyways!": "",
 	"{{.url}} is not accessible: {{.error}}": ""
diff --git a/translations/es.json b/translations/es.json
index bb175f5c33..a664dfaae3 100644
--- a/translations/es.json
+++ b/translations/es.json
@@ -1,15 +1,17 @@
 {
-	"\"{{.minikube_addon}}\" was successfully disabled": "",
-	"\"{{.name}}\" cluster does not exist. Proceeding ahead with cleanup.": "",
+	"\"The '{{.minikube_addon}}' addon is disabled": "",
 	"\"{{.name}}\" profile does not exist": "El perfil \"{{.name}}\" no existe",
+	"\"{{.name}}\" profile does not exist, trying anyways.": "",
 	"\"{{.profile_name}}\" VM does not exist, nothing to stop": "",
 	"\"{{.profile_name}}\" host does not exist, unable to show an IP": "",
 	"\"{{.profile_name}}\" stopped.": "",
 	"'none' driver does not support 'minikube docker-env' command": "",
 	"'none' driver does not support 'minikube mount' command": "",
 	"'none' driver does not support 'minikube ssh' command": "",
+	"'{{.driver}}' driver reported an issue: {{.error}}": "",
+	"- {{.profile}}": "",
 	"A VPN or firewall is interfering with HTTP access to the minikube VM. Alternatively, try a different VM driver: https://minikube.sigs.k8s.io/docs/start/": "",
-	"A firewall is blocking Docker within the minikube VM from reaching the internet. You may need to configure it to use a proxy.": "",
+	"A firewall is blocking Docker the minikube VM from reaching the internet. You may need to configure it to use a proxy.": "",
 	"A firewall is interfering with minikube's ability to make outgoing HTTPS requests. You may need to change the value of the HTTPS_PROXY environment variable.": "",
 	"A firewall is likely blocking minikube from reaching the internet. You may need to configure minikube to use a proxy.": "",
 	"A set of apiserver IP Addresses which are used in the generated certificate for kubernetes.  This can be used if you want to make the apiserver available from outside the machine": "",
@@ -32,21 +34,33 @@
 	"Amount of RAM allocated to the minikube VM (format: \u003cnumber\u003e[\u003cunit\u003e], where unit = b, k, m or g).": "",
 	"Amount of time to wait for a service in seconds": "",
 	"Amount of time to wait for service in seconds": "",
+	"Another hypervisor, such as VirtualBox, is conflicting with KVM. Please stop the other hypervisor, or use --vm-driver to switch to it.": "",
+	"Automatically selected the {{.experimental}}'{{.driver}}' driver": "",
+	"Automatically selected the {{.experimental}}'{{.driver}}' driver (alternates: {{.alternates}})": "",
 	"Available Commands": "",
 	"Basic Commands:": "",
+	"Block until the apiserver is servicing API requests": "",
 	"Cannot find directory {{.path}} for mount": "",
+	"Cannot use both --output and --format options": "",
+	"Check output of 'journalctl -xeu kubelet', try passing --extra-config=kubelet.cgroup-driver=systemd to minikube start": "",
+	"Check that SELinux is disabled, and that the provided apiserver flags are valid": "",
 	"Check that minikube is running and that you have specified the correct namespace (-n flag) if required.": "",
+	"Check that the provided apiserver flags are valid": "",
 	"Check that your --kubernetes-version has a leading 'v'. For example: 'v1.1.14'": "",
-	"Check that your apiserver flags are valid, or run 'minikube delete'": "",
 	"Check your firewall rules for interference, and run 'virt-host-validate' to check for KVM configuration issues. If you are running minikube within a VM, consider using --vm-driver=none": "",
 	"Configuration and Management Commands:": "",
+	"Configure a default route on this Linux host, or use another --vm-driver that does not require it": "",
 	"Configure an external network switch following the official documentation, then add `--hyperv-virtual-switch=\u003cswitch-name\u003e` to `minikube start`": "",
 	"Configures the addon w/ADDON_NAME within minikube (example: minikube addons configure registry-creds). For a list of available addons use: minikube addons list": "",
 	"Configuring local host environment ...": "",
 	"Confirm that you have a working internet connection and that your VM has not run out of resources by using: 'minikube logs'": "",
 	"Confirm that you have supplied the correct value to --hyperv-virtual-switch using the 'Get-VMSwitch' command": "",
+	"Could not get profile flag": "",
+	"Could not process error from failed deletion": "",
+	"Could not process errors from failed deletion": "",
 	"Country code of the image mirror to be used. Leave empty to use the global one. For Chinese mainland users, set it to cn.": "Código de país de la réplica de imagen que quieras utilizar. Déjalo en blanco para usar el valor global. Los usuarios de China continental deben definirlo como cn.",
 	"Created a new profile : {{.profile_name}}": "",
+	"Creating Kubernetes in {{.driver_name}} container with (CPUs={{.number_of_cpus}}), Memory={{.memory_size}}MB ({{.host_memory_size}}MB available) ...": "",
 	"Creating a new profile failed": "",
 	"Creating mount {{.name}} ...": "Creando la activación {{.name}}...",
 	"Creating {{.driver_name}} VM (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...": "",
@@ -82,26 +96,29 @@
 	"ERROR creating `registry-creds-ecr` secret: {{.error}}": "",
 	"ERROR creating `registry-creds-gcr` secret: {{.error}}": "",
 	"Either systemctl is not installed, or Docker is broken. Run 'sudo systemctl start docker' and 'journalctl -u docker'": "",
+	"Enable addons. see `minikube addons list` for a list of valid addon names.": "",
 	"Enable experimental NVIDIA GPU support in minikube": "Permite habilitar la compatibilidad experimental con GPUs NVIDIA en minikube",
 	"Enable host resolver for NAT DNS requests (virtualbox driver only)": "Permite habilitar la resolución del host en las solicitudes DNS con traducción de direcciones de red (NAT) aplicada (solo con el controlador de Virtualbox)",
+	"Enable istio needs {{.minMem}} MB of memory and {{.minCpus}} CPUs.": "",
 	"Enable proxy for NAT DNS requests (virtualbox driver only)": "Permite habilitar el uso de proxies en las solicitudes de DNS con traducción de direcciones de red (NAT) aplicada (solo con el controlador de Virtualbox)",
 	"Enable the default CNI plugin (/etc/cni/net.d/k8s.conf). Used in conjunction with \\\"--network-plugin=cni\\": "Permite habilitar el complemento CNI predeterminado (/etc/cni/net.d/k8s.conf). Se utiliza junto con \"--network-plugin=cni",
 	"Enable the default CNI plugin (/etc/cni/net.d/k8s.conf). Used in conjunction with \\\"--network-plugin=cni\\\".": "",
 	"Enables the addon w/ADDON_NAME within minikube (example: minikube addons enable dashboard). For a list of available addons use: minikube addons list": "",
+	"Enabling '{{.name}}' returned an error: {{.error}}": "",
+	"Enabling addons: {{.addons}}": "",
 	"Enabling dashboard ...": "",
+	"Ensure that CRI-O is installed and healthy: Run 'sudo systemctl start crio' and 'journalctl -u crio'. Alternatively, use --container-runtime=docker": "",
+	"Ensure that Docker is installed and healthy: Run 'sudo systemctl start docker' and 'journalctl -u docker'. Alternatively, select another value for --vm-driver": "",
+	"Ensure that the user listed in /etc/libvirt/qemu.conf has access to your home directory": "",
+	"Ensure that your value for HTTPS_PROXY points to an HTTPS proxy rather than an HTTP proxy": "",
 	"Environment variables to pass to the Docker daemon. (format: key=value)": "Variables de entorno que se transferirán al daemon de Docker. Formato: clave=valor",
 	"Error checking driver version: {{.error}}": "No se ha podido comprobar la versión del controlador: {{.error}}",
-	"Error creating list template": "",
 	"Error creating minikube directory": "",
-	"Error creating status template": "",
 	"Error creating view template": "",
-	"Error executing list template": "",
-	"Error executing status template": "",
 	"Error executing template": "",
 	"Error executing view template": "",
 	"Error finding port for mount": "",
 	"Error getting IP": "",
-	"Error getting bootstrapper": "",
 	"Error getting client": "",
 	"Error getting client: {{.error}}": "",
 	"Error getting cluster": "",
@@ -110,7 +127,7 @@
 	"Error getting host": "",
 	"Error getting host status": "",
 	"Error getting machine logs": "",
-	"Error getting machine status": "",
+	"Error getting profiles to delete": "",
 	"Error getting service status": "",
 	"Error getting service with namespace: {{.namespace}} and labels {{.labelName}}:{{.addonName}}: {{.error}}": "",
 	"Error getting the host IP address to use from within the VM": "",
@@ -124,7 +141,6 @@
 	"Error parsing minikube version: {{.error}}": "No se ha podido analizar la versión de minikube: {{.error}}",
 	"Error parsing vmDriver version: {{.error}}": "No se ha podido analizar la versión de vmDriver: {{.error}}",
 	"Error reading {{.path}}: {{.error}}": "",
-	"Error restarting cluster": "",
 	"Error setting shell variables": "",
 	"Error starting cluster": "",
 	"Error starting mount": "",
@@ -136,12 +152,12 @@
 	"Error: [{{.id}}] {{.error}}": "",
 	"Examples": "",
 	"Exiting": "Saliendo",
-	"Exiting due to driver incompatibility": "",
+	"Exiting.": "",
 	"Failed runtime": "",
 	"Failed to cache ISO": "",
 	"Failed to cache and load images": "",
 	"Failed to cache binaries": "",
-	"Failed to cache images": "",
+	"Failed to cache images to tar": "",
 	"Failed to change permissions for {{.minikube_dir_path}}: {{.error}}": "No se han podido cambiar los permisos de {{.minikube_dir_path}}: {{.error}}",
 	"Failed to check if machine exists": "",
 	"Failed to check main repository and mirrors for images for images": "",
@@ -160,7 +176,7 @@
 	"Failed to get service URL: {{.error}}": "",
 	"Failed to kill mount process: {{.error}}": "No se ha podido detener el proceso de activación: {{.error}}",
 	"Failed to list cached images": "",
-	"Failed to remove profile": "",
+	"Failed to reload cached images": "",
 	"Failed to save config": "",
 	"Failed to set NO_PROXY Env. Please use `export NO_PROXY=$NO_PROXY,{{.ip}}": "No se ha podido definir la variable de entorno NO_PROXY. Utiliza export NO_PROXY=$NO_PROXY,{{.ip}}",
 	"Failed to set NO_PROXY Env. Please use `export NO_PROXY=$NO_PROXY,{{.ip}}`.": "",
@@ -185,14 +201,19 @@
 	"Gets the status of a local kubernetes cluster": "",
 	"Gets the status of a local kubernetes cluster.\n\tExit status contains the status of minikube's VM, cluster and kubernetes encoded on it's bits in this order from right to left.\n\tEg: 7 meaning: 1 (for minikube NOK) + 2 (for cluster NOK) + 4 (for kubernetes NOK)": "",
 	"Gets the value of PROPERTY_NAME from the minikube config file": "",
+	"Getting machine config failed": "",
 	"Global Flags": "",
-	"Go template format string for the addon list output.  The format for Go templates can be found here: https://golang.org/pkg/text/template/\nFor the list of accessible variables for the template, see the struct values here: https://godoc.org/k8s.io/minikube/cmd/minikube/cmd/config#AddonListTemplate": "",
 	"Go template format string for the cache list output.  The format for Go templates can be found here: https://golang.org/pkg/text/template/\nFor the list of accessible variables for the template, see the struct values here: https://godoc.org/k8s.io/minikube/cmd/minikube/cmd#CacheListTemplate": "",
 	"Go template format string for the config view output.  The format for Go templates can be found here: https://golang.org/pkg/text/template/\nFor the list of accessible variables for the template, see the struct values here: https://godoc.org/k8s.io/minikube/cmd/minikube/cmd/config#ConfigViewTemplate": "",
 	"Go template format string for the status output.  The format for Go templates can be found here: https://golang.org/pkg/text/template/\nFor the list accessible variables for the template, see the struct values here: https://godoc.org/k8s.io/minikube/cmd/minikube/cmd#Status": "",
 	"Group ID:     {{.groupID}}": "",
 	"Have you set up libvirt correctly?": "",
 	"Hide the hypervisor signature from the guest in minikube (kvm2 driver only)": "Permite ocultar la firma del hipervisor al invitado en minikube (solo con el controlador de kvm2)",
+	"Hyperkit is broken. Upgrade to the latest hyperkit version and/or Docker for Desktop. Alternatively, you may choose an alternate --vm-driver": "",
+	"Hyperkit networking is broken. Upgrade to the latest hyperkit version and/or Docker for Desktop. Alternatively, you may choose an alternate --vm-driver": "",
+	"If set, automatically updates drivers to the latest version. Defaults to true.": "",
+	"If set, pause all namespaces": "",
+	"If set, unpause all namespaces": "",
 	"If the above advice does not help, please let us know:": "",
 	"If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --vm-driver=none.": "Si el valor es \"true\", las imágenes de Docker del programa previo actual se almacenan en caché y se cargan en la máquina. Siempre es \"false\" si se especifica --vm-driver=none.",
 	"If true, only download and cache files for later use - don't install or start anything.": "Si el valor es \"true\", los archivos solo se descargan y almacenan en caché (no se instala ni inicia nada).",
@@ -206,6 +227,7 @@
 	"Invalid size passed in argument: {{.error}}": "",
 	"IsEnabled failed": "",
 	"Kill the mount process spawned by minikube start": "",
+	"Kubernetes {{.new}} is now available. If you would like to upgrade, specify: --kubernetes-version={{.new}}": "",
 	"Kubernetes {{.version}} is not supported by this release of minikube": "",
 	"Launching Kubernetes ...": "Iniciando Kubernetes...",
 	"Launching proxy ...": "",
@@ -230,9 +252,13 @@
 	"Mounting host path {{.sourcePath}} into VM as {{.destinationPath}} ...": "",
 	"Mounts the specified directory into minikube": "",
 	"Mounts the specified directory into minikube.": "",
+	"Multiple errors deleting profiles": "",
+	"Multiple minikube profiles were found -": "",
+	"NIC Type used for host only network. One of Am79C970A, Am79C973, 82540EM, 82543GC, 82545EM, or virtio (virtualbox driver only)": "",
 	"NOTE: This process must stay alive for the mount to be accessible ...": "",
 	"Networking and Connectivity Commands:": "",
 	"No minikube profile was found. You can create one using `minikube start`.": "",
+	"Node may be unable to resolve external DNS records": "",
 	"None of the known repositories in your location are accessible. Using {{.image_repository_name}} as fallback.": "No se puede acceder a ninguno de los repositorios conocidos de tu ubicación. Se utilizará {{.image_repository_name}} como alternativa.",
 	"None of the known repositories is accessible. Consider specifying an alternative image repository with --image-repository flag": "No se puede acceder a ninguno de los repositorios conocidos. Plantéate indicar un repositorio de imágenes alternativo con la marca --image-repository.",
 	"Not passing {{.name}}={{.value}} to docker env.": "",
@@ -243,11 +269,15 @@
 	"Open the addons URL with https instead of http": "",
 	"Open the service URL with https instead of http": "",
 	"Opening kubernetes service  {{.namespace_name}}/{{.service_name}} in default browser...": "",
+	"Opening service {{.namespace_name}}/{{.service_name}} in default browser...": "",
 	"Opening {{.url}} in your default browser...": "",
 	"Opens the addon w/ADDON_NAME within minikube (example: minikube addons open dashboard). For a list of available addons use: minikube addons list": "",
 	"Options:      {{.options}}": "",
 	"Outputs minikube shell completion for the given shell (bash or zsh)": "",
 	"Outputs minikube shell completion for the given shell (bash or zsh)\n\n\tThis depends on the bash-completion binary.  Example installation instructions:\n\tOS X:\n\t\t$ brew install bash-completion\n\t\t$ source $(brew --prefix)/etc/bash_completion\n\t\t$ minikube completion bash \u003e ~/.minikube-completion  # for bash users\n\t\t$ minikube completion zsh \u003e ~/.minikube-completion  # for zsh users\n\t\t$ source ~/.minikube-completion\n\tUbuntu:\n\t\t$ apt-get install bash-completion\n\t\t$ source /etc/bash-completion\n\t\t$ source \u003c(minikube completion bash) # for bash users\n\t\t$ source \u003c(minikube completion zsh) # for zsh users\n\n\tAdditionally, you may want to output the completion to a file and source in your .bashrc\n\n\tNote for zsh users: [1] zsh completions are only supported in versions of zsh \u003e= 5.2": "",
+	"Pause": "",
+	"Paused kubelet and {{.count}} containers": "",
+	"Paused kubelet and {{.count}} containers in: {{.namespaces}}": "",
 	"Permissions:  {{.octalMode}} ({{.writtenMode}})": "",
 	"Please enter a value:": "",
 	"Please install the minikube hyperkit VM driver, or select an alternative --vm-driver": "",
@@ -264,22 +294,24 @@
 	"Problems detected in {{.entry}}:": "",
 	"Problems detected in {{.name}}:": "",
 	"Profile gets or sets the current minikube profile": "",
+	"Profile name \"{{.profilename}}\" is minikube keyword. To delete profile use command minikube delete -p \u003cprofile name\u003e": "",
 	"Provide VM UUID to restore MAC address (hyperkit driver only)": "Permite especificar un UUID de VM para restaurar la dirección MAC (solo con el controlador de hyperkit)",
 	"Pulling images ...": "",
-	"Reboot to complete VirtualBox installation, and verify that VirtualBox is not blocked by your system": "",
+	"Reboot to complete VirtualBox installation, verify that VirtualBox is not blocked by your system, and/or use another hypervisor": "",
 	"Rebuild libvirt with virt-network support": "",
 	"Received {{.name}} signal": "",
+	"Reconfiguring existing host ...": "",
 	"Registry mirrors to pass to the Docker daemon": "Réplicas del registro que se transferirán al daemon de Docker",
 	"Reinstall VirtualBox and reboot. Alternatively, try the kvm2 driver: https://minikube.sigs.k8s.io/docs/reference/drivers/kvm2/": "",
 	"Reinstall VirtualBox and verify that it is not blocked: System Preferences -\u003e Security \u0026 Privacy -\u003e General -\u003e Some system software was blocked from loading": "",
 	"Related issues:": "",
 	"Relaunching Kubernetes using {{.bootstrapper}} ...": "Reiniciando Kubernetes con {{.bootstrapper}}...",
+	"Removed all traces of the \"{{.name}}\" cluster.": "",
 	"Removing {{.directory}} ...": "Eliminando {{.directory}}...",
-	"Requested CPU count {{.requested_cpus}} is less than the minimum allowed of {{.minimum_cpus}}": "",
+	"Requested cpu count {{.requested_cpus}} is less than the minimum allowed of {{.minimum_cpus}}": "",
 	"Requested disk size {{.requested_size}} is less than minimum of {{.minimum_size}}": "El tamaño de disco de {{.requested_size}} que se ha solicitado es inferior al tamaño mínimo de {{.minimum_size}}",
 	"Requested memory allocation ({{.memory}}MB) is less than the default memory allocation of {{.default_memorysize}}MB. Beware that minikube might not work correctly or crash unexpectedly.": "El valor de la asignación de memoria ({{.memory}} MB) solicitada es inferior a la asignación de memoria predeterminada de {{.default_memorysize}} MB. minikube podría no funcionar correctamente o fallar de manera inesperada.",
 	"Requested memory allocation {{.requested_size}} is less than the minimum allowed of {{.minimum_size}}": "El valor de la asignación de memoria de {{.requested_size}} solicitada es inferior al valor mínimo de {{.minimum_size}}",
-	"Retriable failure: {{.error}}": "",
 	"Retrieve the ssh identity key path of the specified cluster": "",
 	"Retrieve the ssh identity key path of the specified cluster.": "",
 	"Retrieves the IP address of the running cluster": "",
@@ -287,22 +319,27 @@
 	"Retrieves the IP address of the running cluster, checks it\n\t\t\twith IP in kubeconfig, and corrects kubeconfig if incorrect.": "",
 	"Returns the value of PROPERTY_NAME from the minikube config file.  Can be overwritten at runtime by flags or environmental variables.": "",
 	"Run 'kubectl describe pod coredns -n kube-system' and check for a firewall or DNS conflict": "",
-	"Run 'minikube delete' to delete the stale VM": "",
+	"Run 'minikube delete' to delete the stale VM, or and ensure that minikube is running as the same user you are issuing this command with": "",
 	"Run kubectl": "",
 	"Run minikube from the C: drive.": "",
 	"Run the kubernetes client, download it if necessary. Remember -- after kubectl!\n\nExamples:\nminikube kubectl -- --help\nminikube kubectl -- get pods --namespace kube-system": "",
 	"Run the minikube command as an Administrator": "",
+	"Run: 'chmod 600 $HOME/.kube/config'": "",
 	"Running on localhost (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...": "",
+	"Selecting {{.experimental}}'{{.driver}}' driver from existing profile (alternates: {{.alternates}})": "",
+	"Selecting {{.experimental}}'{{.driver}}' driver from user configuration (alternates: {{.alternates}})": "",
 	"Set failed": "",
+	"Set flag to delete all profiles": "",
+	"Set this flag to delete the '.minikube' folder from your user directory.": "",
 	"Sets an individual value in a minikube config file": "",
 	"Sets the PROPERTY_NAME config value to PROPERTY_VALUE\n\tThese values can be overwritten by flags or environment variables at runtime.": "",
 	"Sets up docker env variables; similar to '$(docker-machine env)'": "",
 	"Sets up docker env variables; similar to '$(docker-machine env)'.": "",
 	"Setting profile failed": "",
+	"Show a list of global command-line options (applies to all commands).": "",
 	"Show only log entries which point to known problems": "",
 	"Show only the most recent journal entries, and continuously print new entries as they are appended to the journal.": "",
 	"Skipped switching kubectl context for {{.profile_name}} because --keep-context was set.": "",
-	"Sorry that minikube crashed. If this was unexpected, we would love to hear from you:": "",
 	"Sorry, Kubernetes {{.version}} is not supported by this release of minikube": "",
 	"Sorry, completion support is not yet implemented for {{.name}}": "",
 	"Sorry, the kubeadm.{{.parameter_name}} parameter is currently not supported by --extra-config": "De momento, --extra-config no admite el parámetro kubeadm.{{.parameter_name}}",
@@ -319,16 +356,23 @@
 	"Stopping \"{{.profile_name}}\" in {{.driver_name}} ...": "",
 	"Stops a local kubernetes cluster running in Virtualbox. This command stops the VM\nitself, leaving all files intact. The cluster can be started again with the \"start\" command.": "",
 	"Stops a running local kubernetes cluster": "",
+	"Successfully deleted all profiles": "",
 	"Successfully mounted {{.sourcePath}} to {{.destinationPath}}": "",
+	"Successfully powered off Hyper-V. minikube driver -- {{.driver}}": "",
+	"Successfully purged minikube directory located at - [{{.minikubeDirectory}}]": "",
 	"Suggestion: {{.advice}}": "",
+	"Suggestion: {{.fix}}": "",
 	"Target directory {{.path}} must be an absolute path": "",
 	"The \"{{.driver_name}}\" driver requires root privileges. Please run minikube using 'sudo minikube --vm-driver={{.driver_name}}": "El controlador \"{{.driver_name}}\" requiere privilegios de raíz. Ejecuta minikube mediante sudo minikube --vm-driver={{.driver_name}}",
 	"The \"{{.driver_name}}\" driver requires root privileges. Please run minikube using 'sudo minikube --vm-driver={{.driver_name}}'.": "",
 	"The \"{{.driver_name}}\" driver should not be used with root privileges.": "",
 	"The \"{{.name}}\" cluster has been deleted.": "Se ha eliminado el clúster \"{{.name}}\".",
 	"The \"{{.name}}\" cluster has been deleted.__1": "Se ha eliminado el clúster \"{{.name}}\".",
+	"The 'none' driver does not respect the --cpus flag": "",
+	"The 'none' driver does not respect the --memory flag": "",
 	"The 'none' driver does not support multiple profiles: https://minikube.sigs.k8s.io/docs/reference/drivers/none/": "",
 	"The 'none' driver provides limited isolation and may reduce system security and reliability.": "La opción de controlador \"none\" proporciona un aislamiento limitado y puede reducir la seguridad y la fiabilidad del sistema.",
+	"The '{{.addonName}}' addon is enabled": "",
 	"The '{{.driver}}' driver requires elevated permissions. The following commands will be executed:\\n\\n{{ .example }}\\n": "",
 	"The CIDR to be used for service cluster IPs.": "El CIDR de las IP del clúster de servicio.",
 	"The CIDR to be used for the minikube VM (virtualbox driver only)": "El CIDR de la VM de minikube (solo con el controlador de Virtualbox)",
@@ -351,14 +395,18 @@
 	"The docker host is currently not running": "",
 	"The docker service is currently not active": "",
 	"The driver '{{.driver}}' is not supported on {{.os}}": "El controlador \"{{.driver}}\" no se puede utilizar en {{.os}}",
+	"The driver {{.experimental}} '{{.driver}}' is not supported on {{.os}}": "",
 	"The existing \"{{.profile_name}}\" VM that was created using the \"{{.old_driver}}\" driver, and is incompatible with the \"{{.driver}}\" driver.": "",
 	"The hyperv virtual switch name. Defaults to first found. (hyperv driver only)": "El nombre del conmutador virtual de hyperv. El valor predeterminado será el primer nombre que se encuentre (solo con el controlador de hyperv).",
+	"The hypervisor does not appear to be configured properly. Run 'minikube start --alsologtostderr -v=1' and inspect the error code": "",
 	"The initial time interval for each check that wait performs in seconds": "",
 	"The kubernetes version that the minikube VM will use (ex: v1.2.3)": "La versión de Kubernetes que utilizará la VM de minikube (p. ej.: versión 1.2.3)",
+	"The machine-driver specified is failing to start. Try running 'docker-machine-driver-\u003ctype\u003e version'": "",
 	"The minikube VM is offline. Please run 'minikube start' to start it again.": "",
 	"The name of the network plugin": "El nombre del complemento de red",
 	"The name of the network plugin.": "",
 	"The number of bytes to use for 9p packet payload": "",
+	"The output format. One of 'json', 'table'": "",
 	"The path on the file system where the docs in markdown need to be saved": "",
 	"The service namespace": "",
 	"The services namespace": "",
@@ -367,7 +415,6 @@
 	"The value passed to --format is invalid: {{.error}}": "",
 	"The vmwarefusion driver is deprecated and support for it will be removed in a future release.\n\t\t\tPlease consider switching to the new vmware unified driver, which is intended to replace the vmwarefusion driver.\n\t\t\tSee https://minikube.sigs.k8s.io/docs/reference/drivers/vmware/ for more information.\n\t\t\tTo disable this message, run [minikube config set ShowDriverDeprecationNotification false]": "",
 	"The {{.driver_name}} driver should not be used with root privileges.": "El controlador {{.driver_name}} no se debe utilizar con privilegios de raíz.",
-	"There appears to be another hypervisor conflicting with KVM. Please stop the other hypervisor, or use --vm-driver to switch to it.": "",
 	"There's a new version for '{{.driver_executable}}'. Please consider upgrading. {{.documentation_url}}": "Hay una nueva versión de \"{{.driver_executable}}\". Te recomendamos que realices la actualización. {{.documentation_url}}",
 	"These changes will take effect upon a minikube delete and then a minikube start": "",
 	"This addon does not have an endpoint defined for the 'addons open' command.\nYou can add one by annotating a service with the label {{.labelName}}:{{.addonName}}": "",
@@ -376,38 +423,45 @@
 	"This will start the mount daemon and automatically mount files into minikube": "Se iniciará el daemon de activación y se activarán automáticamente los archivos en minikube",
 	"This will start the mount daemon and automatically mount files into minikube.": "",
 	"Tip: To remove this root owned cluster, run: sudo {{.cmd}} delete": "Para eliminar este clúster de raíz, ejecuta: sudo {{.cmd}} delete",
-	"Tip: Use 'minikube start -p \u003cname\u003e' to create a new cluster, or 'minikube delete' to delete this one.": "",
 	"To connect to this cluster, use: kubectl --context={{.name}}": "Para conectarte a este clúster, usa: kubectl --context={{.name}}",
 	"To connect to this cluster, use: kubectl --context={{.name}}__1": "Para conectarte a este clúster, usa: kubectl --context={{.name}}",
 	"To connect to this cluster, use: kubectl --context={{.profile_name}}": "",
 	"To disable this notice, run: 'minikube config set WantUpdateNotification false'\\n": "",
-	"To proceed, either:\n      1) Delete the existing VM using: '{{.command}} delete'\n      or\n      2) Restart with the existing driver: '{{.command}} start --vm-driver={{.old_driver}}'": "",
+	"To proceed, either:\n\n    1) Delete the existing \"{{.profile_name}}\" cluster using: '{{.command}} delete'\n\n    * or *\n\n    2) Start the existing \"{{.profile_name}}\" cluster using: '{{.command}} start --vm-driver={{.old_driver}}'": "",
+	"To see addons list for other profiles use: `minikube addons -p name list`": "",
 	"To start minikube with HyperV Powershell must be in your PATH`": "",
 	"To use kubectl or minikube commands as your own user, you may need to relocate them. For example, to overwrite your own settings, run:": "Para usar comandos de kubectl o minikube como tu propio usuario, puede que debas reubicarlos. Por ejemplo, para sobrescribir tu configuración, ejecuta:",
 	"Troubleshooting Commands:": "",
+	"Trying to delete invalid profile {{.profile}}": "",
 	"Unable to bind flags": "",
+	"Unable to determine a default driver to use. Try specifying --vm-driver, or see https://minikube.sigs.k8s.io/docs/start/": "",
 	"Unable to enable dashboard": "",
 	"Unable to fetch latest version info": "",
 	"Unable to generate docs": "",
 	"Unable to generate the documentation. Please ensure that the path specified is a directory, exists \u0026 you have permission to write to it.": "",
 	"Unable to get VM IP address": "",
+	"Unable to get addon status for {{.name}}: {{.error}}": "",
 	"Unable to get bootstrapper: {{.error}}": "No se ha podido obtener el programa previo: {{.error}}",
 	"Unable to get current user": "",
 	"Unable to get runtime": "",
-	"Unable to get the status of the cluster.": "",
+	"Unable to get the status of the {{.name}} cluster.": "",
 	"Unable to kill mount process: {{.error}}": "",
 	"Unable to load cached images from config file.": "No se han podido cargar las imágenes almacenadas en caché del archivo de configuración.",
 	"Unable to load cached images: {{.error}}": "",
 	"Unable to load config: {{.error}}": "No se ha podido cargar la configuración: {{.error}}",
 	"Unable to parse \"{{.kubernetes_version}}\": {{.error}}": "No se ha podido analizar la versión \"{{.kubernetes_version}}\": {{.error}}",
+	"Unable to parse default Kubernetes version from constants: {{.error}}": "",
 	"Unable to parse oldest Kubernetes version from constants: {{.error}}": "",
 	"Unable to pull images, which may be OK: {{.error}}": "No se ha podido recuperar imágenes, que podrían estar en buen estado: {{.error}}",
 	"Unable to remove machine directory: %v": "",
-	"Unable to start VM": "",
+	"Unable to start VM. Please investigate and run 'minikube delete' if possible": "",
 	"Unable to stop VM": "",
 	"Unable to update {{.driver}} driver: {{.error}}": "",
+	"Unable to verify SSH connectivity: {{.error}}. Will retry...": "",
 	"Uninstalling Kubernetes {{.kubernetes_version}} using {{.bootstrapper_name}} ...": "Desinstalando Kubernetes {{.kubernetes_version}} mediante {{.bootstrapper_name}}...",
 	"Unmounting {{.path}} ...": "",
+	"Unpaused kubelet and {{.count}} containers": "",
+	"Unpaused kubelet and {{.count}} containers in: {{.namespaces}}": "",
 	"Unset the KUBECONFIG environment variable, or verify that it does not point to an empty or otherwise invalid path": "",
 	"Unset variables instead of setting them": "",
 	"Update server returned an empty list": "",
@@ -416,15 +470,20 @@
 	"Usage": "",
 	"Usage: minikube completion SHELL": "",
 	"Usage: minikube delete": "",
+	"Usage: minikube delete --all --purge": "",
 	"Use \"{{.CommandPath}} [command] --help\" for more information about a command.": "",
+	"Use 'kubect get po -A' to find the correct and namespace name": "",
+	"Use -A to specify all namespaces": "",
 	"Use VirtualBox to remove the conflicting VM and/or network interfaces": "",
 	"Use native Golang SSH client (default true). Set to 'false' to use the command line 'ssh' command when accessing the docker machine. Useful for the machine drivers when they will not start with 'Waiting for SSH'.": "",
 	"User ID:      {{.userID}}": "",
 	"Userspace file server is shutdown": "",
 	"Userspace file server:": "",
 	"Using image repository {{.name}}": "Utilizando el repositorio de imágenes {{.name}}",
+	"Using the '{{.runtime}}' runtime with the 'none' driver is an untested configuration!": "",
 	"Using the running {{.driver_name}} \"{{.profile_name}}\" VM ...": "",
 	"VM driver is one of: %v": "El controlador de la VM es uno de los siguientes: %v",
+	"VM is unable to access {{.repository}}, you may need to configure a proxy or set --image-repository": "",
 	"Verify that your HTTP_PROXY and HTTPS_PROXY environment variables are set correctly.": "",
 	"Verify the IP address of the running cluster in kubeconfig.": "",
 	"Verifying dashboard health ...": "",
@@ -434,12 +493,12 @@
 	"VirtualBox cannot create a network, probably because it conflicts with an existing network that minikube no longer knows about. Try running 'minikube delete'": "",
 	"VirtualBox is broken. Disable real-time anti-virus software, reboot, and reinstall VirtualBox if the problem continues.": "",
 	"VirtualBox is broken. Reinstall VirtualBox, reboot, and run 'minikube delete'.": "",
+	"VirtualBox is unable to find its network interface. Try upgrading to the latest release and rebooting.": "",
+	"Virtualization support is disabled on your computer. If you are running minikube within a VM, try '--vm-driver=none'. Otherwise, consult your systems BIOS manual for how to enable virtualization.": "",
 	"Wait failed": "",
 	"Wait failed: {{.error}}": "",
 	"Wait until Kubernetes core services are healthy before exiting": "Espera hasta que los servicios principales de Kubernetes se encuentren en buen estado antes de salir",
-	"Wait until Kubernetes core services are healthy before exiting.": "",
-	"Waiting for the host to be provisioned ...": "",
-	"Waiting for:": "",
+	"Waiting for cluster to come online ...": "",
 	"Where to root the NFS Shares, defaults to /nfsshares (hyperkit driver only)": "Ruta en la raíz de los recursos compartidos de NFS. Su valor predeterminado es /nfsshares (solo con el controlador de hyperkit)",
 	"You appear to be using a proxy, but your NO_PROXY environment does not include the minikube IP ({{.ip_address}}). Please see {{.documentation_url}} for more details": "Parece que estás usando un proxy, pero tu entorno NO_PROXY no incluye la dirección IP de minikube ({{.ip_address}}). Consulta {{.documentation_url}} para obtener más información",
 	"You can delete them using the following command(s):": "",
@@ -452,50 +511,62 @@
 	"Your minikube vm is not running, try minikube start.": "",
 	"addon '{{.name}}' is currently not enabled.\nTo enable this addon run:\nminikube addons enable {{.name}}": "",
 	"addon '{{.name}}' is not a valid addon packaged with minikube.\nTo see the list of available addons run:\nminikube addons list": "",
-	"addon list failed": "",
-	"addons modifies minikube addons files using subcommands like \"minikube addons enable heapster\"": "",
+	"addons modifies minikube addons files using subcommands like \"minikube addons enable dashboard\"": "",
 	"api load": "",
 	"bash completion failed": "",
-	"browser failed to open url: {{.error}}": "",
 	"call with cleanup=true to remove old tunnels": "",
 	"command runner": "",
 	"config modifies minikube config files using subcommands like \"minikube config set vm-driver kvm\"\nConfigurable fields:\\n\\n": "",
 	"config view failed": "",
 	"dashboard service is not running: {{.error}}": "",
 	"disable failed": "",
+	"dry-run mode. Validates configuration, but does not mutate system state": "",
+	"dry-run validation complete!": "",
 	"enable failed": "",
 	"error creating clientset": "",
 	"error creating machine client": "",
-	"error getting driver": "",
 	"error parsing the input ip address for mount": "",
 	"error starting tunnel": "",
 	"failed to open browser: {{.error}}": "",
 	"if true, will embed the certs in kubeconfig.": "",
+	"kubeadm detected a TCP port conflict with another process: probably another local Kubernetes installation. Run lsof -p\u003cport\u003e to find the process and kill it": "",
 	"kubectl and minikube configuration will be stored in {{.home_folder}}": "La configuración de kubectl y de minikube se almacenará en {{.home_folder}}",
 	"kubectl not found in PATH, but is required for the dashboard. Installation guide: https://kubernetes.io/docs/tasks/tools/install-kubectl/": "",
 	"kubectl proxy": "",
 	"logdir set failed": "",
 	"max time to wait per Kubernetes core services to be healthy.": "",
-	"minikube is not running, so the service cannot be accessed": "",
+	"minikube addons list --output OUTPUT. json, list": "",
+	"minikube is exiting due to an error. If the above message is not useful, open an issue:": "",
 	"minikube is unable to access the Google Container Registry. You may need to configure it to use a HTTP proxy.": "",
+	"minikube is unable to connect to the VM: {{.error}}\n\nThis is likely due to one of two reasons:\n\n- VPN or firewall interference\n- {{.hypervisor}} network configuration issue\n\nSuggested workarounds:\n\n- Disable your local VPN or firewall software\n- Configure your local VPN or firewall to allow access to {{.ip}}\n- Restart or reinstall {{.hypervisor}}\n- Use an alternative --vm-driver": "",
 	"minikube profile was successfully set to {{.profile_name}}": "",
+	"minikube status --output OUTPUT. json, text": "",
 	"minikube {{.version}} is available! Download it: {{.url}}": "",
+	"mkcmp is used to compare performance of two minikube binaries": "",
 	"mount argument \"{{.value}}\" must be in form: \u003csource directory\u003e:\u003ctarget directory\u003e": "",
 	"mount failed": "",
+	"namespaces to pause": "",
+	"namespaces to unpause": "",
 	"not enough arguments ({{.ArgCount}}).\\nusage: minikube config set PROPERTY_NAME PROPERTY_VALUE": "",
+	"pause containers": "",
 	"profile sets the current minikube profile, or gets the current profile if no arguments are provided.  This is used to run and manage multiple minikube instance.  You can return to the default minikube profile by running `minikube profile default`": "",
+	"reload cached images.": "",
+	"reloads images previously added using the 'cache add' subcommand": "",
 	"service {{.namespace_name}}/{{.service_name}} has no node port": "",
 	"stat failed": "",
+	"status json failure": "",
+	"status text failure": "",
 	"toom any arguments ({{.ArgCount}}).\\nusage: minikube config set PROPERTY_NAME PROPERTY_VALUE": "",
-	"tunnel creates a route to services deployed with type LoadBalancer and sets their Ingress to their ClusterIP": "",
+	"tunnel creates a route to services deployed with type LoadBalancer and sets their Ingress to their ClusterIP. for a detailed example see https://minikube.sigs.k8s.io/docs/tasks/loadbalancer": "",
 	"tunnel makes services of type LoadBalancer accessible on localhost": "",
 	"unable to bind flags": "",
+	"unable to delete minikube config folder": "",
 	"unable to set logtostderr": "",
+	"unpause Kubernetes": "",
 	"unset failed": "",
-	"unset minikube profile": "",
 	"unsets PROPERTY_NAME from the minikube config file.  Can be overwritten by flags or environmental variables": "",
 	"unsets an individual value in a minikube config file": "",
-	"unsupported driver: {{.name}}": "",
+	"unsupported or missing driver: {{.name}}": "",
 	"update config": "",
 	"usage: minikube addons configure ADDON_NAME": "",
 	"usage: minikube addons disable ADDON_NAME": "",
@@ -503,15 +574,19 @@
 	"usage: minikube addons list": "",
 	"usage: minikube addons open ADDON_NAME": "",
 	"usage: minikube config unset PROPERTY_NAME": "",
+	"usage: minikube delete": "",
+	"usage: minikube delete --all": "",
 	"usage: minikube profile [MINIKUBE_PROFILE_NAME]": "",
 	"zsh completion failed": "",
-	"{{.addonName}} was successfully enabled": "",
+	"{{.driver}} does not appear to be installed": "",
+	"{{.driver}} does not appear to be installed, but is specified by an existing profile. Please run 'minikube delete' or install {{.driver}}": "",
 	"{{.extra_option_component_name}}.{{.key}}={{.value}}": "",
 	"{{.machine}} IP has been updated to point at {{.ip}}": "",
 	"{{.machine}} IP was already correctly configured for {{.ip}}": "",
 	"{{.name}} cluster does not exist": "",
 	"{{.name}} has no available configuration options": "",
 	"{{.name}} was successfully configured": "",
+	"{{.path}} is version {{.client_version}}, and is incompatible with Kubernetes {{.cluster_version}}. You will need to update {{.path}} or use 'minikube kubectl' to connect with this cluster": "",
 	"{{.prefix}}minikube {{.version}} on {{.platform}}": "{{.prefix}}minikube {{.version}} en {{.platform}}",
 	"{{.type}} is not yet a supported filesystem. We will try anyways!": "",
 	"{{.url}} is not accessible: {{.error}}": ""
diff --git a/translations/fr.json b/translations/fr.json
index 1d760fd5a2..ffb83b230b 100644
--- a/translations/fr.json
+++ b/translations/fr.json
@@ -1,15 +1,17 @@
 {
-	"\"{{.minikube_addon}}\" was successfully disabled": "",
-	"\"{{.name}}\" cluster does not exist. Proceeding ahead with cleanup.": "",
+	"\"The '{{.minikube_addon}}' addon is disabled": "",
 	"\"{{.name}}\" profile does not exist": "Le profil \"{{.name}}\" n'existe pas.",
+	"\"{{.name}}\" profile does not exist, trying anyways.": "",
 	"\"{{.profile_name}}\" VM does not exist, nothing to stop": "",
 	"\"{{.profile_name}}\" host does not exist, unable to show an IP": "",
 	"\"{{.profile_name}}\" stopped.": "\"{{.profile_name}}\" est arrêté.",
 	"'none' driver does not support 'minikube docker-env' command": "",
 	"'none' driver does not support 'minikube mount' command": "",
 	"'none' driver does not support 'minikube ssh' command": "",
+	"'{{.driver}}' driver reported an issue: {{.error}}": "",
+	"- {{.profile}}": "",
 	"A VPN or firewall is interfering with HTTP access to the minikube VM. Alternatively, try a different VM driver: https://minikube.sigs.k8s.io/docs/start/": "",
-	"A firewall is blocking Docker within the minikube VM from reaching the internet. You may need to configure it to use a proxy.": "",
+	"A firewall is blocking Docker the minikube VM from reaching the internet. You may need to configure it to use a proxy.": "",
 	"A firewall is interfering with minikube's ability to make outgoing HTTPS requests. You may need to change the value of the HTTPS_PROXY environment variable.": "",
 	"A firewall is likely blocking minikube from reaching the internet. You may need to configure minikube to use a proxy.": "",
 	"A set of apiserver IP Addresses which are used in the generated certificate for kubernetes.  This can be used if you want to make the apiserver available from outside the machine": "",
@@ -32,22 +34,34 @@
 	"Amount of RAM allocated to the minikube VM (format: \u003cnumber\u003e[\u003cunit\u003e], where unit = b, k, m or g).": "",
 	"Amount of time to wait for a service in seconds": "",
 	"Amount of time to wait for service in seconds": "",
+	"Another hypervisor, such as VirtualBox, is conflicting with KVM. Please stop the other hypervisor, or use --vm-driver to switch to it.": "",
+	"Automatically selected the {{.experimental}}'{{.driver}}' driver": "",
+	"Automatically selected the {{.experimental}}'{{.driver}}' driver (alternates: {{.alternates}})": "",
 	"Available Commands": "",
 	"Basic Commands:": "",
+	"Block until the apiserver is servicing API requests": "",
 	"Cannot find directory {{.path}} for mount": "",
+	"Cannot use both --output and --format options": "",
+	"Check output of 'journalctl -xeu kubelet', try passing --extra-config=kubelet.cgroup-driver=systemd to minikube start": "",
+	"Check that SELinux is disabled, and that the provided apiserver flags are valid": "",
 	"Check that minikube is running and that you have specified the correct namespace (-n flag) if required.": "",
+	"Check that the provided apiserver flags are valid": "",
 	"Check that your --kubernetes-version has a leading 'v'. For example: 'v1.1.14'": "",
-	"Check that your apiserver flags are valid, or run 'minikube delete'": "",
 	"Check your firewall rules for interference, and run 'virt-host-validate' to check for KVM configuration issues. If you are running minikube within a VM, consider using --vm-driver=none": "",
 	"Configuration and Management Commands:": "",
+	"Configure a default route on this Linux host, or use another --vm-driver that does not require it": "",
 	"Configure an external network switch following the official documentation, then add `--hyperv-virtual-switch=\u003cswitch-name\u003e` to `minikube start`": "",
 	"Configures the addon w/ADDON_NAME within minikube (example: minikube addons configure registry-creds). For a list of available addons use: minikube addons list": "",
 	"Configuring environment for Kubernetes {{.k8sVersion}} on {{.runtime}} {{.runtimeVersion}}": "Configuration de l'environment pour Kubernetes {{.k8sVersion}} sur {{.runtime}} {{.runtimeVersion}}",
 	"Configuring local host environment ...": "",
 	"Confirm that you have a working internet connection and that your VM has not run out of resources by using: 'minikube logs'": "",
 	"Confirm that you have supplied the correct value to --hyperv-virtual-switch using the 'Get-VMSwitch' command": "",
+	"Could not get profile flag": "",
+	"Could not process error from failed deletion": "",
+	"Could not process errors from failed deletion": "",
 	"Country code of the image mirror to be used. Leave empty to use the global one. For Chinese mainland users, set it to cn.": "Code pays du miroir d'images à utiliser. Laissez ce paramètre vide pour utiliser le miroir international. Pour les utilisateurs situés en Chine continentale, définissez sa valeur sur \"cn\".",
 	"Created a new profile : {{.profile_name}}": "",
+	"Creating Kubernetes in {{.driver_name}} container with (CPUs={{.number_of_cpus}}), Memory={{.memory_size}}MB ({{.host_memory_size}}MB available) ...": "",
 	"Creating a new profile failed": "",
 	"Creating mount {{.name}} ...": "Création de l'installation {{.name}}…",
 	"Creating {{.driver_name}} VM (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...": "Création d'une VM {{.driver_name}} (CPUs={{.number_of_cpus}}, Mémoire={{.memory_size}}MB, Disque={{.disk_size}}MB)...",
@@ -82,26 +96,29 @@
 	"ERROR creating `registry-creds-ecr` secret: {{.error}}": "",
 	"ERROR creating `registry-creds-gcr` secret: {{.error}}": "",
 	"Either systemctl is not installed, or Docker is broken. Run 'sudo systemctl start docker' and 'journalctl -u docker'": "",
+	"Enable addons. see `minikube addons list` for a list of valid addon names.": "",
 	"Enable experimental NVIDIA GPU support in minikube": "Active l'assistance expérimentale du GPU NVIDIA dans minikube.",
 	"Enable host resolver for NAT DNS requests (virtualbox driver only)": "Active le résolveur d'hôte pour les requêtes DNS NAT (pilote VirtualBox uniquement).",
+	"Enable istio needs {{.minMem}} MB of memory and {{.minCpus}} CPUs.": "",
 	"Enable proxy for NAT DNS requests (virtualbox driver only)": "Active le proxy pour les requêtes DNS NAT (pilote VirtualBox uniquement).",
 	"Enable the default CNI plugin (/etc/cni/net.d/k8s.conf). Used in conjunction with \\\"--network-plugin=cni\\": "Active le plug-in CNI par défaut (/etc/cni/net.d/k8s.conf). Utilisé en association avec \\\"--network-plugin=cni\\\".",
 	"Enable the default CNI plugin (/etc/cni/net.d/k8s.conf). Used in conjunction with \\\"--network-plugin=cni\\\".": "",
 	"Enables the addon w/ADDON_NAME within minikube (example: minikube addons enable dashboard). For a list of available addons use: minikube addons list": "",
+	"Enabling '{{.name}}' returned an error: {{.error}}": "",
+	"Enabling addons: {{.addons}}": "",
 	"Enabling dashboard ...": "",
+	"Ensure that CRI-O is installed and healthy: Run 'sudo systemctl start crio' and 'journalctl -u crio'. Alternatively, use --container-runtime=docker": "",
+	"Ensure that Docker is installed and healthy: Run 'sudo systemctl start docker' and 'journalctl -u docker'. Alternatively, select another value for --vm-driver": "",
+	"Ensure that the user listed in /etc/libvirt/qemu.conf has access to your home directory": "",
+	"Ensure that your value for HTTPS_PROXY points to an HTTPS proxy rather than an HTTP proxy": "",
 	"Environment variables to pass to the Docker daemon. (format: key=value)": "Variables d'environment à transmettre au daemon Docker (format : clé = valeur).",
 	"Error checking driver version: {{.error}}": "Erreur lors de la vérification de la version du driver : {{.error}}",
-	"Error creating list template": "",
 	"Error creating minikube directory": "",
-	"Error creating status template": "",
 	"Error creating view template": "",
-	"Error executing list template": "",
-	"Error executing status template": "",
 	"Error executing template": "",
 	"Error executing view template": "",
 	"Error finding port for mount": "",
 	"Error getting IP": "",
-	"Error getting bootstrapper": "",
 	"Error getting client": "",
 	"Error getting client: {{.error}}": "",
 	"Error getting cluster": "",
@@ -110,7 +127,6 @@
 	"Error getting host": "",
 	"Error getting host status": "",
 	"Error getting machine logs": "",
-	"Error getting machine status": "",
 	"Error getting profiles to delete": "",
 	"Error getting service status": "",
 	"Error getting service with namespace: {{.namespace}} and labels {{.labelName}}:{{.addonName}}: {{.error}}": "",
@@ -125,7 +141,6 @@
 	"Error parsing minikube version: {{.error}}": "Erreur lors de l'analyse de la version de minikube : {{.error}}",
 	"Error parsing vmDriver version: {{.error}}": "Erreur lors de l'analyse de la version du pilote de la VM : {{.error}}",
 	"Error reading {{.path}}: {{.error}}": "",
-	"Error restarting cluster": "",
 	"Error setting shell variables": "",
 	"Error starting cluster": "",
 	"Error starting mount": "",
@@ -137,12 +152,12 @@
 	"Error: [{{.id}}] {{.error}}": "",
 	"Examples": "",
 	"Exiting": "Fermeture…",
-	"Exiting due to driver incompatibility": "",
+	"Exiting.": "",
 	"Failed runtime": "",
 	"Failed to cache ISO": "",
 	"Failed to cache and load images": "",
 	"Failed to cache binaries": "",
-	"Failed to cache images": "",
+	"Failed to cache images to tar": "",
 	"Failed to change permissions for {{.minikube_dir_path}}: {{.error}}": "Échec de la modification des autorisations pour {{.minikube_dir_path}} : {{.error}}",
 	"Failed to check if machine exists": "",
 	"Failed to check main repository and mirrors for images for images": "",
@@ -161,7 +176,7 @@
 	"Failed to get service URL: {{.error}}": "",
 	"Failed to kill mount process: {{.error}}": "Échec de l'arrêt du processus d'installation : {{.error}}",
 	"Failed to list cached images": "",
-	"Failed to remove profile": "",
+	"Failed to reload cached images": "",
 	"Failed to save config": "",
 	"Failed to set NO_PROXY Env. Please use `export NO_PROXY=$NO_PROXY,{{.ip}}": "Échec de la définition de NO_PROXY Env. Veuillez utiliser `export NO_PROXY=$NO_PROXY,{{.ip}}.",
 	"Failed to set NO_PROXY Env. Please use `export NO_PROXY=$NO_PROXY,{{.ip}}`.": "",
@@ -186,14 +201,19 @@
 	"Gets the status of a local kubernetes cluster": "",
 	"Gets the status of a local kubernetes cluster.\n\tExit status contains the status of minikube's VM, cluster and kubernetes encoded on it's bits in this order from right to left.\n\tEg: 7 meaning: 1 (for minikube NOK) + 2 (for cluster NOK) + 4 (for kubernetes NOK)": "",
 	"Gets the value of PROPERTY_NAME from the minikube config file": "",
+	"Getting machine config failed": "",
 	"Global Flags": "",
-	"Go template format string for the addon list output.  The format for Go templates can be found here: https://golang.org/pkg/text/template/\nFor the list of accessible variables for the template, see the struct values here: https://godoc.org/k8s.io/minikube/cmd/minikube/cmd/config#AddonListTemplate": "",
 	"Go template format string for the cache list output.  The format for Go templates can be found here: https://golang.org/pkg/text/template/\nFor the list of accessible variables for the template, see the struct values here: https://godoc.org/k8s.io/minikube/cmd/minikube/cmd#CacheListTemplate": "",
 	"Go template format string for the config view output.  The format for Go templates can be found here: https://golang.org/pkg/text/template/\nFor the list of accessible variables for the template, see the struct values here: https://godoc.org/k8s.io/minikube/cmd/minikube/cmd/config#ConfigViewTemplate": "",
 	"Go template format string for the status output.  The format for Go templates can be found here: https://golang.org/pkg/text/template/\nFor the list accessible variables for the template, see the struct values here: https://godoc.org/k8s.io/minikube/cmd/minikube/cmd#Status": "",
 	"Group ID:     {{.groupID}}": "",
 	"Have you set up libvirt correctly?": "",
 	"Hide the hypervisor signature from the guest in minikube (kvm2 driver only)": "Masque la signature de l'hyperviseur de l'invité dans minikube (pilote kvm2 uniquement).",
+	"Hyperkit is broken. Upgrade to the latest hyperkit version and/or Docker for Desktop. Alternatively, you may choose an alternate --vm-driver": "",
+	"Hyperkit networking is broken. Upgrade to the latest hyperkit version and/or Docker for Desktop. Alternatively, you may choose an alternate --vm-driver": "",
+	"If set, automatically updates drivers to the latest version. Defaults to true.": "",
+	"If set, pause all namespaces": "",
+	"If set, unpause all namespaces": "",
 	"If the above advice does not help, please let us know:": "",
 	"If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --vm-driver=none.": "Si la valeur est \"true\", mettez les images Docker en cache pour l'amorceur actuel et chargez-les dans la machine. La valeur est toujours \"false\" avec --vm-driver=none.",
 	"If true, only download and cache files for later use - don't install or start anything.": "Si la valeur est \"true\", téléchargez les fichiers et mettez-les en cache uniquement pour une utilisation future. Ne lancez pas d'installation et ne commencez aucun processus.",
@@ -207,6 +227,7 @@
 	"Invalid size passed in argument: {{.error}}": "",
 	"IsEnabled failed": "",
 	"Kill the mount process spawned by minikube start": "",
+	"Kubernetes {{.new}} is now available. If you would like to upgrade, specify: --kubernetes-version={{.new}}": "",
 	"Kubernetes {{.version}} is not supported by this release of minikube": "",
 	"Launching Kubernetes ...": "Lancement de Kubernetes...",
 	"Launching proxy ...": "",
@@ -231,9 +252,13 @@
 	"Mounting host path {{.sourcePath}} into VM as {{.destinationPath}} ...": "",
 	"Mounts the specified directory into minikube": "",
 	"Mounts the specified directory into minikube.": "",
+	"Multiple errors deleting profiles": "",
+	"Multiple minikube profiles were found -": "",
+	"NIC Type used for host only network. One of Am79C970A, Am79C973, 82540EM, 82543GC, 82545EM, or virtio (virtualbox driver only)": "",
 	"NOTE: This process must stay alive for the mount to be accessible ...": "",
 	"Networking and Connectivity Commands:": "",
 	"No minikube profile was found. You can create one using `minikube start`.": "",
+	"Node may be unable to resolve external DNS records": "",
 	"None of the known repositories in your location are accessible. Using {{.image_repository_name}} as fallback.": "Aucun dépôt connu dans votre emplacement n'est accessible. {{.image_repository_name}} est utilisé comme dépôt de remplacement.",
 	"None of the known repositories is accessible. Consider specifying an alternative image repository with --image-repository flag": "Aucun dépôt connu n'est accessible. Pensez à spécifier un autre dépôt d'images à l'aide de l'indicateur \"--image-repository\".",
 	"Not passing {{.name}}={{.value}} to docker env.": "",
@@ -244,11 +269,15 @@
 	"Open the addons URL with https instead of http": "",
 	"Open the service URL with https instead of http": "",
 	"Opening kubernetes service  {{.namespace_name}}/{{.service_name}} in default browser...": "",
+	"Opening service {{.namespace_name}}/{{.service_name}} in default browser...": "",
 	"Opening {{.url}} in your default browser...": "",
 	"Opens the addon w/ADDON_NAME within minikube (example: minikube addons open dashboard). For a list of available addons use: minikube addons list": "",
 	"Options:      {{.options}}": "",
 	"Outputs minikube shell completion for the given shell (bash or zsh)": "",
 	"Outputs minikube shell completion for the given shell (bash or zsh)\n\n\tThis depends on the bash-completion binary.  Example installation instructions:\n\tOS X:\n\t\t$ brew install bash-completion\n\t\t$ source $(brew --prefix)/etc/bash_completion\n\t\t$ minikube completion bash \u003e ~/.minikube-completion  # for bash users\n\t\t$ minikube completion zsh \u003e ~/.minikube-completion  # for zsh users\n\t\t$ source ~/.minikube-completion\n\tUbuntu:\n\t\t$ apt-get install bash-completion\n\t\t$ source /etc/bash-completion\n\t\t$ source \u003c(minikube completion bash) # for bash users\n\t\t$ source \u003c(minikube completion zsh) # for zsh users\n\n\tAdditionally, you may want to output the completion to a file and source in your .bashrc\n\n\tNote for zsh users: [1] zsh completions are only supported in versions of zsh \u003e= 5.2": "",
+	"Pause": "",
+	"Paused kubelet and {{.count}} containers": "",
+	"Paused kubelet and {{.count}} containers in: {{.namespaces}}": "",
 	"Permissions:  {{.octalMode}} ({{.writtenMode}})": "",
 	"Please enter a value:": "",
 	"Please install the minikube hyperkit VM driver, or select an alternative --vm-driver": "",
@@ -265,22 +294,24 @@
 	"Problems detected in {{.entry}}:": "",
 	"Problems detected in {{.name}}:": "",
 	"Profile gets or sets the current minikube profile": "",
+	"Profile name \"{{.profilename}}\" is minikube keyword. To delete profile use command minikube delete -p \u003cprofile name\u003e": "",
 	"Provide VM UUID to restore MAC address (hyperkit driver only)": "Fournit l'identifiant unique universel (UUID) de la VM pour restaurer l'adresse MAC (pilote hyperkit uniquement).",
 	"Pulling images ...": "Extraction des images... ",
-	"Reboot to complete VirtualBox installation, and verify that VirtualBox is not blocked by your system": "",
+	"Reboot to complete VirtualBox installation, verify that VirtualBox is not blocked by your system, and/or use another hypervisor": "",
 	"Rebuild libvirt with virt-network support": "",
 	"Received {{.name}} signal": "",
+	"Reconfiguring existing host ...": "",
 	"Registry mirrors to pass to the Docker daemon": "Miroirs de dépôt à transmettre au daemon Docker.",
 	"Reinstall VirtualBox and reboot. Alternatively, try the kvm2 driver: https://minikube.sigs.k8s.io/docs/reference/drivers/kvm2/": "",
 	"Reinstall VirtualBox and verify that it is not blocked: System Preferences -\u003e Security \u0026 Privacy -\u003e General -\u003e Some system software was blocked from loading": "",
 	"Related issues:": "",
 	"Relaunching Kubernetes using {{.bootstrapper}} ...": "Redémarrage de Kubernetes à l'aide de {{.bootstrapper}}…",
+	"Removed all traces of the \"{{.name}}\" cluster.": "",
 	"Removing {{.directory}} ...": "Suppression du répertoire {{.directory}}…",
-	"Requested CPU count {{.requested_cpus}} is less than the minimum allowed of {{.minimum_cpus}}": "",
+	"Requested cpu count {{.requested_cpus}} is less than the minimum allowed of {{.minimum_cpus}}": "",
 	"Requested disk size {{.requested_size}} is less than minimum of {{.minimum_size}}": "La taille de disque demandée ({{.requested_size}}) est inférieure à la taille minimale ({{.minimum_size}}).",
 	"Requested memory allocation ({{.memory}}MB) is less than the default memory allocation of {{.default_memorysize}}MB. Beware that minikube might not work correctly or crash unexpectedly.": "L'allocation de mémoire demandée ({{.memory}} Mo) est inférieure à l'allocation de mémoire par défaut ({{.default_memorysize}} Mo). Sachez que minikube pourrait ne pas fonctionner correctement ou planter de manière inattendue.",
 	"Requested memory allocation {{.requested_size}} is less than the minimum allowed of {{.minimum_size}}": "L'allocation de mémoire demandée ({{.requested_size}}) est inférieure au minimum autorisé ({{.minimum_size}}).",
-	"Retriable failure: {{.error}}": "",
 	"Retrieve the ssh identity key path of the specified cluster": "",
 	"Retrieve the ssh identity key path of the specified cluster.": "",
 	"Retrieves the IP address of the running cluster": "",
@@ -288,22 +319,27 @@
 	"Retrieves the IP address of the running cluster, checks it\n\t\t\twith IP in kubeconfig, and corrects kubeconfig if incorrect.": "",
 	"Returns the value of PROPERTY_NAME from the minikube config file.  Can be overwritten at runtime by flags or environmental variables.": "",
 	"Run 'kubectl describe pod coredns -n kube-system' and check for a firewall or DNS conflict": "",
-	"Run 'minikube delete' to delete the stale VM": "",
+	"Run 'minikube delete' to delete the stale VM, or and ensure that minikube is running as the same user you are issuing this command with": "",
 	"Run kubectl": "",
 	"Run minikube from the C: drive.": "",
 	"Run the kubernetes client, download it if necessary. Remember -- after kubectl!\n\nExamples:\nminikube kubectl -- --help\nminikube kubectl -- get pods --namespace kube-system": "",
 	"Run the minikube command as an Administrator": "",
+	"Run: 'chmod 600 $HOME/.kube/config'": "",
 	"Running on localhost (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...": "",
+	"Selecting {{.experimental}}'{{.driver}}' driver from existing profile (alternates: {{.alternates}})": "",
+	"Selecting {{.experimental}}'{{.driver}}' driver from user configuration (alternates: {{.alternates}})": "",
 	"Set failed": "",
+	"Set flag to delete all profiles": "",
+	"Set this flag to delete the '.minikube' folder from your user directory.": "",
 	"Sets an individual value in a minikube config file": "",
 	"Sets the PROPERTY_NAME config value to PROPERTY_VALUE\n\tThese values can be overwritten by flags or environment variables at runtime.": "",
 	"Sets up docker env variables; similar to '$(docker-machine env)'": "",
 	"Sets up docker env variables; similar to '$(docker-machine env)'.": "",
 	"Setting profile failed": "",
+	"Show a list of global command-line options (applies to all commands).": "",
 	"Show only log entries which point to known problems": "",
 	"Show only the most recent journal entries, and continuously print new entries as they are appended to the journal.": "",
 	"Skipped switching kubectl context for {{.profile_name}} because --keep-context was set.": "",
-	"Sorry that minikube crashed. If this was unexpected, we would love to hear from you:": "",
 	"Sorry, Kubernetes {{.version}} is not supported by this release of minikube": "",
 	"Sorry, completion support is not yet implemented for {{.name}}": "",
 	"Sorry, the kubeadm.{{.parameter_name}} parameter is currently not supported by --extra-config": "Désolé, le paramètre kubeadm.{{.parameter_name}} ne peut actuellement pas être utilisé avec \"--extra-config\".",
@@ -320,15 +356,22 @@
 	"Stopping \"{{.profile_name}}\" in {{.driver_name}} ...": "Arrêt de \"{{.profile_name}}\" sur {{.driver_name}}...",
 	"Stops a local kubernetes cluster running in Virtualbox. This command stops the VM\nitself, leaving all files intact. The cluster can be started again with the \"start\" command.": "",
 	"Stops a running local kubernetes cluster": "",
+	"Successfully deleted all profiles": "",
 	"Successfully mounted {{.sourcePath}} to {{.destinationPath}}": "",
+	"Successfully powered off Hyper-V. minikube driver -- {{.driver}}": "",
+	"Successfully purged minikube directory located at - [{{.minikubeDirectory}}]": "",
 	"Suggestion: {{.advice}}": "",
+	"Suggestion: {{.fix}}": "",
 	"Target directory {{.path}} must be an absolute path": "",
 	"The \"{{.driver_name}}\" driver requires root privileges. Please run minikube using 'sudo minikube --vm-driver={{.driver_name}}": "Le pilote \"{{.driver_name}}\" nécessite de disposer de droits racine. Veuillez exécuter minikube à l'aide de \"sudo minikube --vm-driver={{.driver_name}}\".",
 	"The \"{{.driver_name}}\" driver requires root privileges. Please run minikube using 'sudo minikube --vm-driver={{.driver_name}}'.": "",
 	"The \"{{.driver_name}}\" driver should not be used with root privileges.": "",
 	"The \"{{.name}}\" cluster has been deleted.": "Le cluster \"{{.name}}\" a été supprimé.",
+	"The 'none' driver does not respect the --cpus flag": "",
+	"The 'none' driver does not respect the --memory flag": "",
 	"The 'none' driver does not support multiple profiles: https://minikube.sigs.k8s.io/docs/reference/drivers/none/": "",
 	"The 'none' driver provides limited isolation and may reduce system security and reliability.": "L'isolation fournie par le pilote \"none\" (aucun) est limitée, ce qui peut diminuer la sécurité et la fiabilité du système.",
+	"The '{{.addonName}}' addon is enabled": "",
 	"The '{{.driver}}' driver requires elevated permissions. The following commands will be executed:\\n\\n{{ .example }}\\n": "",
 	"The CIDR to be used for service cluster IPs.": "Méthode CIDR à exploiter pour les adresses IP des clusters du service.",
 	"The CIDR to be used for the minikube VM (virtualbox driver only)": "Méthode CIDR à exploiter pour la VM minikube (pilote virtualbox uniquement).",
@@ -351,14 +394,18 @@
 	"The docker host is currently not running": "",
 	"The docker service is currently not active": "",
 	"The driver '{{.driver}}' is not supported on {{.os}}": "Le pilote \"{{.driver}}\" n'est pas compatible avec {{.os}}.",
+	"The driver {{.experimental}} '{{.driver}}' is not supported on {{.os}}": "",
 	"The existing \"{{.profile_name}}\" VM that was created using the \"{{.old_driver}}\" driver, and is incompatible with the \"{{.driver}}\" driver.": "",
 	"The hyperv virtual switch name. Defaults to first found. (hyperv driver only)": "Nom du commutateur virtuel hyperv. La valeur par défaut affiche le premier commutateur trouvé (pilote hyperv uniquement).",
+	"The hypervisor does not appear to be configured properly. Run 'minikube start --alsologtostderr -v=1' and inspect the error code": "",
 	"The initial time interval for each check that wait performs in seconds": "",
 	"The kubernetes version that the minikube VM will use (ex: v1.2.3)": "Version de Kubernetes qu'utilisera la VM minikube (exemple : v1.2.3).",
+	"The machine-driver specified is failing to start. Try running 'docker-machine-driver-\u003ctype\u003e version'": "",
 	"The minikube VM is offline. Please run 'minikube start' to start it again.": "",
 	"The name of the network plugin": "Nom du plug-in réseau.",
 	"The name of the network plugin.": "",
 	"The number of bytes to use for 9p packet payload": "",
+	"The output format. One of 'json', 'table'": "",
 	"The path on the file system where the docs in markdown need to be saved": "",
 	"The service namespace": "",
 	"The services namespace": "",
@@ -367,7 +414,6 @@
 	"The value passed to --format is invalid: {{.error}}": "",
 	"The vmwarefusion driver is deprecated and support for it will be removed in a future release.\n\t\t\tPlease consider switching to the new vmware unified driver, which is intended to replace the vmwarefusion driver.\n\t\t\tSee https://minikube.sigs.k8s.io/docs/reference/drivers/vmware/ for more information.\n\t\t\tTo disable this message, run [minikube config set ShowDriverDeprecationNotification false]": "",
 	"The {{.driver_name}} driver should not be used with root privileges.": "Le pilote {{.driver_name}} ne doit pas être utilisé avec des droits racine.",
-	"There appears to be another hypervisor conflicting with KVM. Please stop the other hypervisor, or use --vm-driver to switch to it.": "",
 	"There's a new version for '{{.driver_executable}}'. Please consider upgrading. {{.documentation_url}}": "Une nouvelle version de \"{{.driver_executable}}\" est disponible. Pensez à effectuer la mise à niveau. {{.documentation_url}}",
 	"These changes will take effect upon a minikube delete and then a minikube start": "",
 	"This addon does not have an endpoint defined for the 'addons open' command.\nYou can add one by annotating a service with the label {{.labelName}}:{{.addonName}}": "",
@@ -376,38 +422,45 @@
 	"This will start the mount daemon and automatically mount files into minikube": "Cela permet de lancer le daemon d'installation et d'installer automatiquement les fichiers dans minikube.",
 	"This will start the mount daemon and automatically mount files into minikube.": "",
 	"Tip: To remove this root owned cluster, run: sudo {{.cmd}} delete": "Conseil : Pour supprimer ce cluster appartenant à la racine, exécutez la commande \"sudo {{.cmd}} delete\".",
-	"Tip: Use 'minikube start -p \u003cname\u003e' to create a new cluster, or 'minikube delete' to delete this one.": "",
 	"To connect to this cluster, use: kubectl --context={{.name}}": "Pour vous connecter à ce cluster, utilisez la commande \"kubectl --context={{.name}}\".",
 	"To connect to this cluster, use: kubectl --context={{.name}}__1": "Pour vous connecter à ce cluster, utilisez la commande \"kubectl --context={{.name}}\".",
 	"To connect to this cluster, use: kubectl --context={{.profile_name}}": "",
 	"To disable this notice, run: 'minikube config set WantUpdateNotification false'\\n": "",
-	"To proceed, either:\n      1) Delete the existing VM using: '{{.command}} delete'\n      or\n      2) Restart with the existing driver: '{{.command}} start --vm-driver={{.old_driver}}'": "",
+	"To proceed, either:\n\n    1) Delete the existing \"{{.profile_name}}\" cluster using: '{{.command}} delete'\n\n    * or *\n\n    2) Start the existing \"{{.profile_name}}\" cluster using: '{{.command}} start --vm-driver={{.old_driver}}'": "",
+	"To see addons list for other profiles use: `minikube addons -p name list`": "",
 	"To start minikube with HyperV Powershell must be in your PATH`": "",
 	"To use kubectl or minikube commands as your own user, you may need to relocate them. For example, to overwrite your own settings, run:": "Pour utiliser les commandes kubectl ou minikube sous votre propre nom d'utilisateur, vous devrez peut-être les déplacer. Par exemple, pour écraser vos propres paramètres, exécutez la commande suivante :",
 	"Troubleshooting Commands:": "",
+	"Trying to delete invalid profile {{.profile}}": "",
 	"Unable to bind flags": "",
+	"Unable to determine a default driver to use. Try specifying --vm-driver, or see https://minikube.sigs.k8s.io/docs/start/": "",
 	"Unable to enable dashboard": "",
 	"Unable to fetch latest version info": "",
 	"Unable to generate docs": "",
 	"Unable to generate the documentation. Please ensure that the path specified is a directory, exists \u0026 you have permission to write to it.": "",
 	"Unable to get VM IP address": "",
+	"Unable to get addon status for {{.name}}: {{.error}}": "",
 	"Unable to get bootstrapper: {{.error}}": "Impossible d'obtenir l'amorceur : {{.error}}",
 	"Unable to get current user": "",
 	"Unable to get runtime": "",
-	"Unable to get the status of the cluster.": "",
+	"Unable to get the status of the {{.name}} cluster.": "",
 	"Unable to kill mount process: {{.error}}": "",
 	"Unable to load cached images from config file.": "Impossible de charger les images mises en cache depuis le fichier de configuration.",
 	"Unable to load cached images: {{.error}}": "",
 	"Unable to load config: {{.error}}": "Impossible de charger la configuration : {{.error}}",
 	"Unable to parse \"{{.kubernetes_version}}\": {{.error}}": "Impossible d'analyser la version \"{{.kubernetes_version}}\" : {{.error}}",
+	"Unable to parse default Kubernetes version from constants: {{.error}}": "",
 	"Unable to parse oldest Kubernetes version from constants: {{.error}}": "",
 	"Unable to pull images, which may be OK: {{.error}}": "Impossible d'extraire des images, qui sont peut-être au bon format : {{.error}}",
 	"Unable to remove machine directory: %v": "",
-	"Unable to start VM": "",
+	"Unable to start VM. Please investigate and run 'minikube delete' if possible": "",
 	"Unable to stop VM": "",
 	"Unable to update {{.driver}} driver: {{.error}}": "",
+	"Unable to verify SSH connectivity: {{.error}}. Will retry...": "",
 	"Uninstalling Kubernetes {{.kubernetes_version}} using {{.bootstrapper_name}} ...": "Désinstallation de Kubernetes {{.kubernetes_version}} à l'aide de {{.bootstrapper_name}}…",
 	"Unmounting {{.path}} ...": "",
+	"Unpaused kubelet and {{.count}} containers": "",
+	"Unpaused kubelet and {{.count}} containers in: {{.namespaces}}": "",
 	"Unset the KUBECONFIG environment variable, or verify that it does not point to an empty or otherwise invalid path": "",
 	"Unset variables instead of setting them": "",
 	"Update server returned an empty list": "",
@@ -416,15 +469,20 @@
 	"Usage": "Usage",
 	"Usage: minikube completion SHELL": "",
 	"Usage: minikube delete": "",
+	"Usage: minikube delete --all --purge": "",
 	"Use \"{{.CommandPath}} [command] --help\" for more information about a command.": "",
+	"Use 'kubect get po -A' to find the correct and namespace name": "",
+	"Use -A to specify all namespaces": "",
 	"Use VirtualBox to remove the conflicting VM and/or network interfaces": "",
 	"Use native Golang SSH client (default true). Set to 'false' to use the command line 'ssh' command when accessing the docker machine. Useful for the machine drivers when they will not start with 'Waiting for SSH'.": "",
 	"User ID:      {{.userID}}": "",
 	"Userspace file server is shutdown": "",
 	"Userspace file server:": "",
 	"Using image repository {{.name}}": "Utilisation du dépôt d'images {{.name}}…",
+	"Using the '{{.runtime}}' runtime with the 'none' driver is an untested configuration!": "",
 	"Using the running {{.driver_name}} \"{{.profile_name}}\" VM ...": "",
 	"VM driver is one of: %v": "Le pilote de la VM appartient à : %v",
+	"VM is unable to access {{.repository}}, you may need to configure a proxy or set --image-repository": "",
 	"Verify that your HTTP_PROXY and HTTPS_PROXY environment variables are set correctly.": "",
 	"Verify the IP address of the running cluster in kubeconfig.": "",
 	"Verifying dashboard health ...": "",
@@ -435,12 +493,13 @@
 	"VirtualBox cannot create a network, probably because it conflicts with an existing network that minikube no longer knows about. Try running 'minikube delete'": "",
 	"VirtualBox is broken. Disable real-time anti-virus software, reboot, and reinstall VirtualBox if the problem continues.": "",
 	"VirtualBox is broken. Reinstall VirtualBox, reboot, and run 'minikube delete'.": "",
+	"VirtualBox is unable to find its network interface. Try upgrading to the latest release and rebooting.": "",
+	"Virtualization support is disabled on your computer. If you are running minikube within a VM, try '--vm-driver=none'. Otherwise, consult your systems BIOS manual for how to enable virtualization.": "",
 	"Wait failed": "",
 	"Wait failed: {{.error}}": "",
 	"Wait until Kubernetes core services are healthy before exiting": "Avant de quitter, veuillez patienter jusqu'à ce que les principaux services Kubernetes soient opérationnels.",
-	"Wait until Kubernetes core services are healthy before exiting.": "",
 	"Waiting for SSH access ...": "En attente de l'accès SSH...",
-	"Waiting for the host to be provisioned ...": "",
+	"Waiting for cluster to come online ...": "",
 	"Waiting for:": "En attente de :",
 	"Where to root the NFS Shares, defaults to /nfsshares (hyperkit driver only)": "Emplacement permettant d'accéder aux partages NFS en mode root, la valeur par défaut affichant /nfsshares (pilote hyperkit uniquement).",
 	"You appear to be using a proxy, but your NO_PROXY environment does not include the minikube IP ({{.ip_address}}). Please see {{.documentation_url}} for more details": "Il semble que vous utilisiez un proxy, mais votre environment NO_PROXY n'inclut pas l'adresse IP ({{.ip_address}}) de minikube. Consultez la documentation à l'adresse {{.documentation_url}} pour en savoir plus.",
@@ -454,50 +513,62 @@
 	"Your minikube vm is not running, try minikube start.": "",
 	"addon '{{.name}}' is currently not enabled.\nTo enable this addon run:\nminikube addons enable {{.name}}": "",
 	"addon '{{.name}}' is not a valid addon packaged with minikube.\nTo see the list of available addons run:\nminikube addons list": "",
-	"addon list failed": "",
-	"addons modifies minikube addons files using subcommands like \"minikube addons enable heapster\"": "",
+	"addons modifies minikube addons files using subcommands like \"minikube addons enable dashboard\"": "",
 	"api load": "",
 	"bash completion failed": "",
-	"browser failed to open url: {{.error}}": "",
 	"call with cleanup=true to remove old tunnels": "",
 	"command runner": "",
 	"config modifies minikube config files using subcommands like \"minikube config set vm-driver kvm\"\nConfigurable fields:\\n\\n": "",
 	"config view failed": "",
 	"dashboard service is not running: {{.error}}": "",
 	"disable failed": "",
+	"dry-run mode. Validates configuration, but does not mutate system state": "",
+	"dry-run validation complete!": "",
 	"enable failed": "",
 	"error creating clientset": "",
 	"error creating machine client": "",
-	"error getting driver": "",
 	"error parsing the input ip address for mount": "",
 	"error starting tunnel": "",
 	"failed to open browser: {{.error}}": "",
 	"if true, will embed the certs in kubeconfig.": "",
+	"kubeadm detected a TCP port conflict with another process: probably another local Kubernetes installation. Run lsof -p\u003cport\u003e to find the process and kill it": "",
 	"kubectl and minikube configuration will be stored in {{.home_folder}}": "Les configurations kubectl et minikube seront stockées dans le dossier {{.home_folder}}.",
 	"kubectl not found in PATH, but is required for the dashboard. Installation guide: https://kubernetes.io/docs/tasks/tools/install-kubectl/": "",
 	"kubectl proxy": "",
 	"logdir set failed": "",
 	"max time to wait per Kubernetes core services to be healthy.": "",
-	"minikube is not running, so the service cannot be accessed": "",
+	"minikube addons list --output OUTPUT. json, list": "",
+	"minikube is exiting due to an error. If the above message is not useful, open an issue:": "",
 	"minikube is unable to access the Google Container Registry. You may need to configure it to use a HTTP proxy.": "",
+	"minikube is unable to connect to the VM: {{.error}}\n\nThis is likely due to one of two reasons:\n\n- VPN or firewall interference\n- {{.hypervisor}} network configuration issue\n\nSuggested workarounds:\n\n- Disable your local VPN or firewall software\n- Configure your local VPN or firewall to allow access to {{.ip}}\n- Restart or reinstall {{.hypervisor}}\n- Use an alternative --vm-driver": "",
 	"minikube profile was successfully set to {{.profile_name}}": "",
+	"minikube status --output OUTPUT. json, text": "",
 	"minikube {{.version}} is available! Download it: {{.url}}": "",
+	"mkcmp is used to compare performance of two minikube binaries": "",
 	"mount argument \"{{.value}}\" must be in form: \u003csource directory\u003e:\u003ctarget directory\u003e": "",
 	"mount failed": "",
+	"namespaces to pause": "",
+	"namespaces to unpause": "",
 	"not enough arguments ({{.ArgCount}}).\\nusage: minikube config set PROPERTY_NAME PROPERTY_VALUE": "",
+	"pause containers": "",
 	"profile sets the current minikube profile, or gets the current profile if no arguments are provided.  This is used to run and manage multiple minikube instance.  You can return to the default minikube profile by running `minikube profile default`": "",
+	"reload cached images.": "",
+	"reloads images previously added using the 'cache add' subcommand": "",
 	"service {{.namespace_name}}/{{.service_name}} has no node port": "",
 	"stat failed": "",
+	"status json failure": "",
+	"status text failure": "",
 	"toom any arguments ({{.ArgCount}}).\\nusage: minikube config set PROPERTY_NAME PROPERTY_VALUE": "",
-	"tunnel creates a route to services deployed with type LoadBalancer and sets their Ingress to their ClusterIP": "",
+	"tunnel creates a route to services deployed with type LoadBalancer and sets their Ingress to their ClusterIP. for a detailed example see https://minikube.sigs.k8s.io/docs/tasks/loadbalancer": "",
 	"tunnel makes services of type LoadBalancer accessible on localhost": "",
 	"unable to bind flags": "",
+	"unable to delete minikube config folder": "",
 	"unable to set logtostderr": "",
+	"unpause Kubernetes": "",
 	"unset failed": "",
-	"unset minikube profile": "",
 	"unsets PROPERTY_NAME from the minikube config file.  Can be overwritten by flags or environmental variables": "",
 	"unsets an individual value in a minikube config file": "",
-	"unsupported driver: {{.name}}": "",
+	"unsupported or missing driver: {{.name}}": "",
 	"update config": "",
 	"usage: minikube addons configure ADDON_NAME": "",
 	"usage: minikube addons disable ADDON_NAME": "",
@@ -509,14 +580,16 @@
 	"usage: minikube delete --all": "",
 	"usage: minikube profile [MINIKUBE_PROFILE_NAME]": "",
 	"zsh completion failed": "",
-	"{{.addonName}} was successfully enabled": "",
+	"{{.driver}} does not appear to be installed": "",
+	"{{.driver}} does not appear to be installed, but is specified by an existing profile. Please run 'minikube delete' or install {{.driver}}": "",
 	"{{.extra_option_component_name}}.{{.key}}={{.value}}": "",
 	"{{.machine}} IP has been updated to point at {{.ip}}": "",
 	"{{.machine}} IP was already correctly configured for {{.ip}}": "",
 	"{{.name}} cluster does not exist": "",
 	"{{.name}} has no available configuration options": "",
 	"{{.name}} was successfully configured": "",
+	"{{.path}} is version {{.client_version}}, and is incompatible with Kubernetes {{.cluster_version}}. You will need to update {{.path}} or use 'minikube kubectl' to connect with this cluster": "",
 	"{{.prefix}}minikube {{.version}} on {{.platform}}": "{{.prefix}}minikube {{.version}} sur {{.platform}}",
 	"{{.type}} is not yet a supported filesystem. We will try anyways!": "",
 	"{{.url}} is not accessible: {{.error}}": ""
-}
+}
\ No newline at end of file
diff --git a/translations/ja.json b/translations/ja.json
index 9e6637b253..b0a1a213a8 100644
--- a/translations/ja.json
+++ b/translations/ja.json
@@ -1,7 +1,9 @@
 {
+	"\"The '{{.minikube_addon}}' addon is disabled": "",
 	"\"{{.minikube_addon}}\" was successfully disabled": "「{{.minikube_addon}}」が無効化されました",
 	"\"{{.name}}\" cluster does not exist. Proceeding ahead with cleanup.": "「{{.name}}」というクラスターは存在しません。クリーンアップ処理を続行します。",
 	"\"{{.name}}\" profile does not exist": "「{{.name}}」というプロファイルは存在しません",
+	"\"{{.name}}\" profile does not exist, trying anyways.": "",
 	"\"{{.profile_name}}\" VM does not exist, nothing to stop": "「{{.profile_name}}」というVMは存在しません。停止すべき対象がありません",
 	"\"{{.profile_name}}\" host does not exist, unable to show an IP": "「{{.profile_name}}」というホストは存在しません。IPを表示できません",
 	"\"{{.profile_name}}\" stopped.": "「{{.profile_name}}」が停止しました。",
@@ -35,8 +37,8 @@
 	"Amount of time to wait for a service in seconds": "",
 	"Amount of time to wait for service in seconds": "",
 	"Another hypervisor, such as VirtualBox, is conflicting with KVM. Please stop the other hypervisor, or use --vm-driver to switch to it.": "",
-	"Automatically selected the {{.experimental}} '{{.driver}}' driver": "",
-	"Automatically selected the {{.experimental}} '{{.driver}}' driver (alternates: {{.alternates}})": "",
+	"Automatically selected the {{.experimental}}'{{.driver}}' driver": "",
+	"Automatically selected the {{.experimental}}'{{.driver}}' driver (alternates: {{.alternates}})": "",
 	"Available Commands": "",
 	"Basic Commands:": "",
 	"Block until the apiserver is servicing API requests": "",
@@ -104,6 +106,8 @@
 	"Enable the default CNI plugin (/etc/cni/net.d/k8s.conf). Used in conjunction with \\\"--network-plugin=cni\\": "デフォルトの CNI プラグイン（/etc/cni/net.d/k8s.conf）を有効にします。\\\"--network-plugin=cni\\\" と組み合わせて使用されます。",
 	"Enable the default CNI plugin (/etc/cni/net.d/k8s.conf). Used in conjunction with \\\"--network-plugin=cni\\\".": "",
 	"Enables the addon w/ADDON_NAME within minikube (example: minikube addons enable dashboard). For a list of available addons use: minikube addons list": "",
+	"Enabling '{{.name}}' returned an error: {{.error}}": "",
+	"Enabling addons: {{.addons}}": "",
 	"Enabling dashboard ...": "",
 	"Ensure that CRI-O is installed and healthy: Run 'sudo systemctl start crio' and 'journalctl -u crio'. Alternatively, use --container-runtime=docker": "",
 	"Ensure that Docker is installed and healthy: Run 'sudo systemctl start docker' and 'journalctl -u docker'. Alternatively, select another value for --vm-driver": "",
@@ -111,17 +115,12 @@
 	"Ensure that your value for HTTPS_PROXY points to an HTTPS proxy rather than an HTTP proxy": "",
 	"Environment variables to pass to the Docker daemon. (format: key=value)": "Docker デーモンに渡す環境変数（形式: Key=Value）",
 	"Error checking driver version: {{.error}}": "ドライバのバージョンの確認中にエラーが発生しました。{{.error}}",
-	"Error converting status to json": "",
 	"Error creating minikube directory": "",
-	"Error creating status template": "",
 	"Error creating view template": "",
-	"Error executing status template": "",
 	"Error executing template": "",
 	"Error executing view template": "",
 	"Error finding port for mount": "",
 	"Error getting IP": "",
-	"Error getting addons status": "",
-	"Error getting bootstrapper": "",
 	"Error getting client": "",
 	"Error getting client: {{.error}}": "",
 	"Error getting cluster": "",
@@ -215,6 +214,8 @@
 	"Hyperkit is broken. Upgrade to the latest hyperkit version and/or Docker for Desktop. Alternatively, you may choose an alternate --vm-driver": "",
 	"Hyperkit networking is broken. Upgrade to the latest hyperkit version and/or Docker for Desktop. Alternatively, you may choose an alternate --vm-driver": "",
 	"If set, automatically updates drivers to the latest version. Defaults to true.": "",
+	"If set, pause all namespaces": "",
+	"If set, unpause all namespaces": "",
 	"If the above advice does not help, please let us know:": "",
 	"If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --vm-driver=none.": "true の場合、現在のブートストラッパの Docker イメージをキャッシュに保存して、マシンに読み込みます。--vm-driver=none の場合は常に false です。",
 	"If true, only download and cache files for later use - don't install or start anything.": "true の場合、後で使用できるようにファイルのダウンロードとキャッシュ保存だけが行われます。インストールも起動も行われません。",
@@ -259,6 +260,7 @@
 	"NOTE: This process must stay alive for the mount to be accessible ...": "",
 	"Networking and Connectivity Commands:": "",
 	"No minikube profile was found. You can create one using `minikube start`.": "",
+	"Node may be unable to resolve external DNS records": "",
 	"None of the known repositories in your location are accessible. Using {{.image_repository_name}} as fallback.": "使用しているロケーション内で既知のいずれのリポジトリにもアクセスできません。フォールバックとして {{.image_repository_name}} を使用します。",
 	"None of the known repositories is accessible. Consider specifying an alternative image repository with --image-repository flag": "既知のいずれのリポジトリにもアクセスできません。--image-repository フラグとともに代替のイメージ リポジトリを指定することを検討してください。",
 	"Not passing {{.name}}={{.value}} to docker env.": "",
@@ -275,6 +277,9 @@
 	"Options:      {{.options}}": "",
 	"Outputs minikube shell completion for the given shell (bash or zsh)": "",
 	"Outputs minikube shell completion for the given shell (bash or zsh)\n\n\tThis depends on the bash-completion binary.  Example installation instructions:\n\tOS X:\n\t\t$ brew install bash-completion\n\t\t$ source $(brew --prefix)/etc/bash_completion\n\t\t$ minikube completion bash \u003e ~/.minikube-completion  # for bash users\n\t\t$ minikube completion zsh \u003e ~/.minikube-completion  # for zsh users\n\t\t$ source ~/.minikube-completion\n\tUbuntu:\n\t\t$ apt-get install bash-completion\n\t\t$ source /etc/bash-completion\n\t\t$ source \u003c(minikube completion bash) # for bash users\n\t\t$ source \u003c(minikube completion zsh) # for zsh users\n\n\tAdditionally, you may want to output the completion to a file and source in your .bashrc\n\n\tNote for zsh users: [1] zsh completions are only supported in versions of zsh \u003e= 5.2": "",
+	"Pause": "",
+	"Paused kubelet and {{.count}} containers": "",
+	"Paused kubelet and {{.count}} containers in: {{.namespaces}}": "",
 	"Permissions:  {{.octalMode}} ({{.writtenMode}})": "",
 	"Please enter a value:": "",
 	"Please install the minikube hyperkit VM driver, or select an alternative --vm-driver": "",
@@ -297,11 +302,13 @@
 	"Reboot to complete VirtualBox installation, verify that VirtualBox is not blocked by your system, and/or use another hypervisor": "",
 	"Rebuild libvirt with virt-network support": "",
 	"Received {{.name}} signal": "",
+	"Reconfiguring existing host ...": "",
 	"Registry mirrors to pass to the Docker daemon": "Docker デーモンに渡すレジストリ ミラー",
 	"Reinstall VirtualBox and reboot. Alternatively, try the kvm2 driver: https://minikube.sigs.k8s.io/docs/reference/drivers/kvm2/": "",
 	"Reinstall VirtualBox and verify that it is not blocked: System Preferences -\u003e Security \u0026 Privacy -\u003e General -\u003e Some system software was blocked from loading": "",
 	"Related issues:": "",
 	"Relaunching Kubernetes using {{.bootstrapper}} ...": "{{.bootstrapper}} を使用して Kubernetes を再起動しています...",
+	"Removed all traces of the \"{{.name}}\" cluster.": "",
 	"Removing {{.directory}} ...": "{{.directory}} を削除しています...",
 	"Requested cpu count {{.requested_cpus}} is less than the minimum allowed of {{.minimum_cpus}}": "",
 	"Requested disk size {{.requested_size}} is less than minimum of {{.minimum_size}}": "リクエストされたディスクサイズ {{.requested_size}} が最小値 {{.minimum_size}} 未満です",
@@ -321,8 +328,8 @@
 	"Run the minikube command as an Administrator": "",
 	"Run: 'chmod 600 $HOME/.kube/config'": "",
 	"Running on localhost (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...": "",
-	"Selecting {{.experimental}} '{{.driver}}' driver from existing profile (alternates: {{.alternates}})": "",
-	"Selecting {{.experimental}} '{{.driver}}' driver from user configuration (alternates: {{.alternates}})": "",
+	"Selecting {{.experimental}}'{{.driver}}' driver from existing profile (alternates: {{.alternates}})": "",
+	"Selecting {{.experimental}}'{{.driver}}' driver from user configuration (alternates: {{.alternates}})": "",
 	"Set failed": "",
 	"Set flag to delete all profiles": "",
 	"Set this flag to delete the '.minikube' folder from your user directory.": "",
@@ -352,7 +359,6 @@
 	"Stops a local kubernetes cluster running in Virtualbox. This command stops the VM\nitself, leaving all files intact. The cluster can be started again with the \"start\" command.": "",
 	"Stops a running local kubernetes cluster": "",
 	"Successfully deleted all profiles": "",
-	"Successfully deleted profile \\\"{{.name}}\\\"": "",
 	"Successfully mounted {{.sourcePath}} to {{.destinationPath}}": "",
 	"Successfully powered off Hyper-V. minikube driver -- {{.driver}}": "",
 	"Successfully purged minikube directory located at - [{{.minikubeDirectory}}]": "",
@@ -368,6 +374,7 @@
 	"The 'none' driver does not respect the --memory flag": "",
 	"The 'none' driver does not support multiple profiles: https://minikube.sigs.k8s.io/docs/reference/drivers/none/": "",
 	"The 'none' driver provides limited isolation and may reduce system security and reliability.": "ドライバに「none」を指定すると、分離が制限され、システムのセキュリティと信頼性が低下する可能性があります。",
+	"The '{{.addonName}}' addon is enabled": "",
 	"The '{{.driver}}' driver requires elevated permissions. The following commands will be executed:\\n\\n{{ .example }}\\n": "",
 	"The CIDR to be used for service cluster IPs.": "サービス クラスタ IP に使用される CIDR。",
 	"The CIDR to be used for the minikube VM (virtualbox driver only)": "minikube VM に使用される CIDR（virtualbox ドライバのみ）",
@@ -418,7 +425,6 @@
 	"This will start the mount daemon and automatically mount files into minikube": "これによりマウント デーモンが起動し、ファイルが minikube に自動的にマウントされます",
 	"This will start the mount daemon and automatically mount files into minikube.": "",
 	"Tip: To remove this root owned cluster, run: sudo {{.cmd}} delete": "ヒント: この root 所有のクラスタを削除するには、「sudo {{.cmd}} delete」を実行します",
-	"Tip: Use 'minikube start -p \u003cname\u003e' to create a new cluster, or 'minikube delete' to delete this one.": "",
 	"To connect to this cluster, use: kubectl --context={{.name}}": "このクラスタに接続するには、「kubectl --context={{.name}}」を使用します",
 	"To connect to this cluster, use: kubectl --context={{.name}}__1": "このクラスタに接続するには、「kubectl --context={{.name}}」を使用します",
 	"To connect to this cluster, use: kubectl --context={{.profile_name}}": "",
@@ -436,6 +442,7 @@
 	"Unable to generate docs": "",
 	"Unable to generate the documentation. Please ensure that the path specified is a directory, exists \u0026 you have permission to write to it.": "",
 	"Unable to get VM IP address": "",
+	"Unable to get addon status for {{.name}}: {{.error}}": "",
 	"Unable to get bootstrapper: {{.error}}": "ブートストラッパを取得できません。{{.error}}",
 	"Unable to get current user": "",
 	"Unable to get runtime": "",
@@ -455,6 +462,8 @@
 	"Unable to verify SSH connectivity: {{.error}}. Will retry...": "",
 	"Uninstalling Kubernetes {{.kubernetes_version}} using {{.bootstrapper_name}} ...": "{{.bootstrapper_name}} を使用して Kubernetes {{.kubernetes_version}} をアンインストールしています...",
 	"Unmounting {{.path}} ...": "",
+	"Unpaused kubelet and {{.count}} containers": "",
+	"Unpaused kubelet and {{.count}} containers in: {{.namespaces}}": "",
 	"Unset the KUBECONFIG environment variable, or verify that it does not point to an empty or otherwise invalid path": "",
 	"Unset variables instead of setting them": "",
 	"Update server returned an empty list": "",
@@ -466,6 +475,7 @@
 	"Usage: minikube delete --all --purge": "",
 	"Use \"{{.CommandPath}} [command] --help\" for more information about a command.": "",
 	"Use 'kubect get po -A' to find the correct and namespace name": "",
+	"Use -A to specify all namespaces": "",
 	"Use VirtualBox to remove the conflicting VM and/or network interfaces": "",
 	"Use native Golang SSH client (default true). Set to 'false' to use the command line 'ssh' command when accessing the docker machine. Useful for the machine drivers when they will not start with 'Waiting for SSH'.": "",
 	"User ID:      {{.userID}}": "",
@@ -476,7 +486,6 @@
 	"Using the running {{.driver_name}} \"{{.profile_name}}\" VM ...": "",
 	"VM driver is one of: %v": "VM ドライバは次のいずれかです。%v",
 	"VM is unable to access {{.repository}}, you may need to configure a proxy or set --image-repository": "",
-	"VM may be unable to resolve external DNS records": "",
 	"Verify that your HTTP_PROXY and HTTPS_PROXY environment variables are set correctly.": "",
 	"Verify the IP address of the running cluster in kubeconfig.": "",
 	"Verifying dashboard health ...": "",
@@ -492,8 +501,6 @@
 	"Wait failed: {{.error}}": "",
 	"Wait until Kubernetes core services are healthy before exiting": "Kubernetes コアサービスが正常になるまで待機してから終了してください",
 	"Waiting for cluster to come online ...": "",
-	"Waiting for the host to be provisioned ...": "",
-	"Warning: Your kubectl is pointing to stale minikube-vm.\\nTo fix the kubectl context, run `minikube update-context`": "",
 	"Where to root the NFS Shares, defaults to /nfsshares (hyperkit driver only)": "NFS 共有のルートに指定する場所。デフォルトは /nfsshares（hyperkit ドライバのみ）",
 	"You appear to be using a proxy, but your NO_PROXY environment does not include the minikube IP ({{.ip_address}}). Please see {{.documentation_url}} for more details": "プロキシを使用しようとしていますが、現在の NO_PROXY 環境に minikube IP（{{.ip_address}}）は含まれていません。詳細については、{{.documentation_url}} をご覧ください",
 	"You can delete them using the following command(s):": "",
@@ -506,7 +513,6 @@
 	"Your minikube vm is not running, try minikube start.": "",
 	"addon '{{.name}}' is currently not enabled.\nTo enable this addon run:\nminikube addons enable {{.name}}": "",
 	"addon '{{.name}}' is not a valid addon packaged with minikube.\nTo see the list of available addons run:\nminikube addons list": "",
-	"addon enable failed": "",
 	"addons modifies minikube addons files using subcommands like \"minikube addons enable dashboard\"": "",
 	"api load": "",
 	"bash completion failed": "",
@@ -541,18 +547,24 @@
 	"mkcmp is used to compare performance of two minikube binaries": "",
 	"mount argument \"{{.value}}\" must be in form: \u003csource directory\u003e:\u003ctarget directory\u003e": "",
 	"mount failed": "",
+	"namespaces to pause": "",
+	"namespaces to unpause": "",
 	"not enough arguments ({{.ArgCount}}).\\nusage: minikube config set PROPERTY_NAME PROPERTY_VALUE": "",
+	"pause containers": "",
 	"profile sets the current minikube profile, or gets the current profile if no arguments are provided.  This is used to run and manage multiple minikube instance.  You can return to the default minikube profile by running `minikube profile default`": "",
 	"reload cached images.": "",
 	"reloads images previously added using the 'cache add' subcommand": "",
 	"service {{.namespace_name}}/{{.service_name}} has no node port": "",
 	"stat failed": "",
+	"status json failure": "",
+	"status text failure": "",
 	"toom any arguments ({{.ArgCount}}).\\nusage: minikube config set PROPERTY_NAME PROPERTY_VALUE": "",
 	"tunnel creates a route to services deployed with type LoadBalancer and sets their Ingress to their ClusterIP. for a detailed example see https://minikube.sigs.k8s.io/docs/tasks/loadbalancer": "",
 	"tunnel makes services of type LoadBalancer accessible on localhost": "",
 	"unable to bind flags": "",
 	"unable to delete minikube config folder": "",
 	"unable to set logtostderr": "",
+	"unpause Kubernetes": "",
 	"unset failed": "",
 	"unsets PROPERTY_NAME from the minikube config file.  Can be overwritten by flags or environmental variables": "",
 	"unsets an individual value in a minikube config file": "",
@@ -568,7 +580,6 @@
 	"usage: minikube delete --all": "",
 	"usage: minikube profile [MINIKUBE_PROFILE_NAME]": "",
 	"zsh completion failed": "",
-	"{{.addonName}} was successfully enabled": "",
 	"{{.driver}} does not appear to be installed": "",
 	"{{.driver}} does not appear to be installed, but is specified by an existing profile. Please run 'minikube delete' or install {{.driver}}": "",
 	"{{.extra_option_component_name}}.{{.key}}={{.value}}": "",
diff --git a/translations/pl.json b/translations/pl.json
index 41491e9b93..a9b8da947c 100644
--- a/translations/pl.json
+++ b/translations/pl.json
@@ -1,22 +1,22 @@
 {
+	"\"The '{{.minikube_addon}}' addon is disabled": "",
 	"\"{{.minikube_addon}}\" was successfully disabled": "\"{{.minikube_addon}}\" został wyłaczony",
-	"\"{{.name}}\" cluster does not exist. Proceeding ahead with cleanup.": "",
 	"\"{{.name}}\" profile does not exist": "Profil \"{{.name}}\" nie istnieje",
+	"\"{{.name}}\" profile does not exist, trying anyways.": "",
 	"\"{{.profile_name}}\" VM does not exist, nothing to stop": "Maszyna wirtualna \"{{.profile_name}}\" nie istnieje. Nie można zatrzymać",
 	"\"{{.profile_name}}\" host does not exist, unable to show an IP": "Profil \"{{.profile_name}}\" nie istnieje. Nie można wyświetlić adresu IP ",
 	"\"{{.profile_name}}\" stopped.": "Zatrzymano \"{{.profile_name}}\"",
 	"'none' driver does not support 'minikube docker-env' command": "sterownik 'none' nie wspiera komendy 'minikube docker-env'",
 	"'none' driver does not support 'minikube mount' command": "sterownik 'none' nie wspiera komendy 'minikube mount'",
 	"'none' driver does not support 'minikube ssh' command": "sterownik 'none' nie wspiera komendy 'minikube ssh'",
+	"'{{.driver}}' driver reported an issue: {{.error}}": "",
+	"- {{.profile}}": "",
 	"A VPN or firewall is interfering with HTTP access to the minikube VM. Alternatively, try a different VM driver: https://minikube.sigs.k8s.io/docs/start/": "",
-	"A firewall is blocking Docker within the minikube VM from reaching the internet. You may need to configure it to use a proxy.": "",
+	"A firewall is blocking Docker the minikube VM from reaching the internet. You may need to configure it to use a proxy.": "",
 	"A firewall is interfering with minikube's ability to make outgoing HTTPS requests. You may need to change the value of the HTTPS_PROXY environment variable.": "",
 	"A firewall is likely blocking minikube from reaching the internet. You may need to configure minikube to use a proxy.": "",
 	"A set of apiserver IP Addresses which are used in the generated certificate for kubernetes.  This can be used if you want to make the apiserver available from outside the machine": "",
-	"A set of apiserver IP Addresses which are used in the generated certificate for kubernetes. This can be used if you want to make the apiserver available from outside the machine": "",
 	"A set of apiserver names which are used in the generated certificate for kubernetes.  This can be used if you want to make the apiserver available from outside the machine": "",
-	"A set of apiserver names which are used in the generated certificate for kubernetes. This can be used if you want to make the apiserver available from outside the machine": "", 
-	"A set of key=value pairs that describe configuration that may be passed to different components.\nThe key should be '.' separated, and the first part before the dot is the component to apply the configuration to.\nValid components are: kubelet, kubeadm, apiserver, controller-manager, etcd, proxy, scheduler\nValid kubeadm parameters:": "",
 	"A set of key=value pairs that describe feature gates for alpha/experimental features.": "",
 	"Access the kubernetes dashboard running within the minikube cluster": "Dostęp do dashboardu uruchomionego w klastrze kubernetesa w minikube",
 	"Add an image to local cache.": "",
@@ -32,22 +32,34 @@
 	"Amount of RAM allocated to the minikube VM (format: \u003cnumber\u003e[\u003cunit\u003e], where unit = b, k, m or g).": "Ilość zarezerwowanej pamieci RAM dla maszyny wirtualnej minikube (format: \u003cnumber\u003e[\u003cunit\u003e], where unit = b, k, m or )",
 	"Amount of time to wait for a service in seconds": "Czas oczekiwania na serwis w sekundach",
 	"Amount of time to wait for service in seconds": "Czas oczekiwania na servis w sekundach",
+	"Another hypervisor, such as VirtualBox, is conflicting with KVM. Please stop the other hypervisor, or use --vm-driver to switch to it.": "",
+	"Automatically selected the {{.experimental}}'{{.driver}}' driver": "",
+	"Automatically selected the {{.experimental}}'{{.driver}}' driver (alternates: {{.alternates}})": "",
 	"Available Commands": "Dostępne polecenia",
 	"Basic Commands:": "Podstawowe polecenia",
+	"Block until the apiserver is servicing API requests": "",
 	"Cannot find directory {{.path}} for mount": "Nie można odnoleść folderu {{.path}} do zamontowania",
+	"Cannot use both --output and --format options": "",
+	"Check output of 'journalctl -xeu kubelet', try passing --extra-config=kubelet.cgroup-driver=systemd to minikube start": "",
+	"Check that SELinux is disabled, and that the provided apiserver flags are valid": "",
 	"Check that minikube is running and that you have specified the correct namespace (-n flag) if required.": "Upewnij się że minikube zpstało uruchomione i że podano poprawną przestrzeń nazw(-n flag) celem zamontowania",
+	"Check that the provided apiserver flags are valid": "",
 	"Check that your --kubernetes-version has a leading 'v'. For example: 'v1.1.14'": "Upewnij się że --kubernetes-version ma 'v' z przodu. Na przykład `v1.1.14`",
-	"Check that your apiserver flags are valid, or run 'minikube delete'": "",
 	"Check your firewall rules for interference, and run 'virt-host-validate' to check for KVM configuration issues. If you are running minikube within a VM, consider using --vm-driver=none": "",
 	"Configuration and Management Commands:": "Polecenia konfiguracji i zarządzania",
+	"Configure a default route on this Linux host, or use another --vm-driver that does not require it": "",
 	"Configure an external network switch following the official documentation, then add `--hyperv-virtual-switch=\u003cswitch-name\u003e` to `minikube start`": "",
 	"Configures the addon w/ADDON_NAME within minikube (example: minikube addons configure registry-creds). For a list of available addons use: minikube addons list": "",
 	"Configuring environment for Kubernetes {{.k8sVersion}} on {{.runtime}} {{.runtimeVersion}}": "Konfiguruje środowisko dla: Kubernetesa w wersji {{.k8sVersion}} na {{.runtime}} {{.runtimeVersion}}",
 	"Configuring local host environment ...": "Konfiguruje lokalne środowisko hosta",
 	"Confirm that you have a working internet connection and that your VM has not run out of resources by using: 'minikube logs'": "",
 	"Confirm that you have supplied the correct value to --hyperv-virtual-switch using the 'Get-VMSwitch' command": "",
+	"Could not get profile flag": "",
+	"Could not process error from failed deletion": "",
+	"Could not process errors from failed deletion": "",
 	"Country code of the image mirror to be used. Leave empty to use the global one. For Chinese mainland users, set it to cn.": "",
 	"Created a new profile : {{.profile_name}}": "Stworzono nowy profil : {{.profile_name}}",
+	"Creating Kubernetes in {{.driver_name}} container with (CPUs={{.number_of_cpus}}), Memory={{.memory_size}}MB ({{.host_memory_size}}MB available) ...": "",
 	"Creating a new profile failed": "Tworzenie nowego profilu nie powiodło się",
 	"Creating mount {{.name}} ...": "",
 	"Creating {{.driver_name}} VM (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...": "Tworzenie {{.driver_name}} (CPUs={{.number_of_cpus}}, Pamięć={{.memory_size}}MB, Dysk={{.disk_size}}MB)...",
@@ -62,7 +74,6 @@
 	"Disable dynamic memory in your VM manager, or pass in a larger --memory value": "",
 	"Disables the addon w/ADDON_NAME within minikube (example: minikube addons disable dashboard). For a list of available addons use: minikube addons list": "",
 	"Disables the filesystem mounts provided by the hypervisors": "",
-	"Disk size allocated to the minikube VM (format: \u003cnumber\u003e[\u003cunit\u003e], where unit = b, k, m or g)": "",
 	"Disk size allocated to the minikube VM (format: \u003cnumber\u003e[\u003cunit\u003e], where unit = b, k, m or g).": "",
 	"Display dashboard URL instead of opening a browser": "",
 	"Display the kubernetes addons URL in the CLI instead of opening it in the default browser": "",
@@ -82,26 +93,28 @@
 	"ERROR creating `registry-creds-ecr` secret: {{.error}}": "",
 	"ERROR creating `registry-creds-gcr` secret: {{.error}}": "",
 	"Either systemctl is not installed, or Docker is broken. Run 'sudo systemctl start docker' and 'journalctl -u docker'": "",
+	"Enable addons. see `minikube addons list` for a list of valid addon names.": "",
 	"Enable experimental NVIDIA GPU support in minikube": "aktywuj eksperymentalne wsparcie minikube dla NVIDIA GPU",
 	"Enable host resolver for NAT DNS requests (virtualbox driver only)": "",
+	"Enable istio needs {{.minMem}} MB of memory and {{.minCpus}} CPUs.": "",
 	"Enable proxy for NAT DNS requests (virtualbox driver only)": "",
-	"Enable the default CNI plugin (/etc/cni/net.d/k8s.conf). Used in conjunction with \\\"--network-plugin=cni\\": "",
 	"Enable the default CNI plugin (/etc/cni/net.d/k8s.conf). Used in conjunction with \\\"--network-plugin=cni\\\".": "",
 	"Enables the addon w/ADDON_NAME within minikube (example: minikube addons enable dashboard). For a list of available addons use: minikube addons list": "",
+	"Enabling '{{.name}}' returned an error: {{.error}}": "",
+	"Enabling addons: {{.addons}}": "",
 	"Enabling dashboard ...": "",
+	"Ensure that CRI-O is installed and healthy: Run 'sudo systemctl start crio' and 'journalctl -u crio'. Alternatively, use --container-runtime=docker": "",
+	"Ensure that Docker is installed and healthy: Run 'sudo systemctl start docker' and 'journalctl -u docker'. Alternatively, select another value for --vm-driver": "",
+	"Ensure that the user listed in /etc/libvirt/qemu.conf has access to your home directory": "",
+	"Ensure that your value for HTTPS_PROXY points to an HTTPS proxy rather than an HTTP proxy": "",
 	"Environment variables to pass to the Docker daemon. (format: key=value)": "Zmienne środowiskowe do przekazania do demona docker (format: klucz-wartość)",
 	"Error checking driver version: {{.error}}": "Błąd podczas sprawdzania wersij sterownika : {{.error}}",
-	"Error creating list template": "",
 	"Error creating minikube directory": "",
-	"Error creating status template": "",
 	"Error creating view template": "",
-	"Error executing list template": "",
-	"Error executing status template": "",
 	"Error executing template": "",
 	"Error executing view template": "",
 	"Error finding port for mount": "",
 	"Error getting IP": "",
-	"Error getting bootstrapper": "",
 	"Error getting client": "",
 	"Error getting client: {{.error}}": "",
 	"Error getting cluster": "",
@@ -110,7 +123,7 @@
 	"Error getting host": "",
 	"Error getting host status": "",
 	"Error getting machine logs": "",
-	"Error getting machine status": "",
+	"Error getting profiles to delete": "",
 	"Error getting service status": "",
 	"Error getting service with namespace: {{.namespace}} and labels {{.labelName}}:{{.addonName}}: {{.error}}": "",
 	"Error getting the host IP address to use from within the VM": "",
@@ -119,7 +132,6 @@
 	"Error loading api": "",
 	"Error loading profile config": "",
 	"Error loading profile config: {{.error}}": "",
-	"Error loading profile {{.name}}: {{.error}}": "",
 	"Error opening service": "",
 	"Error parsing minikube version: {{.error}}": "Bład parsowania wersji minikube: {{.error}}",
 	"Error parsing vmDriver version: {{.error}}": "Błąd parsowania wersji vmDriver: {{.error}}",
@@ -136,17 +148,16 @@
 	"Error: [{{.id}}] {{.error}}": "",
 	"Examples": "Przykłady",
 	"Exiting": "",
-	"Exiting due to driver incompatibility": "",
+	"Exiting.": "",
 	"Failed runtime": "",
 	"Failed to cache ISO": "",
 	"Failed to cache and load images": "",
 	"Failed to cache binaries": "",
-	"Failed to cache images": "",
+	"Failed to cache images to tar": "",
 	"Failed to change permissions for {{.minikube_dir_path}}: {{.error}}": "Nie udało się zmienić uprawnień pliku {{.minikube_dir_path}}: {{.error}}",
 	"Failed to check if machine exists": "",
 	"Failed to check main repository and mirrors for images for images": "",
 	"Failed to delete cluster: {{.error}}": "",
-	"Failed to delete cluster: {{.error}}__1": "",
 	"Failed to delete images": "",
 	"Failed to delete images from config": "",
 	"Failed to download kubectl": "Pobieranie kubectl nie powiodło się",
@@ -160,9 +171,9 @@
 	"Failed to get service URL: {{.error}}": "",
 	"Failed to kill mount process: {{.error}}": "Zabicie procesu nie powiodło się: {{.error}}",
 	"Failed to list cached images": "",
+	"Failed to reload cached images": "",
 	"Failed to remove profile": "Usunięcie profilu nie powiodło się",
 	"Failed to save config": "Zapisywanie konfiguracji nie powiodło się",
-	"Failed to set NO_PROXY Env. Please use `export NO_PROXY=$NO_PROXY,{{.ip}}": "",
 	"Failed to set NO_PROXY Env. Please use `export NO_PROXY=$NO_PROXY,{{.ip}}`.": "",
 	"Failed to setup certs": "Konfiguracja certyfikatów nie powiodła się",
 	"Failed to setup kubeconfig": "Konfiguracja kubeconfig nie powiodła się",
@@ -173,7 +184,6 @@
 	"Flags": "",
 	"Follow": "",
 	"For best results, install kubectl: https://kubernetes.io/docs/tasks/tools/install-kubectl/": "",
-	"For best results, install kubectl: https://kubernetes.io/docs/tasks/tools/install-kubectl/__1": "",
 	"For more information, see:": "",
 	"Force environment to be configured for a specified shell: [fish, cmd, powershell, tcsh, bash, zsh], default is auto-detect": "",
 	"Force minikube to perform possibly dangerous operations": "Wymuś wykonanie potencjalnie niebezpiecznych operacji",
@@ -185,14 +195,19 @@
 	"Gets the status of a local kubernetes cluster": "Pobiera aktualny status klastra kubernetesa",
 	"Gets the status of a local kubernetes cluster.\n\tExit status contains the status of minikube's VM, cluster and kubernetes encoded on it's bits in this order from right to left.\n\tEg: 7 meaning: 1 (for minikube NOK) + 2 (for cluster NOK) + 4 (for kubernetes NOK)": "",
 	"Gets the value of PROPERTY_NAME from the minikube config file": "",
+	"Getting machine config failed": "",
 	"Global Flags": "",
-	"Go template format string for the addon list output.  The format for Go templates can be found here: https://golang.org/pkg/text/template/\nFor the list of accessible variables for the template, see the struct values here: https://godoc.org/k8s.io/minikube/cmd/minikube/cmd/config#AddonListTemplate": "",
 	"Go template format string for the cache list output.  The format for Go templates can be found here: https://golang.org/pkg/text/template/\nFor the list of accessible variables for the template, see the struct values here: https://godoc.org/k8s.io/minikube/cmd/minikube/cmd#CacheListTemplate": "",
 	"Go template format string for the config view output.  The format for Go templates can be found here: https://golang.org/pkg/text/template/\nFor the list of accessible variables for the template, see the struct values here: https://godoc.org/k8s.io/minikube/cmd/minikube/cmd/config#ConfigViewTemplate": "",
 	"Go template format string for the status output.  The format for Go templates can be found here: https://golang.org/pkg/text/template/\nFor the list accessible variables for the template, see the struct values here: https://godoc.org/k8s.io/minikube/cmd/minikube/cmd#Status": "",
 	"Group ID:     {{.groupID}}": "",
 	"Have you set up libvirt correctly?": "Czy napewno skonfigurowano libvirt w sposób prawidłowy?",
 	"Hide the hypervisor signature from the guest in minikube (kvm2 driver only)": "",
+	"Hyperkit is broken. Upgrade to the latest hyperkit version and/or Docker for Desktop. Alternatively, you may choose an alternate --vm-driver": "",
+	"Hyperkit networking is broken. Upgrade to the latest hyperkit version and/or Docker for Desktop. Alternatively, you may choose an alternate --vm-driver": "",
+	"If set, automatically updates drivers to the latest version. Defaults to true.": "",
+	"If set, pause all namespaces": "",
+	"If set, unpause all namespaces": "",
 	"If the above advice does not help, please let us know:": "",
 	"If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --vm-driver=none.": "",
 	"If true, only download and cache files for later use - don't install or start anything.": "",
@@ -200,12 +215,12 @@
 	"If you are running minikube within a VM, consider using --vm-driver=none:": "",
 	"Images Commands:": "",
 	"Insecure Docker registries to pass to the Docker daemon.  The default service CIDR range will automatically be added.": "",
-	"Insecure Docker registries to pass to the Docker daemon. The default service CIDR range will automatically be added.": "",
 	"Install VirtualBox, or select an alternative value for --vm-driver": "",
 	"Install the latest hyperkit binary, and run 'minikube delete'": "",
 	"Invalid size passed in argument: {{.error}}": "Nieprawidłowy rozmiar przekazany w argumencie: {{.error}}",
 	"IsEnabled failed": "",
 	"Kill the mount process spawned by minikube start": "",
+	"Kubernetes {{.new}} is now available. If you would like to upgrade, specify: --kubernetes-version={{.new}}": "",
 	"Kubernetes {{.version}} is not supported by this release of minikube": "",
 	"Launching Kubernetes ...": "Uruchamianie Kubernetesa...",
 	"Launching proxy ...": "",
@@ -230,9 +245,13 @@
 	"Mounting host path {{.sourcePath}} into VM as {{.destinationPath}} ...": "",
 	"Mounts the specified directory into minikube": "Montuje podany katalog wewnątrz minikube",
 	"Mounts the specified directory into minikube.": "Montuje podany katalog wewnątrz minikube",
+	"Multiple errors deleting profiles": "",
+	"Multiple minikube profiles were found -": "",
+	"NIC Type used for host only network. One of Am79C970A, Am79C973, 82540EM, 82543GC, 82545EM, or virtio (virtualbox driver only)": "",
 	"NOTE: This process must stay alive for the mount to be accessible ...": "",
 	"Networking and Connectivity Commands:": "",
 	"No minikube profile was found. You can create one using `minikube start`.": "",
+	"Node may be unable to resolve external DNS records": "",
 	"None of the known repositories in your location are accessible. Using {{.image_repository_name}} as fallback.": "",
 	"None of the known repositories is accessible. Consider specifying an alternative image repository with --image-repository flag": "",
 	"Not passing {{.name}}={{.value}} to docker env.": "",
@@ -243,11 +262,15 @@
 	"Open the addons URL with https instead of http": "",
 	"Open the service URL with https instead of http": "",
 	"Opening kubernetes service  {{.namespace_name}}/{{.service_name}} in default browser...": "",
+	"Opening service {{.namespace_name}}/{{.service_name}} in default browser...": "",
 	"Opening {{.url}} in your default browser...": "",
 	"Opens the addon w/ADDON_NAME within minikube (example: minikube addons open dashboard). For a list of available addons use: minikube addons list": "",
 	"Options:      {{.options}}": "",
 	"Outputs minikube shell completion for the given shell (bash or zsh)": "Zwraca autouzupełnianie poleceń minikube dla powłoki system(bash, zsh)",
 	"Outputs minikube shell completion for the given shell (bash or zsh)\n\n\tThis depends on the bash-completion binary.  Example installation instructions:\n\tOS X:\n\t\t$ brew install bash-completion\n\t\t$ source $(brew --prefix)/etc/bash_completion\n\t\t$ minikube completion bash \u003e ~/.minikube-completion  # for bash users\n\t\t$ minikube completion zsh \u003e ~/.minikube-completion  # for zsh users\n\t\t$ source ~/.minikube-completion\n\tUbuntu:\n\t\t$ apt-get install bash-completion\n\t\t$ source /etc/bash-completion\n\t\t$ source \u003c(minikube completion bash) # for bash users\n\t\t$ source \u003c(minikube completion zsh) # for zsh users\n\n\tAdditionally, you may want to output the completion to a file and source in your .bashrc\n\n\tNote for zsh users: [1] zsh completions are only supported in versions of zsh \u003e= 5.2": "",
+	"Pause": "",
+	"Paused kubelet and {{.count}} containers": "",
+	"Paused kubelet and {{.count}} containers in: {{.namespaces}}": "",
 	"Permissions:  {{.octalMode}} ({{.writtenMode}})": "",
 	"Please enter a value:": "Wprowadź wartość",
 	"Please install the minikube hyperkit VM driver, or select an alternative --vm-driver": "",
@@ -264,22 +287,24 @@
 	"Problems detected in {{.entry}}:": "Wykryto problem w {{.name}}",
 	"Problems detected in {{.name}}:": "Wykryto problem w {{.name}}",
 	"Profile gets or sets the current minikube profile": "Pobiera lub ustawia aktywny profil minikube",
+	"Profile name \"{{.profilename}}\" is minikube keyword. To delete profile use command minikube delete -p \u003cprofile name\u003e": "",
 	"Provide VM UUID to restore MAC address (hyperkit driver only)": "",
 	"Pulling images ...": "",
 	"Reboot to complete VirtualBox installation, and verify that VirtualBox is not blocked by your system": "Uruchom ponownie komputer aby zakończyć instalacje VirtualBox'a i upewnij się że nie jest on blokowany przez twój system",
+	"Reboot to complete VirtualBox installation, verify that VirtualBox is not blocked by your system, and/or use another hypervisor": "",
 	"Rebuild libvirt with virt-network support": "",
 	"Received {{.name}} signal": "",
+	"Reconfiguring existing host ...": "",
 	"Registry mirrors to pass to the Docker daemon": "",
 	"Reinstall VirtualBox and reboot. Alternatively, try the kvm2 driver: https://minikube.sigs.k8s.io/docs/reference/drivers/kvm2/": "",
 	"Reinstall VirtualBox and verify that it is not blocked: System Preferences -\u003e Security \u0026 Privacy -\u003e General -\u003e Some system software was blocked from loading": "",
 	"Related issues:": "Powiązane problemy",
-	"Relaunching Kubernetes using {{.bootstrapper}} ...": "",
+	"Removed all traces of the \"{{.name}}\" cluster.": "",
 	"Removing {{.directory}} ...": "",
 	"Requested cpu count {{.requested_cpus}} is less than the minimum allowed of {{.minimum_cpus}}": "",
 	"Requested disk size {{.requested_size}} is less than minimum of {{.minimum_size}}": "",
 	"Requested memory allocation ({{.memory}}MB) is less than the default memory allocation of {{.default_memorysize}}MB. Beware that minikube might not work correctly or crash unexpectedly.": "",
 	"Requested memory allocation {{.requested_size}} is less than the minimum allowed of {{.minimum_size}}": "",
-	"Retriable failure: {{.error}}": "",
 	"Retrieve the ssh identity key path of the specified cluster": "Pozyskuje ścieżkę do klucza ssh dla wyspecyfikowanego klastra",
 	"Retrieve the ssh identity key path of the specified cluster.": "Pozyskuje ścieżkę do klucza ssh dla wyspecyfikowanego klastra.",
 	"Retrieves the IP address of the running cluster": "Pobiera adres IP aktualnie uruchomionego klastra",
@@ -287,22 +312,27 @@
 	"Retrieves the IP address of the running cluster, checks it\n\t\t\twith IP in kubeconfig, and corrects kubeconfig if incorrect.": "",
 	"Returns the value of PROPERTY_NAME from the minikube config file.  Can be overwritten at runtime by flags or environmental variables.": "",
 	"Run 'kubectl describe pod coredns -n kube-system' and check for a firewall or DNS conflict": "",
-	"Run 'minikube delete' to delete the stale VM": "",
+	"Run 'minikube delete' to delete the stale VM, or and ensure that minikube is running as the same user you are issuing this command with": "",
 	"Run kubectl": "Uruchamia kubectl",
 	"Run minikube from the C: drive.": "",
 	"Run the kubernetes client, download it if necessary. Remember -- after kubectl!\n\nExamples:\nminikube kubectl -- --help\nminikube kubectl -- get pods --namespace kube-system": "",
 	"Run the minikube command as an Administrator": "",
+	"Run: 'chmod 600 $HOME/.kube/config'": "",
 	"Running on localhost (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...": "",
+	"Selecting {{.experimental}}'{{.driver}}' driver from existing profile (alternates: {{.alternates}})": "",
+	"Selecting {{.experimental}}'{{.driver}}' driver from user configuration (alternates: {{.alternates}})": "",
 	"Set failed": "",
+	"Set flag to delete all profiles": "",
+	"Set this flag to delete the '.minikube' folder from your user directory.": "",
 	"Sets an individual value in a minikube config file": "",
 	"Sets the PROPERTY_NAME config value to PROPERTY_VALUE\n\tThese values can be overwritten by flags or environment variables at runtime.": "",
 	"Sets up docker env variables; similar to '$(docker-machine env)'": "Ustawia zmienne środowiskowe dockera. Podobne do `(docker-machine env)`",
 	"Sets up docker env variables; similar to '$(docker-machine env)'.": "Ustawia zmienne środowiskowe dockera. Podobne do `(docker-machine env)`",
 	"Setting profile failed": "Ustawianie profilu nie powiodło się",
+	"Show a list of global command-line options (applies to all commands).": "",
 	"Show only log entries which point to known problems": "Pokaż logi które wskazują na znane problemy",
 	"Show only the most recent journal entries, and continuously print new entries as they are appended to the journal.": "",
 	"Skipped switching kubectl context for {{.profile_name}} because --keep-context was set.": "Zignorowano zmianę kontekstu kubectl ponieważ --keep-context zostało przekazane",
-	"Sorry that minikube crashed. If this was unexpected, we would love to hear from you:": "",
 	"Sorry, Kubernetes {{.version}} is not supported by this release of minikube": "",
 	"Sorry, completion support is not yet implemented for {{.name}}": "",
 	"Sorry, the kubeadm.{{.parameter_name}} parameter is currently not supported by --extra-config": "",
@@ -319,15 +349,22 @@
 	"Stopping \"{{.profile_name}}\" in {{.driver_name}} ...": "Zatrzymywanie \"{{.profile_name}}\" - {{.driver_name}}...",
 	"Stops a local kubernetes cluster running in Virtualbox. This command stops the VM\nitself, leaving all files intact. The cluster can be started again with the \"start\" command.": "",
 	"Stops a running local kubernetes cluster": "Zatrzymuje lokalny klaster kubernetesa",
+	"Successfully deleted all profiles": "",
 	"Successfully mounted {{.sourcePath}} to {{.destinationPath}}": "Pomyślnie zamontowano {{.sourcePath}} do {{.destinationPath}}",
+	"Successfully powered off Hyper-V. minikube driver -- {{.driver}}": "",
+	"Successfully purged minikube directory located at - [{{.minikubeDirectory}}]": "",
 	"Suggestion: {{.advice}}": "Sugestia: {{.advice}}",
+	"Suggestion: {{.fix}}": "",
 	"Target directory {{.path}} must be an absolute path": "",
 	"The \"{{.cluster_name}}\" cluster has been deleted.": "Klaster \"{{.cluster_name}}\" został usunięty",
 	"The \"{{.driver_name}}\" driver requires root privileges. Please run minikube using 'sudo minikube --vm-driver={{.driver_name}}'.": "Sterownik \"{{.driver_name}}\" wymaga uprawnień root'a. Użyj 'sudo minikube --vm-driver={{.driver_name}}'",
 	"The \"{{.driver_name}}\" driver should not be used with root privileges.": "",
 	"The \"{{.name}}\" cluster has been deleted.": "Klaster \"{{.name}}\" został usunięty",
+	"The 'none' driver does not respect the --cpus flag": "",
+	"The 'none' driver does not respect the --memory flag": "",
 	"The 'none' driver does not support multiple profiles: https://minikube.sigs.k8s.io/docs/reference/drivers/none/": "",
 	"The 'none' driver provides limited isolation and may reduce system security and reliability.": "",
+	"The '{{.addonName}}' addon is enabled": "",
 	"The '{{.driver}}' driver requires elevated permissions. The following commands will be executed:\\n\\n{{ .example }}\\n": "",
 	"The CIDR to be used for service cluster IPs.": "",
 	"The CIDR to be used for the minikube VM (virtualbox driver only)": "",
@@ -339,25 +376,26 @@
 	"The VM that minikube is configured for no longer exists. Run 'minikube delete'": "",
 	"The apiserver listening port": "API nasłuchuje na porcie:",
 	"The apiserver name which is used in the generated certificate for kubernetes.  This can be used if you want to make the apiserver available from outside the machine": "",
-	"The apiserver name which is used in the generated certificate for kubernetes. This can be used if you want to make the apiserver available from outside the machine": "",
-	"The argument to pass the minikube mount command on start": "",
 	"The argument to pass the minikube mount command on start.": "",
 	"The cluster dns domain name used in the kubernetes cluster": "Domena dns clastra użyta przez kubernetesa",
 	"The container runtime to be used (docker, crio, containerd)": "Runtime konteneryzacji (docker, crio, containerd).",
 	"The container runtime to be used (docker, crio, containerd).": "",
-	"The cri socket path to be used": "",
 	"The cri socket path to be used.": "",
 	"The docker host is currently not running": "",
 	"The docker service is currently not active": "Serwis docker jest nieaktywny",
 	"The driver '{{.driver}}' is not supported on {{.os}}": "Sterownik '{{.driver}} jest niewspierany przez system {{.os}}",
+	"The driver {{.experimental}} '{{.driver}}' is not supported on {{.os}}": "",
 	"The existing \"{{.profile_name}}\" VM that was created using the \"{{.old_driver}}\" driver, and is incompatible with the \"{{.driver}}\" driver.": "",
 	"The hyperv virtual switch name. Defaults to first found. (hyperv driver only)": "",
+	"The hypervisor does not appear to be configured properly. Run 'minikube start --alsologtostderr -v=1' and inspect the error code": "",
 	"The initial time interval for each check that wait performs in seconds": "",
 	"The kubernetes version that the minikube VM will use (ex: v1.2.3)": "Wersja kubernetesa która zostanie użyta przez wirtualną maszyna minikube (np. v1.2.3)",
+	"The machine-driver specified is failing to start. Try running 'docker-machine-driver-\u003ctype\u003e version'": "",
 	"The minikube VM is offline. Please run 'minikube start' to start it again.": "",
 	"The name of the network plugin": "Nazwa pluginu sieciowego",
 	"The name of the network plugin.": "Nazwa pluginu sieciowego",
 	"The number of bytes to use for 9p packet payload": "",
+	"The output format. One of 'json', 'table'": "",
 	"The path on the file system where the docs in markdown need to be saved": "",
 	"The service namespace": "",
 	"The services namespace": "",
@@ -366,64 +404,72 @@
 	"The value passed to --format is invalid: {{.error}}": "Wartość przekazana do --format jest nieprawidłowa: {{.error}}",
 	"The vmwarefusion driver is deprecated and support for it will be removed in a future release.\n\t\t\tPlease consider switching to the new vmware unified driver, which is intended to replace the vmwarefusion driver.\n\t\t\tSee https://minikube.sigs.k8s.io/docs/reference/drivers/vmware/ for more information.\n\t\t\tTo disable this message, run [minikube config set ShowDriverDeprecationNotification false]": "",
 	"The {{.driver_name}} driver should not be used with root privileges.": "{{.driver_name}} nie powinien byc używany z przywilejami root'a.",
-	"There appears to be another hypervisor conflicting with KVM. Please stop the other hypervisor, or use --vm-driver to switch to it.": "",
-	"There's a new version for '{{.driver_executable}}'. Please consider upgrading. {{.documentation_url}}": "",
 	"These changes will take effect upon a minikube delete and then a minikube start": "",
 	"This addon does not have an endpoint defined for the 'addons open' command.\nYou can add one by annotating a service with the label {{.labelName}}:{{.addonName}}": "",
 	"This can also be done automatically by setting the env var CHANGE_MINIKUBE_NONE_USER=true": "",
 	"This will keep the existing kubectl context and will create a minikube context.": "",
-	"This will start the mount daemon and automatically mount files into minikube": "",
 	"This will start the mount daemon and automatically mount files into minikube.": "",
 	"Tip: To remove this root owned cluster, run: sudo {{.cmd}} delete": "",
-	"Tip: Use 'minikube start -p \u003cname\u003e' to create a new cluster, or 'minikube delete' to delete this one.": "",
 	"To connect to this cluster, use: kubectl --context={{.name}}": "Aby połączyć się z klastrem użyj: kubectl --context={{.name}}",
 	"To connect to this cluster, use: kubectl --context={{.profile_name}}": "Aby połaczyć się z klastem uzyj: kubectl --context={{.profile_name}}",
 	"To disable this notice, run: 'minikube config set WantUpdateNotification false'": "Aby wyłączyć te notyfikację, użyj: 'minikube config set WantUpdateNotification false'",
 	"To disable this notice, run: 'minikube config set WantUpdateNotification false'\\n": "",
-	"To proceed, either:\n      1) Delete the existing VM using: '{{.command}} delete'\n      or\n      2) Restart with the existing driver: '{{.command}} start --vm-driver={{.old_driver}}'": "",
+	"To proceed, either:\n\n    1) Delete the existing \"{{.profile_name}}\" cluster using: '{{.command}} delete'\n\n    * or *\n\n    2) Start the existing \"{{.profile_name}}\" cluster using: '{{.command}} start --vm-driver={{.old_driver}}'": "",
+	"To see addons list for other profiles use: `minikube addons -p name list`": "",
 	"To start minikube with HyperV Powershell must be in your PATH`": "Aby uruchomić minikube z HyperV Powershell musi znajdować się w zmiennej PATH",
 	"To use kubectl or minikube commands as your own user, you may need to relocate them. For example, to overwrite your own settings, run:": "",
 	"Troubleshooting Commands:": "",
+	"Trying to delete invalid profile {{.profile}}": "",
 	"Unable to bind flags": "",
+	"Unable to determine a default driver to use. Try specifying --vm-driver, or see https://minikube.sigs.k8s.io/docs/start/": "",
 	"Unable to enable dashboard": "",
 	"Unable to fetch latest version info": "",
 	"Unable to generate docs": "",
 	"Unable to generate the documentation. Please ensure that the path specified is a directory, exists \u0026 you have permission to write to it.": "",
 	"Unable to get VM IP address": "",
-	"Unable to get bootstrapper: {{.error}}": "",
+	"Unable to get addon status for {{.name}}: {{.error}}": "",
 	"Unable to get current user": "",
 	"Unable to get runtime": "",
-	"Unable to get the status of the cluster.": "",
+	"Unable to get the status of the {{.name}} cluster.": "",
 	"Unable to kill mount process: {{.error}}": "",
 	"Unable to load cached images from config file.": "",
 	"Unable to load cached images: {{.error}}": "",
 	"Unable to load config: {{.error}}": "",
 	"Unable to parse \"{{.kubernetes_version}}\": {{.error}}": "",
+	"Unable to parse default Kubernetes version from constants: {{.error}}": "",
 	"Unable to parse oldest Kubernetes version from constants: {{.error}}": "",
 	"Unable to pull images, which may be OK: {{.error}}": "",
 	"Unable to remove machine directory: %v": "",
 	"Unable to start VM": "Nie można uruchomić maszyny wirtualnej",
+	"Unable to start VM. Please investigate and run 'minikube delete' if possible": "",
 	"Unable to stop VM": "Nie można zatrzymać maszyny wirtualnej",
 	"Unable to update {{.driver}} driver: {{.error}}": "",
+	"Unable to verify SSH connectivity: {{.error}}. Will retry...": "",
 	"Uninstalling Kubernetes {{.kubernetes_version}} using {{.bootstrapper_name}} ...": "",
 	"Unmounting {{.path}} ...": "",
+	"Unpaused kubelet and {{.count}} containers": "",
+	"Unpaused kubelet and {{.count}} containers in: {{.namespaces}}": "",
 	"Unset the KUBECONFIG environment variable, or verify that it does not point to an empty or otherwise invalid path": "",
 	"Unset variables instead of setting them": "",
 	"Update server returned an empty list": "",
 	"Upgrade to QEMU v3.1.0+, run 'virt-host-validate', or ensure that you are not running in a nested VM environment.": "",
-	"Upgrading from Kubernetes {{.old}} to {{.new}}": "",
 	"Usage": "",
 	"Usage: minikube completion SHELL": "",
 	"Usage: minikube delete": "",
+	"Usage: minikube delete --all --purge": "",
 	"Use \"{{.CommandPath}} [command] --help\" for more information about a command.": "",
+	"Use 'kubect get po -A' to find the correct and namespace name": "",
+	"Use -A to specify all namespaces": "",
 	"Use VirtualBox to remove the conflicting VM and/or network interfaces": "",
 	"Use native Golang SSH client (default true). Set to 'false' to use the command line 'ssh' command when accessing the docker machine. Useful for the machine drivers when they will not start with 'Waiting for SSH'.": "",
 	"User ID:      {{.userID}}": "",
 	"Userspace file server is shutdown": "",
 	"Userspace file server:": "",
 	"Using image repository {{.name}}": "",
+	"Using the '{{.runtime}}' runtime with the 'none' driver is an untested configuration!": "",
 	"Using the running {{.driver_name}} \"{{.profile_name}}\" VM ...": "",
 	"VM driver is one of: %v": "Sterownik wirtualnej maszyny to jeden z: %v",
+	"VM is unable to access {{.repository}}, you may need to configure a proxy or set --image-repository": "",
 	"Verify that your HTTP_PROXY and HTTPS_PROXY environment variables are set correctly.": "Weryfikuję czy zmienne HTTP_PROXY i HTTPS_PROXY sa ustawione poprawnie",
 	"Verify the IP address of the running cluster in kubeconfig.": "Weryfikuję adres IP działającego klastra w kubeconfig",
 	"Verifying dashboard health ...": "Weryfikuję status dashboardu",
@@ -434,13 +480,12 @@
 	"VirtualBox cannot create a network, probably because it conflicts with an existing network that minikube no longer knows about. Try running 'minikube delete'": "",
 	"VirtualBox is broken. Disable real-time anti-virus software, reboot, and reinstall VirtualBox if the problem continues.": "",
 	"VirtualBox is broken. Reinstall VirtualBox, reboot, and run 'minikube delete'.": "",
+	"VirtualBox is unable to find its network interface. Try upgrading to the latest release and rebooting.": "",
 	"Virtualization support is disabled on your computer. If you are running minikube within a VM, try '--vm-driver=none'. Otherwise, consult your systems BIOS manual for how to enable virtualization.": "",
 	"Wait failed": "",
 	"Wait failed: {{.error}}": "",
-	"Wait until Kubernetes core services are healthy before exiting": "",
-	"Wait until Kubernetes core services are healthy before exiting.": "",
 	"Waiting for SSH access ...": "Oczekiwanie na połaczenie SSH...",
-	"Waiting for the host to be provisioned ...": "",
+	"Waiting for cluster to come online ...": "",
 	"Waiting for:": "Oczekiwanie na :",
 	"Where to root the NFS Shares, defaults to /nfsshares (hyperkit driver only)": "",
 	"You appear to be using a proxy, but your NO_PROXY environment does not include the minikube IP ({{.ip_address}}). Please see {{.documentation_url}} for more details": "",
@@ -454,50 +499,63 @@
 	"Your minikube vm is not running, try minikube start.": "",
 	"addon '{{.name}}' is currently not enabled.\nTo enable this addon run:\nminikube addons enable {{.name}}": "",
 	"addon '{{.name}}' is not a valid addon packaged with minikube.\nTo see the list of available addons run:\nminikube addons list": "",
-	"addon list failed": "",
-	"addons modifies minikube addons files using subcommands like \"minikube addons enable heapster\"": "",
+	"addons modifies minikube addons files using subcommands like \"minikube addons enable dashboard\"": "",
 	"api load": "",
 	"bash completion failed": "",
-	"browser failed to open url: {{.error}}": "",
 	"call with cleanup=true to remove old tunnels": "",
 	"command runner": "",
 	"config modifies minikube config files using subcommands like \"minikube config set vm-driver kvm\"\nConfigurable fields:\\n\\n": "",
 	"config view failed": "",
 	"dashboard service is not running: {{.error}}": "",
 	"disable failed": "",
+	"dry-run mode. Validates configuration, but does not mutate system state": "",
+	"dry-run validation complete!": "",
 	"enable failed": "",
 	"error creating clientset": "",
 	"error creating machine client": "",
-	"error getting driver": "",
 	"error parsing the input ip address for mount": "",
 	"error starting tunnel": "",
 	"failed to open browser: {{.error}}": "Nie udało się otworzyć przeglądarki: {{.error}}",
 	"if true, will embed the certs in kubeconfig.": "",
+	"kubeadm detected a TCP port conflict with another process: probably another local Kubernetes installation. Run lsof -p\u003cport\u003e to find the process and kill it": "",
 	"kubectl and minikube configuration will be stored in {{.home_folder}}": "konfiguracja minikube i kubectl będzie przechowywana w katalogu {{.home_dir}}",
 	"kubectl not found in PATH, but is required for the dashboard. Installation guide: https://kubernetes.io/docs/tasks/tools/install-kubectl/": "kubectl nie zostało odnaleźione w zmiennej środowiskowej ${PATH}. Instrukcja instalacji:  https://kubernetes.io/docs/tasks/tools/install-kubectl/",
 	"kubectl proxy": "",
 	"logdir set failed": "",
 	"max time to wait per Kubernetes core services to be healthy.": "",
-	"minikube is not running, so the service cannot be accessed": "",
+	"minikube addons list --output OUTPUT. json, list": "",
+	"minikube is exiting due to an error. If the above message is not useful, open an issue:": "",
 	"minikube is unable to access the Google Container Registry. You may need to configure it to use a HTTP proxy.": "",
+	"minikube is unable to connect to the VM: {{.error}}\n\nThis is likely due to one of two reasons:\n\n- VPN or firewall interference\n- {{.hypervisor}} network configuration issue\n\nSuggested workarounds:\n\n- Disable your local VPN or firewall software\n- Configure your local VPN or firewall to allow access to {{.ip}}\n- Restart or reinstall {{.hypervisor}}\n- Use an alternative --vm-driver": "",
 	"minikube profile was successfully set to {{.profile_name}}": "",
+	"minikube status --output OUTPUT. json, text": "",
 	"minikube {{.version}} is available! Download it: {{.url}}": "minikube {{.version}} jest dostępne! Pobierz je z: {{.url}}",
+	"mkcmp is used to compare performance of two minikube binaries": "",
 	"mount argument \"{{.value}}\" must be in form: \u003csource directory\u003e:\u003ctarget directory\u003e": "",
 	"mount failed": "Montowanie się nie powiodło",
+	"namespaces to pause": "",
+	"namespaces to unpause": "",
 	"not enough arguments ({{.ArgCount}}).\\nusage: minikube config set PROPERTY_NAME PROPERTY_VALUE": "",
+	"pause containers": "",
 	"profile sets the current minikube profile, or gets the current profile if no arguments are provided.  This is used to run and manage multiple minikube instance.  You can return to the default minikube profile by running `minikube profile default`": "",
+	"reload cached images.": "",
+	"reloads images previously added using the 'cache add' subcommand": "",
 	"service {{.namespace_name}}/{{.service_name}} has no node port": "",
 	"stat failed": "",
+	"status json failure": "",
+	"status text failure": "",
 	"toom any arguments ({{.ArgCount}}).\\nusage: minikube config set PROPERTY_NAME PROPERTY_VALUE": "",
-	"tunnel creates a route to services deployed with type LoadBalancer and sets their Ingress to their ClusterIP": "",
+	"tunnel creates a route to services deployed with type LoadBalancer and sets their Ingress to their ClusterIP. for a detailed example see https://minikube.sigs.k8s.io/docs/tasks/loadbalancer": "",
 	"tunnel makes services of type LoadBalancer accessible on localhost": "",
 	"unable to bind flags": "",
+	"unable to delete minikube config folder": "",
 	"unable to set logtostderr": "",
+	"unpause Kubernetes": "",
 	"unset failed": "",
-	"unset minikube profile": "",
 	"unsets PROPERTY_NAME from the minikube config file.  Can be overwritten by flags or environmental variables": "",
 	"unsets an individual value in a minikube config file": "",
 	"unsupported driver: {{.name}}": "nie wspierany sterownik: {{.name}}",
+	"unsupported or missing driver: {{.name}}": "",
 	"update config": "",
 	"usage: minikube addons configure ADDON_NAME": "",
 	"usage: minikube addons disable ADDON_NAME": "",
@@ -505,16 +563,21 @@
 	"usage: minikube addons list": "",
 	"usage: minikube addons open ADDON_NAME": "",
 	"usage: minikube config unset PROPERTY_NAME": "",
+	"usage: minikube delete": "",
+	"usage: minikube delete --all": "",
 	"usage: minikube profile [MINIKUBE_PROFILE_NAME]": "",
 	"zsh completion failed": "",
 	"{{.addonName}} was successfully enabled": "{{.addonName}} został aktywowany pomyślnie",
+	"{{.driver}} does not appear to be installed": "",
+	"{{.driver}} does not appear to be installed, but is specified by an existing profile. Please run 'minikube delete' or install {{.driver}}": "",
 	"{{.extra_option_component_name}}.{{.key}}={{.value}}": "",
 	"{{.machine}} IP has been updated to point at {{.ip}}": "",
 	"{{.machine}} IP was already correctly configured for {{.ip}}": "",
 	"{{.name}} cluster does not exist": "Klaster {{.name}} nie istnieje",
 	"{{.name}} has no available configuration options": "{{.name}} nie posiada opcji configuracji",
 	"{{.name}} was successfully configured": "{{.name}} skonfigurowano pomyślnie",
+	"{{.path}} is version {{.client_version}}, and is incompatible with Kubernetes {{.cluster_version}}. You will need to update {{.path}} or use 'minikube kubectl' to connect with this cluster": "",
 	"{{.prefix}}minikube {{.version}} on {{.platform}}": "{{.prefix}}minikube {{.version}} na {{.platform}}",
 	"{{.type}} is not yet a supported filesystem. We will try anyways!": "{{.type}} nie jest wspierany przez system plików. I tak spróbujemy!",
 	"{{.url}} is not accessible: {{.error}}": ""
-}
+}
\ No newline at end of file
diff --git a/translations/zh-CN.json b/translations/zh-CN.json
index 02b0ab7ad2..45b4b035fe 100644
--- a/translations/zh-CN.json
+++ b/translations/zh-CN.json
@@ -1,7 +1,9 @@
 {
+	"\"The '{{.minikube_addon}}' addon is disabled": "",
 	"\"{{.minikube_addon}}\" was successfully disabled": "已成功禁用 \"{{.minikube_addon}}\"",
 	"\"{{.name}}\" cluster does not exist. Proceeding ahead with cleanup.": "\"{{.name}}\" 集群不存在，将继续清理",
 	"\"{{.name}}\" profile does not exist": "“{{.name}}”配置文件不存在",
+	"\"{{.name}}\" profile does not exist, trying anyways.": "",
 	"\"{{.profile_name}}\" VM does not exist, nothing to stop": "\"{{.profile_name}}\" 虚拟机不存在，没有什么可供停止的",
 	"\"{{.profile_name}}\" host does not exist, unable to show an IP": "\"{{.profile_name}}\" 主机不存在，无法显示其IP",
 	"\"{{.profile_name}}\" stopped.": "\"{{.profile_name}}\" 已停止",
@@ -38,6 +40,8 @@
 	"Another hypervisor, such as VirtualBox, is conflicting with KVM. Please stop the other hypervisor, or use --vm-driver to switch to it.": "另外一个管理程序与 KVM 产生了冲突，如 VirtualBox。请停止其他的管理程序",
 	"Automatically selected the '{{.driver}}' driver": "自动选择 '{{.driver}}' 驱动",
 	"Automatically selected the '{{.driver}}' driver (alternates: {{.alternates}})": "自动选择 '{{.driver}}' 驱动（可选项：{{.alternates}}）",
+	"Automatically selected the {{.experimental}}'{{.driver}}' driver": "",
+	"Automatically selected the {{.experimental}}'{{.driver}}' driver (alternates: {{.alternates}})": "",
 	"Available Commands": "可用命令",
 	"Basic Commands:": "基本命令：",
 	"Block until the apiserver is servicing API requests": "阻塞直到 apiserver 为 API 请求提供服务",
@@ -106,6 +110,8 @@
 	"Enable the default CNI plugin (/etc/cni/net.d/k8s.conf). Used in conjunction with \\\"--network-plugin=cni\\": "启用默认 CNI 插件 (/etc/cni/net.d/k8s.conf)。与“--network-plugin=cni”结合使用",
 	"Enable the default CNI plugin (/etc/cni/net.d/k8s.conf). Used in conjunction with \\\"--network-plugin=cni\\\".": "启用默认 CNI 插件 (/etc/cni/net.d/k8s.conf)。与“--network-plugin=cni”结合使用。",
 	"Enables the addon w/ADDON_NAME within minikube (example: minikube addons enable dashboard). For a list of available addons use: minikube addons list": "启动 minikube 插件 w/ADDON_NAME（例如：minikube addons enable dashboard）。查看相关可用的插件列表，请使用：minikube addons list",
+	"Enabling '{{.name}}' returned an error: {{.error}}": "",
+	"Enabling addons: {{.addons}}": "",
 	"Enabling dashboard ...": "正在开启 dashboard ...",
 	"Ensure that CRI-O is installed and healthy: Run 'sudo systemctl start crio' and 'journalctl -u crio'. Alternatively, use --container-runtime=docker": "确保 CRI-O 已安装且正常运行：执行 'sudo systemctl start crio' and 'journalctl -u crio'。或者使用 --container-runtime=docker",
 	"Ensure that Docker is installed and healthy: Run 'sudo systemctl start docker' and 'journalctl -u docker'. Alternatively, select another value for --vm-driver": "确保 Docker 已安装且正常运行： 执行 'sudo systemctl start docker' and 'journalctl -u docker'。或者为 --vm-driver 指定另外的值",
@@ -215,7 +221,6 @@
 	"Gets the value of PROPERTY_NAME from the minikube config file": "",
 	"Getting machine config failed": "获取机器配置失败",
 	"Global Flags": "",
-	"Go template format string for the addon list output.  The format for Go templates can be found here: https://golang.org/pkg/text/template/\nFor the list of accessible variables for the template, see the struct values here: https://godoc.org/k8s.io/minikube/cmd/minikube/cmd/config#AddonListTemplate": "",
 	"Go template format string for the cache list output.  The format for Go templates can be found here: https://golang.org/pkg/text/template/\nFor the list of accessible variables for the template, see the struct values here: https://godoc.org/k8s.io/minikube/cmd/minikube/cmd#CacheListTemplate": "",
 	"Go template format string for the config view output.  The format for Go templates can be found here: https://golang.org/pkg/text/template/\nFor the list of accessible variables for the template, see the struct values here: https://godoc.org/k8s.io/minikube/cmd/minikube/cmd/config#ConfigViewTemplate": "",
 	"Go template format string for the status output.  The format for Go templates can be found here: https://golang.org/pkg/text/template/\nFor the list accessible variables for the template, see the struct values here: https://godoc.org/k8s.io/minikube/cmd/minikube/cmd#Status": "",
@@ -225,6 +230,8 @@
 	"Hyperkit is broken. Upgrade to the latest hyperkit version and/or Docker for Desktop. Alternatively, you may choose an alternate --vm-driver": "Hyperkit 已损坏。升级到最新的 hyperkit 版本以及/或者 Docker 桌面版。或者，你可以通过 --vm-driver 切换其他选项",
 	"Hyperkit networking is broken. Upgrade to the latest hyperkit version and/or Docker for Desktop. Alternatively, you may choose an alternate --vm-driver": "Hyperkit 的网络挂了。升级到最新的 hyperkit 版本以及/或者 Docker 桌面版。或者，你可以通过 --vm-driver 切换其他选项",
 	"If set, automatically updates drivers to the latest version. Defaults to true.": "如果设置了，将自动更新驱动到最新版本。默认为 true。",
+	"If set, pause all namespaces": "",
+	"If set, unpause all namespaces": "",
 	"If the above advice does not help, please let us know:": "",
 	"If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --vm-driver=none.": "如果为 true，请缓存当前引导程序的 docker 镜像并将其加载到机器中。在 --vm-driver=none 情况下始终为 false。",
 	"If true, only download and cache files for later use - don't install or start anything.": "如果为 true，仅会下载和缓存文件以备后用 - 不会安装或启动任何项。",
@@ -270,6 +277,7 @@
 	"NOTE: This process must stay alive for the mount to be accessible ...": "",
 	"Networking and Connectivity Commands:": "",
 	"No minikube profile was found. You can create one using `minikube start`.": "",
+	"Node may be unable to resolve external DNS records": "",
 	"None of the known repositories in your location are accessible. Using {{.image_repository_name}} as fallback.": "您所在位置的已知存储库都无法访问。正在将 {{.image_repository_name}} 用作后备存储库。",
 	"None of the known repositories is accessible. Consider specifying an alternative image repository with --image-repository flag": "已知存储库都无法访问。请考虑使用 --image-repository 标志指定备选镜像存储库",
 	"Not passing {{.name}}={{.value}} to docker env.": "",
@@ -286,6 +294,9 @@
 	"Options:      {{.options}}": "",
 	"Outputs minikube shell completion for the given shell (bash or zsh)": "",
 	"Outputs minikube shell completion for the given shell (bash or zsh)\n\n\tThis depends on the bash-completion binary.  Example installation instructions:\n\tOS X:\n\t\t$ brew install bash-completion\n\t\t$ source $(brew --prefix)/etc/bash_completion\n\t\t$ minikube completion bash \u003e ~/.minikube-completion  # for bash users\n\t\t$ minikube completion zsh \u003e ~/.minikube-completion  # for zsh users\n\t\t$ source ~/.minikube-completion\n\tUbuntu:\n\t\t$ apt-get install bash-completion\n\t\t$ source /etc/bash-completion\n\t\t$ source \u003c(minikube completion bash) # for bash users\n\t\t$ source \u003c(minikube completion zsh) # for zsh users\n\n\tAdditionally, you may want to output the completion to a file and source in your .bashrc\n\n\tNote for zsh users: [1] zsh completions are only supported in versions of zsh \u003e= 5.2": "",
+	"Pause": "",
+	"Paused kubelet and {{.count}} containers": "",
+	"Paused kubelet and {{.count}} containers in: {{.namespaces}}": "",
 	"Permissions:  {{.octalMode}} ({{.writtenMode}})": "",
 	"Please enter a value:": "",
 	"Please install the minikube hyperkit VM driver, or select an alternative --vm-driver": "",
@@ -308,11 +319,13 @@
 	"Reboot to complete VirtualBox installation, verify that VirtualBox is not blocked by your system, and/or use another hypervisor": "重启以完成 VirtualBox 安装，检查 VirtualBox 未被您的操作系统禁用，或者使用其他的管理程序。",
 	"Rebuild libvirt with virt-network support": "",
 	"Received {{.name}} signal": "",
+	"Reconfiguring existing host ...": "",
 	"Registry mirrors to pass to the Docker daemon": "传递给 Docker 守护进程的注册表镜像",
 	"Reinstall VirtualBox and reboot. Alternatively, try the kvm2 driver: https://minikube.sigs.k8s.io/docs/reference/drivers/kvm2/": "",
 	"Reinstall VirtualBox and verify that it is not blocked: System Preferences -\u003e Security \u0026 Privacy -\u003e General -\u003e Some system software was blocked from loading": "",
 	"Related issues:": "",
 	"Relaunching Kubernetes using {{.bootstrapper}} ...": "正在使用 {{.bootstrapper}} 重新启动 Kubernetes…",
+	"Removed all traces of the \"{{.name}}\" cluster.": "",
 	"Removing {{.directory}} ...": "正在移除 {{.directory}}…",
 	"Requested cpu count {{.requested_cpus}} is less than the minimum allowed of {{.minimum_cpus}}": "请求的 CPU 数量 {{.requested_cpus}} 小于允许的最小值 {{.minimum_cpus}}",
 	"Requested disk size {{.requested_size}} is less than minimum of {{.minimum_size}}": "请求的磁盘大小 {{.requested_size}} 小于最小值 {{.minimum_size}}",
@@ -326,6 +339,7 @@
 	"Returns the value of PROPERTY_NAME from the minikube config file.  Can be overwritten at runtime by flags or environmental variables.": "",
 	"Run 'kubectl describe pod coredns -n kube-system' and check for a firewall or DNS conflict": "",
 	"Run 'minikube delete' to delete the stale VM, or and ensure that minikube is running as the same user you are issuing this command with": "执行 'minikube delete' 以删除过时的虚拟机，或者确保 minikube 以与您发出此命令的用户相同的用户身份运行",
+	"Run kubectl": "",
 	"Run minikube from the C: drive.": "",
 	"Run the kubernetes client, download it if necessary. Remember -- after kubectl!\n\nExamples:\nminikube kubectl -- --help\nminikube kubectl -- get pods --namespace kube-system": "",
 	"Run the minikube command as an Administrator": "",
@@ -333,6 +347,8 @@
 	"Running on localhost (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...": "",
 	"Selecting '{{.driver}}' driver from existing profile (alternates: {{.alternates}})": "从现有配置文件中选择 '{{.driver}}' 驱动程序 （可选：{{.alternates}}）",
 	"Selecting '{{.driver}}' driver from user configuration (alternates: {{.alternates}})": "从用户配置中选择 {{.driver}}' 驱动程序（可选：{{.alternates}}）",
+	"Selecting {{.experimental}}'{{.driver}}' driver from existing profile (alternates: {{.alternates}})": "",
+	"Selecting {{.experimental}}'{{.driver}}' driver from user configuration (alternates: {{.alternates}})": "",
 	"Set failed": "",
 	"Set flag to delete all profiles": "设置标志以删除所有配置文件",
 	"Set this flag to delete the '.minikube' folder from your user directory.": "设置这个标志来删除您用户目录下的 '.minikube' 文件夹。",
@@ -378,6 +394,7 @@
 	"The 'none' driver does not respect the --memory flag": "'none' 驱动程序不遵循 --memory 标志",
 	"The 'none' driver does not support multiple profiles: https://minikube.sigs.k8s.io/docs/reference/drivers/none/": "",
 	"The 'none' driver provides limited isolation and may reduce system security and reliability.": "“none”驱动程序提供有限的隔离功能，并且可能会降低系统安全性和可靠性。",
+	"The '{{.addonName}}' addon is enabled": "",
 	"The '{{.driver}}' driver requires elevated permissions. The following commands will be executed:\\n\\n{{ .example }}\\n": "",
 	"The CIDR to be used for service cluster IPs.": "需要用于服务集群 IP 的 CIDR。",
 	"The CIDR to be used for the minikube VM (virtualbox driver only)": "需要用于 minikube 虚拟机的 CIDR（仅限 virtualbox 驱动程序）",
@@ -400,6 +417,7 @@
 	"The docker host is currently not running": "",
 	"The docker service is currently not active": "",
 	"The driver '{{.driver}}' is not supported on {{.os}}": "{{.os}} 不支持驱动程序“{{.driver}}”",
+	"The driver {{.experimental}} '{{.driver}}' is not supported on {{.os}}": "",
 	"The existing \"{{.profile_name}}\" VM that was created using the \"{{.old_driver}}\" driver, and is incompatible with the \"{{.driver}}\" driver.": "",
 	"The hyperv virtual switch name. Defaults to first found. (hyperv driver only)": "hyperv 虚拟交换机名称。默认为找到的第一个 hyperv 虚拟交换机。（仅限 hyperv 驱动程序）",
 	"The hypervisor does not appear to be configured properly. Run 'minikube start --alsologtostderr -v=1' and inspect the error code": "管理程序似乎配置的不正确。执行 'minikube start --alsologtostderr -v=1' 并且检查错误代码",
@@ -427,12 +445,12 @@
 	"This will start the mount daemon and automatically mount files into minikube": "这将启动装载守护进程并将文件自动装载到 minikube 中",
 	"This will start the mount daemon and automatically mount files into minikube.": "",
 	"Tip: To remove this root owned cluster, run: sudo {{.cmd}} delete": "提示：要移除这个由根用户拥有的集群，请运行 sudo {{.cmd}} delete",
-	"Tip: Use 'minikube start -p \u003cname\u003e' to create a new cluster, or 'minikube delete' to delete this one.": "",
 	"To connect to this cluster, use: kubectl --context={{.name}}": "如需连接到此集群，请使用 kubectl --context={{.name}}",
 	"To connect to this cluster, use: kubectl --context={{.name}}__1": "如需连接到此集群，请使用 kubectl --context={{.name}}",
 	"To connect to this cluster, use: kubectl --context={{.profile_name}}": "",
 	"To disable this notice, run: 'minikube config set WantUpdateNotification false'\\n": "",
 	"To proceed, either:\n\n    1) Delete the existing \"{{.profile_name}}\" cluster using: '{{.command}} delete'\n\n    * or *\n\n    2) Start the existing \"{{.profile_name}}\" cluster using: '{{.command}} start --vm-driver={{.old_driver}}'": "",
+	"To see addons list for other profiles use: `minikube addons -p name list`": "",
 	"To start minikube with HyperV Powershell must be in your PATH`": "",
 	"To use kubectl or minikube commands as your own user, you may need to relocate them. For example, to overwrite your own settings, run:": "如需以您自己的用户身份使用 kubectl 或 minikube 命令，您可能需要重新定位该命令。例如，如需覆盖您的自定义设置，请运行：",
 	"Troubleshooting Commands:": "",
@@ -444,6 +462,7 @@
 	"Unable to generate docs": "",
 	"Unable to generate the documentation. Please ensure that the path specified is a directory, exists \u0026 you have permission to write to it.": "",
 	"Unable to get VM IP address": "",
+	"Unable to get addon status for {{.name}}: {{.error}}": "",
 	"Unable to get bootstrapper: {{.error}}": "无法获取引导程序：{{.error}}",
 	"Unable to get current user": "",
 	"Unable to get runtime": "",
@@ -463,6 +482,8 @@
 	"Unable to verify SSH connectivity: {{.error}}. Will retry...": "无法验证 SSH 连接： {{.error}}。即将重试...",
 	"Uninstalling Kubernetes {{.kubernetes_version}} using {{.bootstrapper_name}} ...": "正在使用 {{.bootstrapper_name}} 卸载 Kubernetes {{.kubernetes_version}}…",
 	"Unmounting {{.path}} ...": "",
+	"Unpaused kubelet and {{.count}} containers": "",
+	"Unpaused kubelet and {{.count}} containers in: {{.namespaces}}": "",
 	"Unset the KUBECONFIG environment variable, or verify that it does not point to an empty or otherwise invalid path": "",
 	"Unset variables instead of setting them": "",
 	"Update server returned an empty list": "",
@@ -474,6 +495,7 @@
 	"Usage: minikube delete --all --purge": "使用方法：minikube delete --all --purge",
 	"Use \"{{.CommandPath}} [command] --help\" for more information about a command.": "",
 	"Use 'kubect get po -A' to find the correct and namespace name": "使用 'kubect get po -A' 来查询正确的命名空间名称",
+	"Use -A to specify all namespaces": "",
 	"Use VirtualBox to remove the conflicting VM and/or network interfaces": "",
 	"Use native Golang SSH client (default true). Set to 'false' to use the command line 'ssh' command when accessing the docker machine. Useful for the machine drivers when they will not start with 'Waiting for SSH'.": "",
 	"User ID:      {{.userID}}": "",
@@ -525,6 +547,8 @@
 	"config view failed": "",
 	"dashboard service is not running: {{.error}}": "",
 	"disable failed": "",
+	"dry-run mode. Validates configuration, but does not mutate system state": "",
+	"dry-run validation complete!": "",
 	"enable failed": "",
 	"error creating clientset": "",
 	"error creating machine client": "",
@@ -548,18 +572,24 @@
 	"mkcmp is used to compare performance of two minikube binaries": "mkcmp 用于对比两个 minikube 二进制的性能",
 	"mount argument \"{{.value}}\" must be in form: \u003csource directory\u003e:\u003ctarget directory\u003e": "",
 	"mount failed": "",
+	"namespaces to pause": "",
+	"namespaces to unpause": "",
 	"not enough arguments ({{.ArgCount}}).\\nusage: minikube config set PROPERTY_NAME PROPERTY_VALUE": "",
+	"pause containers": "",
 	"profile sets the current minikube profile, or gets the current profile if no arguments are provided.  This is used to run and manage multiple minikube instance.  You can return to the default minikube profile by running `minikube profile default`": "",
 	"reload cached images.": "重新加载缓存的镜像",
 	"reloads images previously added using the 'cache add' subcommand": "重新加载之前通过子命令 'cache add' 添加的镜像",
 	"service {{.namespace_name}}/{{.service_name}} has no node port": "",
 	"stat failed": "",
+	"status json failure": "",
+	"status text failure": "",
 	"toom any arguments ({{.ArgCount}}).\\nusage: minikube config set PROPERTY_NAME PROPERTY_VALUE": "",
-	"tunnel creates a route to services deployed with type LoadBalancer and sets their Ingress to their ClusterIP": "",
+	"tunnel creates a route to services deployed with type LoadBalancer and sets their Ingress to their ClusterIP. for a detailed example see https://minikube.sigs.k8s.io/docs/tasks/loadbalancer": "",
 	"tunnel makes services of type LoadBalancer accessible on localhost": "",
 	"unable to bind flags": "",
 	"unable to delete minikube config folder": "无法删除 minikube 配置目录",
 	"unable to set logtostderr": "",
+	"unpause Kubernetes": "",
 	"unset failed": "",
 	"unsets PROPERTY_NAME from the minikube config file.  Can be overwritten by flags or environmental variables": "",
 	"unsets an individual value in a minikube config file": "",
@@ -575,7 +605,6 @@
 	"usage: minikube delete --all": "",
 	"usage: minikube profile [MINIKUBE_PROFILE_NAME]": "",
 	"zsh completion failed": "",
-	"{{.addonName}} was successfully enabled": "",
 	"{{.driver}} does not appear to be installed": "似乎并未安装 {{.driver}}",
 	"{{.driver}} does not appear to be installed, but is specified by an existing profile. Please run 'minikube delete' or install {{.driver}}": "似乎并未安装 {{.driver}}，但已被当前的配置文件指定。请执行 'minikube delete' 或者安装 {{.driver}}",
 	"{{.extra_option_component_name}}.{{.key}}={{.value}}": "",

From aea610b510b23413dde1cd713d2086aa6c26a2eb Mon Sep 17 00:00:00 2001
From: tstromberg <tstromberg@google.com>
Date: Tue, 4 Feb 2020 13:19:17 -0800
Subject: [PATCH 231/250] Add file sync support back to minikube

---
 pkg/minikube/assets/addons.go          | 60 ----------------
 pkg/minikube/bootstrapper/certs.go     | 15 +++-
 pkg/minikube/cluster/filesync.go       | 98 ++++++++++++++++++++++++++
 pkg/minikube/cluster/start.go          |  8 +--
 pkg/minikube/command/command_runner.go |  8 +++
 pkg/minikube/command/ssh_runner.go     |  5 +-
 6 files changed, 122 insertions(+), 72 deletions(-)
 create mode 100644 pkg/minikube/cluster/filesync.go

diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go
index 3bdcb758aa..d2a50ed24c 100644
--- a/pkg/minikube/assets/addons.go
+++ b/pkg/minikube/assets/addons.go
@@ -17,19 +17,13 @@ limitations under the License.
 package assets
 
 import (
-	"fmt"
-	"os"
-	"path"
-	"path/filepath"
 	"runtime"
 
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
-	"k8s.io/minikube/pkg/minikube/localpath"
 	"k8s.io/minikube/pkg/minikube/vmpath"
-	"k8s.io/minikube/pkg/util"
 )
 
 // Addon is a named list of assets, that can be enabled
@@ -340,60 +334,6 @@ var Addons = map[string]*Addon{
 	}, false, "ingress-dns"),
 }
 
-// AddMinikubeDirAssets adds all addons and files to the list
-// of files to be copied to the vm.
-func AddMinikubeDirAssets(assets *[]CopyableFile) error {
-	if err := addMinikubeDirToAssets(localpath.MakeMiniPath("addons"), vmpath.GuestAddonsDir, assets); err != nil {
-		return errors.Wrap(err, "adding addons folder to assets")
-	}
-	if err := addMinikubeDirToAssets(localpath.MakeMiniPath("files"), "", assets); err != nil {
-		return errors.Wrap(err, "adding files rootfs to assets")
-	}
-
-	return nil
-}
-
-// AddMinikubeDirToAssets adds all the files in the basedir argument to the list
-// of files to be copied to the vm.  If vmpath is left blank, the files will be
-// transferred to the location according to their relative minikube folder path.
-func addMinikubeDirToAssets(basedir, vmpath string, assets *[]CopyableFile) error {
-	return filepath.Walk(basedir, func(hostpath string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-		isDir, err := util.IsDirectory(hostpath)
-		if err != nil {
-			return errors.Wrapf(err, "checking if %s is directory", hostpath)
-		}
-		if !isDir {
-			vmdir := vmpath
-			if vmdir == "" {
-				rPath, err := filepath.Rel(basedir, hostpath)
-				if err != nil {
-					return errors.Wrap(err, "generating relative path")
-				}
-				rPath = filepath.Dir(rPath)
-				rPath = filepath.ToSlash(rPath)
-				vmdir = path.Join("/", rPath)
-			}
-			permString := fmt.Sprintf("%o", info.Mode().Perm())
-			// The conversion will strip the leading 0 if present, so add it back
-			// if we need to.
-			if len(permString) == 3 {
-				permString = fmt.Sprintf("0%s", permString)
-			}
-
-			f, err := NewFileAsset(hostpath, vmdir, filepath.Base(hostpath), permString)
-			if err != nil {
-				return errors.Wrapf(err, "creating file asset for %s", hostpath)
-			}
-			*assets = append(*assets, f)
-		}
-
-		return nil
-	})
-}
-
 // GenerateTemplateData generates template data for template assets
 func GenerateTemplateData(cfg config.KubernetesConfig) interface{} {
 
diff --git a/pkg/minikube/bootstrapper/certs.go b/pkg/minikube/bootstrapper/certs.go
index f7b3d3a7fe..b8bfb60f6a 100644
--- a/pkg/minikube/bootstrapper/certs.go
+++ b/pkg/minikube/bootstrapper/certs.go
@@ -221,8 +221,8 @@ func generateCerts(k8s config.KubernetesConfig, n config.Node) error {
 	}
 
 	for _, caCertSpec := range caCertSpecs {
-		if !(util.CanReadFile(caCertSpec.certPath) &&
-			util.CanReadFile(caCertSpec.keyPath)) {
+		if !(canReadFile(caCertSpec.certPath) &&
+			canReadFile(caCertSpec.keyPath)) {
 			if err := util.GenerateCACert(
 				caCertSpec.certPath, caCertSpec.keyPath, caCertSpec.subject,
 			); err != nil {
@@ -361,3 +361,14 @@ func configureCACerts(cr command.Runner, caCerts map[string]string) error {
 	}
 	return nil
 }
+
+// canReadFile returns true if the file represented
+// by path exists and is readable, otherwise false.
+func canReadFile(path string) bool {
+	f, err := os.Open(path)
+	if err != nil {
+		return false
+	}
+	defer f.Close()
+	return true
+}
diff --git a/pkg/minikube/cluster/filesync.go b/pkg/minikube/cluster/filesync.go
new file mode 100644
index 0000000000..072fc00b9c
--- /dev/null
+++ b/pkg/minikube/cluster/filesync.go
@@ -0,0 +1,98 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cluster
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+
+	"github.com/golang/glog"
+	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/minikube/assets"
+	"k8s.io/minikube/pkg/minikube/command"
+	"k8s.io/minikube/pkg/minikube/localpath"
+	"k8s.io/minikube/pkg/minikube/vmpath"
+)
+
+// syncLocalAssets syncs files from MINIKUBE_HOME into the cluster
+func syncLocalAssets(cr command.Runner) error {
+	fs, err := localAssets()
+	if err != nil {
+		return err
+	}
+
+	for _, f := range fs {
+		err := cr.Copy(f)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// localAssets returns local files and addons from the minikube home directory
+func localAssets() ([]assets.CopyableFile, error) {
+	fs, err := assetsFromDir(localpath.MakeMiniPath("addons"), vmpath.GuestAddonsDir)
+	if err != nil {
+		return fs, errors.Wrap(err, "addons dir")
+	}
+
+	localFiles, err := assetsFromDir(localpath.MakeMiniPath("files"), "/")
+	if err != nil {
+		return fs, errors.Wrap(err, "files dir")
+	}
+
+	fs = append(fs, localFiles...)
+	return fs, nil
+}
+
+// assetsFromDir generates assets from a local filepath
+func assetsFromDir(localRoot string, destRoot string) ([]assets.CopyableFile, error) {
+	glog.Infof("Scanning %s for local assets ...", localRoot)
+	fs := []assets.CopyableFile{}
+	err := filepath.Walk(localRoot, func(localPath string, fi os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if fi.IsDir() {
+			return nil
+		}
+
+		rel, err := filepath.Rel(localRoot, localPath)
+		if err != nil {
+			return err
+		}
+
+		// The conversion will strip the leading 0 if present, so add it back if necessary
+		ps := fmt.Sprintf("%o", fi.Mode().Perm())
+		if len(ps) == 3 {
+			ps = fmt.Sprintf("0%s", ps)
+		}
+
+		dest := path.Join(destRoot, rel)
+		f, err := assets.NewFileAsset(localPath, path.Dir(dest), path.Base(dest), ps)
+		if err != nil {
+			return errors.Wrapf(err, "creating file asset for %s", localPath)
+		}
+		glog.Infof("found asset to sync: %s -> %s", localPath, dest)
+		fs = append(fs, f)
+		return nil
+	})
+	return fs, err
+}
diff --git a/pkg/minikube/cluster/start.go b/pkg/minikube/cluster/start.go
index 4895ff9484..26307ac948 100644
--- a/pkg/minikube/cluster/start.go
+++ b/pkg/minikube/cluster/start.go
@@ -158,13 +158,7 @@ func postStartSetup(h *host.Host, mc config.MachineConfig) error {
 		showLocalOsRelease()
 	}
 
-	if !driver.BareMetal(mc.VMDriver) && !driver.IsKIC(mc.VMDriver) {
-		showRemoteOsRelease(h.Driver)
-		if err := ensureSyncedGuestClock(h); err != nil {
-			return err
-		}
-	}
-	return nil
+	return syncLocalAssets(r)
 }
 
 // commandRunner returns best available command runner for this host
diff --git a/pkg/minikube/command/command_runner.go b/pkg/minikube/command/command_runner.go
index 309c8f6133..dfd30c5233 100644
--- a/pkg/minikube/command/command_runner.go
+++ b/pkg/minikube/command/command_runner.go
@@ -26,6 +26,14 @@ import (
 	"k8s.io/minikube/pkg/minikube/assets"
 )
 
+var (
+	// ErrPrefix notes an error
+	ErrPrefix = "! "
+
+	// OutPrefix notes output
+	OutPrefix = "> "
+)
+
 // RunResult holds the results of a Runner
 type RunResult struct {
 	Stdout   bytes.Buffer
diff --git a/pkg/minikube/command/ssh_runner.go b/pkg/minikube/command/ssh_runner.go
index 41e4ba1323..66473fd312 100644
--- a/pkg/minikube/command/ssh_runner.go
+++ b/pkg/minikube/command/ssh_runner.go
@@ -34,7 +34,6 @@ import (
 	"golang.org/x/crypto/ssh"
 	"golang.org/x/sync/errgroup"
 	"k8s.io/minikube/pkg/minikube/assets"
-	"k8s.io/minikube/pkg/util"
 )
 
 var (
@@ -80,13 +79,13 @@ func teeSSH(s *ssh.Session, cmd string, outB io.Writer, errB io.Writer) error {
 	wg.Add(2)
 
 	go func() {
-		if err := teePrefix(util.ErrPrefix, errPipe, errB, glog.V(8).Infof); err != nil {
+		if err := teePrefix(ErrPrefix, errPipe, errB, glog.V(8).Infof); err != nil {
 			glog.Errorf("tee stderr: %v", err)
 		}
 		wg.Done()
 	}()
 	go func() {
-		if err := teePrefix(util.OutPrefix, outPipe, outB, glog.V(8).Infof); err != nil {
+		if err := teePrefix(OutPrefix, outPipe, outB, glog.V(8).Infof); err != nil {
 			glog.Errorf("tee stdout: %v", err)
 		}
 		wg.Done()

From 3db7713399e0b45a420662cd9734663a2ebd6379 Mon Sep 17 00:00:00 2001
From: tstromberg <tstromberg@google.com>
Date: Tue, 4 Feb 2020 13:20:37 -0800
Subject: [PATCH 232/250] Remove more functions from the util package

---
 pkg/provision/provision.go | 28 +++++++++++--
 pkg/util/utils.go          | 82 --------------------------------------
 2 files changed, 25 insertions(+), 85 deletions(-)

diff --git a/pkg/provision/provision.go b/pkg/provision/provision.go
index 0ef9f69664..b0873615ce 100644
--- a/pkg/provision/provision.go
+++ b/pkg/provision/provision.go
@@ -39,7 +39,6 @@ import (
 	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/sshutil"
-	"k8s.io/minikube/pkg/util"
 )
 
 // generic interface for minikube provisioner
@@ -246,7 +245,30 @@ func rootFileSystemType(p provision.SSHCommander) (string, error) {
 // (see systemd man pages for more info). This is not supported by minikube, thus needs to be escaped.
 func escapeSystemdDirectives(engineConfigContext *provision.EngineConfigContext) {
 	// escape '%' in Environment option so that it does not evaluate into a template specifier
-	engineConfigContext.EngineOptions.Env = util.ReplaceChars(engineConfigContext.EngineOptions.Env, systemdSpecifierEscaper)
+	engineConfigContext.EngineOptions.Env = replaceChars(engineConfigContext.EngineOptions.Env, systemdSpecifierEscaper)
 	// input might contain whitespaces, wrap it in quotes
-	engineConfigContext.EngineOptions.Env = util.ConcatStrings(engineConfigContext.EngineOptions.Env, "\"", "\"")
+	engineConfigContext.EngineOptions.Env = concatStrings(engineConfigContext.EngineOptions.Env, "\"", "\"")
+}
+
+// replaceChars returns a copy of the src slice with each string modified by the replacer
+func replaceChars(src []string, replacer *strings.Replacer) []string {
+	ret := make([]string, len(src))
+	for i, s := range src {
+		ret[i] = replacer.Replace(s)
+	}
+	return ret
+}
+
+// concatStrings concatenates each string in the src slice with prefix and postfix and returns a new slice
+func concatStrings(src []string, prefix string, postfix string) []string {
+	var buf bytes.Buffer
+	ret := make([]string, len(src))
+	for i, s := range src {
+		buf.WriteString(prefix)
+		buf.WriteString(s)
+		buf.WriteString(postfix)
+		ret[i] = buf.String()
+		buf.Reset()
+	}
+	return ret
 }
diff --git a/pkg/util/utils.go b/pkg/util/utils.go
index 54ec5d4101..9ab0a57bb3 100644
--- a/pkg/util/utils.go
+++ b/pkg/util/utils.go
@@ -17,15 +17,11 @@ limitations under the License.
 package util
 
 import (
-	"bytes"
 	"fmt"
-	"io"
 	"os"
 	"os/user"
 	"path/filepath"
 	"strconv"
-	"strings"
-	"time"
 
 	units "github.com/docker/go-units"
 	"github.com/pkg/errors"
@@ -34,12 +30,6 @@ import (
 )
 
 const (
-	// ErrPrefix notes an error
-	ErrPrefix = "! "
-
-	// OutPrefix notes output
-	OutPrefix = "> "
-
 	downloadURL = "https://storage.googleapis.com/minikube/releases/%s/minikube-%s-amd64%s"
 )
 
@@ -57,46 +47,6 @@ func CalculateSizeInMB(humanReadableSize string) int {
 	return int(size / units.MB)
 }
 
-// Until endlessly loops the provided function until a message is received on the done channel.
-// The function will wait the duration provided in sleep between function calls. Errors will be sent on provider Writer.
-func Until(fn func() error, w io.Writer, name string, sleep time.Duration, done <-chan struct{}) {
-	var exitErr error
-	for {
-		select {
-		case <-done:
-			return
-		default:
-			exitErr = fn()
-			if exitErr == nil {
-				fmt.Fprintf(w, Pad("%s: Exited with no errors.\n"), name)
-			} else {
-				fmt.Fprintf(w, Pad("%s: Exit with error: %v"), name, exitErr)
-			}
-
-			// wait provided duration before trying again
-			time.Sleep(sleep)
-		}
-	}
-}
-
-// Pad pads the string with newlines
-func Pad(str string) string {
-	return fmt.Sprintf("\n%s\n", str)
-}
-
-// CanReadFile returns true if the file represented
-// by path exists and is readable, otherwise false.
-func CanReadFile(path string) bool {
-	f, err := os.Open(path)
-	if err != nil {
-		return false
-	}
-
-	defer f.Close()
-
-	return true
-}
-
 // GetBinaryDownloadURL returns a suitable URL for the platform
 func GetBinaryDownloadURL(version, platform string) string {
 	switch platform {
@@ -107,15 +57,6 @@ func GetBinaryDownloadURL(version, platform string) string {
 	}
 }
 
-// IsDirectory checks if path is a directory
-func IsDirectory(path string) (bool, error) {
-	fileInfo, err := os.Stat(path)
-	if err != nil {
-		return false, errors.Wrapf(err, "Error calling os.Stat on file %s", path)
-	}
-	return fileInfo.IsDir(), nil
-}
-
 // ChownR does a recursive os.Chown
 func ChownR(path string, uid, gid int) error {
 	return filepath.Walk(path, func(name string, info os.FileInfo, err error) error {
@@ -148,26 +89,3 @@ func MaybeChownDirRecursiveToMinikubeUser(dir string) error {
 	}
 	return nil
 }
-
-// ReplaceChars returns a copy of the src slice with each string modified by the replacer
-func ReplaceChars(src []string, replacer *strings.Replacer) []string {
-	ret := make([]string, len(src))
-	for i, s := range src {
-		ret[i] = replacer.Replace(s)
-	}
-	return ret
-}
-
-// ConcatStrings concatenates each string in the src slice with prefix and postfix and returns a new slice
-func ConcatStrings(src []string, prefix string, postfix string) []string {
-	var buf bytes.Buffer
-	ret := make([]string, len(src))
-	for i, s := range src {
-		buf.WriteString(prefix)
-		buf.WriteString(s)
-		buf.WriteString(postfix)
-		ret[i] = buf.String()
-		buf.Reset()
-	}
-	return ret
-}

From 3937bae26988403c0fbf1e17a9683687a60e1fbb Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Tue, 4 Feb 2020 13:23:26 -0800
Subject: [PATCH 233/250] add new env var for docker-env command

---
 cmd/minikube/cmd/env.go             | 76 ++++++++++++++++-------------
 pkg/minikube/constants/constants.go | 10 ++++
 2 files changed, 53 insertions(+), 33 deletions(-)

diff --git a/cmd/minikube/cmd/env.go b/cmd/minikube/cmd/env.go
index 01ca87260a..92c2e23095 100644
--- a/cmd/minikube/cmd/env.go
+++ b/cmd/minikube/cmd/env.go
@@ -36,14 +36,15 @@ import (
 	"github.com/spf13/viper"
 	"k8s.io/minikube/pkg/minikube/cluster"
 	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/driver"
 	"k8s.io/minikube/pkg/minikube/exit"
 	"k8s.io/minikube/pkg/minikube/machine"
 )
 
-const (
-	envTmpl = `{{ .Prefix }}DOCKER_TLS_VERIFY{{ .Delimiter }}{{ .DockerTLSVerify }}{{ .Suffix }}{{ .Prefix }}DOCKER_HOST{{ .Delimiter }}{{ .DockerHost }}{{ .Suffix }}{{ .Prefix }}DOCKER_CERT_PATH{{ .Delimiter }}{{ .DockerCertPath }}{{ .Suffix }}{{ if .NoProxyVar }}{{ .Prefix }}{{ .NoProxyVar }}{{ .Delimiter }}{{ .NoProxyValue }}{{ .Suffix }}{{end}}{{ .UsageHint }}`
+var envTmpl = fmt.Sprintf("{{ .Prefix }}%s{{ .Delimiter }}{{ .DockerTLSVerify }}{{ .Suffix }}{{ .Prefix }}%s{{ .Delimiter }}{{ .DockerHost }}{{ .Suffix }}{{ .Prefix }}%s{{ .Delimiter }}{{ .DockerCertPath }}{{ .Suffix }}{{ .Prefix }}%s{{ .Delimiter }}{{ .MinikubeDockerdProfile }}{{ .Suffix }}{{ if .NoProxyVar }}{{ .Prefix }}{{ .NoProxyVar }}{{ .Delimiter }}{{ .NoProxyValue }}{{ .Suffix }}{{end}}{{ .UsageHint }}", constants.DockerTLSVerifyEnv, constants.DockerHostEnv, constants.DockerCertPathEnv, constants.MinikubeActiveDockerdEnv)
 
+const (
 	fishSetPfx   = "set -gx "
 	fishSetSfx   = "\";\n"
 	fishSetDelim = " \""
@@ -89,35 +90,18 @@ const (
 	noneDelim = "="
 )
 
-var usageHintMap = map[string]string{
-	"bash": `# Run this command to configure your shell:
-# eval $(minikube docker-env)
-`,
-	"fish": `# Run this command to configure your shell:
-# eval (minikube docker-env)
-`,
-	"powershell": `# Run this command to configure your shell:
-# & minikube docker-env | Invoke-Expression
-`,
-	"cmd": `REM Run this command to configure your shell:
-REM @FOR /f "tokens=*" %i IN ('minikube docker-env') DO @%i
-`,
-	"emacs": `;; Run this command to configure your shell:
-;; (with-temp-buffer (shell-command "minikube docker-env" (current-buffer)) (eval-buffer))
-`,
-}
-
 // ShellConfig represents the shell config
 type ShellConfig struct {
-	Prefix          string
-	Delimiter       string
-	Suffix          string
-	DockerCertPath  string
-	DockerHost      string
-	DockerTLSVerify string
-	UsageHint       string
-	NoProxyVar      string
-	NoProxyValue    string
+	Prefix                 string
+	Delimiter              string
+	Suffix                 string
+	DockerCertPath         string
+	DockerHost             string
+	DockerTLSVerify        string
+	MinikubeDockerdProfile string
+	UsageHint              string
+	NoProxyVar             string
+	NoProxyValue           string
 }
 
 var (
@@ -145,6 +129,31 @@ type NoProxyGetter interface {
 type EnvNoProxyGetter struct{}
 
 func generateUsageHint(userShell string) string {
+	const usgPlz = "Please run command bellow to point your shell to minikube's docker-daemon :"
+	var usgCmd = fmt.Sprintf("minikube -p %s docker-env", viper.GetString(config.MachineProfile))
+	var usageHintMap = map[string]string{
+		"bash": fmt.Sprintf(`
+# %s
+# eval $(%s)
+`, usgPlz, usgCmd),
+		"fish": fmt.Sprintf(`
+# %s
+# eval (%s)
+`, usgPlz, usgCmd),
+		"powershell": fmt.Sprintf(`
+# %s
+# & %s | Invoke-Expression
+	`, usgPlz, usgCmd),
+		"cmd": fmt.Sprintf(`
+REM %s
+REM @FOR /f "tokens=*" %%i IN ('%s') DO @%%i
+	`, usgPlz, usgCmd),
+		"emacs": fmt.Sprintf(`
+;; %s
+;; (with-temp-buffer (shell-command "%s" (current-buffer)) (eval-buffer))
+	`, usgPlz, usgCmd),
+	}
+
 	hint, ok := usageHintMap[userShell]
 	if !ok {
 		return usageHintMap["bash"]
@@ -165,10 +174,11 @@ func shellCfgSet(api libmachine.API) (*ShellConfig, error) {
 	}
 
 	shellCfg := &ShellConfig{
-		DockerCertPath:  envMap["DOCKER_CERT_PATH"],
-		DockerHost:      envMap["DOCKER_HOST"],
-		DockerTLSVerify: envMap["DOCKER_TLS_VERIFY"],
-		UsageHint:       generateUsageHint(userShell),
+		DockerCertPath:         envMap["DOCKER_CERT_PATH"],
+		DockerHost:             envMap["DOCKER_HOST"],
+		DockerTLSVerify:        envMap["DOCKER_TLS_VERIFY"],
+		MinikubeDockerdProfile: envMap["MINIKUBE_ACTIVE_DOCKERD"],
+		UsageHint:              generateUsageHint(userShell),
 	}
 
 	if noProxy {
diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go
index a05042939a..0db6d641d7 100644
--- a/pkg/minikube/constants/constants.go
+++ b/pkg/minikube/constants/constants.go
@@ -39,6 +39,16 @@ const (
 	ClusterDNSDomain = "cluster.local"
 	// DefaultServiceCIDR is The CIDR to be used for service cluster IPs
 	DefaultServiceCIDR = "10.96.0.0/12"
+
+	// DockerTLSVerifyEnv is used for docker daemon settings
+	DockerTLSVerifyEnv = "DOCKER_TLS_VERIFY"
+	// DockerHostEnv is used for docker daemon settings
+	DockerHostEnv = "DOCKER_HOST"
+	// DockerCertPathEnv is used for docker daemon settings
+	DockerCertPathEnv = "DOCKER_CERT_PATH"
+	// MinikubeActiveDockerdEnv holds the docker daemon which user's shell is pointing at
+	// value would be profile or empty if pointing to the user's host daemon.
+	MinikubeActiveDockerdEnv = "MINIKUBE_ACTIVE_DOCKERD"
 )
 
 // DefaultMinipath is the default Minikube path (under the home directory)

From f6c704839213753e35cb07345362a222f287c7db Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Tue, 4 Feb 2020 13:30:23 -0800
Subject: [PATCH 234/250] use constants instead of hardcode

---
 cmd/minikube/cmd/env.go              | 8 ++++----
 pkg/minikube/cluster/cluster_test.go | 7 ++++---
 pkg/minikube/cluster/docker_env.go   | 6 +++---
 3 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/cmd/minikube/cmd/env.go b/cmd/minikube/cmd/env.go
index 92c2e23095..bed87b4f1c 100644
--- a/cmd/minikube/cmd/env.go
+++ b/cmd/minikube/cmd/env.go
@@ -174,10 +174,10 @@ func shellCfgSet(api libmachine.API) (*ShellConfig, error) {
 	}
 
 	shellCfg := &ShellConfig{
-		DockerCertPath:         envMap["DOCKER_CERT_PATH"],
-		DockerHost:             envMap["DOCKER_HOST"],
-		DockerTLSVerify:        envMap["DOCKER_TLS_VERIFY"],
-		MinikubeDockerdProfile: envMap["MINIKUBE_ACTIVE_DOCKERD"],
+		DockerCertPath:         envMap[constants.DockerCertPathEnv],
+		DockerHost:             envMap[constants.DockerHostEnv],
+		DockerTLSVerify:        envMap[constants.DockerTLSVerifyEnv],
+		MinikubeDockerdProfile: envMap[constants.MinikubeActiveDockerdEnv],
 		UsageHint:              generateUsageHint(userShell),
 	}
 
diff --git a/pkg/minikube/cluster/cluster_test.go b/pkg/minikube/cluster/cluster_test.go
index d91c469031..b983436416 100644
--- a/pkg/minikube/cluster/cluster_test.go
+++ b/pkg/minikube/cluster/cluster_test.go
@@ -362,9 +362,10 @@ func TestGetHostDockerEnv(t *testing.T) {
 	}
 
 	dockerEnvKeys := [...]string{
-		"DOCKER_TLS_VERIFY",
-		"DOCKER_HOST",
-		"DOCKER_CERT_PATH",
+		constants.DockerTLSVerifyEnv,
+		constants.DockerHostEnv,
+		constants.DockerCertPathEnv,
+		constants.MinikubeActiveDockerdEnv,
 	}
 	for _, dockerEnvKey := range dockerEnvKeys {
 		if _, hasKey := envMap[dockerEnvKey]; !hasKey {
diff --git a/pkg/minikube/cluster/docker_env.go b/pkg/minikube/cluster/docker_env.go
index 6c349a4128..9db3855608 100644
--- a/pkg/minikube/cluster/docker_env.go
+++ b/pkg/minikube/cluster/docker_env.go
@@ -58,9 +58,9 @@ func GetHostDockerEnv(api libmachine.API) (map[string]string, error) {
 	}
 
 	envMap := map[string]string{
-		"DOCKER_TLS_VERIFY": "1",
-		"DOCKER_HOST":       tcpPrefix + net.JoinHostPort(ip, fmt.Sprint(port)),
-		"DOCKER_CERT_PATH":  localpath.MakeMiniPath("certs"),
+		constants.DockerTLSVerifyEnv: "1",
+		constants.DockerHostEnv:      tcpPrefix + net.JoinHostPort(ip, fmt.Sprint(port)),
+		constants.DockerCertPathEnv:  localpath.MakeMiniPath("certs"),
 	}
 	return envMap, nil
 }

From f41a10a98fb4b255bcfbc9ee42f41f9a842282c1 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Tue, 4 Feb 2020 15:01:40 -0800
Subject: [PATCH 235/250] Fix up some unit tests

---
 cmd/minikube/cmd/config/set_test.go | 37 ++++++++++++++++++++++++++++-
 pkg/minikube/cluster/fix.go         |  5 ++++
 2 files changed, 41 insertions(+), 1 deletion(-)

diff --git a/cmd/minikube/cmd/config/set_test.go b/cmd/minikube/cmd/config/set_test.go
index 9a6adde384..4e76b36ee0 100644
--- a/cmd/minikube/cmd/config/set_test.go
+++ b/cmd/minikube/cmd/config/set_test.go
@@ -16,9 +16,18 @@ limitations under the License.
 
 package config
 
-import "testing"
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/localpath"
+)
 
 func TestNotFound(t *testing.T) {
+	createTestProfile(t)
 	err := Set("nonexistent", "10")
 	if err == nil {
 		t.Fatalf("Set did not return error for unknown property")
@@ -26,6 +35,7 @@ func TestNotFound(t *testing.T) {
 }
 
 func TestSetNotAllowed(t *testing.T) {
+	createTestProfile(t)
 	err := Set("vm-driver", "123456")
 	if err == nil || err.Error() != "[driver \"123456\" is not supported]" {
 		t.Fatalf("Set did not return error for unallowed value")
@@ -33,7 +43,9 @@ func TestSetNotAllowed(t *testing.T) {
 }
 
 func TestSetOK(t *testing.T) {
+	createTestProfile(t)
 	err := Set("vm-driver", "virtualbox")
+	defer Unset("vm-driver")
 	if err != nil {
 		t.Fatalf("Set returned error for valid property value")
 	}
@@ -45,3 +57,26 @@ func TestSetOK(t *testing.T) {
 		t.Fatalf("Get returned %s, expected \"virtualbox\"", val)
 	}
 }
+
+func createTestProfile(t *testing.T) string {
+	t.Helper()
+	td, err := ioutil.TempDir("", "profile")
+	if err != nil {
+		t.Fatalf("tempdir: %v", err)
+	}
+
+	err = os.Setenv(localpath.MinikubeHome, td)
+	if err != nil {
+		t.Errorf("error setting up test environment. could not set %s", localpath.MinikubeHome)
+	}
+
+	// Not necessary, but it is a handy random alphanumeric
+	name := filepath.Base(td)
+	if err := os.MkdirAll(config.ProfileFolderPath(name), 0777); err != nil {
+		t.Fatalf("error creating temporary directory")
+	}
+	if err := config.DefaultLoader.WriteConfigToFile(name, &config.MachineConfig{}); err != nil {
+		t.Fatalf("error creating temporary profile config: %v", err)
+	}
+	return name
+}
diff --git a/pkg/minikube/cluster/fix.go b/pkg/minikube/cluster/fix.go
index 1a5e9c6cea..bd625394ee 100644
--- a/pkg/minikube/cluster/fix.go
+++ b/pkg/minikube/cluster/fix.go
@@ -30,6 +30,7 @@ import (
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
 	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/driver"
 	"k8s.io/minikube/pkg/minikube/out"
 	"k8s.io/minikube/pkg/util/retry"
 )
@@ -90,6 +91,10 @@ func fixHost(api libmachine.API, mc config.MachineConfig) (*host.Host, error) {
 		}
 	}
 
+	if h.DriverName == driver.Mock {
+		return h, nil
+	}
+
 	if err := postStartSetup(h, mc); err != nil {
 		return h, errors.Wrap(err, "post-start")
 	}

From 6f6663ec7787313f103f47023ebe31de0b4dca4e Mon Sep 17 00:00:00 2001
From: tstromberg <tstromberg@google.com>
Date: Tue, 4 Feb 2020 15:06:43 -0800
Subject: [PATCH 236/250] Make filesync unit tests excellent again

---
 pkg/minikube/assets/addons_test.go | 159 -----------------------------
 pkg/minikube/cluster/filesync.go   |  17 +--
 pkg/minikube/cluster/fix.go        |   8 +-
 pkg/minikube/cluster/info.go       |  14 +--
 pkg/minikube/cluster/start.go      |   9 +-
 pkg/minikube/driver/driver.go      |  13 +++
 pkg/util/utils_test.go             |  41 --------
 7 files changed, 43 insertions(+), 218 deletions(-)
 delete mode 100644 pkg/minikube/assets/addons_test.go

diff --git a/pkg/minikube/assets/addons_test.go b/pkg/minikube/assets/addons_test.go
deleted file mode 100644
index 9e7e745f06..0000000000
--- a/pkg/minikube/assets/addons_test.go
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
-Copyright 2016 The Kubernetes Authors All rights reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package assets
-
-import (
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"testing"
-
-	"github.com/google/go-cmp/cmp"
-	"k8s.io/minikube/pkg/minikube/localpath"
-	"k8s.io/minikube/pkg/minikube/vmpath"
-)
-
-func setupTestDir() (string, error) {
-	path, err := ioutil.TempDir("", "minipath")
-	if err != nil {
-		return "", err
-	}
-
-	os.Setenv(localpath.MinikubeHome, path)
-	return path, err
-}
-
-func TestAddMinikubeDirAssets(t *testing.T) {
-
-	tests := []struct {
-		description string
-		baseDir     string
-		vmPath      string
-		files       []struct {
-			relativePath string
-			expectedPath string
-		}
-	}{
-		{
-			description: "relative path assets",
-			baseDir:     "/files",
-			files: []struct {
-				relativePath string
-				expectedPath string
-			}{
-				{
-					relativePath: "/dir1/file1.txt",
-					expectedPath: vmpath.GuestAddonsDir,
-				},
-				{
-					relativePath: "/dir1/file2.txt",
-					expectedPath: vmpath.GuestAddonsDir,
-				},
-				{
-					relativePath: "/dir2/file1.txt",
-					expectedPath: vmpath.GuestAddonsDir,
-				},
-			},
-			vmPath: vmpath.GuestAddonsDir,
-		},
-		{
-			description: "absolute path assets",
-			baseDir:     "/files",
-			files: []struct {
-				relativePath string
-				expectedPath string
-			}{
-				{
-					relativePath: "/dir1/file1.txt",
-					expectedPath: "/dir1",
-				},
-				{
-					relativePath: "/dir1/file2.txt",
-					expectedPath: "/dir1",
-				},
-				{
-					relativePath: "/dir2/file1.txt",
-					expectedPath: "/dir2",
-				},
-			},
-			vmPath: "",
-		},
-	}
-	var testDirs = make([]string, 0)
-	defer func() {
-		for _, testDir := range testDirs {
-			err := os.RemoveAll(testDir)
-			if err != nil {
-				t.Logf("got unexpected error removing test dir: %v", err)
-			}
-		}
-	}()
-
-	for _, test := range tests {
-		t.Run(test.description, func(t *testing.T) {
-			testDir, err := setupTestDir()
-			if err != nil {
-				t.Errorf("got unexpected error creating test dir: %v", err)
-				return
-			}
-
-			testDirs = append(testDirs, testDir)
-			testFileBaseDir := filepath.Join(testDir, test.baseDir)
-			want := make(map[string]string)
-			for _, fileDef := range test.files {
-				err := func() error {
-					path := filepath.Join(testFileBaseDir, fileDef.relativePath)
-					err := os.MkdirAll(filepath.Dir(path), 0755)
-					want[path] = fileDef.expectedPath
-					if err != nil {
-						return err
-					}
-
-					file, err := os.Create(path)
-					if err != nil {
-						return err
-					}
-
-					defer file.Close()
-
-					_, err = file.WriteString("test")
-					return err
-				}()
-				if err != nil {
-					t.Errorf("unable to create file on fs: %v", err)
-					return
-				}
-			}
-
-			var actualFiles []CopyableFile
-			err = addMinikubeDirToAssets(testFileBaseDir, test.vmPath, &actualFiles)
-			if err != nil {
-				t.Errorf("got unexpected error adding minikube dir assets: %v", err)
-				return
-			}
-
-			got := make(map[string]string)
-			for _, actualFile := range actualFiles {
-				got[actualFile.GetAssetName()] = actualFile.GetTargetDir()
-			}
-			if diff := cmp.Diff(want, got); diff != "" {
-				t.Errorf("files differ: (-want +got)\n%s", diff)
-			}
-		})
-	}
-
-}
diff --git a/pkg/minikube/cluster/filesync.go b/pkg/minikube/cluster/filesync.go
index 072fc00b9c..6bf4559335 100644
--- a/pkg/minikube/cluster/filesync.go
+++ b/pkg/minikube/cluster/filesync.go
@@ -48,12 +48,12 @@ func syncLocalAssets(cr command.Runner) error {
 
 // localAssets returns local files and addons from the minikube home directory
 func localAssets() ([]assets.CopyableFile, error) {
-	fs, err := assetsFromDir(localpath.MakeMiniPath("addons"), vmpath.GuestAddonsDir)
+	fs, err := assetsFromDir(localpath.MakeMiniPath("addons"), vmpath.GuestAddonsDir, true)
 	if err != nil {
 		return fs, errors.Wrap(err, "addons dir")
 	}
 
-	localFiles, err := assetsFromDir(localpath.MakeMiniPath("files"), "/")
+	localFiles, err := assetsFromDir(localpath.MakeMiniPath("files"), "/", false)
 	if err != nil {
 		return fs, errors.Wrap(err, "files dir")
 	}
@@ -62,8 +62,8 @@ func localAssets() ([]assets.CopyableFile, error) {
 	return fs, nil
 }
 
-// assetsFromDir generates assets from a local filepath
-func assetsFromDir(localRoot string, destRoot string) ([]assets.CopyableFile, error) {
+// assetsFromDir generates assets from a local filepath, with/without a flattened hierarchy
+func assetsFromDir(localRoot string, destRoot string, flatten bool) ([]assets.CopyableFile, error) {
 	glog.Infof("Scanning %s for local assets ...", localRoot)
 	fs := []assets.CopyableFile{}
 	err := filepath.Walk(localRoot, func(localPath string, fi os.FileInfo, err error) error {
@@ -86,11 +86,16 @@ func assetsFromDir(localRoot string, destRoot string) ([]assets.CopyableFile, er
 		}
 
 		dest := path.Join(destRoot, rel)
-		f, err := assets.NewFileAsset(localPath, path.Dir(dest), path.Base(dest), ps)
+		targetDir := path.Dir(dest)
+		targetName := path.Base(dest)
+		if flatten {
+			targetDir = destRoot
+		}
+		glog.Infof("local asset: %s -> %s in %s", localPath, targetName, targetDir)
+		f, err := assets.NewFileAsset(localPath, targetDir, targetName, ps)
 		if err != nil {
 			return errors.Wrapf(err, "creating file asset for %s", localPath)
 		}
-		glog.Infof("found asset to sync: %s -> %s", localPath, dest)
 		fs = append(fs, f)
 		return nil
 	})
diff --git a/pkg/minikube/cluster/fix.go b/pkg/minikube/cluster/fix.go
index 1a5e9c6cea..0437a5cffd 100644
--- a/pkg/minikube/cluster/fix.go
+++ b/pkg/minikube/cluster/fix.go
@@ -30,6 +30,7 @@ import (
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
 	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/driver"
 	"k8s.io/minikube/pkg/minikube/out"
 	"k8s.io/minikube/pkg/util/retry"
 )
@@ -98,11 +99,14 @@ func fixHost(api libmachine.API, mc config.MachineConfig) (*host.Host, error) {
 	if err := h.ConfigureAuth(); err != nil {
 		return h, &retry.RetriableError{Err: errors.Wrap(err, "Error configuring auth on host")}
 	}
-	return h, ensureSyncedGuestClock(h)
+	return h, ensureSyncedGuestClock(h, mc.VMDriver)
 }
 
 // ensureGuestClockSync ensures that the guest system clock is relatively in-sync
-func ensureSyncedGuestClock(h hostRunner) error {
+func ensureSyncedGuestClock(h hostRunner, drv string) error {
+	if !driver.IsVM(drv) {
+		return nil
+	}
 	d, err := guestClockDelta(h, time.Now())
 	if err != nil {
 		glog.Warningf("Unable to measure system clock delta: %v", err)
diff --git a/pkg/minikube/cluster/info.go b/pkg/minikube/cluster/info.go
index 49c133502c..cad29adb16 100644
--- a/pkg/minikube/cluster/info.go
+++ b/pkg/minikube/cluster/info.go
@@ -81,9 +81,9 @@ func showLocalOsRelease() {
 	out.T(out.Provisioner, "OS release is {{.pretty_name}}", out.V{"pretty_name": osReleaseInfo.PrettyName})
 }
 
-// showRemoteOsRelease shows systemd information about the current linux distribution, on the remote VM
-func showRemoteOsRelease(driver drivers.Driver) {
-	provisioner, err := provision.DetectProvisioner(driver)
+// logRemoteOsRelease shows systemd information about the current linux distribution, on the remote VM
+func logRemoteOsRelease(drv drivers.Driver) {
+	provisioner, err := provision.DetectProvisioner(drv)
 	if err != nil {
 		glog.Errorf("DetectProvisioner: %v", err)
 		return
@@ -105,12 +105,14 @@ func showHostInfo(cfg config.MachineConfig) {
 		if err == nil {
 			out.T(out.StartingNone, "Running on localhost (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...", out.V{"number_of_cpus": info.CPUs, "memory_size": info.Memory, "disk_size": info.DiskSize})
 		}
-	} else if driver.IsKIC(cfg.VMDriver) {
+		return
+	}
+	if driver.IsKIC(cfg.VMDriver) {
 		info, err := getHostInfo() // TODO medyagh: get docker-machine info for non linux
 		if err == nil {
 			out.T(out.StartingVM, "Creating Kubernetes in {{.driver_name}} container with (CPUs={{.number_of_cpus}}), Memory={{.memory_size}}MB ({{.host_memory_size}}MB available) ...", out.V{"driver_name": cfg.VMDriver, "number_of_cpus": cfg.CPUs, "number_of_host_cpus": info.CPUs, "memory_size": cfg.Memory, "host_memory_size": info.Memory})
 		}
-	} else {
-		out.T(out.StartingVM, "Creating {{.driver_name}} VM (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...", out.V{"driver_name": cfg.VMDriver, "number_of_cpus": cfg.CPUs, "memory_size": cfg.Memory, "disk_size": cfg.DiskSize})
+		return
 	}
+	out.T(out.StartingVM, "Creating {{.driver_name}} VM (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...", out.V{"driver_name": cfg.VMDriver, "number_of_cpus": cfg.CPUs, "memory_size": cfg.Memory, "disk_size": cfg.DiskSize})
 }
diff --git a/pkg/minikube/cluster/start.go b/pkg/minikube/cluster/start.go
index 26307ac948..2607c879ef 100644
--- a/pkg/minikube/cluster/start.go
+++ b/pkg/minikube/cluster/start.go
@@ -138,8 +138,7 @@ func createHost(api libmachine.API, cfg config.MachineConfig) (*host.Host, error
 
 // postStart are functions shared between startHost and fixHost
 func postStartSetup(h *host.Host, mc config.MachineConfig) error {
-	if h.DriverName == driver.Mock {
-		glog.Infof("mock driver: skipping postStart")
+	if driver.IsMock(h.DriverName) {
 		return nil
 	}
 
@@ -157,13 +156,15 @@ func postStartSetup(h *host.Host, mc config.MachineConfig) error {
 	if driver.BareMetal(mc.VMDriver) {
 		showLocalOsRelease()
 	}
-
+	if driver.IsVM(mc.VMDriver) {
+		logRemoteOsRelease(h.Driver)
+	}
 	return syncLocalAssets(r)
 }
 
 // commandRunner returns best available command runner for this host
 func commandRunner(h *host.Host) (command.Runner, error) {
-	if h.DriverName == driver.Mock {
+	if driver.IsMock(h.DriverName) {
 		glog.Errorf("commandRunner: returning unconfigured FakeCommandRunner, commands will fail!")
 		return &command.FakeCommandRunner{}, nil
 	}
diff --git a/pkg/minikube/driver/driver.go b/pkg/minikube/driver/driver.go
index 6af4933423..b666c648b5 100644
--- a/pkg/minikube/driver/driver.go
+++ b/pkg/minikube/driver/driver.go
@@ -88,6 +88,19 @@ func IsKIC(name string) bool {
 	return name == Docker
 }
 
+// IsMock checks if the driver is a mock
+func IsMock(name string) bool {
+	return name == Mock
+}
+
+// IsVM checks if the driver is a VM
+func IsVM(name string) bool {
+	if IsKIC(name) || IsMock(name) || BareMetal(name) {
+		return false
+	}
+	return true
+}
+
 // BareMetal returns if this driver is unisolated
 func BareMetal(name string) bool {
 	return name == None || name == Mock
diff --git a/pkg/util/utils_test.go b/pkg/util/utils_test.go
index 1be9f669fc..b8a3514d28 100644
--- a/pkg/util/utils_test.go
+++ b/pkg/util/utils_test.go
@@ -17,7 +17,6 @@ limitations under the License.
 package util
 
 import (
-	"strings"
 	"testing"
 )
 
@@ -41,46 +40,6 @@ func TestGetBinaryDownloadURL(t *testing.T) {
 
 }
 
-func TestReplaceChars(t *testing.T) {
-	testData := []struct {
-		src         []string
-		replacer    *strings.Replacer
-		expectedRes []string
-	}{
-		{[]string{"abc%def", "%Y%"}, strings.NewReplacer("%", "X"), []string{"abcXdef", "XYX"}},
-	}
-
-	for _, tt := range testData {
-		res := ReplaceChars(tt.src, tt.replacer)
-		for i, val := range res {
-			if val != tt.expectedRes[i] {
-				t.Fatalf("Expected '%s' but got '%s'", tt.expectedRes, res)
-			}
-		}
-	}
-}
-
-func TestConcatStrings(t *testing.T) {
-	testData := []struct {
-		src         []string
-		prefix      string
-		postfix     string
-		expectedRes []string
-	}{
-		{[]string{"abc", ""}, "xx", "yy", []string{"xxabcyy", "xxyy"}},
-		{[]string{"abc", ""}, "", "", []string{"abc", ""}},
-	}
-
-	for _, tt := range testData {
-		res := ConcatStrings(tt.src, tt.prefix, tt.postfix)
-		for i, val := range res {
-			if val != tt.expectedRes[i] {
-				t.Fatalf("Expected '%s' but got '%s'", tt.expectedRes, res)
-			}
-		}
-	}
-}
-
 func TestCalculateSizeInMB(t *testing.T) {
 	testData := []struct {
 		size           string

From 2d2b60f95beee29e8fa45209ec293c7ff07887ce Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Tue, 4 Feb 2020 15:12:55 -0800
Subject: [PATCH 237/250] fix lint

---
 cmd/minikube/cmd/config/set_test.go | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/cmd/minikube/cmd/config/set_test.go b/cmd/minikube/cmd/config/set_test.go
index 4e76b36ee0..f236445ffc 100644
--- a/cmd/minikube/cmd/config/set_test.go
+++ b/cmd/minikube/cmd/config/set_test.go
@@ -45,7 +45,12 @@ func TestSetNotAllowed(t *testing.T) {
 func TestSetOK(t *testing.T) {
 	createTestProfile(t)
 	err := Set("vm-driver", "virtualbox")
-	defer Unset("vm-driver")
+	defer func() {
+		err = Unset("vm-driver")
+		if err != nil {
+			t.Errorf("failed to unset vm-driver")
+		}
+	}()
 	if err != nil {
 		t.Fatalf("Set returned error for valid property value")
 	}
@@ -58,7 +63,7 @@ func TestSetOK(t *testing.T) {
 	}
 }
 
-func createTestProfile(t *testing.T) string {
+func createTestProfile(t *testing.T) {
 	t.Helper()
 	td, err := ioutil.TempDir("", "profile")
 	if err != nil {
@@ -78,5 +83,4 @@ func createTestProfile(t *testing.T) string {
 	if err := config.DefaultLoader.WriteConfigToFile(name, &config.MachineConfig{}); err != nil {
 		t.Fatalf("error creating temporary profile config: %v", err)
 	}
-	return name
 }

From 6189e9e6f762ebf6c5efd4835b77ed09706a2008 Mon Sep 17 00:00:00 2001
From: tstromberg <tstromberg@google.com>
Date: Tue, 4 Feb 2020 15:20:13 -0800
Subject: [PATCH 238/250] Add unit tests back

---
 pkg/minikube/cluster/filesync_test.go | 160 ++++++++++++++++++++++++++
 1 file changed, 160 insertions(+)
 create mode 100644 pkg/minikube/cluster/filesync_test.go

diff --git a/pkg/minikube/cluster/filesync_test.go b/pkg/minikube/cluster/filesync_test.go
new file mode 100644
index 0000000000..5898afbf66
--- /dev/null
+++ b/pkg/minikube/cluster/filesync_test.go
@@ -0,0 +1,160 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cluster
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"k8s.io/minikube/pkg/minikube/localpath"
+	"k8s.io/minikube/pkg/minikube/vmpath"
+)
+
+func setupTestDir() (string, error) {
+	path, err := ioutil.TempDir("", "minipath")
+	if err != nil {
+		return "", err
+	}
+
+	os.Setenv(localpath.MinikubeHome, path)
+	return path, err
+}
+
+func TestAssetsFromDir(t *testing.T) {
+	tests := []struct {
+		description string
+		baseDir     string
+		vmPath      string
+		flatten     bool
+		files       []struct {
+			relativePath string
+			expectedPath string
+		}
+	}{
+		{
+			description: "relative path assets",
+			baseDir:     "/addons",
+			flatten:     true,
+			files: []struct {
+				relativePath string
+				expectedPath string
+			}{
+				{
+					relativePath: "/dir1/file1.txt",
+					expectedPath: vmpath.GuestAddonsDir,
+				},
+				{
+					relativePath: "/dir1/file2.txt",
+					expectedPath: vmpath.GuestAddonsDir,
+				},
+				{
+					relativePath: "/dir2/file1.txt",
+					expectedPath: vmpath.GuestAddonsDir,
+				},
+			},
+			vmPath: vmpath.GuestAddonsDir,
+		},
+		{
+			description: "absolute path assets",
+			baseDir:     "/files",
+			flatten:     false,
+			files: []struct {
+				relativePath string
+				expectedPath string
+			}{
+				{
+					relativePath: "/dir1/file1.txt",
+					expectedPath: "/dir1",
+				},
+				{
+					relativePath: "/dir1/file2.txt",
+					expectedPath: "/dir1",
+				},
+				{
+					relativePath: "/dir2/file1.txt",
+					expectedPath: "/dir2",
+				},
+			},
+			vmPath: "/",
+		},
+	}
+	var testDirs = make([]string, 0)
+	defer func() {
+		for _, testDir := range testDirs {
+			err := os.RemoveAll(testDir)
+			if err != nil {
+				t.Logf("got unexpected error removing test dir: %v", err)
+			}
+		}
+	}()
+
+	for _, test := range tests {
+		t.Run(test.description, func(t *testing.T) {
+			testDir, err := setupTestDir()
+			if err != nil {
+				t.Errorf("got unexpected error creating test dir: %v", err)
+				return
+			}
+
+			testDirs = append(testDirs, testDir)
+			testFileBaseDir := filepath.Join(testDir, test.baseDir)
+			want := make(map[string]string)
+			for _, fileDef := range test.files {
+				err := func() error {
+					path := filepath.Join(testFileBaseDir, fileDef.relativePath)
+					err := os.MkdirAll(filepath.Dir(path), 0755)
+					want[path] = fileDef.expectedPath
+					if err != nil {
+						return err
+					}
+
+					file, err := os.Create(path)
+					if err != nil {
+						return err
+					}
+
+					defer file.Close()
+
+					_, err = file.WriteString("test")
+					return err
+				}()
+				if err != nil {
+					t.Errorf("unable to create file on fs: %v", err)
+					return
+				}
+			}
+
+			actualFiles, err := assetsFromDir(testFileBaseDir, test.vmPath, test.flatten)
+			if err != nil {
+				t.Errorf("got unexpected error adding minikube dir assets: %v", err)
+				return
+			}
+
+			got := make(map[string]string)
+			for _, actualFile := range actualFiles {
+				got[actualFile.GetAssetName()] = actualFile.GetTargetDir()
+			}
+			if diff := cmp.Diff(want, got); diff != "" {
+				t.Errorf("files differ: (-want +got)\n%s", diff)
+			}
+		})
+	}
+
+}

From 4f78a3ce888aa3d135bd6de9d2d0815086e3ad2f Mon Sep 17 00:00:00 2001
From: tstromberg <tstromberg@google.com>
Date: Tue, 4 Feb 2020 15:28:09 -0800
Subject: [PATCH 239/250] Use driver.IsMock

---
 pkg/minikube/cluster/fix.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/minikube/cluster/fix.go b/pkg/minikube/cluster/fix.go
index 3ff3fd3011..5a53680f34 100644
--- a/pkg/minikube/cluster/fix.go
+++ b/pkg/minikube/cluster/fix.go
@@ -91,7 +91,7 @@ func fixHost(api libmachine.API, mc config.MachineConfig) (*host.Host, error) {
 		}
 	}
 
-	if h.DriverName == driver.Mock {
+	if driver.IsMock(h.DriverName) {
 		return h, nil
 	}
 

From c86b20ade23e3722719d553bf08ba2aec6d1415b Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Tue, 4 Feb 2020 15:33:04 -0800
Subject: [PATCH 240/250] add option to exec runner to point to host docker
 daemon

---
 cmd/minikube/cmd/env.go              |  2 +-
 pkg/drivers/none/none.go             |  2 +-
 pkg/minikube/cluster/cluster_test.go |  8 ++--
 pkg/minikube/cluster/docker_env.go   | 11 ++---
 pkg/minikube/cluster/start.go        |  2 +-
 pkg/minikube/command/exec_runner.go  | 62 ++++++++++++++++++++++++++--
 pkg/minikube/constants/constants.go  |  5 ++-
 pkg/minikube/machine/client.go       |  2 +-
 pkg/provision/provision.go           |  2 +-
 9 files changed, 77 insertions(+), 19 deletions(-)

diff --git a/cmd/minikube/cmd/env.go b/cmd/minikube/cmd/env.go
index bed87b4f1c..2a8ab45e1b 100644
--- a/cmd/minikube/cmd/env.go
+++ b/cmd/minikube/cmd/env.go
@@ -163,7 +163,7 @@ REM @FOR /f "tokens=*" %%i IN ('%s') DO @%%i
 
 func shellCfgSet(api libmachine.API) (*ShellConfig, error) {
 
-	envMap, err := cluster.GetHostDockerEnv(api)
+	envMap, err := cluster.GetNodeDockerEnv(api)
 	if err != nil {
 		return nil, err
 	}
diff --git a/pkg/drivers/none/none.go b/pkg/drivers/none/none.go
index 3fd00b76b9..a2a43e8855 100644
--- a/pkg/drivers/none/none.go
+++ b/pkg/drivers/none/none.go
@@ -64,7 +64,7 @@ type Config struct {
 
 // NewDriver returns a fully configured None driver
 func NewDriver(c Config) *Driver {
-	runner := &command.ExecRunner{}
+	runner := command.NewExecRunner()
 	runtime, err := cruntime.New(cruntime.Config{Type: c.ContainerRuntime, Runner: runner})
 	// Libraries shouldn't panic, but there is no way for drivers to return error :(
 	if err != nil {
diff --git a/pkg/minikube/cluster/cluster_test.go b/pkg/minikube/cluster/cluster_test.go
index b983436416..4c9490ecb8 100644
--- a/pkg/minikube/cluster/cluster_test.go
+++ b/pkg/minikube/cluster/cluster_test.go
@@ -338,7 +338,7 @@ func TestGetHostStatus(t *testing.T) {
 	checkState(state.Stopped.String())
 }
 
-func TestGetHostDockerEnv(t *testing.T) {
+func TestGetNodeDockerEnv(t *testing.T) {
 	RegisterMockDriver(t)
 	tempDir := tests.MakeTempDir()
 	defer os.RemoveAll(tempDir)
@@ -356,7 +356,7 @@ func TestGetHostDockerEnv(t *testing.T) {
 	}
 	h.Driver = d
 
-	envMap, err := GetHostDockerEnv(api)
+	envMap, err := GetNodeDockerEnv(api)
 	if err != nil {
 		t.Fatalf("Unexpected error getting env: %v", err)
 	}
@@ -374,7 +374,7 @@ func TestGetHostDockerEnv(t *testing.T) {
 	}
 }
 
-func TestGetHostDockerEnvIPv6(t *testing.T) {
+func TestGetNodeDockerEnvIPv6(t *testing.T) {
 	RegisterMockDriver(t)
 
 	tempDir := tests.MakeTempDir()
@@ -393,7 +393,7 @@ func TestGetHostDockerEnvIPv6(t *testing.T) {
 	}
 	h.Driver = d
 
-	envMap, err := GetHostDockerEnv(api)
+	envMap, err := GetNodeDockerEnv(api)
 	if err != nil {
 		t.Fatalf("Unexpected error getting env: %v", err)
 	}
diff --git a/pkg/minikube/cluster/docker_env.go b/pkg/minikube/cluster/docker_env.go
index 9db3855608..d3feac7fca 100644
--- a/pkg/minikube/cluster/docker_env.go
+++ b/pkg/minikube/cluster/docker_env.go
@@ -31,8 +31,8 @@ import (
 	"k8s.io/minikube/pkg/minikube/localpath"
 )
 
-// GetHostDockerEnv gets the necessary docker env variables to allow the use of docker through minikube's vm
-func GetHostDockerEnv(api libmachine.API) (map[string]string, error) {
+// GetNodeDockerEnv gets the necessary docker env variables to allow the use of docker through minikube's vm
+func GetNodeDockerEnv(api libmachine.API) (map[string]string, error) {
 	pName := viper.GetString(config.MachineProfile)
 	host, err := CheckIfHostExistsAndLoad(api, pName)
 	if err != nil {
@@ -58,9 +58,10 @@ func GetHostDockerEnv(api libmachine.API) (map[string]string, error) {
 	}
 
 	envMap := map[string]string{
-		constants.DockerTLSVerifyEnv: "1",
-		constants.DockerHostEnv:      tcpPrefix + net.JoinHostPort(ip, fmt.Sprint(port)),
-		constants.DockerCertPathEnv:  localpath.MakeMiniPath("certs"),
+		constants.DockerTLSVerifyEnv:       "1",
+		constants.DockerHostEnv:            tcpPrefix + net.JoinHostPort(ip, fmt.Sprint(port)),
+		constants.DockerCertPathEnv:        localpath.MakeMiniPath("certs"),
+		constants.MinikubeActiveDockerdEnv: pName,
 	}
 	return envMap, nil
 }
diff --git a/pkg/minikube/cluster/start.go b/pkg/minikube/cluster/start.go
index 4895ff9484..86c14d24d2 100644
--- a/pkg/minikube/cluster/start.go
+++ b/pkg/minikube/cluster/start.go
@@ -174,7 +174,7 @@ func commandRunner(h *host.Host) (command.Runner, error) {
 		return &command.FakeCommandRunner{}, nil
 	}
 	if driver.BareMetal(h.Driver.DriverName()) {
-		return &command.ExecRunner{}, nil
+		return command.NewExecRunner(), nil
 	}
 	if h.Driver.DriverName() == driver.Docker {
 		return command.NewKICRunner(h.Name, "docker"), nil
diff --git a/pkg/minikube/command/exec_runner.go b/pkg/minikube/command/exec_runner.go
index ed7740b5b6..eebf685735 100644
--- a/pkg/minikube/command/exec_runner.go
+++ b/pkg/minikube/command/exec_runner.go
@@ -30,17 +30,36 @@ import (
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
 	"k8s.io/minikube/pkg/minikube/assets"
+	"k8s.io/minikube/pkg/minikube/constants"
 )
 
 // ExecRunner runs commands using the os/exec package.
 //
 // It implements the CommandRunner interface.
-type ExecRunner struct{}
+type execRunner struct {
+	EnsureUsingHostDaemon bool // if set it ensures that the command runs against the host docker (podman) daemon not the one inside minikube
+}
+
+// NewExecRunner returns a kicRunner implementor of runner which runs cmds inside a container
+func NewExecRunner(ensureHostDaemon ...bool) Runner {
+	if len(ensureHostDaemon) > 0 {
+		return &execRunner{
+			EnsureUsingHostDaemon: ensureHostDaemon[0]}
+	} else {
+		return &execRunner{}
+	}
+}
 
 // RunCmd implements the Command Runner interface to run a exec.Cmd object
-func (*ExecRunner) RunCmd(cmd *exec.Cmd) (*RunResult, error) {
+func (e *execRunner) RunCmd(cmd *exec.Cmd) (*RunResult, error) {
 	rr := &RunResult{Args: cmd.Args}
 	glog.Infof("Run: %v", rr.Command())
+	if e.EnsureUsingHostDaemon {
+		err := pointToHostDockerDaemon()
+		if err != nil {
+			return rr, errors.Wrap(err, "pointing to correct docker-daemon")
+		}
+	}
 
 	var outb, errb io.Writer
 	if cmd.Stdout == nil {
@@ -79,7 +98,13 @@ func (*ExecRunner) RunCmd(cmd *exec.Cmd) (*RunResult, error) {
 }
 
 // Copy copies a file and its permissions
-func (*ExecRunner) Copy(f assets.CopyableFile) error {
+func (e *execRunner) Copy(f assets.CopyableFile) error {
+	if e.EnsureUsingHostDaemon {
+		err := pointToHostDockerDaemon()
+		if err != nil {
+			return errors.Wrap(err, "pointing to correct docker-daemon")
+		}
+	}
 	targetPath := path.Join(f.GetTargetDir(), f.GetTargetName())
 	if _, err := os.Stat(targetPath); err == nil {
 		if err := os.Remove(targetPath); err != nil {
@@ -108,7 +133,36 @@ do you have the correct permissions?`,
 }
 
 // Remove removes a file
-func (e *ExecRunner) Remove(f assets.CopyableFile) error {
+func (e *execRunner) Remove(f assets.CopyableFile) error {
+	if e.EnsureUsingHostDaemon {
+		err := pointToHostDockerDaemon()
+		if err != nil {
+			return errors.Wrap(err, "pointing to correct docker-daemon")
+		}
+	}
 	targetPath := filepath.Join(f.GetTargetDir(), f.GetTargetName())
 	return os.Remove(targetPath)
 }
+
+// pointToHostDockerDaemon will unset env variables that point to docker inside minikube
+// to make sure it points to the docker daemon installed by user.
+func pointToHostDockerDaemon() error {
+	start := time.Now()
+	p := os.Getenv(constants.MinikubeActiveDockerdEnv)
+	if p != "" {
+		fmt.Println("medya dbg: shell is pointing to docker inside minikube will unset")
+	}
+
+	for i := range constants.DockerDaemonEnvs {
+		e := constants.DockerDaemonEnvs[i]
+		fmt.Printf("medya dbg: setting env %s", e)
+		err := os.Setenv(e, "")
+		if err != nil {
+			return errors.Wrapf(err, "resetting %s env", e)
+		}
+
+	}
+	elapsed := time.Since(start)
+	fmt.Printf("Done: (%s)\n", elapsed)
+	return nil
+}
diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go
index 0db6d641d7..edf14a90e4 100644
--- a/pkg/minikube/constants/constants.go
+++ b/pkg/minikube/constants/constants.go
@@ -39,7 +39,6 @@ const (
 	ClusterDNSDomain = "cluster.local"
 	// DefaultServiceCIDR is The CIDR to be used for service cluster IPs
 	DefaultServiceCIDR = "10.96.0.0/12"
-
 	// DockerTLSVerifyEnv is used for docker daemon settings
 	DockerTLSVerifyEnv = "DOCKER_TLS_VERIFY"
 	// DockerHostEnv is used for docker daemon settings
@@ -48,9 +47,13 @@ const (
 	DockerCertPathEnv = "DOCKER_CERT_PATH"
 	// MinikubeActiveDockerdEnv holds the docker daemon which user's shell is pointing at
 	// value would be profile or empty if pointing to the user's host daemon.
+	// DockerDaemonEnvs has list of environment variables to control docker daemon shell is using
+
 	MinikubeActiveDockerdEnv = "MINIKUBE_ACTIVE_DOCKERD"
 )
 
+var DockerDaemonEnvs = [3]string{DockerHostEnv, DockerTLSVerifyEnv, DockerCertPathEnv}
+
 // DefaultMinipath is the default Minikube path (under the home directory)
 var DefaultMinipath = filepath.Join(homedir.HomeDir(), ".minikube")
 
diff --git a/pkg/minikube/machine/client.go b/pkg/minikube/machine/client.go
index a07cda9d22..ba79fe4745 100644
--- a/pkg/minikube/machine/client.go
+++ b/pkg/minikube/machine/client.go
@@ -152,7 +152,7 @@ func CommandRunner(h *host.Host) (command.Runner, error) {
 		return &command.FakeCommandRunner{}, nil
 	}
 	if driver.BareMetal(h.Driver.DriverName()) {
-		return &command.ExecRunner{}, nil
+		return command.NewExecRunner(), nil
 	}
 	if h.Driver.DriverName() == driver.Docker {
 		return command.NewKICRunner(h.Name, "docker"), nil
diff --git a/pkg/provision/provision.go b/pkg/provision/provision.go
index 0ef9f69664..b781fec47a 100644
--- a/pkg/provision/provision.go
+++ b/pkg/provision/provision.go
@@ -117,7 +117,7 @@ func configureAuth(p miniProvisioner) error {
 
 func copyHostCerts(authOptions auth.Options) error {
 	log.Infof("copyHostCerts")
-	execRunner := &command.ExecRunner{}
+	execRunner := command.NewExecRunner()
 	hostCerts := map[string]string{
 		authOptions.CaCertPath:     path.Join(authOptions.StorePath, "ca.pem"),
 		authOptions.ClientCertPath: path.Join(authOptions.StorePath, "cert.pem"),

From 6c131469873496ee4d522e82b5465ad01d808155 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Tue, 4 Feb 2020 15:48:20 -0800
Subject: [PATCH 241/250] revert some changes

---
 pkg/minikube/command/exec_runner.go | 54 ++---------------------------
 1 file changed, 2 insertions(+), 52 deletions(-)

diff --git a/pkg/minikube/command/exec_runner.go b/pkg/minikube/command/exec_runner.go
index eebf685735..014723a873 100644
--- a/pkg/minikube/command/exec_runner.go
+++ b/pkg/minikube/command/exec_runner.go
@@ -30,37 +30,22 @@ import (
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
 	"k8s.io/minikube/pkg/minikube/assets"
-	"k8s.io/minikube/pkg/minikube/constants"
 )
 
 // ExecRunner runs commands using the os/exec package.
 //
 // It implements the CommandRunner interface.
 type execRunner struct {
-	EnsureUsingHostDaemon bool // if set it ensures that the command runs against the host docker (podman) daemon not the one inside minikube
 }
 
 // NewExecRunner returns a kicRunner implementor of runner which runs cmds inside a container
-func NewExecRunner(ensureHostDaemon ...bool) Runner {
-	if len(ensureHostDaemon) > 0 {
-		return &execRunner{
-			EnsureUsingHostDaemon: ensureHostDaemon[0]}
-	} else {
-		return &execRunner{}
-	}
+func NewExecRunner() Runner {
+	return &execRunner{}
 }
 
 // RunCmd implements the Command Runner interface to run a exec.Cmd object
 func (e *execRunner) RunCmd(cmd *exec.Cmd) (*RunResult, error) {
 	rr := &RunResult{Args: cmd.Args}
-	glog.Infof("Run: %v", rr.Command())
-	if e.EnsureUsingHostDaemon {
-		err := pointToHostDockerDaemon()
-		if err != nil {
-			return rr, errors.Wrap(err, "pointing to correct docker-daemon")
-		}
-	}
-
 	var outb, errb io.Writer
 	if cmd.Stdout == nil {
 		var so bytes.Buffer
@@ -99,12 +84,6 @@ func (e *execRunner) RunCmd(cmd *exec.Cmd) (*RunResult, error) {
 
 // Copy copies a file and its permissions
 func (e *execRunner) Copy(f assets.CopyableFile) error {
-	if e.EnsureUsingHostDaemon {
-		err := pointToHostDockerDaemon()
-		if err != nil {
-			return errors.Wrap(err, "pointing to correct docker-daemon")
-		}
-	}
 	targetPath := path.Join(f.GetTargetDir(), f.GetTargetName())
 	if _, err := os.Stat(targetPath); err == nil {
 		if err := os.Remove(targetPath); err != nil {
@@ -134,35 +113,6 @@ do you have the correct permissions?`,
 
 // Remove removes a file
 func (e *execRunner) Remove(f assets.CopyableFile) error {
-	if e.EnsureUsingHostDaemon {
-		err := pointToHostDockerDaemon()
-		if err != nil {
-			return errors.Wrap(err, "pointing to correct docker-daemon")
-		}
-	}
 	targetPath := filepath.Join(f.GetTargetDir(), f.GetTargetName())
 	return os.Remove(targetPath)
 }
-
-// pointToHostDockerDaemon will unset env variables that point to docker inside minikube
-// to make sure it points to the docker daemon installed by user.
-func pointToHostDockerDaemon() error {
-	start := time.Now()
-	p := os.Getenv(constants.MinikubeActiveDockerdEnv)
-	if p != "" {
-		fmt.Println("medya dbg: shell is pointing to docker inside minikube will unset")
-	}
-
-	for i := range constants.DockerDaemonEnvs {
-		e := constants.DockerDaemonEnvs[i]
-		fmt.Printf("medya dbg: setting env %s", e)
-		err := os.Setenv(e, "")
-		if err != nil {
-			return errors.Wrapf(err, "resetting %s env", e)
-		}
-
-	}
-	elapsed := time.Since(start)
-	fmt.Printf("Done: (%s)\n", elapsed)
-	return nil
-}

From 4b9d15a34c7ea04810a85eaa0675197f71056eea Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Tue, 4 Feb 2020 16:09:43 -0800
Subject: [PATCH 242/250] point to host docker

---
 pkg/drivers/kic/kic.go     |  4 +++
 pkg/drivers/kic/oci/oci.go | 66 ++++++++++++++++++++++++++++++++++----
 2 files changed, 64 insertions(+), 6 deletions(-)

diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go
index cfc6b33f33..13d7de7fb6 100644
--- a/pkg/drivers/kic/kic.go
+++ b/pkg/drivers/kic/kic.go
@@ -180,6 +180,10 @@ func (d *Driver) GetURL() (string, error) {
 
 // GetState returns the state that the host is in (running, stopped, etc)
 func (d *Driver) GetState() (state.State, error) {
+	if err := oci.PointToHostDockerDaemon(); err != nil {
+		return state.Error, errors.Wrap(err, "point host docker-daemon")
+	}
+
 	cmd := exec.Command(d.NodeConfig.OCIBinary, "inspect", "-f", "{{.State.Status}}", d.MachineName)
 	out, err := cmd.CombinedOutput()
 	o := strings.Trim(string(out), "\n")
diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go
index 1ea1f53c3c..9f8ddd6048 100644
--- a/pkg/drivers/kic/oci/oci.go
+++ b/pkg/drivers/kic/oci/oci.go
@@ -23,7 +23,9 @@ import (
 	"bufio"
 	"bytes"
 
+	"github.com/golang/glog"
 	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/minikube/constants"
 
 	"fmt"
 	"os/exec"
@@ -32,6 +34,10 @@ import (
 
 // CreateContainerNode creates a new container node
 func CreateContainerNode(p CreateParams) error {
+	if err := PointToHostDockerDaemon(); err != nil {
+		return errors.Wrap(err, "point host docker-daemon")
+	}
+
 	runArgs := []string{
 		fmt.Sprintf("--cpus=%s", p.CPUs),
 		fmt.Sprintf("--memory=%s", p.Memory),
@@ -65,13 +71,17 @@ func CreateContainerNode(p CreateParams) error {
 	// adds node specific args
 	runArgs = append(runArgs, p.ExtraArgs...)
 
-	if isUsernsRemapEnabled(p.OCIBinary) {
+	enabled, err := isUsernsRemapEnabled(p.OCIBinary)
+	if err != nil {
+		glog.Warningf("Failed to detect if userns is enabled: %v", err)
+	}
+	if enabled {
 		// We need this argument in order to make this command work
 		// in systems that have userns-remap enabled on the docker daemon
 		runArgs = append(runArgs, "--userns=host")
 	}
 
-	_, err := createContainer(p.OCIBinary,
+	_, err = createContainer(p.OCIBinary,
 		p.Image,
 		withRunArgs(runArgs...),
 		withMounts(p.Mounts),
@@ -85,6 +95,10 @@ func CreateContainerNode(p CreateParams) error {
 
 // CreateContainer creates a container with "docker/podman run"
 func createContainer(ociBinary string, image string, opts ...createOpt) ([]string, error) {
+	if err := PointToHostDockerDaemon(); err != nil {
+		return nil, errors.Wrap(err, "point host docker-daemon")
+	}
+
 	o := &createOpts{}
 	for _, opt := range opts {
 		o = opt(o)
@@ -121,6 +135,9 @@ func createContainer(ociBinary string, image string, opts ...createOpt) ([]strin
 
 // Copy copies a local asset into the container
 func Copy(ociBinary string, ociID string, targetDir string, fName string) error {
+	if err := PointToHostDockerDaemon(); err != nil {
+		return errors.Wrap(err, "point host docker-daemon")
+	}
 	if _, err := os.Stat(fName); os.IsNotExist(err) {
 		return errors.Wrapf(err, "error source %s does not exist", fName)
 	}
@@ -139,6 +156,9 @@ func Copy(ociBinary string, ociID string, targetDir string, fName string) error
 // 32769, nil
 // only supports TCP ports
 func HostPortBinding(ociBinary string, ociID string, contPort int) (int, error) {
+	if err := PointToHostDockerDaemon(); err != nil {
+		return 0, errors.Wrap(err, "point host docker-daemon")
+	}
 	cmd := exec.Command(ociBinary, "inspect", "-f", fmt.Sprintf("'{{(index (index .NetworkSettings.Ports \"%d/tcp\") 0).HostPort}}'", contPort), ociID)
 	out, err := cmd.CombinedOutput()
 	if err != nil {
@@ -155,6 +175,9 @@ func HostPortBinding(ociBinary string, ociID string, contPort int) (int, error)
 
 // ContainerIPs returns ipv4,ipv6, error of a container by their name
 func ContainerIPs(ociBinary string, name string) (string, string, error) {
+	if err := PointToHostDockerDaemon(); err != nil {
+		return "", "", errors.Wrap(err, "point host docker-daemon")
+	}
 	// retrieve the IP address of the node using docker inspect
 	lines, err := inspect(ociBinary, name, "{{range .NetworkSettings.Networks}}{{.IPAddress}},{{.GlobalIPv6Address}}{{end}}")
 	if err != nil {
@@ -173,6 +196,9 @@ func ContainerIPs(ociBinary string, name string) (string, string, error) {
 
 // ContainerID returns id of a container name
 func ContainerID(ociBinary string, nameOrID string) (string, error) {
+	if err := PointToHostDockerDaemon(); err != nil {
+		return "", errors.Wrap(err, "point host docker-daemon")
+	}
 	cmd := exec.Command(ociBinary, "inspect", "-f", "{{.Id}}", nameOrID)
 	id, err := cmd.CombinedOutput()
 	if err != nil {
@@ -188,6 +214,9 @@ func ListOwnedContainers(ociBinary string) ([]string, error) {
 
 // inspect return low-level information on containers
 func inspect(ociBinary string, containerNameOrID, format string) ([]string, error) {
+	if err := PointToHostDockerDaemon(); err != nil {
+		return nil, errors.Wrap(err, "point host docker-daemon")
+	}
 	cmd := exec.Command(ociBinary, "inspect",
 		"-f", format,
 		containerNameOrID) // ... against the "node" container
@@ -250,7 +279,10 @@ func generateMountBindings(mounts ...Mount) []string {
 }
 
 // isUsernsRemapEnabled checks if userns-remap is enabled in docker
-func isUsernsRemapEnabled(ociBinary string) bool {
+func isUsernsRemapEnabled(ociBinary string) (bool, error) {
+	if err := PointToHostDockerDaemon(); err != nil {
+		return false, errors.Wrap(err, "point host docker-daemon")
+	}
 	cmd := exec.Command(ociBinary, "info", "--format", "'{{json .SecurityOptions}}'")
 	var buff bytes.Buffer
 	cmd.Stdout = &buff
@@ -262,14 +294,14 @@ func isUsernsRemapEnabled(ociBinary string) bool {
 		lines = append(lines, scanner.Text())
 	}
 	if err != nil {
-		return false
+		return false, nil
 	}
 	if len(lines) > 0 {
 		if strings.Contains(lines[0], "name=userns") {
-			return true
+			return true, nil
 		}
 	}
-	return false
+	return false, nil
 }
 
 func generatePortMappings(portMappings ...PortMapping) []string {
@@ -311,6 +343,9 @@ func withPortMappings(portMappings []PortMapping) createOpt {
 // listContainersByLabel lists all the containres that kic driver created on user's machine using a label
 // io.x-k8s.kic.cluster
 func listContainersByLabel(ociBinary string, label string) ([]string, error) {
+	if err := PointToHostDockerDaemon(); err != nil {
+		return nil, errors.Wrap(err, "point host docker-daemon")
+	}
 	cmd := exec.Command(ociBinary, "ps", "-a", "--filter", fmt.Sprintf("label=%s", label), "--format", "{{.Names}}")
 	var b bytes.Buffer
 	cmd.Stdout = &b
@@ -323,3 +358,22 @@ func listContainersByLabel(ociBinary string, label string) ([]string, error) {
 	}
 	return lines, err
 }
+
+// PointToHostDockerDaemon will unset env variables that point to docker inside minikube
+// to make sure it points to the docker daemon installed by user.
+func PointToHostDockerDaemon() error {
+	p := os.Getenv(constants.MinikubeActiveDockerdEnv)
+	if p != "" {
+		glog.Infof("shell is pointing to docker inside minikube. will unset to use host")
+	}
+
+	for i := range constants.DockerDaemonEnvs {
+		e := constants.DockerDaemonEnvs[i]
+		err := os.Setenv(e, "")
+		if err != nil {
+			return errors.Wrapf(err, "resetting %s env", e)
+		}
+
+	}
+	return nil
+}

From 435010a3fc229496afa028fbce89beff9a290a08 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Tue, 4 Feb 2020 16:35:51 -0800
Subject: [PATCH 243/250] pass profilename for testability

---
 cmd/minikube/cmd/env.go             |  8 ++++----
 cmd/minikube/cmd/env_test.go        | 25 +++++++++++++------------
 pkg/minikube/command/exec_runner.go |  6 +++---
 3 files changed, 20 insertions(+), 19 deletions(-)

diff --git a/cmd/minikube/cmd/env.go b/cmd/minikube/cmd/env.go
index 2a8ab45e1b..546a6a5cc5 100644
--- a/cmd/minikube/cmd/env.go
+++ b/cmd/minikube/cmd/env.go
@@ -128,9 +128,9 @@ type NoProxyGetter interface {
 // EnvNoProxyGetter gets the no_proxy variable, using environment
 type EnvNoProxyGetter struct{}
 
-func generateUsageHint(userShell string) string {
+func generateUsageHint(profile string, userShell string) string {
 	const usgPlz = "Please run command bellow to point your shell to minikube's docker-daemon :"
-	var usgCmd = fmt.Sprintf("minikube -p %s docker-env", viper.GetString(config.MachineProfile))
+	var usgCmd = fmt.Sprintf("minikube -p %s docker-env", profile)
 	var usageHintMap = map[string]string{
 		"bash": fmt.Sprintf(`
 # %s
@@ -178,7 +178,7 @@ func shellCfgSet(api libmachine.API) (*ShellConfig, error) {
 		DockerHost:             envMap[constants.DockerHostEnv],
 		DockerTLSVerify:        envMap[constants.DockerTLSVerifyEnv],
 		MinikubeDockerdProfile: envMap[constants.MinikubeActiveDockerdEnv],
-		UsageHint:              generateUsageHint(userShell),
+		UsageHint:              generateUsageHint(viper.GetString(config.MachineProfile), userShell),
 	}
 
 	if noProxy {
@@ -247,7 +247,7 @@ func shellCfgUnset() (*ShellConfig, error) {
 	}
 
 	shellCfg := &ShellConfig{
-		UsageHint: generateUsageHint(userShell),
+		UsageHint: generateUsageHint(viper.GetString(config.MachineProfile), userShell),
 	}
 
 	if noProxy {
diff --git a/cmd/minikube/cmd/env_test.go b/cmd/minikube/cmd/env_test.go
index 9060563067..781dfec88d 100644
--- a/cmd/minikube/cmd/env_test.go
+++ b/cmd/minikube/cmd/env_test.go
@@ -62,7 +62,7 @@ func newShellCfg(shell, prefix, suffix, delim string) *ShellConfig {
 		DockerCertPath:  localpath.MakeMiniPath("certs"),
 		DockerTLSVerify: "1",
 		DockerHost:      "tcp://127.0.0.1:2376",
-		UsageHint:       generateUsageHint(shell),
+		UsageHint:       generateUsageHint("minikube", shell),
 		Prefix:          prefix,
 		Suffix:          suffix,
 		Delimiter:       delim,
@@ -144,7 +144,7 @@ func TestShellCfgSet(t *testing.T) {
 				DockerCertPath:  localpath.MakeMiniPath("certs"),
 				DockerTLSVerify: "1",
 				DockerHost:      "tcp://127.0.0.1:2376",
-				UsageHint:       usageHintMap["bash"],
+				UsageHint:       generateUsageHint("minikube", "bash"),
 				Prefix:          bashSetPfx,
 				Suffix:          bashSetSfx,
 				Delimiter:       bashSetDelim,
@@ -163,7 +163,7 @@ func TestShellCfgSet(t *testing.T) {
 				DockerCertPath:  localpath.MakeMiniPath("certs"),
 				DockerTLSVerify: "1",
 				DockerHost:      "tcp://127.0.0.1:2376",
-				UsageHint:       usageHintMap["bash"],
+				UsageHint:       generateUsageHint("minikube", "bash"),
 				Prefix:          bashSetPfx,
 				Suffix:          bashSetSfx,
 				Delimiter:       bashSetDelim,
@@ -182,7 +182,7 @@ func TestShellCfgSet(t *testing.T) {
 				DockerCertPath:  localpath.MakeMiniPath("certs"),
 				DockerTLSVerify: "1",
 				DockerHost:      "tcp://127.0.0.1:2376",
-				UsageHint:       usageHintMap["bash"],
+				UsageHint:       generateUsageHint("minikube", "bash"),
 				Prefix:          bashSetPfx,
 				Suffix:          bashSetSfx,
 				Delimiter:       bashSetDelim,
@@ -201,7 +201,7 @@ func TestShellCfgSet(t *testing.T) {
 				DockerCertPath:  localpath.MakeMiniPath("certs"),
 				DockerTLSVerify: "1",
 				DockerHost:      "tcp://127.0.0.1:2376",
-				UsageHint:       usageHintMap["bash"],
+				UsageHint:       generateUsageHint("minikube", "bash"),
 				Prefix:          bashSetPfx,
 				Suffix:          bashSetSfx,
 				Delimiter:       bashSetDelim,
@@ -220,7 +220,7 @@ func TestShellCfgSet(t *testing.T) {
 				DockerCertPath:  localpath.MakeMiniPath("certs"),
 				DockerTLSVerify: "1",
 				DockerHost:      "tcp://127.0.0.1:2376",
-				UsageHint:       usageHintMap["bash"],
+				UsageHint:       generateUsageHint("minikube", "bash"),
 				Prefix:          bashSetPfx,
 				Suffix:          bashSetSfx,
 				Delimiter:       bashSetDelim,
@@ -254,6 +254,7 @@ func TestShellCfgSet(t *testing.T) {
 }
 
 func TestShellCfgUnset(t *testing.T) {
+
 	var tests = []struct {
 		description      string
 		shell            string
@@ -266,7 +267,7 @@ func TestShellCfgUnset(t *testing.T) {
 				Prefix:    bashUnsetPfx,
 				Suffix:    bashUnsetSfx,
 				Delimiter: bashUnsetDelim,
-				UsageHint: usageHintMap["bash"],
+				UsageHint: generateUsageHint("minikube", "bash"),
 			},
 		},
 		{
@@ -276,7 +277,7 @@ func TestShellCfgUnset(t *testing.T) {
 				Prefix:    bashUnsetPfx,
 				Suffix:    bashUnsetSfx,
 				Delimiter: bashUnsetDelim,
-				UsageHint: usageHintMap["bash"],
+				UsageHint: generateUsageHint("minikube", "bash"),
 			},
 		},
 		{
@@ -286,7 +287,7 @@ func TestShellCfgUnset(t *testing.T) {
 				Prefix:    fishUnsetPfx,
 				Suffix:    fishUnsetSfx,
 				Delimiter: fishUnsetDelim,
-				UsageHint: usageHintMap["fish"],
+				UsageHint: generateUsageHint("minikube", "fish"),
 			},
 		},
 		{
@@ -296,7 +297,7 @@ func TestShellCfgUnset(t *testing.T) {
 				Prefix:    psUnsetPfx,
 				Suffix:    psUnsetSfx,
 				Delimiter: psUnsetDelim,
-				UsageHint: usageHintMap["powershell"],
+				UsageHint: generateUsageHint("minikube", "powershell"),
 			},
 		},
 		{
@@ -306,7 +307,7 @@ func TestShellCfgUnset(t *testing.T) {
 				Prefix:    cmdUnsetPfx,
 				Suffix:    cmdUnsetSfx,
 				Delimiter: cmdUnsetDelim,
-				UsageHint: usageHintMap["cmd"],
+				UsageHint: generateUsageHint("minikube", "cmd"),
 			},
 		},
 		{
@@ -316,7 +317,7 @@ func TestShellCfgUnset(t *testing.T) {
 				Prefix:    emacsUnsetPfx,
 				Suffix:    emacsUnsetSfx,
 				Delimiter: emacsUnsetDelim,
-				UsageHint: usageHintMap["emacs"],
+				UsageHint: generateUsageHint("minikube", "emacs"),
 			},
 		},
 	}
diff --git a/pkg/minikube/command/exec_runner.go b/pkg/minikube/command/exec_runner.go
index 014723a873..5dc5b81a64 100644
--- a/pkg/minikube/command/exec_runner.go
+++ b/pkg/minikube/command/exec_runner.go
@@ -44,7 +44,7 @@ func NewExecRunner() Runner {
 }
 
 // RunCmd implements the Command Runner interface to run a exec.Cmd object
-func (e *execRunner) RunCmd(cmd *exec.Cmd) (*RunResult, error) {
+func (*execRunner) RunCmd(cmd *exec.Cmd) (*RunResult, error) {
 	rr := &RunResult{Args: cmd.Args}
 	var outb, errb io.Writer
 	if cmd.Stdout == nil {
@@ -83,7 +83,7 @@ func (e *execRunner) RunCmd(cmd *exec.Cmd) (*RunResult, error) {
 }
 
 // Copy copies a file and its permissions
-func (e *execRunner) Copy(f assets.CopyableFile) error {
+func (*execRunner) Copy(f assets.CopyableFile) error {
 	targetPath := path.Join(f.GetTargetDir(), f.GetTargetName())
 	if _, err := os.Stat(targetPath); err == nil {
 		if err := os.Remove(targetPath); err != nil {
@@ -112,7 +112,7 @@ do you have the correct permissions?`,
 }
 
 // Remove removes a file
-func (e *execRunner) Remove(f assets.CopyableFile) error {
+func (*execRunner) Remove(f assets.CopyableFile) error {
 	targetPath := filepath.Join(f.GetTargetDir(), f.GetTargetName())
 	return os.Remove(targetPath)
 }

From 12e53f4d1fe6fa3181524bc7de497f852910faf8 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Tue, 4 Feb 2020 16:45:27 -0800
Subject: [PATCH 244/250] fix unit tests

---
 cmd/minikube/cmd/env_test.go | 110 ++++++++++++++++++-----------------
 1 file changed, 58 insertions(+), 52 deletions(-)

diff --git a/cmd/minikube/cmd/env_test.go b/cmd/minikube/cmd/env_test.go
index 781dfec88d..f304471e6f 100644
--- a/cmd/minikube/cmd/env_test.go
+++ b/cmd/minikube/cmd/env_test.go
@@ -59,13 +59,14 @@ var defaultAPI = &tests.MockAPI{
 // Most of the shell cfg isn't configurable
 func newShellCfg(shell, prefix, suffix, delim string) *ShellConfig {
 	return &ShellConfig{
-		DockerCertPath:  localpath.MakeMiniPath("certs"),
-		DockerTLSVerify: "1",
-		DockerHost:      "tcp://127.0.0.1:2376",
-		UsageHint:       generateUsageHint("minikube", shell),
-		Prefix:          prefix,
-		Suffix:          suffix,
-		Delimiter:       delim,
+		DockerCertPath:         localpath.MakeMiniPath("certs"),
+		DockerTLSVerify:        "1",
+		DockerHost:             "tcp://127.0.0.1:2376",
+		UsageHint:              generateUsageHint("minikube", shell),
+		Prefix:                 prefix,
+		Suffix:                 suffix,
+		Delimiter:              delim,
+		MinikubeDockerdProfile: "minikube",
 	}
 }
 
@@ -141,15 +142,16 @@ func TestShellCfgSet(t *testing.T) {
 			noProxyValue: "",
 			noProxyFlag:  true,
 			expectedShellCfg: &ShellConfig{
-				DockerCertPath:  localpath.MakeMiniPath("certs"),
-				DockerTLSVerify: "1",
-				DockerHost:      "tcp://127.0.0.1:2376",
-				UsageHint:       generateUsageHint("minikube", "bash"),
-				Prefix:          bashSetPfx,
-				Suffix:          bashSetSfx,
-				Delimiter:       bashSetDelim,
-				NoProxyVar:      "NO_PROXY",
-				NoProxyValue:    "127.0.0.1",
+				DockerCertPath:         localpath.MakeMiniPath("certs"),
+				DockerTLSVerify:        "1",
+				DockerHost:             "tcp://127.0.0.1:2376",
+				UsageHint:              generateUsageHint("minikube", "bash"),
+				Prefix:                 bashSetPfx,
+				Suffix:                 bashSetSfx,
+				Delimiter:              bashSetDelim,
+				NoProxyVar:             "NO_PROXY",
+				NoProxyValue:           "127.0.0.1",
+				MinikubeDockerdProfile: "minikube",
 			},
 		},
 		{
@@ -160,15 +162,16 @@ func TestShellCfgSet(t *testing.T) {
 			noProxyValue: "",
 			noProxyFlag:  true,
 			expectedShellCfg: &ShellConfig{
-				DockerCertPath:  localpath.MakeMiniPath("certs"),
-				DockerTLSVerify: "1",
-				DockerHost:      "tcp://127.0.0.1:2376",
-				UsageHint:       generateUsageHint("minikube", "bash"),
-				Prefix:          bashSetPfx,
-				Suffix:          bashSetSfx,
-				Delimiter:       bashSetDelim,
-				NoProxyVar:      "no_proxy",
-				NoProxyValue:    "127.0.0.1",
+				DockerCertPath:         localpath.MakeMiniPath("certs"),
+				DockerTLSVerify:        "1",
+				DockerHost:             "tcp://127.0.0.1:2376",
+				UsageHint:              generateUsageHint("minikube", "bash"),
+				Prefix:                 bashSetPfx,
+				Suffix:                 bashSetSfx,
+				Delimiter:              bashSetDelim,
+				NoProxyVar:             "no_proxy",
+				NoProxyValue:           "127.0.0.1",
+				MinikubeDockerdProfile: "minikube",
 			},
 		},
 		{
@@ -179,15 +182,16 @@ func TestShellCfgSet(t *testing.T) {
 			noProxyValue: "127.0.0.1",
 			noProxyFlag:  true,
 			expectedShellCfg: &ShellConfig{
-				DockerCertPath:  localpath.MakeMiniPath("certs"),
-				DockerTLSVerify: "1",
-				DockerHost:      "tcp://127.0.0.1:2376",
-				UsageHint:       generateUsageHint("minikube", "bash"),
-				Prefix:          bashSetPfx,
-				Suffix:          bashSetSfx,
-				Delimiter:       bashSetDelim,
-				NoProxyVar:      "no_proxy",
-				NoProxyValue:    "127.0.0.1",
+				DockerCertPath:         localpath.MakeMiniPath("certs"),
+				DockerTLSVerify:        "1",
+				DockerHost:             "tcp://127.0.0.1:2376",
+				UsageHint:              generateUsageHint("minikube", "bash"),
+				Prefix:                 bashSetPfx,
+				Suffix:                 bashSetSfx,
+				Delimiter:              bashSetDelim,
+				NoProxyVar:             "no_proxy",
+				NoProxyValue:           "127.0.0.1",
+				MinikubeDockerdProfile: "minikube",
 			},
 		},
 		{
@@ -198,15 +202,16 @@ func TestShellCfgSet(t *testing.T) {
 			noProxyValue: "0.0.0.0",
 			noProxyFlag:  true,
 			expectedShellCfg: &ShellConfig{
-				DockerCertPath:  localpath.MakeMiniPath("certs"),
-				DockerTLSVerify: "1",
-				DockerHost:      "tcp://127.0.0.1:2376",
-				UsageHint:       generateUsageHint("minikube", "bash"),
-				Prefix:          bashSetPfx,
-				Suffix:          bashSetSfx,
-				Delimiter:       bashSetDelim,
-				NoProxyVar:      "no_proxy",
-				NoProxyValue:    "0.0.0.0,127.0.0.1",
+				DockerCertPath:         localpath.MakeMiniPath("certs"),
+				DockerTLSVerify:        "1",
+				DockerHost:             "tcp://127.0.0.1:2376",
+				UsageHint:              generateUsageHint("minikube", "bash"),
+				Prefix:                 bashSetPfx,
+				Suffix:                 bashSetSfx,
+				Delimiter:              bashSetDelim,
+				NoProxyVar:             "no_proxy",
+				NoProxyValue:           "0.0.0.0,127.0.0.1",
+				MinikubeDockerdProfile: "minikube",
 			},
 		},
 		{
@@ -217,15 +222,16 @@ func TestShellCfgSet(t *testing.T) {
 			noProxyValue: "0.0.0.0,127.0.0.1",
 			noProxyFlag:  true,
 			expectedShellCfg: &ShellConfig{
-				DockerCertPath:  localpath.MakeMiniPath("certs"),
-				DockerTLSVerify: "1",
-				DockerHost:      "tcp://127.0.0.1:2376",
-				UsageHint:       generateUsageHint("minikube", "bash"),
-				Prefix:          bashSetPfx,
-				Suffix:          bashSetSfx,
-				Delimiter:       bashSetDelim,
-				NoProxyVar:      "no_proxy",
-				NoProxyValue:    "0.0.0.0,127.0.0.1",
+				DockerCertPath:         localpath.MakeMiniPath("certs"),
+				DockerTLSVerify:        "1",
+				DockerHost:             "tcp://127.0.0.1:2376",
+				UsageHint:              generateUsageHint("minikube", "bash"),
+				Prefix:                 bashSetPfx,
+				Suffix:                 bashSetSfx,
+				Delimiter:              bashSetDelim,
+				NoProxyVar:             "no_proxy",
+				NoProxyValue:           "0.0.0.0,127.0.0.1",
+				MinikubeDockerdProfile: "minikube",
 			},
 		},
 	}

From 8a7d8dc7154232e34e0dee8a076076bf3741e885 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Tue, 4 Feb 2020 18:59:00 -0800
Subject: [PATCH 245/250] Add debugging log statements to cluster startup

---
 pkg/minikube/cluster/start.go | 31 ++++++++++++++++++++++++++-----
 1 file changed, 26 insertions(+), 5 deletions(-)

diff --git a/pkg/minikube/cluster/start.go b/pkg/minikube/cluster/start.go
index 2607c879ef..e6edd1012a 100644
--- a/pkg/minikube/cluster/start.go
+++ b/pkg/minikube/cluster/start.go
@@ -94,6 +94,12 @@ func engineOptions(cfg config.MachineConfig) *engine.Options {
 }
 
 func createHost(api libmachine.API, cfg config.MachineConfig) (*host.Host, error) {
+	glog.Infof("createHost starting for %q (driver=%q)", cfg.Name, cfg.VMDriver)
+	start := time.Now()
+	defer func() {
+		glog.Infof("createHost completed in %s", time.Since(start))
+	}()
+
 	if cfg.VMDriver == driver.VMwareFusion && viper.GetBool(config.ShowDriverDeprecationNotification) {
 		out.WarningT(`The vmwarefusion driver is deprecated and support for it will be removed in a future release.
 			Please consider switching to the new vmware unified driver, which is intended to replace the vmwarefusion driver.
@@ -120,11 +126,14 @@ func createHost(api libmachine.API, cfg config.MachineConfig) (*host.Host, error
 	h.HostOptions.AuthOptions.StorePath = localpath.MiniPath()
 	h.HostOptions.EngineOptions = engineOptions(cfg)
 
+	cstart := time.Now()
+	glog.Infof("libmachine.API.Create for %q (driver=%q)", cfg.Name, cfg.VMDriver)
 	if err := api.Create(h); err != nil {
 		// Wait for all the logs to reach the client
 		time.Sleep(2 * time.Second)
 		return nil, errors.Wrap(err, "create")
 	}
+	glog.Infof("libmachine.API.Create for %q took %s", cfg.Name, time.Since(cstart))
 
 	if err := postStartSetup(h, cfg); err != nil {
 		return h, errors.Wrap(err, "post-start")
@@ -138,6 +147,12 @@ func createHost(api libmachine.API, cfg config.MachineConfig) (*host.Host, error
 
 // postStart are functions shared between startHost and fixHost
 func postStartSetup(h *host.Host, mc config.MachineConfig) error {
+	glog.Infof("post-start starting for %q (driver=%q)", h.Name, h.DriverName)
+	start := time.Now()
+	defer func() {
+		glog.Infof("post-start completed in %s", time.Since(start))
+	}()
+
 	if driver.IsMock(h.DriverName) {
 		return nil
 	}
@@ -164,19 +179,25 @@ func postStartSetup(h *host.Host, mc config.MachineConfig) error {
 
 // commandRunner returns best available command runner for this host
 func commandRunner(h *host.Host) (command.Runner, error) {
-	if driver.IsMock(h.DriverName) {
-		glog.Errorf("commandRunner: returning unconfigured FakeCommandRunner, commands will fail!")
+	d := h.Driver.DriverName()
+	glog.V(1).Infof("determining appropriate runner for %q", d)
+	if driver.IsMock(d) {
+		glog.Infof("returning FakeCommandRunner for %q driver", d)
 		return &command.FakeCommandRunner{}, nil
 	}
-	if driver.BareMetal(h.Driver.DriverName()) {
+	if driver.BareMetal(d) {
+		glog.Infof("returning ExecRunner for %q driver", d)
 		return &command.ExecRunner{}, nil
 	}
-	if h.Driver.DriverName() == driver.Docker {
+	if driver.IsKIC(d) {
+		glog.Infof("Returning KICRunner for %q driver", d)
 		return command.NewKICRunner(h.Name, "docker"), nil
 	}
+
+	glog.Infof("Creating SSH client and returning SSHRunner for %q driver", d)
 	client, err := sshutil.NewSSHClient(h.Driver)
 	if err != nil {
-		return nil, errors.Wrap(err, "getting ssh client for bootstrapper")
+		return nil, errors.Wrap(err, "ssh client")
 	}
 	return command.NewSSHRunner(client), nil
 }

From 478502a217c1c2740bad92917b221d4e16161926 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Tue, 4 Feb 2020 20:16:40 -0800
Subject: [PATCH 246/250] Skip ahead if we are running on bare metal

---
 pkg/minikube/cluster/fix.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/pkg/minikube/cluster/fix.go b/pkg/minikube/cluster/fix.go
index 5a53680f34..8ba2dd83c7 100644
--- a/pkg/minikube/cluster/fix.go
+++ b/pkg/minikube/cluster/fix.go
@@ -99,6 +99,11 @@ func fixHost(api libmachine.API, mc config.MachineConfig) (*host.Host, error) {
 		return h, errors.Wrap(err, "post-start")
 	}
 
+	if driver.BareMetal(h.Driver.DriverName()) {
+		glog.Infof("%s is local, skipping auth/time setup (requires ssh)", h.Driver.DriverName())
+		return h, nil
+	}
+
 	glog.Infof("Configuring auth for driver %s ...", h.Driver.DriverName())
 	if err := h.ConfigureAuth(); err != nil {
 		return h, &retry.RetriableError{Err: errors.Wrap(err, "Error configuring auth on host")}

From 87cdbc10f9f72622cf97528f70fab38b1593a370 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Tue, 4 Feb 2020 21:47:14 -0800
Subject: [PATCH 247/250] Version bump to v1.7.0

---
 CHANGELOG.md | 27 +++++++++++++++------------
 Makefile     |  2 +-
 2 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 957e2e1025..5956d185fb 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,15 +1,27 @@
 # Release Notes
 
+## Version 1.7.0 - 2020-02-04
+
+* Support --force for overriding the ssh check [#6237](https://github.com/kubernetes/minikube/pull/6237)
+* Update translation files with new strings [#6491](https://github.com/kubernetes/minikube/pull/6491)
+* fix docker-env for kic drivers [#6487](https://github.com/kubernetes/minikube/pull/6487)
+* Fix bugs that prevented previously-enabled addons from starting up [#6471](https://github.com/kubernetes/minikube/pull/6471)
+* Fix none driver bugs with "pause"  [#6452](https://github.com/kubernetes/minikube/pull/6452)
+
+Thank you to those brave souls who made the final push toward this release:
+
+- Medya Gh
+- Priya Wadhwa
+- Sharif Elgamal
+- Thomas Strömberg
+
 ## Version 1.7.0-beta.2 - 2020-01-31
 
 * Add docker run-time for kic driver [#6436](https://github.com/kubernetes/minikube/pull/6436)
-* Add addon enablement to start [#6440](https://github.com/kubernetes/minikube/pull/6440)
 * Configure etcd and kube-proxy metrics to listen on minikube node IP [#6322](https://github.com/kubernetes/minikube/pull/6322)
 * add container runtime info to profile list [#6409](https://github.com/kubernetes/minikube/pull/6409)
-* Allow 'profiles list' to work even if a profile has no control plane [#6450](https://github.com/kubernetes/minikube/pull/6450)
 * status: Explicitly state that the cluster does not exist [#6438](https://github.com/kubernetes/minikube/pull/6438)
 * Do not use an arch suffix for the coredns name [#6243](https://github.com/kubernetes/minikube/pull/6243)
-* Allow building storage-provisioner for other arch [#6257](https://github.com/kubernetes/minikube/pull/6257)
 * Prevent registry-creds configure from failing when a secret does not exist.  [#6380](https://github.com/kubernetes/minikube/pull/6380)
 * improve checking modprob netfilter [#6427](https://github.com/kubernetes/minikube/pull/6427)
 
@@ -31,16 +43,13 @@ Huge thank you for this release towards our contributors:
 * kic driver: add multiple profiles and ssh [#6390](https://github.com/kubernetes/minikube/pull/6390)
 * Update DefaultKubernetesVersion to v1.17.2 [#6392](https://github.com/kubernetes/minikube/pull/6392)
 * Add varlink program for using with podman-remote [#6349](https://github.com/kubernetes/minikube/pull/6349)
-* Batch file manipulation calls to reduce number of redundant commands [#6385](https://github.com/kubernetes/minikube/pull/6385)
 * Update Kubernetes libraries to v1.17.2 [#6374](https://github.com/kubernetes/minikube/pull/6374)
-* Fix dashboard by using kubectl apply/delete instead of prune for addons [#6376](https://github.com/kubernetes/minikube/pull/6376)
 * Remove addon manager [#6334](https://github.com/kubernetes/minikube/pull/6334)
 * Remove unnecessary crio restart to improve start latency [#6369](https://github.com/kubernetes/minikube/pull/6369)
 * Check for nil ref and img before passing them into go-containerregistry [#6236](https://github.com/kubernetes/minikube/pull/6236)
 * Change the compression methods used on the iso [#6341](https://github.com/kubernetes/minikube/pull/6341)
 * Update the crio.conf instead of overwriting it [#6219](https://github.com/kubernetes/minikube/pull/6219)
 * Update Japanese translation [#6339](https://github.com/kubernetes/minikube/pull/6339)
-* Fix syntax of minikube kubectl example command [#6255](https://github.com/kubernetes/minikube/pull/6255)
 * Stop minikube dashboard from crashing at start [#6325](https://github.com/kubernetes/minikube/pull/6325)
 
 Thanks you to the following contributors:
@@ -56,16 +65,12 @@ Thanks you to the following contributors:
 
 ## Version 1.7.0-beta.0 - 2020-01-15
 
-* add the ability to mark drivers as experimental  [#6326](https://github.com/kubernetes/minikube/pull/6326)
 * Use CGroupDriver function from cruntime for kubelet [#6287](https://github.com/kubernetes/minikube/pull/6287)
 * Experimental Docker support (kic) using the Kind image [#6151](https://github.com/kubernetes/minikube/pull/6151)
 * disable istio provisioner by default [#6315](https://github.com/kubernetes/minikube/pull/6315)
 * Add --dry-run option to start [#6256](https://github.com/kubernetes/minikube/pull/6256)
-* add addons for kic [#6277](https://github.com/kubernetes/minikube/pull/6277)
 * Improve "addon list" by viewing as a table  [#6274](https://github.com/kubernetes/minikube/pull/6274)
-* Handle empty k8s version in existing profile [#6292](https://github.com/kubernetes/minikube/pull/6292)
 * Disable IPv6 in the minikube VM until it can be properly supported [#6241](https://github.com/kubernetes/minikube/pull/6241)
-* Rebuild docker images for each release [#6235](https://github.com/kubernetes/minikube/pull/6235)
 * Fixes IPv6 address handling in kubeadm [#6214](https://github.com/kubernetes/minikube/pull/6214)
 * Upgrade crio to 1.16.1 [#6210](https://github.com/kubernetes/minikube/pull/6210)
 * Upgrade podman to 1.6.4 [#6208](https://github.com/kubernetes/minikube/pull/6208)
@@ -74,7 +79,6 @@ Thanks you to the following contributors:
 * Add systemd patch for booting on AMD Ryzen [#6183](https://github.com/kubernetes/minikube/pull/6183)
 * update zh translation [#6176](https://github.com/kubernetes/minikube/pull/6176)
 * Add istio addon for minikube [#6154](https://github.com/kubernetes/minikube/pull/6154)
-* implemented options command, fixes #5036 [#6144](https://github.com/kubernetes/minikube/pull/6144)
 
 Huge thank you for this release towards our contributors:
 - Anders Björklund
@@ -94,7 +98,6 @@ Huge thank you for this release towards our contributors:
 - Zhongcheng Lao
 - Zhou Hao
 
-
 ## Version 1.6.2  - 2019-12-19
 
 * Offline: always transfer image if lookup fails, always download drivers [#6111](https://github.com/kubernetes/minikube/pull/6111)
diff --git a/Makefile b/Makefile
index 6c4f73b826..97aa025297 100755
--- a/Makefile
+++ b/Makefile
@@ -15,7 +15,7 @@
 # Bump these on release - and please check ISO_VERSION for correctness.
 VERSION_MAJOR ?= 1
 VERSION_MINOR ?= 7
-VERSION_BUILD ?= 0-beta.2
+VERSION_BUILD ?= 0
 RAW_VERSION=$(VERSION_MAJOR).$(VERSION_MINOR).${VERSION_BUILD}
 VERSION ?= v$(RAW_VERSION)
 

From 9854e3067fcc1b188854f364361956baf3a90c86 Mon Sep 17 00:00:00 2001
From: tstromberg <tstromberg@google.com>
Date: Wed, 5 Feb 2020 10:04:00 -0800
Subject: [PATCH 248/250] Add Azure PR

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5956d185fb..76b953035c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,7 @@
 
 ## Version 1.7.0 - 2020-02-04
 
+* Add Azure Container Registry support [#6483](https://github.com/kubernetes/minikube/pull/6483)
 * Support --force for overriding the ssh check [#6237](https://github.com/kubernetes/minikube/pull/6237)
 * Update translation files with new strings [#6491](https://github.com/kubernetes/minikube/pull/6491)
 * fix docker-env for kic drivers [#6487](https://github.com/kubernetes/minikube/pull/6487)

From 4776bb7a2de614b74ede6e2ebb23fc530c5a800e Mon Sep 17 00:00:00 2001
From: tstromberg <tstromberg@google.com>
Date: Wed, 5 Feb 2020 10:18:17 -0800
Subject: [PATCH 249/250] Set GOPATH for the entire script

---
 hack/jenkins/release_build_and_upload.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/hack/jenkins/release_build_and_upload.sh b/hack/jenkins/release_build_and_upload.sh
index 0f5fc8339d..6e76c4c4a1 100755
--- a/hack/jenkins/release_build_and_upload.sh
+++ b/hack/jenkins/release_build_and_upload.sh
@@ -35,11 +35,11 @@ grep -E "^VERSION_MAJOR \\?=" Makefile | grep "${VERSION_MAJOR}"
 grep -E "^VERSION_MINOR \\?=" Makefile | grep "${VERSION_MINOR}"
 grep -E "^VERSION_BUILD \\?=" Makefile | grep "${VERSION_BUILD}"
 
-# Diagnostics
-go env GOPATH
+# Force go packages to the Jekins home directory
+export GOPATH=$HOME/go
 
 # Build and upload
-env GOPATH=$HOME/go BUILD_IN_DOCKER=y \
+env BUILD_IN_DOCKER=y \
   make -j 16 \
   all \
   out/minikube-installer.exe \

From e0dee4864da74f8a770dce5ad50f0a704d7b4a0d Mon Sep 17 00:00:00 2001
From: minikube-bot <minikube-bot@google.com>
Date: Wed, 5 Feb 2020 10:26:21 -0800
Subject: [PATCH 250/250] Update releases.json to include v1.7.0

---
 deploy/minikube/releases.json | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/deploy/minikube/releases.json b/deploy/minikube/releases.json
index fc12766099..4a53dcc373 100644
--- a/deploy/minikube/releases.json
+++ b/deploy/minikube/releases.json
@@ -1,4 +1,12 @@
 [
+  {
+      "name": "v1.7.0",
+      "checksums": {
+          "darwin": "1f3785e9521eabe241df0481fa41887a6a3873307bac8a89fd0e48aa7612be29",
+          "linux": "f1fcab9f161a64f19b618a901e50488ed6f1c6ab20695c82623586a701d2d261",
+          "windows": "2f4448f32e505bf38ba52cd3678c73622a6bb452c63a4179d590f6da26520c68"
+      }
+  },
   {
       "name": "v1.6.2",
       "checksums": {