Merge branch 'master' into cleaner-int-logs

2020-01-29 17:02:38 -08:00 · 2020-01-29 17:02:38 -08:00 · dad1ebaff4
parent 5ba779f74d a9af9d0bd3
commit dad1ebaff4
206 changed files with 4495 additions and 2537 deletions
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@ -0,0 +1,121 @@
+name: CI
+
+on: [push]
+
+jobs:
+  docker_ubuntu:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: build binaries
+      run : |
+        make minikube-linux-amd64
+        make e2e-linux-amd64
+        mkdir -p report
+    - name: run integration test
+      run: |
+        echo running docker driver intergration test on ubuntu
+        ./out/e2e-linux-amd64 -minikube-start-args=--vm-driver=docker -expected-default-driver= -test.timeout=70m -test.v -binary=out/minikube-linux-amd64 2>&1 | tee ./report/testout.txt
+    - name: install gopogh
+      run: |
+        cd /tmp
+        GO111MODULE="on" go get github.com/medyagh/gopogh@v0.0.17 || true
+        cd -
+    - name: generate gopogh report
+      run: |
+        export PATH=${PATH}:`go env GOPATH`/bin
+        go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
+        gopogh -in ./report/testout.json -out ./report/testout.html -name "docker ubuntu" -repo github.com/kubernetes/minikube/  || true
+    - uses: actions/upload-artifact@v1
+      with:
+        name: docker_on_ubuntu_report
+        path: report
+
+  docker_macos:
+
+    runs-on: macos-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: build binaries
+      run : |
+        make minikube-darwin-amd64
+        make e2e-darwin-amd64
+        mkdir -p report
+    - name: run integration test
+      run: |
+        echo running docker driver intergration test on ubuntu
+        ./out/e2e-darwin-amd64 -minikube-start-args=--vm-driver=docker -expected-default-driver= -test.timeout=70m -test.v -binary=out/minikube-darwin-amd64 2>&1 | tee ./report/testout.txt
+    - name: install gopogh
+      run: |
+        cd /tmp
+        GO111MODULE="on" go get github.com/medyagh/gopogh@v0.0.17 || true
+        cd -
+    - name: generate gopogh report
+      run: |
+        export PATH=${PATH}:`go env GOPATH`/bin
+        go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
+        gopogh -in ./report/testout.json -out ./report/testout.html -name "docker macos" -repo github.com/kubernetes/minikube/  || true
+    - uses: actions/upload-artifact@v1
+      with:
+        name: docker_on_macos_report
+        path: ./report
+
+  none_ubuntu:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: build binaries
+      run : |
+        make minikube-linux-amd64
+        make e2e-linux-amd64
+        mkdir -p report
+    - name: run integration test
+      run: |
+        echo running docker driver intergration test on ubuntu
+        ./out/e2e-linux-amd64 -minikube-start-args=--vm-driver=none -expected-default-driver= -test.timeout=70m -test.v -binary=out/minikube-linux-amd64 2>&1 | tee ./report/testout.txt
+    - name: install gopogh
+      run: |
+        cd /tmp
+        GO111MODULE="on" go get github.com/medyagh/gopogh@v0.0.17 || true
+        cd -
+    - name: generate gopogh report
+      run: |
+        export PATH=${PATH}:`go env GOPATH`/bin
+        go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
+        gopogh -in ./report/testout.json -out ./report/testout.html -name "docker ubuntu" -repo github.com/kubernetes/minikube/  || true
+    - uses: actions/upload-artifact@v1
+      with:
+        name: none_on_ubuntu_latest_report
+        path: report
+
+  none_ubuntu16_04:
+    runs-on: ubuntu-16.04
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: build binaries
+      run : |
+        make minikube-linux-amd64
+        make e2e-linux-amd64
+        mkdir -p report
+    - name: run integration test
+      run: |
+        echo running docker driver intergration test on ubuntu
+        ./out/e2e-linux-amd64 -minikube-start-args=--vm-driver=none -expected-default-driver= -test.timeout=70m -test.v -binary=out/minikube-linux-amd64 2>&1 | tee ./report/testout.txt
+    - name: install gopogh
+      run: |
+        cd /tmp
+        GO111MODULE="on" go get github.com/medyagh/gopogh@v0.0.17 || true
+        cd -
+    - name: generate gopogh report
+      run: |
+        export PATH=${PATH}:`go env GOPATH`/bin
+        go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
+        gopogh -in ./report/testout.json -out ./report/testout.html -name "docker ubuntu" -repo github.com/kubernetes/minikube/  || true
+    - uses: actions/upload-artifact@v1
+      with:
+        name: none_on_ubuntu_16_04
+        path: report
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -1,5 +1,74 @@
 # Release Notes

+## Version 1.7.0-beta.1 - 2020-01-24
+
+* Add 'pause' command to freeze Kubernetes cluster [#5962](https://github.com/kubernetes/minikube/pull/5962)
+* kic driver: add multiple profiles and ssh [#6390](https://github.com/kubernetes/minikube/pull/6390)
+* Update DefaultKubernetesVersion to v1.17.2 [#6392](https://github.com/kubernetes/minikube/pull/6392)
+* Add varlink program for using with podman-remote [#6349](https://github.com/kubernetes/minikube/pull/6349)
+* Batch file manipulation calls to reduce number of redundant commands [#6385](https://github.com/kubernetes/minikube/pull/6385)
+* Update Kubernetes libraries to v1.17.2 [#6374](https://github.com/kubernetes/minikube/pull/6374)
+* Fix dashboard by using kubectl apply/delete instead of prune for addons [#6376](https://github.com/kubernetes/minikube/pull/6376)
+* Remove addon manager [#6334](https://github.com/kubernetes/minikube/pull/6334)
+* Remove unnecessary crio restart to improve start latency [#6369](https://github.com/kubernetes/minikube/pull/6369)
+* Check for nil ref and img before passing them into go-containerregistry [#6236](https://github.com/kubernetes/minikube/pull/6236)
+* Change the compression methods used on the iso [#6341](https://github.com/kubernetes/minikube/pull/6341)
+* Update the crio.conf instead of overwriting it [#6219](https://github.com/kubernetes/minikube/pull/6219)
+* Update Japanese translation [#6339](https://github.com/kubernetes/minikube/pull/6339)
+* Fix syntax of minikube kubectl example command [#6255](https://github.com/kubernetes/minikube/pull/6255)
+* Stop minikube dashboard from crashing at start [#6325](https://github.com/kubernetes/minikube/pull/6325)
+
+Thanks you to the following contributors:
+
+- Anders F Björklund
+- inductor
+- Medya Ghazizadeh
+- Naoki Oketani
+- Priya Wadhwa
+- Sharif Elgamal
+- sshukun
+- Thomas Strömberg
+
+## Version 1.7.0-beta.0 - 2020-01-15
+
+* add the ability to mark drivers as experimental  [#6326](https://github.com/kubernetes/minikube/pull/6326)
+* Use CGroupDriver function from cruntime for kubelet [#6287](https://github.com/kubernetes/minikube/pull/6287)
+* Experimental Docker support (kic) using the Kind image [#6151](https://github.com/kubernetes/minikube/pull/6151)
+* disable istio provisioner by default [#6315](https://github.com/kubernetes/minikube/pull/6315)
+* Add --dry-run option to start [#6256](https://github.com/kubernetes/minikube/pull/6256)
+* add addons for kic [#6277](https://github.com/kubernetes/minikube/pull/6277)
+* Improve "addon list" by viewing as a table  [#6274](https://github.com/kubernetes/minikube/pull/6274)
+* Handle empty k8s version in existing profile [#6292](https://github.com/kubernetes/minikube/pull/6292)
+* Disable IPv6 in the minikube VM until it can be properly supported [#6241](https://github.com/kubernetes/minikube/pull/6241)
+* Rebuild docker images for each release [#6235](https://github.com/kubernetes/minikube/pull/6235)
+* Fixes IPv6 address handling in kubeadm [#6214](https://github.com/kubernetes/minikube/pull/6214)
+* Upgrade crio to 1.16.1 [#6210](https://github.com/kubernetes/minikube/pull/6210)
+* Upgrade podman to 1.6.4 [#6208](https://github.com/kubernetes/minikube/pull/6208)
+* Enable or disable addons per profile [#6124](https://github.com/kubernetes/minikube/pull/6124)
+* Upgrade buildroot minor version [#6199](https://github.com/kubernetes/minikube/pull/6199)
+* Add systemd patch for booting on AMD Ryzen [#6183](https://github.com/kubernetes/minikube/pull/6183)
+* update zh translation [#6176](https://github.com/kubernetes/minikube/pull/6176)
+* Add istio addon for minikube [#6154](https://github.com/kubernetes/minikube/pull/6154)
+* implemented options command, fixes #5036 [#6144](https://github.com/kubernetes/minikube/pull/6144)
+
+Huge thank you for this release towards our contributors:
+- Anders Björklund
+- andylibrian
+- Dao Cong Tien
+- Dominic Yin
+- fenglixa
+- GennadySpb
+- Kenta Iso
+- Kim Bao Long
+- Medya Ghazizadeh
+- Nguyen Hai Truong
+- Priya Wadhwa
+- Sharif Elgamal
+- Thomas Strömberg
+- ttonline6
+- Zhongcheng Lao
+- Zhou Hao
+

 ## Version 1.6.2  - 2019-12-19

@ -11,7 +80,7 @@
 * start: Remove create/delete retry loop [#6129](https://github.com/kubernetes/minikube/pull/6129)
 * Change error text to encourage better issue reports [#6121](https://github.com/kubernetes/minikube/pull/6121)

-Huge thank you for this release towards our contributors: 
+Huge thank you for this release towards our contributors:
 - Anukul Sangwan
 - Aresforchina
 - Curtis Carter
--- a/16
+++ b/16
@ -14,13 +14,13 @@

 # Bump these on release - and please check ISO_VERSION for correctness.
 VERSION_MAJOR ?= 1
-VERSION_MINOR ?= 6
-VERSION_BUILD ?= 2
+VERSION_MINOR ?= 7
+VERSION_BUILD ?= 0-beta.1
 RAW_VERSION=$(VERSION_MAJOR).$(VERSION_MINOR).${VERSION_BUILD}
 VERSION ?= v$(RAW_VERSION)

 # Default to .0 for higher cache hit rates, as build increments typically don't require new ISO versions
-ISO_VERSION ?= v$(VERSION_MAJOR).$(VERSION_MINOR).0
+ISO_VERSION ?= v$(VERSION_MAJOR).$(VERSION_MINOR).$(VERSION_BUILD)
 # Dashes are valid in semver, but not Linux packaging. Use ~ to delimit alpha/beta
 DEB_VERSION ?= $(subst -,~,$(RAW_VERSION))
 RPM_VERSION ?= $(DEB_VERSION)
@ -473,7 +473,7 @@ $(ISO_BUILD_IMAGE): deploy/iso/minikube-iso/Dockerfile
 	@echo "$(@) successfully built"

 out/storage-provisioner:
-	GOOS=linux go build -o $@ -ldflags=$(PROVISIONER_LDFLAGS) cmd/storage-provisioner/main.go
+	CGO_ENABLED=0 GOOS=linux go build -o $@ -ldflags=$(PROVISIONER_LDFLAGS) cmd/storage-provisioner/main.go

 .PHONY: storage-provisioner-image
 storage-provisioner-image: out/storage-provisioner ## Build storage-provisioner docker image
@ -483,6 +483,13 @@ else
 	docker build -t $(REGISTRY)/storage-provisioner-$(GOARCH):$(STORAGE_PROVISIONER_TAG) -f deploy/storage-provisioner/Dockerfile-$(GOARCH) .
 endif

+.PHONY: kic-base-image
+kic-base-image: ## builds the base image used for kic.
+	docker rmi -f $(REGISTRY)/kicbase:v0.0.2-snapshot || true
+	docker build -f ./hack/images/kicbase.Dockerfile -t $(REGISTRY)/kicbase:v0.0.2-snapshot  --build-arg COMMIT_SHA=${VERSION}-$(COMMIT)  .
+
+
+
 .PHONY: push-storage-provisioner-image
 push-storage-provisioner-image: storage-provisioner-image ## Push storage-provisioner docker image using gcloud
 ifeq ($(GOARCH),amd64)
@ -591,6 +598,7 @@ out/mkcmp:
 out/performance-monitor:
 	GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o $@ cmd/performance/monitor/monitor.go

+
 .PHONY: help
 help:
 	@printf "\033[1mAvailable targets for minikube ${VERSION}\033[21m\n"
--- a/cmd/minikube/cmd/config/addons_list.go
+++ b/cmd/minikube/cmd/config/addons_list.go
@ -22,17 +22,17 @@ import (
 	"os"
 	"sort"
 	"strings"
-	"text/template"

+	"github.com/golang/glog"
+	"github.com/olekukonko/tablewriter"
 	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
 	"k8s.io/minikube/pkg/minikube/assets"
+	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/exit"
 	"k8s.io/minikube/pkg/minikube/out"
 )

-const defaultAddonListFormat = "- {{.AddonName}}: {{.AddonStatus}}\n"
-
-var addonListFormat string
 var addonListOutput string

 // AddonListTemplate represents the addon list template
@ -50,10 +50,6 @@ var addonsListCmd = &cobra.Command{
 			exit.UsageT("usage: minikube addons list")
 		}

-		if addonListOutput != "list" && addonListFormat != defaultAddonListFormat {
-			exit.UsageT("Cannot use both --output and --format options")
-		}
-
 		switch strings.ToLower(addonListOutput) {
 		case "list":
 			printAddonsList()
@ -66,14 +62,6 @@ var addonsListCmd = &cobra.Command{
 }

 func init() {
-	addonsListCmd.Flags().StringVarP(
-		&addonListFormat,
-		"format",
-		"f",
-		defaultAddonListFormat,
-		`Go template format string for the addon list output.  The format for Go templates can be found here: https://golang.org/pkg/text/template/
-For the list of accessible variables for the template, see the struct values here: https://godoc.org/k8s.io/minikube/cmd/minikube/cmd/config#AddonListTemplate`)
-
 	addonsListCmd.Flags().StringVarP(
 		&addonListOutput,
 		"output",
@ -84,6 +72,13 @@ For the list of accessible variables for the template, see the struct values her
 	AddonsCmd.AddCommand(addonsListCmd)
 }

+var iconFromStatus = func(addonStatus bool) string {
+	if addonStatus {
+		return "✅"
+	}
+	return "   " // because emoji indentation is different
+}
+
 var stringFromStatus = func(addonStatus bool) string {
 	if addonStatus {
 		return "enabled"
@ -97,6 +92,13 @@ var printAddonsList = func() {
 		addonNames = append(addonNames, addonName)
 	}
 	sort.Strings(addonNames)
+	var tData [][]string
+	table := tablewriter.NewWriter(os.Stdout)
+	table.SetHeader([]string{"Addon Name", "Profile", "Status"})
+	table.SetAutoFormatHeaders(true)
+	table.SetBorders(tablewriter.Border{Left: true, Top: true, Right: true, Bottom: true})
+	table.SetCenterSeparator("|")
+	pName := viper.GetString(config.MachineProfile)

 	for _, addonName := range addonNames {
 		addonBundle := assets.Addons[addonName]
@ -104,20 +106,25 @@ var printAddonsList = func() {
 		if err != nil {
 			exit.WithError("Error getting addons status", err)
 		}
-		tmpl, err := template.New("list").Parse(addonListFormat)
-		if err != nil {
-			exit.WithError("Error creating list template", err)
-		}
-		listTmplt := AddonListTemplate{addonName, stringFromStatus(addonStatus)}
-		err = tmpl.Execute(os.Stdout, listTmplt)
-		if err != nil {
-			exit.WithError("Error executing list template", err)
-		}
+		tData = append(tData, []string{addonName, pName, fmt.Sprintf("%s %s", stringFromStatus(addonStatus), iconFromStatus(addonStatus))})
 	}
+
+	table.AppendBulk(tData)
+	table.Render()
+
+	v, _, err := config.ListProfiles()
+	if err != nil {
+		glog.Infof("error getting list of porfiles: %v", err)
+	}
+	if len(v) > 1 {
+		out.T(out.Tip, "To see addons list for other profiles use: `minikube addons -p name list`")
+	}
+
 }

 var printAddonsJSON = func() {
 	addonNames := make([]string, 0, len(assets.Addons))
+	pName := viper.GetString(config.MachineProfile)
 	for addonName := range assets.Addons {
 		addonNames = append(addonNames, addonName)
 	}
@ -134,7 +141,8 @@ var printAddonsJSON = func() {
 		}

 		addonsMap[addonName] = map[string]interface{}{
-			"Status": stringFromStatus(addonStatus),
+			"Status":  stringFromStatus(addonStatus),
+			"Profile": pName,
 		}
 	}
 	jsonString, _ := json.Marshal(addonsMap)
--- a/cmd/minikube/cmd/config/profile_list.go
+++ b/cmd/minikube/cmd/config/profile_list.go
@ -59,7 +59,6 @@ var profileListCmd = &cobra.Command{
 var printProfilesTable = func() {

 	var validData [][]string
-
 	table := tablewriter.NewWriter(os.Stdout)
 	table.SetHeader([]string{"Profile", "VM Driver", "NodeIP", "Node Port", "Kubernetes Version", "Status"})
 	table.SetAutoFormatHeaders(false)
@ -81,7 +80,11 @@ var printProfilesTable = func() {
 		if err != nil {
 			glog.Infof("error getting host status for %v", err)
 		}
-		validData = append(validData, []string{p.Name, p.Config[0].VMDriver, p.Config[0].KubernetesConfig.NodeIP, strconv.Itoa(p.Config[0].KubernetesConfig.NodePort), p.Config[0].KubernetesConfig.KubernetesVersion, p.Status})
+		cp, err := config.PrimaryControlPlane(*p.Config)
+		if err != nil {
+			exit.WithError("profile has no control plane", err)
+		}
+		validData = append(validData, []string{p.Name, p.Config.VMDriver, cp.IP, strconv.Itoa(cp.Port), p.Config.KubernetesConfig.KubernetesVersion, p.Status})
 	}

 	table.AppendBulk(validData)
--- a/cmd/minikube/cmd/dashboard.go
+++ b/cmd/minikube/cmd/dashboard.go
@ -59,11 +59,17 @@ var dashboardCmd = &cobra.Command{
 	Short: "Access the kubernetes dashboard running within the minikube cluster",
 	Long:  `Access the kubernetes dashboard running within the minikube cluster`,
 	Run: func(cmd *cobra.Command, args []string) {
-		cc, err := pkg_config.Load(viper.GetString(config.MachineProfile))
+		profileName := viper.GetString(pkg_config.MachineProfile)
+		cc, err := pkg_config.Load(profileName)
 		if err != nil && !os.IsNotExist(err) {
 			exit.WithError("Error loading profile config", err)
 		}

+		if err != nil {
+			out.ErrT(out.Meh, `"{{.name}}" profile does not exist`, out.V{"name": profileName})
+			os.Exit(1)
+		}
+
 		api, err := machine.NewAPIClient()
 		defer func() {
 			err := api.Close()
@ -85,9 +91,11 @@ var dashboardCmd = &cobra.Command{
 			}
 		}

-		err = proxy.ExcludeIP(cc.KubernetesConfig.NodeIP) // to be used for http get calls
-		if err != nil {
-			glog.Errorf("Error excluding IP from proxy: %s", err)
+		for _, n := range cc.Nodes {
+			err = proxy.ExcludeIP(n.IP) // to be used for http get calls
+			if err != nil {
+				glog.Errorf("Error excluding IP from proxy: %s", err)
+			}
 		}

 		kubectl, err := exec.LookPath("kubectl")
--- a/cmd/minikube/cmd/kubectl.go
+++ b/cmd/minikube/cmd/kubectl.go
@ -38,10 +38,11 @@ import (
 var kubectlCmd = &cobra.Command{
 	Use:   "kubectl",
 	Short: "Run kubectl",
-	Long: `Run the kubernetes client, download it if necessary.
+	Long: `Run the kubernetes client, download it if necessary. Remember -- after kubectl!
+
 Examples:
 minikube kubectl -- --help
-kubectl get pods --namespace kube-system`,
+minikube kubectl -- get pods --namespace kube-system`,
 	Run: func(cmd *cobra.Command, args []string) {
 		api, err := machine.NewAPIClient()
 		if err != nil {
--- a/cmd/minikube/cmd/mount.go
+++ b/cmd/minikube/cmd/mount.go
@ -108,7 +108,6 @@ var mountCmd = &cobra.Command{
 			exit.WithError("Error getting config", err)
 		}
 		host, err := api.Load(cc.Name)
-
 		if err != nil {
 			exit.WithError("Error loading api", err)
 		}
--- a/cmd/minikube/cmd/pause.go
+++ b/cmd/minikube/cmd/pause.go
@ -0,0 +1,103 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cmd
+
+import (
+	"os"
+	"strings"
+
+	"github.com/golang/glog"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+
+	"k8s.io/minikube/pkg/minikube/cluster"
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/cruntime"
+	"k8s.io/minikube/pkg/minikube/exit"
+	"k8s.io/minikube/pkg/minikube/machine"
+	"k8s.io/minikube/pkg/minikube/out"
+)
+
+var (
+	namespaces    []string
+	allNamespaces bool
+)
+
+// pauseCmd represents the docker-pause command
+var pauseCmd = &cobra.Command{
+	Use:   "pause",
+	Short: "pause containers",
+	Run:   runPause,
+}
+
+func runPause(cmd *cobra.Command, args []string) {
+	cname := viper.GetString(config.MachineProfile)
+	api, err := machine.NewAPIClient()
+	if err != nil {
+		exit.WithError("Error getting client", err)
+	}
+	defer api.Close()
+	cc, err := config.Load(cname)
+
+	if err != nil && !os.IsNotExist(err) {
+		exit.WithError("Error loading profile config", err)
+	}
+
+	if err != nil {
+		out.ErrT(out.Meh, `"{{.name}}" profile does not exist`, out.V{"name": cname})
+		os.Exit(1)
+	}
+
+	glog.Infof("config: %+v", cc)
+	host, err := cluster.CheckIfHostExistsAndLoad(api, cname)
+	if err != nil {
+		exit.WithError("Error getting host", err)
+	}
+
+	r, err := machine.CommandRunner(host)
+	if err != nil {
+		exit.WithError("Failed to get command runner", err)
+	}
+
+	cr, err := cruntime.New(cruntime.Config{Type: cc.KubernetesConfig.ContainerRuntime, Runner: r})
+	if err != nil {
+		exit.WithError("Failed runtime", err)
+	}
+
+	glog.Infof("namespaces: %v keys: %v", namespaces, viper.AllSettings())
+	if allNamespaces {
+		namespaces = nil //all
+	} else if len(namespaces) == 0 {
+		exit.WithCodeT(exit.BadUsage, "Use -A to specify all namespaces")
+	}
+
+	ids, err := cluster.Pause(cr, r, namespaces)
+	if err != nil {
+		exit.WithError("Pause", err)
+	}
+
+	if namespaces == nil {
+		out.T(out.Unpause, "Paused kubelet and {{.count}} containers", out.V{"count": len(ids)})
+	} else {
+		out.T(out.Unpause, "Paused kubelet and {{.count}} containers in: {{.namespaces}}", out.V{"count": len(ids), "namespaces": strings.Join(namespaces, ", ")})
+	}
+}
+
+func init() {
+	pauseCmd.Flags().StringSliceVarP(&namespaces, "--namespaces", "n", cluster.DefaultNamespaces, "namespaces to pause")
+	pauseCmd.Flags().BoolVarP(&allNamespaces, "all-namespaces", "A", false, "If set, pause all namespaces")
+}
--- a/cmd/minikube/cmd/root.go
+++ b/cmd/minikube/cmd/root.go
@ -32,7 +32,6 @@ import (
 	"k8s.io/kubectl/pkg/util/templates"
 	configCmd "k8s.io/minikube/cmd/minikube/cmd/config"
 	"k8s.io/minikube/pkg/minikube/bootstrapper"
-	"k8s.io/minikube/pkg/minikube/bootstrapper/kicbs"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/kubeadm"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
@ -173,6 +172,8 @@ func init() {
 				stopCmd,
 				deleteCmd,
 				dashboardCmd,
+				pauseCmd,
+				unpauseCmd,
 			},
 		},
 		{
@ -274,16 +275,9 @@ func getClusterBootstrapper(api libmachine.API, bootstrapperName string) (bootst
 		if err != nil {
 			return nil, errors.Wrap(err, "getting a new kubeadm bootstrapper")
 		}
-	case bootstrapper.KIC:
-		b, err = kicbs.NewBootstrapper(api)
-		if err != nil {
-			return nil, errors.Wrap(err, "getting a new kic bootstrapper")
-		}
-
 	default:
 		return nil, fmt.Errorf("unknown bootstrapper: %s", bootstrapperName)
 	}
-
 	return b, nil
 }

--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@ -53,7 +53,6 @@ import (
 	"k8s.io/minikube/pkg/minikube/cluster"
 	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/minikube/config"
-	cfg "k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/cruntime"
 	"k8s.io/minikube/pkg/minikube/driver"
@ -65,6 +64,7 @@ import (
 	"k8s.io/minikube/pkg/minikube/notify"
 	"k8s.io/minikube/pkg/minikube/out"
 	"k8s.io/minikube/pkg/minikube/proxy"
+	"k8s.io/minikube/pkg/minikube/registry"
 	"k8s.io/minikube/pkg/minikube/translate"
 	pkgutil "k8s.io/minikube/pkg/util"
 	"k8s.io/minikube/pkg/util/lock"
@ -116,6 +116,7 @@ const (
 	hostDNSResolver       = "host-dns-resolver"
 	waitUntilHealthy      = "wait"
 	force                 = "force"
+	dryRun                = "dry-run"
 	interactive           = "interactive"
 	waitTimeout           = "wait-timeout"
 	nativeSSH             = "native-ssh"
@ -135,7 +136,7 @@ var (
 	apiServerNames   []string
 	addonList        []string
 	apiServerIPs     []net.IP
-	extraOptions     cfg.ExtraOptionSlice
+	extraOptions     config.ExtraOptionSlice
 )

 func init() {
@ -158,6 +159,7 @@ func initMinikubeFlags() {

 	startCmd.Flags().Bool(force, false, "Force minikube to perform possibly dangerous operations")
 	startCmd.Flags().Bool(interactive, true, "Allow user prompts for more information")
+	startCmd.Flags().Bool(dryRun, false, "dry-run mode. Validates configuration, but does does not mutate system state")

 	startCmd.Flags().Int(cpus, 2, "Number of CPUs allocated to the minikube VM.")
 	startCmd.Flags().String(memory, defaultMemorySize, "Amount of RAM allocated to the minikube VM (format: <number>[<unit>], where unit = b, k, m or g).")
@ -198,7 +200,7 @@ func initKubernetesFlags() {

 // initDriverFlags inits the commandline flags for vm drivers
 func initDriverFlags() {
-	startCmd.Flags().String("vm-driver", "", fmt.Sprintf("Driver is one of: %v (defaults to auto-detect)", driver.SupportedDrivers()))
+	startCmd.Flags().String("vm-driver", "", fmt.Sprintf("Driver is one of: %v (defaults to auto-detect)", driver.DisplaySupportedDrivers()))
 	startCmd.Flags().Bool(disableDriverMounts, false, "Disables the filesystem mounts provided by the hypervisors")

 	// kvm2
@ -232,7 +234,7 @@ func initNetworkingFlags() {
 	startCmd.Flags().StringSliceVar(&registryMirror, "registry-mirror", nil, "Registry mirrors to pass to the Docker daemon")
 	startCmd.Flags().String(imageRepository, "", "Alternative image repository to pull docker images from. This can be used when you have limited access to gcr.io. Set it to \"auto\" to let minikube decide one for you. For Chinese mainland users, you may use local gcr.io mirrors such as registry.cn-hangzhou.aliyuncs.com/google_containers")
 	startCmd.Flags().String(imageMirrorCountry, "", "Country code of the image mirror to be used. Leave empty to use the global one. For Chinese mainland users, set it to cn.")
-	startCmd.Flags().String(serviceCIDR, pkgutil.DefaultServiceCIDR, "The CIDR to be used for service cluster IPs.")
+	startCmd.Flags().String(serviceCIDR, constants.DefaultServiceCIDR, "The CIDR to be used for service cluster IPs.")
 	startCmd.Flags().StringArrayVar(&dockerEnv, "docker-env", nil, "Environment variables to pass to the Docker daemon. (format: key=value)")
 	startCmd.Flags().StringArrayVar(&dockerOpt, "docker-opt", nil, "Specify arbitrary flags to pass to the Docker daemon. (format: key=value)")
 }
@ -268,7 +270,11 @@ func platform() string {

 	// This environment is exotic, let's output a bit more.
 	if vrole == "guest" || runtime.GOARCH != "amd64" {
-		s.WriteString(fmt.Sprintf(" (%s/%s)", vsys, runtime.GOARCH))
+		if vsys != "" {
+			s.WriteString(fmt.Sprintf(" (%s/%s)", vsys, runtime.GOARCH))
+		} else {
+			s.WriteString(fmt.Sprintf(" (%s)", runtime.GOARCH))
+		}
 	}
 	return s.String()
 }
@ -289,7 +295,7 @@ func runStart(cmd *cobra.Command, args []string) {
 		registryMirror = viper.GetStringSlice("registry_mirror")
 	}

-	existing, err := cfg.Load(viper.GetString(config.MachineProfile))
+	existing, err := config.Load(viper.GetString(config.MachineProfile))
 	if err != nil && !os.IsNotExist(err) {
 		exit.WithCodeT(exit.Data, "Unable to load config: {{.error}}", out.V{"error": err})
 	}
@ -306,15 +312,23 @@ func runStart(cmd *cobra.Command, args []string) {
 	validateUser(driverName)

 	// Download & update the driver, even in --download-only mode
-	updateDriver(driverName)
+	if !viper.GetBool(dryRun) {
+		updateDriver(driverName)
+	}

 	k8sVersion, isUpgrade := getKubernetesVersion(existing)
-	config, err := generateCfgFromFlags(cmd, k8sVersion, driverName)
+	mc, n, err := generateCfgFromFlags(cmd, k8sVersion, driverName)
 	if err != nil {
 		exit.WithError("Failed to generate config", err)
 	}

-	cacheISO(&config, driverName)
+	// This is about as far as we can go without overwriting config files
+	if viper.GetBool(dryRun) {
+		out.T(out.DryRun, `dry-run validation complete!`)
+		return
+	}
+
+	cacheISO(&mc, driverName)

 	if viper.GetBool(nativeSSH) {
 		ssh.SetDefaultClient(ssh.Native)
@ -324,34 +338,34 @@ func runStart(cmd *cobra.Command, args []string) {

 	// Now that the ISO is downloaded, pull images in the background while the VM boots.
 	var cacheGroup errgroup.Group
-	beginCacheRequiredImages(&cacheGroup, config.KubernetesConfig.ImageRepository, k8sVersion)
+	beginCacheRequiredImages(&cacheGroup, mc.KubernetesConfig.ImageRepository, k8sVersion)

 	// Abstraction leakage alert: startHost requires the config to be saved, to satistfy pkg/provision/buildroot.
 	// Hence, saveConfig must be called before startHost, and again afterwards when we know the IP.
-	if err := saveConfig(&config); err != nil {
+	if err := saveConfig(&mc); err != nil {
 		exit.WithError("Failed to save config", err)
 	}

 	// exits here in case of --download-only option.
 	handleDownloadOnly(&cacheGroup, k8sVersion)
-	mRunner, preExists, machineAPI, host := startMachine(&config)
+	mRunner, preExists, machineAPI, host := startMachine(&mc, &n)
 	defer machineAPI.Close()
 	// configure the runtime (docker, containerd, crio)
-	cr := configureRuntimes(mRunner, driverName, config.KubernetesConfig)
+	cr := configureRuntimes(mRunner, driverName, mc.KubernetesConfig)
 	showVersionInfo(k8sVersion, cr)
 	waitCacheRequiredImages(&cacheGroup)

 	// Must be written before bootstrap, otherwise health checks may flake due to stale IP
-	kubeconfig, err := setupKubeconfig(host, &config, config.Name)
+	kubeconfig, err := setupKubeconfig(host, &mc, &n, mc.Name)
 	if err != nil {
 		exit.WithError("Failed to setup kubeconfig", err)
 	}

 	// setup kubeadm (must come after setupKubeconfig)
-	bs := setupKubeAdm(machineAPI, config)
+	bs := setupKubeAdm(machineAPI, mc, n)

 	// pull images or restart cluster
-	bootstrapCluster(bs, cr, mRunner, config.KubernetesConfig, preExists, isUpgrade)
+	bootstrapCluster(bs, cr, mRunner, mc, preExists, isUpgrade)
 	configureMounts()

 	// enable addons with start command
@ -368,11 +382,11 @@ func runStart(cmd *cobra.Command, args []string) {

 	// Skip pre-existing, because we already waited for health
 	if viper.GetBool(waitUntilHealthy) && !preExists {
-		if err := bs.WaitForCluster(config.KubernetesConfig, viper.GetDuration(waitTimeout)); err != nil {
+		if err := bs.WaitForCluster(mc, viper.GetDuration(waitTimeout)); err != nil {
 			exit.WithError("Wait failed", err)
 		}
 	}
-	if err := showKubectlInfo(kubeconfig, k8sVersion, config.Name); err != nil {
+	if err := showKubectlInfo(kubeconfig, k8sVersion, mc.Name); err != nil {
 		glog.Errorf("kubectl info: %v", err)
 	}
 }
@ -386,9 +400,9 @@ func updateDriver(driverName string) {
 	}
 }

-func cacheISO(config *cfg.MachineConfig, driverName string) {
+func cacheISO(cfg *config.MachineConfig, driverName string) {
 	if !driver.BareMetal(driverName) && !driver.IsKIC(driverName) {
-		if err := cluster.CacheISO(*config); err != nil {
+		if err := cluster.CacheISO(*cfg); err != nil {
 			exit.WithError("Failed to cache ISO", err)
 		}
 	}
@ -405,8 +419,8 @@ func enableAddons() {

 func displayVersion(version string) {
 	prefix := ""
-	if viper.GetString(cfg.MachineProfile) != constants.DefaultMachineName {
-		prefix = fmt.Sprintf("[%s] ", viper.GetString(cfg.MachineProfile))
+	if viper.GetString(config.MachineProfile) != constants.DefaultMachineName {
+		prefix = fmt.Sprintf("[%s] ", viper.GetString(config.MachineProfile))
 	}

 	versionState := out.Happy
@ -429,17 +443,19 @@ func displayEnviron(env []string) {
 	}
 }

-func setupKubeconfig(h *host.Host, c *cfg.MachineConfig, clusterName string) (*kubeconfig.Settings, error) {
+func setupKubeconfig(h *host.Host, c *config.MachineConfig, n *config.Node, clusterName string) (*kubeconfig.Settings, error) {
 	addr, err := h.Driver.GetURL()
 	if err != nil {
 		exit.WithError("Failed to get driver URL", err)
 	}
-	addr = strings.Replace(addr, "tcp://", "https://", -1)
-	addr = strings.Replace(addr, ":2376", ":"+strconv.Itoa(c.KubernetesConfig.NodePort), -1)
-	if c.KubernetesConfig.APIServerName != constants.APIServerName {
-		addr = strings.Replace(addr, c.KubernetesConfig.NodeIP, c.KubernetesConfig.APIServerName, -1)
+	if !driver.IsKIC(h.DriverName) {
+		addr = strings.Replace(addr, "tcp://", "https://", -1)
+		addr = strings.Replace(addr, ":2376", ":"+strconv.Itoa(n.Port), -1)
 	}

+	if c.KubernetesConfig.APIServerName != constants.APIServerName {
+		addr = strings.Replace(addr, n.IP, c.KubernetesConfig.APIServerName, -1)
+	}
 	kcs := &kubeconfig.Settings{
 		ClusterName:          clusterName,
 		ClusterServerAddress: addr,
@ -474,12 +490,12 @@ func handleDownloadOnly(cacheGroup *errgroup.Group, k8sVersion string) {

 }

-func startMachine(config *cfg.MachineConfig) (runner command.Runner, preExists bool, machineAPI libmachine.API, host *host.Host) {
+func startMachine(cfg *config.MachineConfig, node *config.Node) (runner command.Runner, preExists bool, machineAPI libmachine.API, host *host.Host) {
 	m, err := machine.NewAPIClient()
 	if err != nil {
 		exit.WithError("Failed to get machine client", err)
 	}
-	host, preExists = startHost(m, *config)
+	host, preExists = startHost(m, *cfg)
 	runner, err = machine.CommandRunner(host)
 	if err != nil {
 		exit.WithError("Failed to get command runner", err)
@ -493,8 +509,9 @@ func startMachine(config *cfg.MachineConfig) (runner command.Runner, preExists b
 		out.ErrT(out.FailureType, "Failed to set NO_PROXY Env. Please use `export NO_PROXY=$NO_PROXY,{{.ip}}`.", out.V{"ip": ip})
 	}
 	// Save IP to configuration file for subsequent use
-	config.KubernetesConfig.NodeIP = ip
-	if err := saveConfig(config); err != nil {
+	node.IP = ip
+
+	if err := saveNodeToConfig(cfg, node); err != nil {
 		exit.WithError("Failed to save config", err)
 	}

@ -556,28 +573,37 @@ func showKubectlInfo(kcs *kubeconfig.Settings, k8sVersion string, machineName st
 	return nil
 }

-func selectDriver(existing *cfg.MachineConfig) string {
+func selectDriver(existing *config.MachineConfig) string {
 	name := viper.GetString("vm-driver")
 	glog.Infof("selectDriver: flag=%q, old=%v", name, existing)
+
+	driver.SetLibvirtURI(viper.GetString(kvmQemuURI))
 	options := driver.Choices()
 	pick, alts := driver.Choose(name, options)
+	exp := ""
+	if pick.Priority == registry.Experimental {
+		exp = "experimental "
+	}

 	if name != "" {
-		out.T(out.Sparkle, `Selecting '{{.driver}}' driver from user configuration (alternates: {{.alternates}})`, out.V{"driver": name, "alternates": alts})
+		out.T(out.Sparkle, `Selecting {{.experimental}}'{{.driver}}' driver from user configuration (alternates: {{.alternates}})`, out.V{"experimental": exp, "driver": name, "alternates": alts})
 		return name
 	}

 	// By default, the driver is whatever we used last time
 	if existing != nil {
 		pick, alts := driver.Choose(existing.VMDriver, options)
-		out.T(out.Sparkle, `Selecting '{{.driver}}' driver from existing profile (alternates: {{.alternates}})`, out.V{"driver": existing.VMDriver, "alternates": alts})
+		if pick.Priority == registry.Experimental {
+			exp = "experimental"
+		}
+		out.T(out.Sparkle, `Selecting {{.experimental}} '{{.driver}}' driver from existing profile (alternates: {{.alternates}})`, out.V{"experimental": exp, "driver": existing.VMDriver, "alternates": alts})
 		return pick.Name
 	}

 	if len(options) > 1 {
-		out.T(out.Sparkle, `Automatically selected the '{{.driver}}' driver (alternates: {{.alternates}})`, out.V{"driver": pick.Name, "alternates": alts})
+		out.T(out.Sparkle, `Automatically selected the {{.experimental}} '{{.driver}}' driver (alternates: {{.alternates}})`, out.V{"experimental": exp, "driver": pick.Name, "alternates": alts})
 	} else {
-		out.T(out.Sparkle, `Automatically selected the '{{.driver}}' driver`, out.V{"driver": pick.Name})
+		out.T(out.Sparkle, `Automatically selected the {{.experimental}} '{{.driver}}' driver`, out.V{"experimental": exp, "driver": pick.Name})
 	}

 	if pick.Name == "" {
@ -587,10 +613,10 @@ func selectDriver(existing *cfg.MachineConfig) string {
 }

 // validateDriver validates that the selected driver appears sane, exits if not
-func validateDriver(name string, existing *cfg.MachineConfig) {
+func validateDriver(name string, existing *config.MachineConfig) {
 	glog.Infof("validating driver %q against %+v", name, existing)
 	if !driver.Supported(name) {
-		exit.WithCodeT(exit.Unavailable, "The driver '{{.driver}}' is not supported on {{.os}}", out.V{"driver": name, "os": runtime.GOOS})
+		exit.WithCodeT(exit.Unavailable, "The driver {{.experimental}} '{{.driver}}' is not supported on {{.os}}", out.V{"driver": name, "os": runtime.GOOS})
 	}

 	st := driver.Status(name)
@ -624,7 +650,7 @@ func validateDriver(name string, existing *cfg.MachineConfig) {
 		return
 	}

-	machineName := viper.GetString(cfg.MachineProfile)
+	machineName := viper.GetString(config.MachineProfile)
 	h, err := api.Load(machineName)
 	if err != nil {
 		glog.Warningf("selectDriver api.Load: %v", err)
@ -703,8 +729,8 @@ func selectImageRepository(mirrorCountry string) (bool, string, error) {

 // Return a minikube command containing the current profile name
 func minikubeCmd() string {
-	if viper.GetString(cfg.MachineProfile) != constants.DefaultMachineName {
-		return fmt.Sprintf("minikube -p %s", cfg.MachineProfile)
+	if viper.GetString(config.MachineProfile) != constants.DefaultMachineName {
+		return fmt.Sprintf("minikube -p %s", config.MachineProfile)
 	}
 	return "minikube"
 }
@ -734,7 +760,7 @@ func validateUser(drvName string) {
 	if !useForce {
 		os.Exit(exit.Permissions)
 	}
-	_, err = cfg.Load(viper.GetString(config.MachineProfile))
+	_, err = config.Load(viper.GetString(config.MachineProfile))
 	if err == nil || !os.IsNotExist(err) {
 		out.T(out.Tip, "Tip: To remove this root owned cluster, run: sudo {{.cmd}} delete", out.V{"cmd": minikubeCmd()})
 	}
@ -755,7 +781,7 @@ func validateDiskSize() {
 func validateMemorySize() {
 	memorySizeMB := pkgutil.CalculateSizeInMB(viper.GetString(memory))
 	if memorySizeMB < pkgutil.CalculateSizeInMB(minimumMemorySize) && !viper.GetBool(force) {
-		exit.UsageT("Requested memory allocation {{.requested_size}} is less than the minimum allowed of {{.minimum_size}}", out.V{"requested_size": memorySizeMB, "minimum_size": pkgutil.CalculateSizeInMB(minimumMemorySize)})
+		exit.WithCodeT(exit.Config, "Requested memory allocation {{.requested_size}} is less than the minimum allowed of {{.minimum_size}}", out.V{"requested_size": memorySizeMB, "minimum_size": pkgutil.CalculateSizeInMB(minimumMemorySize)})
 	}
 	if memorySizeMB < pkgutil.CalculateSizeInMB(defaultMemorySize) && !viper.GetBool(force) {
 		out.T(out.Notice, "Requested memory allocation ({{.memory}}MB) is less than the default memory allocation of {{.default_memorysize}}MB. Beware that minikube might not work correctly or crash unexpectedly.",
@ -788,7 +814,7 @@ func validateFlags(cmd *cobra.Command, drvName string) {
 	validateMemorySize()

 	if driver.BareMetal(drvName) {
-		if viper.GetString(cfg.MachineProfile) != constants.DefaultMachineName {
+		if viper.GetString(config.MachineProfile) != constants.DefaultMachineName {
 			exit.WithCodeT(exit.Config, "The 'none' driver does not support multiple profiles: https://minikube.sigs.k8s.io/docs/reference/drivers/none/")
 		}

@ -809,8 +835,8 @@ func validateFlags(cmd *cobra.Command, drvName string) {

 	// check that kubeadm extra args contain only whitelisted parameters
 	for param := range extraOptions.AsMap().Get(bsutil.Kubeadm) {
-		if !cfg.ContainsParam(bsutil.KubeadmExtraArgsWhitelist[bsutil.KubeadmCmdParam], param) &&
-			!cfg.ContainsParam(bsutil.KubeadmExtraArgsWhitelist[bsutil.KubeadmConfigParam], param) {
+		if !config.ContainsParam(bsutil.KubeadmExtraArgsWhitelist[bsutil.KubeadmCmdParam], param) &&
+			!config.ContainsParam(bsutil.KubeadmExtraArgsWhitelist[bsutil.KubeadmConfigParam], param) {
 			exit.UsageT("Sorry, the kubeadm.{{.parameter_name}} parameter is currently not supported by --extra-config", out.V{"parameter_name": param})
 		}
 	}
@ -862,11 +888,11 @@ func waitCacheRequiredImages(g *errgroup.Group) {
 	}
 }

-// generateCfgFromFlags generates cfg.Config based on flags and supplied arguments
-func generateCfgFromFlags(cmd *cobra.Command, k8sVersion string, drvName string) (cfg.MachineConfig, error) {
+// generateCfgFromFlags generates config.Config based on flags and supplied arguments
+func generateCfgFromFlags(cmd *cobra.Command, k8sVersion string, drvName string) (config.MachineConfig, config.Node, error) {
 	r, err := cruntime.New(cruntime.Config{Type: viper.GetString(containerRuntime)})
 	if err != nil {
-		return cfg.MachineConfig{}, err
+		return config.MachineConfig{}, config.Node{}, err
 	}

 	// Pick good default values for --network-plugin and --enable-default-cni based on runtime.
@ -907,8 +933,17 @@ func generateCfgFromFlags(cmd *cobra.Command, k8sVersion string, drvName string)
 		out.T(out.SuccessType, "Using image repository {{.name}}", out.V{"name": repository})
 	}

-	cfg := cfg.MachineConfig{
-		Name:                viper.GetString(cfg.MachineProfile),
+	// Create the initial node, which will necessarily be a control plane
+	cp := config.Node{
+		Port:              viper.GetInt(apiServerPort),
+		KubernetesVersion: k8sVersion,
+		Name:              constants.DefaultNodeName,
+		ControlPlane:      true,
+		Worker:            true,
+	}
+
+	cfg := config.MachineConfig{
+		Name:                viper.GetString(config.MachineProfile),
 		KeepContext:         viper.GetBool(keepContext),
 		EmbedCerts:          viper.GetBool(embedCerts),
 		MinikubeISO:         viper.GetString(isoURL),
@ -916,7 +951,6 @@ func generateCfgFromFlags(cmd *cobra.Command, k8sVersion string, drvName string)
 		CPUs:                viper.GetInt(cpus),
 		DiskSize:            pkgutil.CalculateSizeInMB(viper.GetString(humanReadableDiskSize)),
 		VMDriver:            drvName,
-		ContainerRuntime:    viper.GetString(containerRuntime),
 		HyperkitVpnKitSock:  viper.GetString(vpnkitSock),
 		HyperkitVSockPorts:  viper.GetStringSlice(vsockPorts),
 		NFSShare:            viper.GetStringSlice(nfsShare),
@ -939,10 +973,8 @@ func generateCfgFromFlags(cmd *cobra.Command, k8sVersion string, drvName string)
 		HostDNSResolver:     viper.GetBool(hostDNSResolver),
 		HostOnlyNicType:     viper.GetString(hostOnlyNicType),
 		NatNicType:          viper.GetString(natNicType),
-		KubernetesConfig: cfg.KubernetesConfig{
+		KubernetesConfig: config.KubernetesConfig{
 			KubernetesVersion:      k8sVersion,
-			NodePort:               viper.GetInt(apiServerPort),
-			NodeName:               constants.DefaultNodeName,
 			APIServerName:          viper.GetString(apiServerName),
 			APIServerNames:         apiServerNames,
 			APIServerIPs:           apiServerIPs,
@ -957,8 +989,9 @@ func generateCfgFromFlags(cmd *cobra.Command, k8sVersion string, drvName string)
 			ShouldLoadCachedImages: viper.GetBool(cacheImages),
 			EnableDefaultCNI:       selectedEnableDefaultCNI,
 		},
+		Nodes: []config.Node{cp},
 	}
-	return cfg, nil
+	return cfg, cp, nil
 }

 // setDockerProxy sets the proxy environment variables in the docker environment.
@ -980,35 +1013,41 @@ func setDockerProxy() {
 }

 // autoSetDriverOptions sets the options needed for specific vm-driver automatically.
-func autoSetDriverOptions(cmd *cobra.Command, drvName string) error {
+func autoSetDriverOptions(cmd *cobra.Command, drvName string) (err error) {
+	err = nil
 	hints := driver.FlagDefaults(drvName)
-	if !cmd.Flags().Changed("extra-config") && hints.ExtraOptions != "" {
-		return extraOptions.Set(hints.ExtraOptions)
+	if !cmd.Flags().Changed("extra-config") && len(hints.ExtraOptions) > 0 {
+		for _, eo := range hints.ExtraOptions {
+			glog.Infof("auto setting extra-config to %q.", eo)
+			err = extraOptions.Set(eo)
+			if err != nil {
+				err = errors.Wrapf(err, "setting extra option %s", eo)
+			}
+		}
 	}

 	if !cmd.Flags().Changed(cacheImages) {
 		viper.Set(cacheImages, hints.CacheImages)
 	}

-	// currently only used for kic
 	if !cmd.Flags().Changed(containerRuntime) && hints.ContainerRuntime != "" {
 		viper.Set(containerRuntime, hints.ContainerRuntime)
-		glog.Infof("auto set container runtime to %s for kic driver.", hints.ContainerRuntime)
-
+		glog.Infof("auto set %s to %q.", containerRuntime, hints.ContainerRuntime)
 	}
-	if !cmd.Flags().Changed("bootstrapper") && hints.Bootstrapper != "" {
+
+	if !cmd.Flags().Changed(cmdcfg.Bootstrapper) && hints.Bootstrapper != "" {
 		viper.Set(cmdcfg.Bootstrapper, hints.Bootstrapper)
-		glog.Infof("auto set bootstrapper to %s for kic driver.", hints.Bootstrapper)
+		glog.Infof("auto set %s to %q.", cmdcfg.Bootstrapper, hints.Bootstrapper)

 	}

-	return nil
+	return err
 }

 // prepareNone prepares the user and host for the joy of the "none" driver
 func prepareNone() {
 	out.T(out.StartingNone, "Configuring local host environment ...")
-	if viper.GetBool(cfg.WantNoneDriverWarning) {
+	if viper.GetBool(config.WantNoneDriverWarning) {
 		out.T(out.Empty, "")
 		out.WarningT("The 'none' driver provides limited isolation and may reduce system security and reliability.")
 		out.WarningT("For more information, see:")
@ -1035,7 +1074,7 @@ func prepareNone() {
 }

 // startHost starts a new minikube host using a VM or None
-func startHost(api libmachine.API, mc cfg.MachineConfig) (*host.Host, bool) {
+func startHost(api libmachine.API, mc config.MachineConfig) (*host.Host, bool) {
 	exists, err := api.Exists(mc.Name)
 	if err != nil {
 		exit.WithError("Failed to check if machine exists", err)
@ -1115,7 +1154,7 @@ Suggested workarounds:

 func tryLookup(r command.Runner) {
 	// DNS check
-	if rr, err := r.RunCmd(exec.Command("nslookup", "-querytype=ns", "kubernetes.io")); err != nil {
+	if rr, err := r.RunCmd(exec.Command("nslookup", "kubernetes.io")); err != nil {
 		glog.Warningf("%s failed: %v", rr.Args, err)
 		out.WarningT("VM may be unable to resolve external DNS records")
 	}
@ -1142,14 +1181,15 @@ func tryRegistry(r command.Runner) {
 }

 // getKubernetesVersion ensures that the requested version is reasonable
-func getKubernetesVersion(old *cfg.MachineConfig) (string, bool) {
+func getKubernetesVersion(old *config.MachineConfig) (string, bool) {
 	paramVersion := viper.GetString(kubernetesVersion)
 	isUpgrade := false

 	if paramVersion == "" { // if the user did not specify any version then ...
-		if old != nil { // .. use the old version from config
+		if old != nil { // .. use the old version from config (if any)
 			paramVersion = old.KubernetesConfig.KubernetesVersion
-		} else { // .. otherwise use the default version
+		}
+		if paramVersion == "" { // .. otherwise use the default version
 			paramVersion = constants.DefaultKubernetesVersion
 		}
 	}
@ -1211,7 +1251,7 @@ func getKubernetesVersion(old *cfg.MachineConfig) (string, bool) {
 }

 // setupKubeAdm adds any requested files into the VM before Kubernetes is started
-func setupKubeAdm(mAPI libmachine.API, config cfg.MachineConfig) bootstrapper.Bootstrapper {
+func setupKubeAdm(mAPI libmachine.API, cfg config.MachineConfig, node config.Node) bootstrapper.Bootstrapper {
 	bs, err := getClusterBootstrapper(mAPI, viper.GetString(cmdcfg.Bootstrapper))
 	if err != nil {
 		exit.WithError("Failed to get bootstrapper", err)
@ -1220,17 +1260,17 @@ func setupKubeAdm(mAPI libmachine.API, config cfg.MachineConfig) bootstrapper.Bo
 		out.T(out.Option, "{{.extra_option_component_name}}.{{.key}}={{.value}}", out.V{"extra_option_component_name": eo.Component, "key": eo.Key, "value": eo.Value})
 	}
 	// Loads cached images, generates config files, download binaries
-	if err := bs.UpdateCluster(config); err != nil {
+	if err := bs.UpdateCluster(cfg); err != nil {
 		exit.WithError("Failed to update cluster", err)
 	}
-	if err := bs.SetupCerts(config.KubernetesConfig); err != nil {
+	if err := bs.SetupCerts(cfg.KubernetesConfig, node); err != nil {
 		exit.WithError("Failed to setup certs", err)
 	}
 	return bs
 }

 // configureRuntimes does what needs to happen to get a runtime going.
-func configureRuntimes(runner cruntime.CommandRunner, drvName string, k8s cfg.KubernetesConfig) cruntime.Manager {
+func configureRuntimes(runner cruntime.CommandRunner, drvName string, k8s config.KubernetesConfig) cruntime.Manager {
 	config := cruntime.Config{Type: viper.GetString(containerRuntime), Runner: runner, ImageRepository: k8s.ImageRepository, KubernetesVersion: k8s.KubernetesVersion}
 	cr, err := cruntime.New(config)
 	if err != nil {
@ -1250,16 +1290,16 @@ func configureRuntimes(runner cruntime.CommandRunner, drvName string, k8s cfg.Ku
 }

 // bootstrapCluster starts Kubernetes using the chosen bootstrapper
-func bootstrapCluster(bs bootstrapper.Bootstrapper, r cruntime.Manager, runner command.Runner, kc cfg.KubernetesConfig, preexisting bool, isUpgrade bool) {
+func bootstrapCluster(bs bootstrapper.Bootstrapper, r cruntime.Manager, runner command.Runner, mc config.MachineConfig, preexisting bool, isUpgrade bool) {
 	if isUpgrade || !preexisting {
 		out.T(out.Pulling, "Pulling images ...")
-		if err := bs.PullImages(kc); err != nil {
+		if err := bs.PullImages(mc.KubernetesConfig); err != nil {
 			out.T(out.FailureType, "Unable to pull images, which may be OK: {{.error}}", out.V{"error": err})
 		}
 	}

 	out.T(out.Launch, "Launching Kubernetes ... ")
-	if err := bs.StartCluster(kc); err != nil {
+	if err := bs.StartCluster(mc); err != nil {
 		exit.WithLogEntries("Error starting cluster", err, logs.FindProblems(r, bs, runner))
 	}
 }
@ -1291,6 +1331,16 @@ func configureMounts() {
 }

 // saveConfig saves profile cluster configuration in $MINIKUBE_HOME/profiles/<profilename>/config.json
-func saveConfig(clusterCfg *cfg.MachineConfig) error {
-	return cfg.CreateProfile(viper.GetString(cfg.MachineProfile), clusterCfg)
+func saveConfig(clusterCfg *config.MachineConfig) error {
+	return config.SaveProfile(viper.GetString(config.MachineProfile), clusterCfg)
+}
+
+func saveNodeToConfig(cfg *config.MachineConfig, node *config.Node) error {
+	for i, n := range cfg.Nodes {
+		if n.Name == node.Name {
+			cfg.Nodes[i] = *node
+			break
+		}
+	}
+	return saveConfig(cfg)
 }
--- a/cmd/minikube/cmd/start_test.go
+++ b/cmd/minikube/cmd/start_test.go
@ -121,7 +121,7 @@ func TestGenerateCfgFromFlagsHTTPProxyHandling(t *testing.T) {
 			if err := os.Setenv("HTTP_PROXY", test.proxy); err != nil {
 				t.Fatalf("Unexpected error setting HTTP_PROXY: %v", err)
 			}
-			config, err := generateCfgFromFlags(cmd, k8sVersion, "none")
+			config, _, err := generateCfgFromFlags(cmd, k8sVersion, "none")
 			if err != nil {
 				t.Fatalf("Got unexpected error %v during config generation", err)
 			}
--- a/cmd/minikube/cmd/status.go
+++ b/cmd/minikube/cmd/status.go
@ -19,12 +19,15 @@ package cmd
 import (
 	"encoding/json"
 	"fmt"
+	"io"
 	"os"
 	"strings"
 	"text/template"

+	"github.com/docker/machine/libmachine"
 	"github.com/docker/machine/libmachine/state"
 	"github.com/golang/glog"
+	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config"
@ -40,16 +43,13 @@ import (
 var statusFormat string
 var output string

-// KubeconfigStatus represents the kubeconfig status
-var KubeconfigStatus = struct {
-	Configured    string
-	Misconfigured string
-}{
-	Configured:    `Configured`,
-	Misconfigured: `Misconfigured`,
-}
+const (
+	// Additional states used by kubeconfig
+	Configured    = "Configured"    // analogous to state.Saved
+	Misconfigured = "Misconfigured" // analogous to state.Error
+)

-// Status represents the status
+// Status holds string representations of component states
 type Status struct {
 	Host       string
 	Kubelet    string
@ -81,7 +81,6 @@ var statusCmd = &cobra.Command{
 			exit.UsageT("Cannot use both --output and --format options")
 		}

-		var returnCode = 0
 		api, err := machine.NewAPIClient()
 		if err != nil {
 			exit.WithCodeT(exit.Unavailable, "Error getting client: {{.error}}", out.V{"error": err})
@ -89,81 +88,94 @@ var statusCmd = &cobra.Command{
 		defer api.Close()

 		machineName := viper.GetString(config.MachineProfile)
-
-		hostSt, err := cluster.GetHostStatus(api, machineName)
+		st, err := status(api, machineName)
 		if err != nil {
-			exit.WithError("Error getting host status", err)
-		}
-
-		kubeletSt := state.None.String()
-		kubeconfigSt := state.None.String()
-		apiserverSt := state.None.String()
-
-		if hostSt == state.Running.String() {
-			clusterBootstrapper, err := getClusterBootstrapper(api, viper.GetString(cmdcfg.Bootstrapper))
-			if err != nil {
-				exit.WithError("Error getting bootstrapper", err)
-			}
-			kubeletSt, err = clusterBootstrapper.GetKubeletStatus()
-			if err != nil {
-				glog.Warningf("kubelet err: %v", err)
-				returnCode |= clusterNotRunningStatusFlag
-			} else if kubeletSt != state.Running.String() {
-				returnCode |= clusterNotRunningStatusFlag
-			}
-
-			ip, err := cluster.GetHostDriverIP(api, machineName)
-			if err != nil {
-				glog.Errorln("Error host driver ip status:", err)
-			}
-
-			apiserverPort, err := kubeconfig.Port(machineName)
-			if err != nil {
-				// Fallback to presuming default apiserver port
-				apiserverPort = constants.APIServerPort
-			}
-
-			apiserverSt, err = clusterBootstrapper.GetAPIServerStatus(ip, apiserverPort)
-			if err != nil {
-				glog.Errorln("Error apiserver status:", err)
-			} else if apiserverSt != state.Running.String() {
-				returnCode |= clusterNotRunningStatusFlag
-			}
-
-			ks, err := kubeconfig.IsClusterInConfig(ip, machineName)
-			if err != nil {
-				glog.Errorln("Error kubeconfig status:", err)
-			}
-			if ks {
-				kubeconfigSt = KubeconfigStatus.Configured
-			} else {
-				kubeconfigSt = KubeconfigStatus.Misconfigured
-				returnCode |= k8sNotRunningStatusFlag
-			}
-		} else {
-			returnCode |= minikubeNotRunningStatusFlag
-		}
-
-		status := Status{
-			Host:       hostSt,
-			Kubelet:    kubeletSt,
-			APIServer:  apiserverSt,
-			Kubeconfig: kubeconfigSt,
+			glog.Errorf("status error: %v", err)
 		}

 		switch strings.ToLower(output) {
 		case "text":
-			printStatusText(status)
+			if err := statusText(st, os.Stdout); err != nil {
+				exit.WithError("status text failure", err)
+			}
 		case "json":
-			printStatusJSON(status)
+			if err := statusJSON(st, os.Stdout); err != nil {
+				exit.WithError("status json failure", err)
+			}
 		default:
 			exit.WithCodeT(exit.BadUsage, fmt.Sprintf("invalid output format: %s. Valid values: 'text', 'json'", output))
 		}

-		os.Exit(returnCode)
+		os.Exit(exitCode(st))
 	},
 }

+func exitCode(st *Status) int {
+	c := 0
+	if st.Host != state.Running.String() {
+		c |= minikubeNotRunningStatusFlag
+	}
+	if st.APIServer != state.Running.String() || st.Kubelet != state.Running.String() {
+		c |= clusterNotRunningStatusFlag
+	}
+	if st.Kubeconfig != Configured {
+		c |= k8sNotRunningStatusFlag
+	}
+	return c
+}
+
+func status(api libmachine.API, name string) (*Status, error) {
+	st := &Status{}
+	hs, err := cluster.GetHostStatus(api, name)
+	if err != nil {
+		return st, errors.Wrap(err, "host")
+	}
+	st.Host = hs
+	if st.Host != state.Running.String() {
+		return st, nil
+	}
+
+	bs, err := getClusterBootstrapper(api, viper.GetString(cmdcfg.Bootstrapper))
+	if err != nil {
+		return st, errors.Wrap(err, "bootstrapper")
+	}
+
+	st.Kubelet, err = bs.GetKubeletStatus()
+	if err != nil {
+		glog.Warningf("kubelet err: %v", err)
+		st.Kubelet = state.Error.String()
+	}
+
+	ip, err := cluster.GetHostDriverIP(api, name)
+	if err != nil {
+		glog.Errorln("Error host driver ip status:", err)
+		st.APIServer = state.Error.String()
+		return st, err
+	}
+
+	port, err := kubeconfig.Port(name)
+	if err != nil {
+		glog.Warningf("unable to get port: %v", err)
+		port = constants.APIServerPort
+	}
+
+	st.APIServer, err = bs.GetAPIServerStatus(ip, port)
+	if err != nil {
+		glog.Errorln("Error apiserver status:", err)
+		st.APIServer = state.Error.String()
+	}
+
+	st.Kubeconfig = Misconfigured
+	ks, err := kubeconfig.IsClusterInConfig(ip, name)
+	if err != nil {
+		glog.Errorln("Error kubeconfig status:", err)
+	}
+	if ks {
+		st.Kubeconfig = Configured
+	}
+	return st, nil
+}
+
 func init() {
 	statusCmd.Flags().StringVarP(&statusFormat, "format", "f", defaultStatusFormat,
 		`Go template format string for the status output.  The format for Go templates can be found here: https://golang.org/pkg/text/template/
@ -172,25 +184,26 @@ For the list accessible variables for the template, see the struct values here:
 		`minikube status --output OUTPUT. json, text`)
 }

-var printStatusText = func(status Status) {
+func statusText(st *Status, w io.Writer) error {
 	tmpl, err := template.New("status").Parse(statusFormat)
 	if err != nil {
-		exit.WithError("Error creating status template", err)
+		return err
 	}
-	err = tmpl.Execute(os.Stdout, status)
-	if err != nil {
-		exit.WithError("Error executing status template", err)
+	if err := tmpl.Execute(w, st); err != nil {
+		return err
 	}
-	if status.Kubeconfig == KubeconfigStatus.Misconfigured {
-		out.WarningT("Warning: Your kubectl is pointing to stale minikube-vm.\nTo fix the kubectl context, run `minikube update-context`")
+	if st.Kubeconfig == Misconfigured {
+		_, err := w.Write([]byte("\nWARNING: Your kubectl is pointing to stale minikube-vm.\nTo fix the kubectl context, run `minikube update-context`\n"))
+		return err
 	}
+	return nil
 }

-var printStatusJSON = func(status Status) {
-
-	jsonString, err := json.Marshal(status)
+func statusJSON(st *Status, w io.Writer) error {
+	js, err := json.Marshal(st)
 	if err != nil {
-		exit.WithError("Error converting status to json", err)
+		return err
 	}
-	out.String(string(jsonString))
+	_, err = w.Write(js)
+	return err
 }
--- a/cmd/minikube/cmd/tunnel.go
+++ b/cmd/minikube/cmd/tunnel.go
@ -38,7 +38,7 @@ var cleanup bool
 var tunnelCmd = &cobra.Command{
 	Use:   "tunnel",
 	Short: "tunnel makes services of type LoadBalancer accessible on localhost",
-	Long:  `tunnel creates a route to services deployed with type LoadBalancer and sets their Ingress to their ClusterIP`,
+	Long:  `tunnel creates a route to services deployed with type LoadBalancer and sets their Ingress to their ClusterIP. for a detailed example see https://minikube.sigs.k8s.io/docs/tasks/loadbalancer`,
 	PersistentPreRun: func(cmd *cobra.Command, args []string) {
 		RootCmd.PersistentPreRun(cmd, args)
 	},
--- a/cmd/minikube/cmd/unpause.go
+++ b/cmd/minikube/cmd/unpause.go
@ -0,0 +1,98 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cmd
+
+import (
+	"os"
+	"strings"
+
+	"github.com/golang/glog"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+
+	"k8s.io/minikube/pkg/minikube/cluster"
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/cruntime"
+	"k8s.io/minikube/pkg/minikube/exit"
+	"k8s.io/minikube/pkg/minikube/machine"
+	"k8s.io/minikube/pkg/minikube/out"
+)
+
+// unpauseCmd represents the docker-pause command
+var unpauseCmd = &cobra.Command{
+	Use:   "unpause",
+	Short: "unpause Kubernetes",
+	Run: func(cmd *cobra.Command, args []string) {
+		cname := viper.GetString(config.MachineProfile)
+		api, err := machine.NewAPIClient()
+		if err != nil {
+			exit.WithError("Error getting client", err)
+		}
+		defer api.Close()
+		cc, err := config.Load(cname)
+
+		if err != nil && !os.IsNotExist(err) {
+			exit.WithError("Error loading profile config", err)
+		}
+
+		if err != nil {
+			out.ErrT(out.Meh, `"{{.name}}" profile does not exist`, out.V{"name": cname})
+			os.Exit(1)
+		}
+		glog.Infof("config: %+v", cc)
+		host, err := cluster.CheckIfHostExistsAndLoad(api, cname)
+		if err != nil {
+			exit.WithError("Error getting host", err)
+		}
+
+		r, err := machine.CommandRunner(host)
+		if err != nil {
+			exit.WithError("Failed to get command runner", err)
+		}
+
+		cr, err := cruntime.New(cruntime.Config{Type: cc.KubernetesConfig.ContainerRuntime, Runner: r})
+		if err != nil {
+			exit.WithError("Failed runtime", err)
+		}
+
+		glog.Infof("namespaces: %v keys: %v", namespaces, viper.AllSettings())
+		if allNamespaces {
+			namespaces = nil //all
+		} else {
+			if len(namespaces) == 0 {
+				exit.WithCodeT(exit.BadUsage, "Use -A to specify all namespaces")
+			}
+		}
+
+		ids, err := cluster.Unpause(cr, r, namespaces)
+		if err != nil {
+			exit.WithError("Pause", err)
+		}
+
+		if namespaces == nil {
+			out.T(out.Pause, "Unpaused kubelet and {{.count}} containers", out.V{"count": len(ids)})
+		} else {
+			out.T(out.Pause, "Unpaused kubelet and {{.count}} containers in: {{.namespaces}}", out.V{"count": len(ids), "namespaces": strings.Join(namespaces, ", ")})
+		}
+
+	},
+}
+
+func init() {
+	unpauseCmd.Flags().StringSliceVarP(&namespaces, "--namespaces", "n", cluster.DefaultNamespaces, "namespaces to unpause")
+	unpauseCmd.Flags().BoolVarP(&allNamespaces, "all-namespaces", "A", false, "If set, unpause all namespaces")
+}
--- a/deploy/addons/addon-manager.yaml.tmpl
+++ b/deploy/addons/addon-manager.yaml.tmpl
@ -1,56 +0,0 @@
-# Copyright 2016 The Kubernetes Authors All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-apiVersion: v1
-kind: Pod
-metadata:
-  name: kube-addon-manager
-  namespace: kube-system
-  labels:
-    component: kube-addon-manager
-    version: v9.0.2
-    kubernetes.io/minikube-addons: addon-manager
-spec:
-  hostNetwork: true
-  containers:
-  - name: kube-addon-manager
-    image: {{default "k8s.gcr.io" .ImageRepository}}/kube-addon-manager{{.ExoticArch}}:v9.0.2
-    env:
-    - name: KUBECONFIG
-      value: /var/lib/minikube/kubeconfig
-    - name: TEST_ADDON_CHECK_INTERVAL_SEC
-      value: "5"
-    - name: ADDON_MANAGER_LEADER_ELECTION
-      value: "false"
-    - name: KUBECTL_EXTRA_PRUNE_WHITELIST
-      value: install.istio.io/v1alpha2/IstioControlPlane
-    imagePullPolicy: IfNotPresent
-    resources:
-      requests:
-        cpu: 5m
-        memory: 50Mi
-    volumeMounts:
-    - mountPath: /etc/kubernetes/
-      name: addons
-      readOnly: true
-    - mountPath: /var/lib/minikube/
-      name: kubeconfig
-      readOnly: true
-  volumes:
-  - hostPath:
-      path: /etc/kubernetes/
-    name: addons
-  - hostPath:
-      path: /var/lib/minikube/
-    name: kubeconfig
--- a/deploy/addons/istio/README.md
+++ b/deploy/addons/istio/README.md
@ -1,11 +1,16 @@
 ## istio Addon
 [istio](https://istio.io/docs/setup/getting-started/) - Cloud platforms provide a wealth of benefits for the organizations that use them.

-### Enabling istio
-Propose to startup minikube with at least 8192 MB of memory and 4 CPUs to enable istio.
-To enable this addon, simply run:
+### Enable istio on minikube
+Make sure to start minikube with at least 8192 MB of memory and 4 CPUs.

 ```shell script
+minikube start --memory=8000mb --cpus=4 
+```
+
+To enable this addon, simply run:
+```shell script
+minikube addons enable istio-provisioner
 minikube addons enable istio
 ```

@ -19,8 +24,9 @@ kubectl get po -n istio-system

 If everything went well you shouldn't get any errors about istio being installed in your cluster. If you haven't deployed any releases `kubectl get po -n istio-system` won't return anything.

-### Deprecation of istio
+### Disable istio
 To disable this addon, simply run:
 ```shell script
+minikube addons disable istio-provisioner
 minikube addons disable istio
-```
+```
--- a/deploy/iso/minikube-iso/board/coreos/minikube/rootfs-overlay/etc/sysctl.d/disable-ipv6.conf
+++ b/deploy/iso/minikube-iso/board/coreos/minikube/rootfs-overlay/etc/sysctl.d/disable-ipv6.conf
@ -0,0 +1,2 @@
+net.ipv6.conf.all.disable_ipv6=1
+net.ipv6.conf.default.disable_ipv6=1
--- a/deploy/iso/minikube-iso/configs/minikube_defconfig
+++ b/deploy/iso/minikube-iso/configs/minikube_defconfig
@ -19,6 +19,8 @@ BR2_ROOTFS_USERS_TABLES="$(BR2_EXTERNAL_MINIKUBE_PATH)/board/coreos/minikube/use
 BR2_ROOTFS_OVERLAY="$(BR2_EXTERNAL_MINIKUBE_PATH)/board/coreos/minikube/rootfs-overlay"
 BR2_GLOBAL_PATCH_DIR="$(BR2_EXTERNAL_MINIKUBE_PATH)/board/coreos/minikube/patches"
 BR2_LINUX_KERNEL=y
+BR2_LINUX_KERNEL_BZIMAGE=y
+BR2_LINUX_KERNEL_LZ4=y
 BR2_LINUX_KERNEL_USE_CUSTOM_CONFIG=y
 BR2_LINUX_KERNEL_CUSTOM_CONFIG_FILE="$(BR2_EXTERNAL_MINIKUBE_PATH)/board/coreos/minikube/linux_defconfig"
 BR2_LINUX_KERNEL_NEEDS_HOST_LIBELF=y
@ -55,7 +57,8 @@ BR2_PACKAGE_SYSTEMD_MACHINED=y
 BR2_PACKAGE_SYSTEMD_VCONSOLE=y
 BR2_PACKAGE_UTIL_LINUX_NSENTER=y
 BR2_PACKAGE_UTIL_LINUX_SCHEDUTILS=y
-BR2_TARGET_ROOTFS_CPIO_BZIP2=y
+BR2_TARGET_ROOTFS_CPIO=y
+BR2_TARGET_ROOTFS_CPIO_GZIP=y
 BR2_TARGET_ROOTFS_ISO9660=y
 BR2_TARGET_ROOTFS_ISO9660_BOOT_MENU="$(BR2_EXTERNAL_MINIKUBE_PATH)/board/coreos/minikube/isolinux.cfg"
 BR2_TARGET_SYSLINUX=y
--- a/deploy/iso/minikube-iso/package/Config.in
+++ b/deploy/iso/minikube-iso/package/Config.in
@ -1,6 +1,7 @@
 menu "System tools"
    source "$BR2_EXTERNAL_MINIKUBE_PATH/package/runc-master/Config.in"
    source "$BR2_EXTERNAL_MINIKUBE_PATH/package/podman/Config.in"
+    source "$BR2_EXTERNAL_MINIKUBE_PATH/package/varlink/Config.in"
    source "$BR2_EXTERNAL_MINIKUBE_PATH/package/conmon-master/Config.in"
    source "$BR2_EXTERNAL_MINIKUBE_PATH/package/crio-bin/Config.in"
    source "$BR2_EXTERNAL_MINIKUBE_PATH/package/crictl-bin/Config.in"
--- a/deploy/iso/minikube-iso/package/automount/minikube-automount
+++ b/deploy/iso/minikube-iso/package/automount/minikube-automount
@ -29,7 +29,7 @@ if [ ! -n "$BOOT2DOCKER_DATA" ]; then
            # Let kernel re-read partition table
            partprobe
            # Add the data partition
-            (echo n; echo p; echo 1; echo ; echo ; echo w) | fdisk $UNPARTITIONED_HD
+            (echo g; echo n; echo 1; echo ; echo ; echo w) | fdisk $UNPARTITIONED_HD
            # Let kernel re-read partition table
            partprobe
            # wait for the partition to actually exist, timeout after about 5 seconds
@ -57,7 +57,7 @@ if [ ! -n "$BOOT2DOCKER_DATA" ]; then
                if [ $NON_NUL == 0 ]; then
                    # Create the partition, format it and then mount it
                    echo "NEW VMware boot2docker managed disk image ($UNPARTITIONED_HD): formatting it for use"
-                    (echo n; echo p; echo 1; echo ; echo ; echo w) | fdisk $UNPARTITIONED_HD
+                    (echo g; echo n; echo 1; echo ; echo ; echo w) | fdisk $UNPARTITIONED_HD
                    BOOT2DOCKER_DATA=`echo "${UNPARTITIONED_HD}1"`
                    mkfs.ext4 -i 2048 -L $LABEL $BOOT2DOCKER_DATA
                else
--- a/deploy/iso/minikube-iso/package/varlink/Config.in
+++ b/deploy/iso/minikube-iso/package/varlink/Config.in
@ -0,0 +1,3 @@
+config BR2_PACKAGE_VARLINK
+	bool "varlink"
+	default y
--- a/deploy/iso/minikube-iso/package/varlink/varlink.hash
+++ b/deploy/iso/minikube-iso/package/varlink/varlink.hash
@ -0,0 +1,3 @@
+sha256 3857f109574750403b233b5fdf73f1852d8decc33dac8f73bd49f2003b69ad22 16.tar.gz
+sha256 0dcb451f32033154c56710c216e67f245923fe2b011321271f6670e5a2285ce6 17.tar.gz
+sha256 7a32543643116ad105da4ddb2f8030de7dcad1cdb3feb1a214ae5e7b65a6a198 18.tar.gz
--- a/deploy/iso/minikube-iso/package/varlink/varlink.mk
+++ b/deploy/iso/minikube-iso/package/varlink/varlink.mk
@ -0,0 +1,7 @@
+VARLINK_VERSION = 18
+VARLINK_SITE = https://github.com/varlink/libvarlink/archive
+VARLINK_SOURCE = $(VARLINK_VERSION).tar.gz
+VARLINK_LICENSE = Apache-2.0
+VARLINK_LICENSE_FILES = LICENSE
+
+$(eval $(meson-package))
--- a/go.mod
+++ b/go.mod
@ -2,10 +2,6 @@ module k8s.io/minikube

 go 1.13

-require github.com/google/go-containerregistry v0.0.0-20180731221751-697ee0b3d46e
-
-require k8s.io/kubernetes v1.15.2
-
 require (
 	github.com/Parallels/docker-machine-parallels v1.3.0
 	github.com/Sirupsen/logrus v0.0.0-20170822132746-89742aefa4b2 // indirect
@ -17,7 +13,7 @@ require (
 	github.com/cloudfoundry-attic/jibber_jabber v0.0.0-20151120183258-bcc4c8345a21
 	github.com/cloudfoundry/jibber_jabber v0.0.0-20151120183258-bcc4c8345a21 // indirect
 	github.com/docker/docker v1.13.1
-	github.com/docker/go-units v0.3.3
+	github.com/docker/go-units v0.4.0
 	github.com/docker/machine v0.7.1-0.20190718054102-a555e4f7a8f5 // version is 0.7.1 to pin to a555e4f7a8f5
 	github.com/elazarl/goproxy v0.0.0-20190421051319-9d40249d3c2f
 	github.com/elazarl/goproxy/ext v0.0.0-20190421051319-9d40249d3c2f // indirect
@ -26,14 +22,16 @@ require (
 	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
 	github.com/google/go-cmp v0.3.0
+	github.com/google/go-containerregistry v0.0.0-20180731221751-697ee0b3d46e
 	github.com/googleapis/gnostic v0.3.0 // indirect
-	github.com/gorilla/mux v1.7.1 // indirect
+	github.com/gorilla/mux v1.7.3 // indirect
 	github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce // indirect
 	github.com/hashicorp/go-getter v1.4.0
 	github.com/hashicorp/go-multierror v0.0.0-20160811015721-8c5f0ad93604 // indirect
 	github.com/hashicorp/go-retryablehttp v0.5.4
 	github.com/hooklift/assert v0.0.0-20170704181755-9d1defd6d214 // indirect
 	github.com/hooklift/iso9660 v0.0.0-20170318115843-1cf07e5970d8
+	github.com/imdario/mergo v0.3.8 // indirect
 	github.com/intel-go/cpuid v0.0.0-20181003105527-1a4a6f06a1c6 // indirect
 	github.com/jimmidyson/go-download v0.0.0-20161028105827-7f9a90c8c95b
 	github.com/johanneswuerbach/nfsexports v0.0.0-20181204082207-1aa528dcb345
@ -48,10 +46,10 @@ require (
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/libvirt/libvirt-go v3.4.0+incompatible
 	github.com/machine-drivers/docker-machine-driver-vmware v0.1.1
-	github.com/mattn/go-isatty v0.0.8
+	github.com/mattn/go-isatty v0.0.9
 	github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936
 	github.com/moby/hyperkit v0.0.0-20171020124204-a12cd7250bcd
-	github.com/olekukonko/tablewriter v0.0.0-20160923125401-bdcc175572fd
+	github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5
 	github.com/otiai10/copy v1.0.2
 	github.com/pborman/uuid v1.2.0
 	github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2
@ -61,30 +59,26 @@ require (
 	github.com/pmezard/go-difflib v1.0.0
 	github.com/samalba/dockerclient v0.0.0-20160414174713-91d7393ff859 // indirect
 	github.com/shirou/gopsutil v2.18.12+incompatible
-	github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 // indirect
-	github.com/sirupsen/logrus v1.4.1 // indirect
 	github.com/spf13/cobra v0.0.5
-	github.com/spf13/pflag v1.0.3
+	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.3.2
 	github.com/xeipuuv/gojsonpointer v0.0.0-20151027082146-e0fe6f683076 // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20150808065054-e02fc20de94c // indirect
 	github.com/xeipuuv/gojsonschema v0.0.0-20160623135812-c539bca196be
 	github.com/zchee/go-vmnet v0.0.0-20161021174912-97ebf9174097
-	golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d
-	golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4
-	golang.org/x/sync v0.0.0-20190423024810-112230192c58
-	golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb
+	golang.org/x/build v0.0.0-20190927031335-2835ba2e683f
+	golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586
+	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
+	golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456
 	golang.org/x/text v0.3.2
-	gopkg.in/airbrake/gobrake.v2 v2.0.9 // indirect
-	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
-	gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 // indirect
 	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
 	gotest.tools v2.2.0+incompatible
-	k8s.io/api v0.0.0
-	k8s.io/apimachinery v0.0.0
+	k8s.io/api v0.17.2
+	k8s.io/apimachinery v0.17.2
 	k8s.io/client-go v11.0.0+incompatible
-	k8s.io/klog v0.3.3 // indirect
-	k8s.io/kubectl v0.0.0-00010101000000-000000000000
+	k8s.io/kubectl v0.0.0
+	k8s.io/kubernetes v1.17.2
+	k8s.io/utils v0.0.0-20200122174043-1e243dd1a584 // indirect
 	sigs.k8s.io/sig-storage-lib-external-provisioner v4.0.0+incompatible
 )

@ -92,28 +86,28 @@ replace (
 	git.apache.org/thrift.git => github.com/apache/thrift v0.0.0-20180902110319-2566ecd5d999
 	github.com/docker/machine => github.com/machine-drivers/machine v0.7.1-0.20191109154235-b39d5b50de51
 	github.com/hashicorp/go-getter => github.com/afbjorklund/go-getter v1.4.1-0.20190910175809-eb9f6c26742c
-	k8s.io/api => k8s.io/kubernetes/staging/src/k8s.io/api v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/apiextensions-apiserver => k8s.io/kubernetes/staging/src/k8s.io/apiextensions-apiserver v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/apimachinery => k8s.io/kubernetes/staging/src/k8s.io/apimachinery v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/apiserver => k8s.io/kubernetes/staging/src/k8s.io/apiserver v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/cli-runtime => k8s.io/kubernetes/staging/src/k8s.io/cli-runtime v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/client-go => k8s.io/kubernetes/staging/src/k8s.io/client-go v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/cloud-provider => k8s.io/kubernetes/staging/src/k8s.io/cloud-provider v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/cluster-bootstrap => k8s.io/kubernetes/staging/src/k8s.io/cluster-bootstrap v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/code-generator => k8s.io/kubernetes/staging/src/k8s.io/code-generator v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/component-base => k8s.io/kubernetes/staging/src/k8s.io/component-base v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/cri-api => k8s.io/kubernetes/staging/src/k8s.io/cri-api v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/csi-translation-lib => k8s.io/kubernetes/staging/src/k8s.io/csi-translation-lib v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/kube-aggregator => k8s.io/kubernetes/staging/src/k8s.io/kube-aggregator v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/kube-controller-manager => k8s.io/kubernetes/staging/src/k8s.io/kube-controller-manager v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/kube-proxy => k8s.io/kubernetes/staging/src/k8s.io/kube-proxy v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/kube-scheduler => k8s.io/kubernetes/staging/src/k8s.io/kube-scheduler v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/kubectl => k8s.io/kubernetes/staging/src/k8s.io/kubectl v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/kubelet => k8s.io/kubernetes/staging/src/k8s.io/kubelet v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/legacy-cloud-providers => k8s.io/kubernetes/staging/src/k8s.io/legacy-cloud-providers v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/metrics => k8s.io/kubernetes/staging/src/k8s.io/metrics v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/node-api => k8s.io/kubernetes/staging/src/k8s.io/node-api v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/sample-apiserver => k8s.io/kubernetes/staging/src/k8s.io/sample-apiserver v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/sample-cli-plugin => k8s.io/kubernetes/staging/src/k8s.io/sample-cli-plugin v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/sample-controller => k8s.io/kubernetes/staging/src/k8s.io/sample-controller v0.0.0-20190623232353-8c3b7d7679cc
+	k8s.io/api => k8s.io/api v0.17.2
+	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.17.2
+	k8s.io/apimachinery => k8s.io/apimachinery v0.17.2
+	k8s.io/apiserver => k8s.io/apiserver v0.17.2
+	k8s.io/cli-runtime => k8s.io/cli-runtime v0.17.2
+	k8s.io/client-go => k8s.io/client-go v0.17.2
+	k8s.io/cloud-provider => k8s.io/cloud-provider v0.17.2
+	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.17.2
+	k8s.io/code-generator => k8s.io/code-generator v0.17.2
+	k8s.io/component-base => k8s.io/component-base v0.17.2
+	k8s.io/cri-api => k8s.io/cri-api v0.17.2
+	k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.17.2
+	k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.17.2
+	k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.17.2
+	k8s.io/kube-proxy => k8s.io/kube-proxy v0.17.2
+	k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.17.2
+	k8s.io/kubectl => k8s.io/kubectl v0.17.2
+	k8s.io/kubelet => k8s.io/kubelet v0.17.2
+	k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.17.2
+	k8s.io/metrics => k8s.io/metrics v0.17.2
+	k8s.io/node-api => k8s.io/node-api v0.17.2
+	k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.17.2
+	k8s.io/sample-cli-plugin => k8s.io/sample-cli-plugin v0.17.2
+	k8s.io/sample-controller => k8s.io/sample-controller v0.17.2
 )
--- a/go.sum
+++ b/go.sum
--- a/hack/images/kicbase.Dockerfile
+++ b/hack/images/kicbase.Dockerfile
@ -0,0 +1,36 @@
+ARG COMMIT_SHA
+# for now using node image created by kind https://github.com/kubernetes-sigs/kind
+# could be changed to slim ubuntu with systemd. 
+FROM kindest/node:v1.16.2
+USER root
+RUN apt-get update && apt-get install -y \
+  sudo \
+  dnsutils \
+  openssh-server \
+  && apt-get clean -y 
+# based on https://github.com/rastasheep/ubuntu-sshd/blob/master/18.04/Dockerfile
+# making SSH work for docker container 
+RUN mkdir /var/run/sshd
+RUN echo 'root:root' |chpasswd
+RUN sed -ri 's/^#?PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config
+RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config
+EXPOSE 22
+# Deleting all "kind" related stuff from the image.
+RUN rm -rf \
+    /var/cache/debconf/* \
+    /var/lib/apt/lists/* \
+    /var/log/* \
+    /tmp/* \
+    /var/tmp/* \
+    /usr/share/doc/* \
+    /usr/share/man/* \
+    /usr/share/local/* \
+    /kind/bin/kubeadm /kind/bin/kubelet /kind/systemd /kind/images /kind/manifests
+RUN echo "kic! Build: ${COMMIT_SHA} Time :$(date)" > "/kic.txt"
+# for minikube ssh. to match VM using docker username
+RUN adduser --disabled-password --gecos '' docker
+RUN adduser docker sudo
+RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+USER docker
+RUN mkdir /home/docker/.ssh
+USER root
--- a/hack/jenkins/common.sh
+++ b/hack/jenkins/common.sh
@ -319,7 +319,9 @@ touch "${JSON_OUT}"
 echo ">> Running go test2json"
 go tool test2json -t < "${TEST_OUT}" > "${JSON_OUT}" || true
 echo ">> Installing gopogh"
+cd /tmp
 GO111MODULE="on" go get -u github.com/medyagh/gopogh@v0.0.17 || true
+cd -
 echo ">> Running gopogh"
 if test -f "${HTML_OUT}"; then
    rm "${HTML_OUT}" || true # clean up previous runs of same build
--- a/hack/jenkins/linux_integration_tests_docker.sh
+++ b/hack/jenkins/linux_integration_tests_docker.sh
@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Copyright 2019 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# This script runs the integration tests on a Linux machine for the KVM Driver
+
+# The script expects the following env variables:
+# MINIKUBE_LOCATION: GIT_COMMIT from upstream build.
+# COMMIT: Actual commit ID from upstream build
+# EXTRA_BUILD_ARGS (optional): Extra args to be passed into the minikube integrations tests
+# access_token: The Github API access token. Injected by the Jenkins credential provider. 
+
+set -e
+
+OS_ARCH="linux-amd64"
+VM_DRIVER="docker"
+JOB_NAME="Docker_Linux"
+
+mkdir -p cron && gsutil -qm rsync "gs://minikube-builds/${MINIKUBE_LOCATION}/cron" cron || echo "FAILED TO GET CRON FILES"
+sudo install cron/cleanup_and_reboot_Linux.sh /etc/cron.hourly/cleanup_and_reboot || echo "FAILED TO INSTALL CLEANUP"
+
+source ./common.sh
--- a/hack/jenkins/minikube_set_pending.sh
+++ b/hack/jenkins/minikube_set_pending.sh
@ -40,6 +40,7 @@ jobs=(
     # 'KVM-GPU_Linux' - Disabled
     'KVM_Linux'
     'none_Linux'
+     'Docker_Linux'
 )

 # retry_github_status provides reliable github status updates
--- a/hack/jenkins/release_build_and_upload.sh
+++ b/hack/jenkins/release_build_and_upload.sh
@ -43,10 +43,10 @@ env GOPATH=$HOME/go BUILD_IN_DOCKER=y \
  make -j 16 \
  all \
  out/minikube-installer.exe \
-  "out/minikube_${DEB_VERSION}.deb" \
-  "out/minikube-${RPM_VERSION}.rpm" \
-  "out/docker-machine-driver-kvm2_${DEB_VERSION}.deb" \
-  "out/docker-machine-driver-kvm2-${RPM_VERSION}.rpm"
+  "out/minikube_${DEB_VERSION}-0_amd64.deb" \
+  "out/minikube-${RPM_VERSION}-0.x86_64.rpm" \
+  "out/docker-machine-driver-kvm2_${DEB_VERSION}-0_amd64.deb" \
+  "out/docker-machine-driver-kvm2-${RPM_VERSION}-0.x86_64.rpm"

 make checksum

@ -55,7 +55,11 @@ gsutil -m cp out/* "gs://$BUCKET/releases/$TAGNAME/"
 # Update "latest" release for non-beta/non-alpha builds
 if ! [[ ${VERSION_BUILD} =~ ^[0-9]+$ ]]; then
  echo "NOTE: ${VERSION} appears to be a non-standard release, not updating /releases/latest"
-else
-  echo "Updating latest bucket for ${VERSION} release"
-  gsutil cp -r "gs://${BUCKET}/releases/${TAGNAME}/*" "gs://${BUCKET}/releases/latest/"
-fi
+  exit 0
+fi
+
+echo "Updating Docker images ..."
+make push-gvisor-addon-image push-storage-provisioner-image
+
+echo "Updating latest bucket for ${VERSION} release ..."
+gsutil cp -r "gs://${BUCKET}/releases/${TAGNAME}/*" "gs://${BUCKET}/releases/latest/"
--- a/hack/jenkins/release_github_page.sh
+++ b/hack/jenkins/release_github_page.sh
@ -61,14 +61,14 @@ See [Getting Started](https://minikube.sigs.k8s.io/docs/start/)

 # ================================================================================
 # Deleting release from github before creating new one
-github-release delete \
+github-release -v delete \
  --user "${GITHUB_ORGANIZATION}" \
  --repo "${GITHUB_REPO}" \
  --tag "${TAGNAME}" \
  || true

 # Creating a new release in github
-github-release release ${RELEASE_FLAGS} \
+github-release -v release ${RELEASE_FLAGS} \
    --user "${GITHUB_ORGANIZATION}" \
    --repo "${GITHUB_REPO}" \
    --tag "${TAGNAME}" \
@ -84,8 +84,8 @@ FILES_TO_UPLOAD=(
    'minikube-windows-amd64.exe'
    'minikube-windows-amd64.exe.sha256'
    'minikube-installer.exe'
-    "minikube_${DEB_VERSION}.deb"
-    "minikube-${RPM_VERSION}.rpm"
+    "minikube_${DEB_VERSION}-0_amd64.deb"
+    "minikube-${RPM_VERSION}-0.x86_64.rpm"
    'docker-machine-driver-kvm2'
    'docker-machine-driver-kvm2.sha256'
    'docker-machine-driver-hyperkit'
@ -106,7 +106,7 @@ do
    n=0
    until [ $n -ge 5 ]
    do
-        github-release upload \
+        github-release -v upload \
          --user "${GITHUB_ORGANIZATION}" \
          --repo "${GITHUB_REPO}" \
          --tag "${TAGNAME}" \
--- a/pkg/addons/addons.go
+++ b/pkg/addons/addons.go
@ -19,8 +19,10 @@ package addons
 import (
 	"fmt"
 	"os"
+	"path/filepath"
 	"strconv"

+	"github.com/golang/glog"
 	"github.com/pkg/errors"
 	"github.com/spf13/viper"
 	"k8s.io/minikube/pkg/minikube/assets"
@ -37,6 +39,7 @@ import (
 // defaultStorageClassProvisioner is the name of the default storage class provisioner
 const defaultStorageClassProvisioner = "standard"

+// Set sets a value
 func Set(name, value, profile string) error {
 	a, valid := isAddonValid(name)
 	if !valid {
@ -153,7 +156,7 @@ func enableOrDisableAddon(name, val, profile string) error {
 	}

 	data := assets.GenerateTemplateData(cfg.KubernetesConfig)
-	return enableOrDisableAddonInternal(addon, cmd, data, enable)
+	return enableOrDisableAddonInternal(addon, cmd, data, enable, profile)
 }

 func isAddonAlreadySet(addon *assets.Addon, enable bool) (bool, error) {
@ -172,16 +175,11 @@ func isAddonAlreadySet(addon *assets.Addon, enable bool) (bool, error) {
 	return false, nil
 }

-func enableOrDisableAddonInternal(addon *assets.Addon, cmd command.Runner, data interface{}, enable bool) error {
-	var err error
-
-	updateFile := cmd.Copy
-	if !enable {
-		updateFile = cmd.Remove
-	}
-
+func enableOrDisableAddonInternal(addon *assets.Addon, cmd command.Runner, data interface{}, enable bool, profile string) error {
+	files := []string{}
 	for _, addon := range addon.Assets {
 		var addonFile assets.CopyableFile
+		var err error
 		if addon.IsTemplate() {
 			addonFile, err = addon.Evaluate(data)
 			if err != nil {
@ -191,9 +189,25 @@ func enableOrDisableAddonInternal(addon *assets.Addon, cmd command.Runner, data
 		} else {
 			addonFile = addon
 		}
-		if err := updateFile(addonFile); err != nil {
-			return errors.Wrapf(err, "updating addon %s", addon.AssetName)
+		if enable {
+			if err := cmd.Copy(addonFile); err != nil {
+				return err
+			}
+		} else {
+			defer func() {
+				if err := cmd.Remove(addonFile); err != nil {
+					glog.Warningf("error removing %s; addon should still be disabled as expected", addonFile)
+				}
+			}()
 		}
+		files = append(files, filepath.Join(addonFile.GetTargetDir(), addonFile.GetTargetName()))
+	}
+	command, err := kubectlCommand(profile, files, enable)
+	if err != nil {
+		return err
+	}
+	if result, err := cmd.RunCmd(command); err != nil {
+		return errors.Wrapf(err, "error updating addon:\n%s", result.Output())
 	}
 	return nil
 }
--- a/pkg/addons/config.go
+++ b/pkg/addons/config.go
@ -30,11 +30,6 @@ type Addon struct {

 // Addons is a list of all addons
 var Addons = []*Addon{
-	{
-		name:      "addon-manager",
-		set:       SetBool,
-		callbacks: []setFn{enableOrDisableAddon},
-	},
 	{
 		name:      "dashboard",
 		set:       SetBool,
--- a/pkg/addons/kubectl.go
+++ b/pkg/addons/kubectl.go
@ -0,0 +1,68 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package addons
+
+import (
+	"os"
+	"os/exec"
+	"path"
+
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/constants"
+)
+
+var (
+	// For testing
+	k8sVersion = kubernetesVersion
+)
+
+func kubectlCommand(profile string, files []string, enable bool) (*exec.Cmd, error) {
+	v, err := k8sVersion(profile)
+	if err != nil {
+		return nil, err
+	}
+	kubectlBinary := kubectlBinaryPath(v)
+
+	kubectlAction := "apply"
+	if !enable {
+		kubectlAction = "delete"
+	}
+
+	args := []string{"KUBECONFIG=/var/lib/minikube/kubeconfig", kubectlBinary, kubectlAction}
+	for _, f := range files {
+		args = append(args, []string{"-f", f}...)
+	}
+
+	cmd := exec.Command("sudo", args...)
+	return cmd, nil
+}
+
+func kubernetesVersion(profile string) (string, error) {
+	cc, err := config.Load(profile)
+	if err != nil && !os.IsNotExist(err) {
+		return "", err
+	}
+	version := constants.DefaultKubernetesVersion
+	if cc != nil {
+		version = cc.KubernetesConfig.KubernetesVersion
+	}
+	return version, nil
+}
+
+func kubectlBinaryPath(version string) string {
+	return path.Join("/var/lib/minikube/binaries", version, "kubectl")
+}
--- a/pkg/addons/kubectl_test.go
+++ b/pkg/addons/kubectl_test.go
@ -0,0 +1,63 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package addons
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestKubectlCommand(t *testing.T) {
+	tests := []struct {
+		description string
+		files       []string
+		enable      bool
+		expected    string
+	}{
+		{
+			description: "enable an addon",
+			files:       []string{"a", "b"},
+			enable:      true,
+			expected:    "sudo KUBECONFIG=/var/lib/minikube/kubeconfig /var/lib/minikube/binaries/v1.17.0/kubectl apply -f a -f b",
+		}, {
+			description: "disable an addon",
+			files:       []string{"a", "b"},
+			enable:      false,
+			expected:    "sudo KUBECONFIG=/var/lib/minikube/kubeconfig /var/lib/minikube/binaries/v1.17.0/kubectl delete -f a -f b",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.description, func(t *testing.T) {
+			originalK8sVersion := k8sVersion
+			defer func() { k8sVersion = originalK8sVersion }()
+			k8sVersion = func(_ string) (string, error) {
+				return "v1.17.0", nil
+			}
+
+			command, err := kubectlCommand("", test.files, test.enable)
+			if err != nil {
+				t.Fatalf("error getting kubectl command: %v", err)
+			}
+			actual := strings.Join(command.Args, " ")
+
+			if actual != test.expected {
+				t.Fatalf("expected does not match actual\nExpected: %s\nActual: %s", test.expected, actual)
+			}
+		})
+	}
+}
--- a/pkg/drivers/kic/kic.go
+++ b/pkg/drivers/kic/kic.go
@ -18,20 +18,37 @@ package kic

 import (
 	"fmt"
+	"net"
 	"os/exec"
 	"strconv"
 	"strings"

 	"github.com/docker/machine/libmachine/drivers"
+	"github.com/docker/machine/libmachine/ssh"
 	"github.com/docker/machine/libmachine/state"
+	"github.com/golang/glog"
 	"github.com/pkg/errors"
 	pkgdrivers "k8s.io/minikube/pkg/drivers"
 	"k8s.io/minikube/pkg/drivers/kic/node"
 	"k8s.io/minikube/pkg/drivers/kic/oci"
+	"k8s.io/minikube/pkg/minikube/assets"
 	"k8s.io/minikube/pkg/minikube/command"
+	"k8s.io/minikube/pkg/minikube/constants"
 )

-// https://minikube.sigs.k8s.io/docs/reference/drivers/kic/
+// DefaultPodCIDR is The CIDR to be used for pods inside the node.
+const DefaultPodCIDR = "10.244.0.0/16"
+
+// DefaultBindIPV4 is The default IP the container will bind to.
+const DefaultBindIPV4 = "127.0.0.1"
+
+// BaseImage is the base image is used to spin up kic containers created by kind.
+const BaseImage = "gcr.io/k8s-minikube/kicbase:v0.0.2@sha256:8f531b90901721a7bd4e67ceffbbc7ee6c4292b0e6d1a9d6eb59f117d57bc4e9"
+
+// OverlayImage is the cni plugin used for overlay image, created by kind.
+const OverlayImage = "kindest/kindnetd:0.5.3"
+
+// Driver represents a kic driver https://minikube.sigs.k8s.io/docs/reference/drivers/kic/
 type Driver struct {
 	*drivers.BaseDriver
 	*pkgdrivers.CommonDriver
@ -46,11 +63,11 @@ type Config struct {
 	MachineName   string            // maps to the container name being created
 	CPU           int               // Number of CPU cores assigned to the container
 	Memory        int               // max memory in MB
-	StorePath     string            // lib machine store path
+	StorePath     string            // libmachine store path
 	OCIBinary     string            // oci tool to use (docker, podman,...)
 	ImageDigest   string            // image name with sha to use for the node
-	APIServerPort int32             // port to connect to forward from container to user's machine
 	Mounts        []oci.Mount       // mounts
+	APIServerPort int               // kubernetes api server port inside the container
 	PortMappings  []oci.PortMapping // container port mappings
 	Envs          map[string]string // key,value of environment variables passed to the node
 }
@ -64,6 +81,7 @@ func NewDriver(c Config) *Driver {
 		},
 		exec:       command.NewKICRunner(c.MachineName, c.OCIBinary),
 		NodeConfig: c,
+		OCIBinary:  c.OCIBinary,
 	}
 	return d
 }
@ -71,27 +89,58 @@ func NewDriver(c Config) *Driver {
 // Create a host using the driver's config
 func (d *Driver) Create() error {
 	params := node.CreateConfig{
-		Name:         d.NodeConfig.MachineName,
-		Image:        d.NodeConfig.ImageDigest,
-		ClusterLabel: node.ClusterLabelKey + "=" + d.MachineName,
-		CPUs:         strconv.Itoa(d.NodeConfig.CPU),
-		Memory:       strconv.Itoa(d.NodeConfig.Memory) + "mb",
-		Envs:         d.NodeConfig.Envs,
-		ExtraArgs:    []string{"--expose", fmt.Sprintf("%d", d.NodeConfig.APIServerPort)},
-		OCIBinary:    d.NodeConfig.OCIBinary,
+		Name:          d.NodeConfig.MachineName,
+		Image:         d.NodeConfig.ImageDigest,
+		ClusterLabel:  node.ClusterLabelKey + "=" + d.MachineName,
+		CPUs:          strconv.Itoa(d.NodeConfig.CPU),
+		Memory:        strconv.Itoa(d.NodeConfig.Memory) + "mb",
+		Envs:          d.NodeConfig.Envs,
+		ExtraArgs:     []string{"--expose", fmt.Sprintf("%d", d.NodeConfig.APIServerPort)},
+		OCIBinary:     d.NodeConfig.OCIBinary,
+		APIServerPort: d.NodeConfig.APIServerPort,
 	}

 	// control plane specific options
 	params.PortMappings = append(params.PortMappings, oci.PortMapping{
-		ListenAddress: "127.0.0.1",
-		HostPort:      d.NodeConfig.APIServerPort,
-		ContainerPort: 6443,
-	})
-
+		ListenAddress: DefaultBindIPV4,
+		ContainerPort: constants.APIServerPort,
+	},
+		oci.PortMapping{
+			ListenAddress: DefaultBindIPV4,
+			ContainerPort: constants.SSHPort,
+		},
+	)
 	_, err := node.CreateNode(params)
 	if err != nil {
 		return errors.Wrap(err, "create kic node")
 	}
+
+	if err := d.prepareSSH(); err != nil {
+		return errors.Wrap(err, "prepare kic ssh")
+	}
+	return nil
+}
+
+// prepareSSH will generate keys and copy to the container so minikube ssh works
+func (d *Driver) prepareSSH() error {
+	keyPath := d.GetSSHKeyPath()
+	glog.Infof("Creating ssh key for kic: %s...", keyPath)
+	if err := ssh.GenerateSSHKey(keyPath); err != nil {
+		return errors.Wrap(err, "generate ssh key")
+	}
+
+	cmder := command.NewKICRunner(d.NodeConfig.MachineName, d.NodeConfig.OCIBinary)
+	f, err := assets.NewFileAsset(d.GetSSHKeyPath()+".pub", "/home/docker/.ssh/", "authorized_keys", "0644")
+	if err != nil {
+		return errors.Wrap(err, "create pubkey assetfile ")
+	}
+	if err := cmder.Copy(f); err != nil {
+		return errors.Wrap(err, "copying pub key")
+	}
+	if rr, err := cmder.RunCmd(exec.Command("chown", "docker:docker", "/home/docker/.ssh/authorized_keys")); err != nil {
+		return errors.Wrapf(err, "apply authorized_keys file ownership, output %s", rr.Output())
+	}
+
 	return nil
 }

@ -115,17 +164,39 @@ func (d *Driver) GetIP() (string, error) {

 // GetSSHHostname returns hostname for use with ssh
 func (d *Driver) GetSSHHostname() (string, error) {
-	return "", fmt.Errorf("driver does not have SSHHostName")
+	return DefaultBindIPV4, nil
 }

 // GetSSHPort returns port for use with ssh
 func (d *Driver) GetSSHPort() (int, error) {
-	return 0, fmt.Errorf("driver does not support GetSSHPort")
+	p, err := oci.HostPortBinding(d.OCIBinary, d.MachineName, constants.SSHPort)
+	if err != nil {
+		return p, errors.Wrap(err, "get ssh host-port")
+	}
+	return p, nil
+}
+
+// GetSSHUsername returns the ssh username
+func (d *Driver) GetSSHUsername() string {
+	return "docker"
+}
+
+// GetSSHKeyPath returns the ssh key path
+func (d *Driver) GetSSHKeyPath() string {
+	if d.SSHKeyPath == "" {
+		d.SSHKeyPath = d.ResolveStorePath("id_rsa")
+	}
+	return d.SSHKeyPath
 }

 // GetURL returns ip of the container running kic control-panel
 func (d *Driver) GetURL() (string, error) {
-	return d.GetIP()
+	p, err := oci.HostPortBinding(d.NodeConfig.OCIBinary, d.MachineName, d.NodeConfig.APIServerPort)
+	url := fmt.Sprintf("https://%s", net.JoinHostPort("127.0.0.1", fmt.Sprint(p)))
+	if err != nil {
+		return url, errors.Wrap(err, "api host port binding")
+	}
+	return url, nil
 }

 // GetState returns the state that the host is in (running, stopped, etc)
@ -136,23 +207,20 @@ func (d *Driver) GetState() (state.State, error) {
 	if err != nil {
 		return state.Error, errors.Wrapf(err, "error stop node %s", d.MachineName)
 	}
-	if o == "running" {
+	switch o {
+	case "running":
 		return state.Running, nil
-	}
-	if o == "exited" {
+	case "exited":
 		return state.Stopped, nil
-	}
-	if o == "paused" {
+	case "paused":
 		return state.Paused, nil
-	}
-	if o == "restarting" {
+	case "restarting":
 		return state.Starting, nil
-	}
-	if o == "dead" {
+	case "dead":
 		return state.Error, nil
+	default:
+		return state.None, fmt.Errorf("unknown state")
 	}
-	return state.None, fmt.Errorf("unknown state")
-
 }

 // Kill stops a host forcefully, including any containers that we are managing.
@ -250,32 +318,3 @@ func (d *Driver) nodeID(nameOrID string) (string, error) {
 	}
 	return string(id), err
 }
-
-func ImageForVersion(ver string) (string, error) {
-	switch ver {
-	case "v1.11.10":
-		return "medyagh/kic:v1.11.10@sha256:23bb7f5e8dd2232ec829132172e87f7b9d8de65269630989e7dac1e0fe993b74", nil
-	case "v1.12.8":
-		return "medyagh/kic:v1.12.8@sha256:c74bc5f3efe3539f6e1ad7f11bf7c09f3091c0547cb28071f4e43067053e5898", nil
-	case "v1.12.9":
-		return "medyagh/kic:v1.12.9@sha256:ff82f58e18dcb22174e8eb09dae14f7edd82d91a83c7ef19e33298d0eba6a0e3", nil
-	case "v1.12.10":
-		return "medyagh/kic:v1.12.10@sha256:2d174bae7c20698e59791e7cca9b6db234053d1a92a009d5bb124e482540c70b", nil
-	case "v1.13.6":
-		return "medyagh/kic:v1.13.6@sha256:cf63e50f824fe17b90374d38d64c5964eb9fe6b3692669e1201fcf4b29af4964", nil
-	case "v1.13.7":
-		return "medyagh/kic:v1.13.7@sha256:1a6a5e1c7534cf3012655e99df680496df9bcf0791a304adb00617d5061233fa", nil
-	case "v1.14.3":
-		return "medyagh/kic:v1.14.3@sha256:cebec21f6af23d5dfa3465b88ddf4a1acb94c2c20a0a6ff8cc1c027b0a4e2cec", nil
-	case "v1.15.0":
-		return "medyagh/kic:v1.15.0@sha256:40d433d00a2837c8be829bd3cb0576988e377472062490bce0b18281c7f85303", nil
-	case "v1.15.3":
-		return "medyagh/kic:v1.15.3@sha256:f05ce52776a86c6ead806942d424de7076af3f115b0999332981a446329e6cf1", nil
-	case "v1.16.1":
-		return "medyagh/kic:v1.16.1@sha256:e74530d22e6a04442a97a09bdbba885ad693fcc813a0d1244da32666410d1ad1", nil
-	case "v1.16.2":
-		return "medyagh/kic:v1.16.2@sha256:3374a30971bf5b0011441a227fa56ef990b76125b36ca0ab8316a3c7e4f137a3", nil
-	default:
-		return "medyagh/kic:v1.16.2@sha256:3374a30971bf5b0011441a227fa56ef990b76125b36ca0ab8316a3c7e4f137a3", nil
-	}
-}
--- a/pkg/drivers/kic/node/node.go
+++ b/pkg/drivers/kic/node/node.go
@ -39,24 +39,25 @@ const (
 // Node represents a handle to a kic node
 // This struct must be created by one of: CreateControlPlane
 type Node struct {
-	// must be one of docker container ID or name
-	name      string
+	id        string         // container id
+	name      string         // container name
 	r         command.Runner // Runner
 	ociBinary string
 }

 type CreateConfig struct {
-	Name         string            // used for container name and hostname
-	Image        string            // container image to use to create the node.
-	ClusterLabel string            // label the containers we create using minikube so we can clean up
-	Role         string            // currently only role supported is control-plane
-	Mounts       []oci.Mount       // volume mounts
-	PortMappings []oci.PortMapping // ports to map to container from host
-	CPUs         string            // number of cpu cores assign to container
-	Memory       string            // memory (mbs) to assign to the container
-	Envs         map[string]string // environment variables to pass to the container
-	ExtraArgs    []string          // a list of any extra option to pass to oci binary during creation time, for example --expose 8080...
-	OCIBinary    string            // docker or podman
+	Name          string            // used for container name and hostname
+	Image         string            // container image to use to create the node.
+	ClusterLabel  string            // label the containers we create using minikube so we can clean up
+	Role          string            // currently only role supported is control-plane
+	Mounts        []oci.Mount       // volume mounts
+	APIServerPort int               // kubernetes api server port
+	PortMappings  []oci.PortMapping // ports to map to container from host
+	CPUs          string            // number of cpu cores assign to container
+	Memory        string            // memory (mbs) to assign to the container
+	Envs          map[string]string // environment variables to pass to the container
+	ExtraArgs     []string          // a list of any extra option to pass to oci binary during creation time, for example --expose 8080...
+	OCIBinary     string            // docker or podman
 }

 // CreateNode creates a new container node
@ -123,13 +124,15 @@ func CreateNode(p CreateConfig) (*Node, error) {

 // Find finds a node
 func Find(ociBinary string, name string, cmder command.Runner) (*Node, error) {
-	_, err := oci.Inspect(ociBinary, name, "{{.Id}}")
+	n, err := oci.Inspect(ociBinary, name, "{{.Id}}")
 	if err != nil {
 		return nil, fmt.Errorf("can't find node %v", err)
 	}
 	return &Node{
-		name: name,
-		r:    cmder,
+		ociBinary: ociBinary,
+		id:        n[0],
+		name:      name,
+		r:         cmder,
 	}, nil
 }

--- a/pkg/drivers/kic/oci/oci.go
+++ b/pkg/drivers/kic/oci/oci.go
@ -18,6 +18,7 @@ package oci

 import (
 	"os"
+	"strconv"

 	"github.com/docker/machine/libmachine/state"
 	"k8s.io/minikube/pkg/minikube/assets"
@ -28,7 +29,6 @@ import (
 	"github.com/pkg/errors"

 	"fmt"
-	"net"
 	"os/exec"
 	"strings"
 	"time"
@ -53,17 +53,14 @@ func Status(ociBinary string, ociID string) (state.State, error) {
 	out, err := cmd.CombinedOutput()
 	o := strings.Trim(string(out), "\n")
 	s := state.Error
-	if o == "running" {
+	switch o {
+	case "running":
 		s = state.Running
-	}
-	if o == "exited" {
+	case "exited":
 		s = state.Stopped
-	}
-
-	if o == "paused" {
+	case "paused":
 		s = state.Paused
-	}
-	if o == "restaring" {
+	case "restaring":
 		s = state.Starting
 	}

@ -111,7 +108,6 @@ func Pause(ociBinary string, ociID string) error {

 // Inspect return low-level information on containers
 func Inspect(ociBinary string, containerNameOrID, format string) ([]string, error) {
-
 	cmd := exec.Command(ociBinary, "inspect",
 		"-f", format,
 		containerNameOrID) // ... against the "node" container
@ -283,13 +279,9 @@ func UsernsRemap(ociBinary string) bool {
 func generatePortMappings(portMappings ...PortMapping) []string {
 	result := make([]string, 0, len(portMappings))
 	for _, pm := range portMappings {
-		var hostPortBinding string
-		if pm.ListenAddress != "" {
-			hostPortBinding = net.JoinHostPort(pm.ListenAddress, fmt.Sprintf("%d", pm.HostPort))
-		} else {
-			hostPortBinding = fmt.Sprintf("%d", pm.HostPort)
-		}
-		publish := fmt.Sprintf("--publish=%s:%d", hostPortBinding, pm.ContainerPort)
+		// let docker pick a host port by leaving it as ::
+		// example --publish=127.0.0.17::8443 will get a random host port for 8443
+		publish := fmt.Sprintf("--publish=%s::%d", pm.ListenAddress, pm.ContainerPort)
 		result = append(result, publish)
 	}
 	return result
@ -353,16 +345,9 @@ func CreateContainer(ociBinary string, image string, opts ...CreateOpt) ([]strin
 	for scanner.Scan() {
 		output = append(output, scanner.Text())
 	}
-	// TODO : check for exist status 125 that means it alread exists, we can re-start it
-	// example error:
-	// $ docker run --cpus=2 --memory=2000m -d -t --privileged --security-opt seccomp=unconfined --tmpfs /tmp --tmpfs /run -v /lib/modules:/lib/modules:ro --hostname p1control-plane --name p1control-plane --label io.k8s.sigs.kic.clusterp1 --label io.k8s.sigs.kic.role=control-plane --expose 50182 --publish=127.0.0.1:50182:6443 medyagh/kic:v1.15.0@sha256:1f03b3168ffe8ab43ce170a5729e31b0d53fb3a1af88e1ad1bdf4626fad8a91c
-	//		 docker: Error response from daemon: Conflict. The container name "/p1control-plane" is already in use by container "0204dcf3ca51c874b6c7dac989beae9d98dd44af53e0a17312f4d3480c1f6191". You have to remove (or rename) that container to be able to reuse that name.
-	// 		 See 'docker run --help'.
-	// $ echo $?
-	// 125

 	if err != nil {
-		return output, errors.Wrapf(err, "CreateContainer %v ", args)
+		return output, errors.Wrapf(err, "args: %v  output: %s ", args, output)
 	}
 	return output, nil
 }
@ -405,3 +390,23 @@ func Copy(ociBinary string, ociID string, asset assets.CopyableFile) error {
 	}
 	return nil
 }
+
+// HostPortBinding will return port mapping for a container using cli.
+// example : HostPortBinding("docker", "minikube", "22")
+// will return the docker assigned port:
+// 32769, nil
+// only supports TCP ports
+func HostPortBinding(ociBinary string, ociID string, contPort int) (int, error) {
+	cmd := exec.Command(ociBinary, "inspect", "-f", fmt.Sprintf("'{{(index (index .NetworkSettings.Ports \"%d/tcp\") 0).HostPort}}'", contPort), ociID)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return 0, errors.Wrapf(err, "getting host-bind port %d for container ID %q, output %s", contPort, ociID, out)
+	}
+	o := strings.Trim(string(out), "\n")
+	o = strings.Trim(o, "'")
+	p, err := strconv.Atoi(o)
+	if err != nil {
+		return p, errors.Wrapf(err, "convert host-port %q to number", p)
+	}
+	return p, nil
+}
--- a/pkg/drivers/kic/oci/types.go
+++ b/pkg/drivers/kic/oci/types.go
@ -16,6 +16,11 @@ limitations under the License.

 package oci

+const (
+	Docker = "docker"
+	Podman = "podman"
+)
+
 /*
 These types are from
 https://github.com/kubernetes/kubernetes/blob/063e7ff358fdc8b0916e6f39beedc0d025734cb1/pkg/kubelet/apis/cri/runtime/v1alpha2/api.pb.go#L183
--- a/pkg/drivers/none/none.go
+++ b/pkg/drivers/none/none.go
@ -94,7 +94,7 @@ func (d *Driver) DriverName() string {

 // GetIP returns an IP or hostname that this host is available at
 func (d *Driver) GetIP() (string, error) {
-	ip, err := net.ChooseBindAddress(nil)
+	ip, err := net.ChooseHostInterface()
 	if err != nil {
 		return "", err
 	}
@ -137,7 +137,7 @@ func (d *Driver) Kill() error {
 	}

 	// First try to gracefully stop containers
-	containers, err := d.runtime.ListContainers("")
+	containers, err := d.runtime.ListContainers(cruntime.ListOptions{})
 	if err != nil {
 		return errors.Wrap(err, "containers")
 	}
@ -149,7 +149,7 @@ func (d *Driver) Kill() error {
 		return errors.Wrap(err, "stop")
 	}

-	containers, err = d.runtime.ListContainers("")
+	containers, err = d.runtime.ListContainers(cruntime.ListOptions{})
 	if err != nil {
 		return errors.Wrap(err, "containers")
 	}
@ -199,7 +199,7 @@ func (d *Driver) Stop() error {
 	if err := stopKubelet(d.exec); err != nil {
 		return err
 	}
-	containers, err := d.runtime.ListContainers("")
+	containers, err := d.runtime.ListContainers(cruntime.ListOptions{})
 	if err != nil {
 		return errors.Wrap(err, "containers")
 	}
--- a/pkg/minikube/assets/addons.go
+++ b/pkg/minikube/assets/addons.go
@ -68,20 +68,13 @@ func (a *Addon) IsEnabled() (bool, error) {
 // Addons is the list of addons
 // TODO: Make dynamically loadable: move this data to a .yaml file within each addon directory
 var Addons = map[string]*Addon{
-	"addon-manager": NewAddon([]*BinAsset{
-		MustBinAsset(
-			"deploy/addons/addon-manager.yaml.tmpl",
-			vmpath.GuestManifestsDir,
-			"addon-manager.yaml.tmpl",
-			"0640",
-			true),
-	}, true, "addon-manager"),
 	"dashboard": NewAddon([]*BinAsset{
+		// We want to create the kubernetes-dashboard ns first so that every subsequent object can be created
+		MustBinAsset("deploy/addons/dashboard/dashboard-ns.yaml", vmpath.GuestAddonsDir, "dashboard-ns.yaml", "0640", false),
 		MustBinAsset("deploy/addons/dashboard/dashboard-clusterrole.yaml", vmpath.GuestAddonsDir, "dashboard-clusterrole.yaml", "0640", false),
 		MustBinAsset("deploy/addons/dashboard/dashboard-clusterrolebinding.yaml", vmpath.GuestAddonsDir, "dashboard-clusterrolebinding.yaml", "0640", false),
 		MustBinAsset("deploy/addons/dashboard/dashboard-configmap.yaml", vmpath.GuestAddonsDir, "dashboard-configmap.yaml", "0640", false),
 		MustBinAsset("deploy/addons/dashboard/dashboard-dp.yaml", vmpath.GuestAddonsDir, "dashboard-dp.yaml", "0640", false),
-		MustBinAsset("deploy/addons/dashboard/dashboard-ns.yaml", vmpath.GuestAddonsDir, "dashboard-ns.yaml", "0640", false),
 		MustBinAsset("deploy/addons/dashboard/dashboard-role.yaml", vmpath.GuestAddonsDir, "dashboard-role.yaml", "0640", false),
 		MustBinAsset("deploy/addons/dashboard/dashboard-rolebinding.yaml", vmpath.GuestAddonsDir, "dashboard-rolebinding.yaml", "0640", false),
 		MustBinAsset("deploy/addons/dashboard/dashboard-sa.yaml", vmpath.GuestAddonsDir, "dashboard-sa.yaml", "0640", false),
@ -195,7 +188,7 @@ var Addons = map[string]*Addon{
 			"istio-operator.yaml",
 			"0640",
 			true),
-	}, true, "istio-provisioner"),
+	}, false, "istio-provisioner"),
 	"istio": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/istio/istio-default-profile.yaml.tmpl",
--- a/pkg/minikube/assets/vm_assets.go
+++ b/pkg/minikube/assets/vm_assets.go
@ -38,6 +38,7 @@ type CopyableFile interface {
 	GetTargetName() string
 	GetPermissions() string
 	GetModTime() (time.Time, error)
+	Seek(int64, int) (int64, error)
 }

 // BaseAsset is the base asset class
@ -76,7 +77,7 @@ func (b *BaseAsset) GetModTime() (time.Time, error) {
 // FileAsset is an asset using a file
 type FileAsset struct {
 	BaseAsset
-	reader io.Reader
+	reader io.ReadSeeker
 }

 // NewMemoryAssetTarget creates a new MemoryAsset, with target
@ -91,6 +92,11 @@ func NewFileAsset(src, targetDir, targetName, permissions string) (*FileAsset, e
 	if err != nil {
 		return nil, errors.Wrapf(err, "Error opening file asset: %s", src)
 	}
+	info, err := os.Stat(src)
+	if err != nil {
+		return nil, errors.Wrapf(err, "Error getting info for %s", src)
+	}
+	r := io.NewSectionReader(f, 0, info.Size())
 	return &FileAsset{
 		BaseAsset: BaseAsset{
 			AssetName:   src,
@ -98,7 +104,7 @@ func NewFileAsset(src, targetDir, targetName, permissions string) (*FileAsset, e
 			TargetName:  targetName,
 			Permissions: permissions,
 		},
-		reader: f,
+		reader: r,
 	}, nil
 }

@ -117,6 +123,7 @@ func (f *FileAsset) GetModTime() (time.Time, error) {
 	return fi.ModTime(), err
 }

+// Read reads the asset
 func (f *FileAsset) Read(p []byte) (int, error) {
 	if f.reader == nil {
 		return 0, errors.New("Error attempting FileAsset.Read, FileAsset.reader uninitialized")
@ -124,10 +131,15 @@ func (f *FileAsset) Read(p []byte) (int, error) {
 	return f.reader.Read(p)
 }

+// Seek resets the reader to offset
+func (f *FileAsset) Seek(offset int64, whence int) (int64, error) {
+	return f.reader.Seek(offset, whence)
+}
+
 // MemoryAsset is a memory-based asset
 type MemoryAsset struct {
 	BaseAsset
-	reader io.Reader
+	reader io.ReadSeeker
 	length int
 }

@ -141,6 +153,11 @@ func (m *MemoryAsset) Read(p []byte) (int, error) {
 	return m.reader.Read(p)
 }

+// Seek resets the reader to offset
+func (m *MemoryAsset) Seek(offset int64, whence int) (int64, error) {
+	return m.reader.Seek(offset, whence)
+}
+
 // NewMemoryAsset creates a new MemoryAsset
 func NewMemoryAsset(d []byte, targetDir, targetName, permissions string) *MemoryAsset {
 	return &MemoryAsset{
@ -157,7 +174,7 @@ func NewMemoryAsset(d []byte, targetDir, targetName, permissions string) *Memory
 // BinAsset is a bindata (binary data) asset
 type BinAsset struct {
 	BaseAsset
-	reader   io.Reader
+	reader   io.ReadSeeker
 	template *template.Template
 	length   int
 }
@ -253,3 +270,8 @@ func (m *BinAsset) Read(p []byte) (int, error) {
 	}
 	return m.reader.Read(p)
 }
+
+// Seek resets the reader to offset
+func (m *BinAsset) Seek(offset int64, whence int) (int64, error) {
+	return m.reader.Seek(offset, whence)
+}
--- a/pkg/minikube/bootstrapper/bootstrapper.go
+++ b/pkg/minikube/bootstrapper/bootstrapper.go
@ -37,13 +37,13 @@ type LogOptions struct {
 type Bootstrapper interface {
 	// PullImages pulls images necessary for a cluster. Success should not be required.
 	PullImages(config.KubernetesConfig) error
-	StartCluster(config.KubernetesConfig) error
+	StartCluster(config.MachineConfig) error
 	UpdateCluster(config.MachineConfig) error
 	DeleteCluster(config.KubernetesConfig) error
-	WaitForCluster(config.KubernetesConfig, time.Duration) error
+	WaitForCluster(config.MachineConfig, time.Duration) error
 	// LogCommands returns a map of log type to a command which will display that log.
 	LogCommands(LogOptions) map[string]string
-	SetupCerts(cfg config.KubernetesConfig) error
+	SetupCerts(config.KubernetesConfig, config.Node) error
 	GetKubeletStatus() (string, error)
 	GetAPIServerStatus(net.IP, int) (string, error)
 }
@ -51,27 +51,14 @@ type Bootstrapper interface {
 const (
 	// Kubeadm is the kubeadm bootstrapper type
 	Kubeadm = "kubeadm"
-	KIC     = "kic"
 )

 // GetCachedBinaryList returns the list of binaries
 func GetCachedBinaryList(bootstrapper string) []string {
-	switch bootstrapper {
-	case Kubeadm:
-		return constants.KubeadmBinaries
-	default:
-		return []string{}
-	}
+	return constants.KubernetesReleaseBinaries
 }

 // GetCachedImageList returns the list of images for a version
 func GetCachedImageList(imageRepository string, version string, bootstrapper string) ([]string, error) {
-	switch bootstrapper {
-	case Kubeadm:
-		return images.Kubeadm(imageRepository, version)
-	case KIC:
-		return []string{"alpine"}, nil // for testing purpose just caching alpine for kicbs
-	default:
-		return []string{}, nil
-	}
+	return images.Kubeadm(imageRepository, version)
 }
--- a/pkg/minikube/bootstrapper/bsutil/binaries.go
+++ b/pkg/minikube/bootstrapper/bsutil/binaries.go
@ -14,10 +14,11 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */

-// bsutil package will eventually be renamed to kubeadm package after getting rid of older one
+// Package bsutil package will eventually be renamed to kubeadm package after getting rid of older one
 package bsutil

 import (
+	"os/exec"
 	"path"
 	"runtime"

@ -32,8 +33,14 @@ import (

 // TransferBinaries transfers all required Kubernetes binaries
 func TransferBinaries(cfg config.KubernetesConfig, c command.Runner) error {
+	dir := binRoot(cfg.KubernetesVersion)
+	_, err := c.RunCmd(exec.Command("sudo", "mkdir", "-p", dir))
+	if err != nil {
+		return err
+	}
+
 	var g errgroup.Group
-	for _, name := range constants.KubeadmBinaries {
+	for _, name := range constants.KubernetesReleaseBinaries {
 		name := name
 		g.Go(func() error {
 			src, err := machine.CacheBinary(name, cfg.KubernetesVersion, "linux", runtime.GOARCH)
@ -41,7 +48,7 @@ func TransferBinaries(cfg config.KubernetesConfig, c command.Runner) error {
 				return errors.Wrapf(err, "downloading %s", name)
 			}

-			dst := path.Join(binRoot(cfg.KubernetesVersion), name)
+			dst := path.Join(dir, name)
 			if err := machine.CopyBinary(c, src, dst); err != nil {
 				return errors.Wrapf(err, "copybinary %s -> %s", src, dst)
 			}
--- a/pkg/minikube/bootstrapper/bsutil/extraconfig.go
+++ b/pkg/minikube/bootstrapper/bsutil/extraconfig.go
@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */

-// bsutil package will eventually be renamed to kubeadm package after getting rid of older one
+// Package bsutil will eventually be renamed to kubeadm package after getting rid of older one
 package bsutil

 import (
--- a/pkg/minikube/bootstrapper/bsutil/featuregates.go
+++ b/pkg/minikube/bootstrapper/bsutil/featuregates.go
@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */

-// bsutil package will eventually be renamed to kubeadm package after getting rid of older one
+// Package bsutil will eventually be renamed to kubeadm package after getting rid of older one
 package bsutil

 import (
--- a/pkg/minikube/bootstrapper/bsutil/featuregates_test.go
+++ b/pkg/minikube/bootstrapper/bsutil/featuregates_test.go
@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */

-// bsutil package will eventually be renamed to kubeadm package after getting rid of older one
+// Package bsutil will eventually be renamed to kubeadm package after getting rid of older one
 package bsutil

 import (
--- a/pkg/minikube/bootstrapper/bsutil/files.go
+++ b/pkg/minikube/bootstrapper/bsutil/files.go
@ -14,12 +14,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */

-// bsutil package will eventually be renamed to kubeadm package after getting rid of older one
+// Package bsutil will eventually be renamed to kubeadm package after getting rid of older one
 package bsutil

 import (
 	"path"

+	"github.com/pkg/errors"
 	"k8s.io/minikube/pkg/minikube/assets"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/vmpath"
@ -42,7 +43,6 @@ func ConfigFileAssets(cfg config.KubernetesConfig, kubeadm []byte, kubelet []byt
 		assets.NewMemoryAssetTarget(kubeadm, KubeadmYamlPath, "0640"),
 		assets.NewMemoryAssetTarget(kubelet, KubeletSystemdConfFile, "0644"),
 		assets.NewMemoryAssetTarget(kubeletSvc, KubeletServiceFile, "0644"),
-		assets.NewMemoryAssetTarget(defaultCNIConfig, DefaultCNIConfigPath, "0644"),
 	}
 	// Copy the default CNI config (k8s.conf), so that kubelet can successfully
 	// start a Pod in the case a user hasn't manually installed any CNI plugin
@ -52,3 +52,33 @@ func ConfigFileAssets(cfg config.KubernetesConfig, kubeadm []byte, kubelet []byt
 	}
 	return fs
 }
+
+// AddAddons adds addons to list of files
+func AddAddons(files *[]assets.CopyableFile, data interface{}) error {
+	// add addons to file list
+	// custom addons
+	if err := assets.AddMinikubeDirAssets(files); err != nil {
+		return errors.Wrap(err, "adding minikube dir assets")
+	}
+	// bundled addons
+	for _, addonBundle := range assets.Addons {
+		if isEnabled, err := addonBundle.IsEnabled(); err == nil && isEnabled {
+			for _, addon := range addonBundle.Assets {
+				if addon.IsTemplate() {
+					addonFile, err := addon.Evaluate(data)
+					if err != nil {
+						return errors.Wrapf(err, "evaluate bundled addon %s asset", addon.GetAssetName())
+					}
+
+					*files = append(*files, addonFile)
+				} else {
+					*files = append(*files, addon)
+				}
+			}
+		} else if err != nil {
+			return nil
+		}
+	}
+
+	return nil
+}
--- a/pkg/minikube/bootstrapper/bsutil/template/kubelet.go
+++ b/pkg/minikube/bootstrapper/bsutil/template/kubelet.go
@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */

-package template
+package ktmpl

 import "text/template"

--- a/pkg/minikube/bootstrapper/bsutil/ktmpl/template.go
+++ b/pkg/minikube/bootstrapper/bsutil/ktmpl/template.go
@ -0,0 +1,41 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ktmpl
+
+import (
+	"fmt"
+	"sort"
+)
+
+// printMapInOrder sorts the keys and prints the map in order, combining key
+// value pairs with the separator character
+//
+// Note: this is not necessary, but makes testing easy
+func printMapInOrder(m map[string]string, sep string) []string {
+	if m == nil {
+		return nil
+	}
+	keys := []string{}
+	for k := range m {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for i, k := range keys {
+		keys[i] = fmt.Sprintf("%s%s\"%s\"", k, sep, m[k])
+	}
+	return keys
+}
--- a/pkg/minikube/bootstrapper/bsutil/template/template_test.go
+++ b/pkg/minikube/bootstrapper/bsutil/template/template_test.go
@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */

-package template
+package ktmpl

 import (
 	"reflect"
--- a/pkg/minikube/bootstrapper/bsutil/ktmpl/v1alpha1.go
+++ b/pkg/minikube/bootstrapper/bsutil/ktmpl/v1alpha1.go
@ -0,0 +1,45 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ktmpl
+
+import "text/template"
+
+// V1Alpha1 is for Kubernetes v1.11
+var V1Alpha1 = template.Must(template.New("configTmpl-v1alpha1").Funcs(template.FuncMap{
+	"printMapInOrder": printMapInOrder,
+}).Parse(`apiVersion: kubeadm.k8s.io/v1alpha1
+kind: MasterConfiguration
+{{if .NoTaintMaster}}noTaintMaster: true{{end}}
+api:
+  advertiseAddress: {{.AdvertiseAddress}}
+  bindPort: {{.APIServerPort}}
+  controlPlaneEndpoint: localhost
+kubernetesVersion: {{.KubernetesVersion}}
+certificatesDir: {{.CertDir}}
+networking:
+  serviceSubnet: {{.ServiceCIDR}}
+etcd:
+  dataDir: {{.EtcdDataDir}}
+nodeName: {{.NodeName}}
+apiServerCertSANs: ["127.0.0.1", "localhost", "{{.AdvertiseAddress}}"]
+{{if .ImageRepository}}imageRepository: {{.ImageRepository}}
+{{end}}{{if .CRISocket}}criSocket: {{.CRISocket}}
+{{end}}{{range .ExtraArgs}}{{.Component}}ExtraArgs:{{range $i, $val := printMapInOrder .Options ": " }}
+  {{$val}}{{end}}
+{{end}}{{if .FeatureArgs}}featureGates: {{range $i, $val := .FeatureArgs}}
+  {{$i}}: {{$val}}{{end}}
+{{end}}`))
--- a/pkg/minikube/bootstrapper/bsutil/ktmpl/v1alpha3.go
+++ b/pkg/minikube/bootstrapper/bsutil/ktmpl/v1alpha3.go
@ -0,0 +1,69 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ktmpl
+
+import "text/template"
+
+// V1Alpha3 is for Kubernetes v1.12
+var V1Alpha3 = template.Must(template.New("configTmpl-v1alpha3").Funcs(template.FuncMap{
+	"printMapInOrder": printMapInOrder,
+}).Parse(`apiVersion: kubeadm.k8s.io/v1alpha3
+kind: InitConfiguration
+apiEndpoint:
+  advertiseAddress: {{.AdvertiseAddress}}
+  bindPort: {{.APIServerPort}}
+bootstrapTokens:
+  - groups:
+      - system:bootstrappers:kubeadm:default-node-token
+    ttl: 24h0m0s
+    usages:
+      - signing
+      - authentication
+nodeRegistration:
+  criSocket: {{if .CRISocket}}{{.CRISocket}}{{else}}/var/run/dockershim.sock{{end}}
+  name: {{.NodeName}}
+  taints: []
+---
+apiVersion: kubeadm.k8s.io/v1alpha3
+kind: ClusterConfiguration
+{{if .ImageRepository}}imageRepository: {{.ImageRepository}}
+{{end}}{{range .ExtraArgs}}{{.Component}}ExtraArgs:{{range $i, $val := printMapInOrder .Options ": " }}
+  {{$val}}{{end}}
+{{end -}}
+{{if .FeatureArgs}}featureGates: {{range $i, $val := .FeatureArgs}}
+  {{$i}}: {{$val}}{{end}}
+{{end -}}
+certificatesDir: {{.CertDir}}
+clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "{{.AdvertiseAddress}}"]
+controlPlaneEndpoint: localhost:{{.APIServerPort}}
+etcd:
+  local:
+    dataDir: {{.EtcdDataDir}}
+kubernetesVersion: {{.KubernetesVersion}}
+networking:
+  dnsDomain: {{if .DNSDomain}}{{.DNSDomain}}{{else}}cluster.local{{end}}
+  podSubnet: "{{ .PodSubnet }}"
+  serviceSubnet: {{.ServiceCIDR}}
+---
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+evictionHard:
+  nodefs.available: "0%"
+  nodefs.inodesFree: "0%"
+  imagefs.available: "0%"
+`))
--- a/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta1.go
+++ b/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta1.go
@ -0,0 +1,82 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ktmpl
+
+import "text/template"
+
+// V1Beta1 is kubeadm config template for Kubernetes v1.13+
+var V1Beta1 = template.Must(template.New("configTmpl-v1beta1").Funcs(template.FuncMap{
+	"printMapInOrder": printMapInOrder,
+}).Parse(`apiVersion: kubeadm.k8s.io/v1beta1
+kind: InitConfiguration
+localAPIEndpoint:
+  advertiseAddress: {{.AdvertiseAddress}}
+  bindPort: {{.APIServerPort}}
+bootstrapTokens:
+  - groups:
+      - system:bootstrappers:kubeadm:default-node-token
+    ttl: 24h0m0s
+    usages:
+      - signing
+      - authentication
+nodeRegistration:
+  criSocket: {{if .CRISocket}}{{.CRISocket}}{{else}}/var/run/dockershim.sock{{end}}
+  name: {{.NodeName}}
+  taints: []
+---
+apiVersion: kubeadm.k8s.io/v1beta1
+kind: ClusterConfiguration
+{{ if .ImageRepository}}imageRepository: {{.ImageRepository}}
+{{end}}{{range .ExtraArgs}}{{.Component}}:
+  extraArgs:
+{{- range $i, $val := printMapInOrder .Options ": " }}
+    {{$val}}
+{{- end}}
+{{end -}}
+{{if .FeatureArgs}}featureGates:
+{{range $i, $val := .FeatureArgs}}{{$i}}: {{$val}}
+{{end -}}{{end -}}
+certificatesDir: {{.CertDir}}
+clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "{{.AdvertiseAddress}}"]
+controlPlaneEndpoint: localhost:{{.APIServerPort}}
+dns:
+  type: CoreDNS
+etcd:
+  local:
+    dataDir: {{.EtcdDataDir}}
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://{{.AdvertiseAddress}}:2381
+kubernetesVersion: {{.KubernetesVersion}}
+networking:
+  dnsDomain: {{if .DNSDomain}}{{.DNSDomain}}{{else}}cluster.local{{end}}
+  podSubnet: "{{.PodSubnet }}"
+  serviceSubnet: {{.ServiceCIDR}}
+---
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+imageGCHighThresholdPercent: 100
+evictionHard:
+  nodefs.available: "0%"
+  nodefs.inodesFree: "0%"
+  imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: {{.AdvertiseAddress}}:10249
+`))
--- a/pkg/minikube/bootstrapper/bsutil/kubeadm.go
+++ b/pkg/minikube/bootstrapper/bsutil/kubeadm.go
@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */

-// bsutil package will eventually be renamed to kubeadm package after getting rid of older one
+// Package bsutil will eventually be renamed to kubeadm package after getting rid of older one
 package bsutil

 import (
@ -24,19 +24,19 @@ import (

 	"github.com/blang/semver"
 	"github.com/pkg/errors"
-	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/template"
+	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/ktmpl"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/cruntime"
 	"k8s.io/minikube/pkg/minikube/vmpath"
-	"k8s.io/minikube/pkg/util"
 )

 // Container runtimes
 const remoteContainerRuntime = "remote"

 // GenerateKubeadmYAML generates the kubeadm.yaml file
-func GenerateKubeadmYAML(k8s config.KubernetesConfig, r cruntime.Manager) ([]byte, error) {
+func GenerateKubeadmYAML(mc config.MachineConfig, r cruntime.Manager) ([]byte, error) {
+	k8s := mc.KubernetesConfig
 	version, err := ParseKubernetesVersion(k8s.KubernetesVersion)
 	if err != nil {
 		return nil, errors.Wrap(err, "parsing kubernetes version")
@ -53,8 +53,12 @@ func GenerateKubeadmYAML(k8s config.KubernetesConfig, r cruntime.Manager) ([]byt
 		return nil, errors.Wrap(err, "generating extra component config for kubeadm")
 	}

-	// In case of no port assigned, use util.APIServerPort
-	nodePort := k8s.NodePort
+	// In case of no port assigned, use default
+	cp, err := config.PrimaryControlPlane(mc)
+	if err != nil {
+		return nil, errors.Wrap(err, "getting control plane")
+	}
+	nodePort := cp.Port
 	if nodePort <= 0 {
 		nodePort = constants.APIServerPort
 	}
@ -76,13 +80,13 @@ func GenerateKubeadmYAML(k8s config.KubernetesConfig, r cruntime.Manager) ([]byt
 		NoTaintMaster     bool
 	}{
 		CertDir:           vmpath.GuestCertsDir,
-		ServiceCIDR:       util.DefaultServiceCIDR,
+		ServiceCIDR:       constants.DefaultServiceCIDR,
 		PodSubnet:         k8s.ExtraOptions.Get("pod-network-cidr", Kubeadm),
-		AdvertiseAddress:  k8s.NodeIP,
+		AdvertiseAddress:  cp.IP,
 		APIServerPort:     nodePort,
 		KubernetesVersion: k8s.KubernetesVersion,
 		EtcdDataDir:       EtcdDataDir(),
-		NodeName:          k8s.NodeName,
+		NodeName:          cp.Name,
 		CRISocket:         r.SocketPath(),
 		ImageRepository:   k8s.ImageRepository,
 		ExtraArgs:         extraComponentConfig,
@ -97,13 +101,13 @@ func GenerateKubeadmYAML(k8s config.KubernetesConfig, r cruntime.Manager) ([]byt

 	opts.NoTaintMaster = true
 	b := bytes.Buffer{}
-	configTmpl := template.KubeAdmConfigTmplV1Alpha1
+	configTmpl := ktmpl.V1Alpha1
 	if version.GTE(semver.MustParse("1.12.0")) {
-		configTmpl = template.KubeAdmConfigTmplV1Alpha3
+		configTmpl = ktmpl.V1Alpha3
 	}
 	// v1beta1 works in v1.13, but isn't required until v1.14.
 	if version.GTE(semver.MustParse("1.14.0-alpha.0")) {
-		configTmpl = template.KubeAdmConfigTmplV1Beta1
+		configTmpl = ktmpl.V1Beta1
 	}
 	if err := configTmpl.Execute(&b, opts); err != nil {
 		return nil, err
--- a/pkg/minikube/bootstrapper/bsutil/kubeadm_test.go
+++ b/pkg/minikube/bootstrapper/bsutil/kubeadm_test.go
@ -101,14 +101,14 @@ This test case has only 1 thing to test and that is the
 nnetworking/dnsDomain value
 */
 func TestGenerateKubeadmYAMLDNS(t *testing.T) {
-	versions := []string{"v1.16", "v1.15", "v1.14", "v1.13", "v1.12"}
+	versions := []string{"v1.17", "v1.16", "v1.15", "v1.14", "v1.13", "v1.12"}
 	tests := []struct {
 		name      string
 		runtime   string
 		shouldErr bool
-		cfg       config.KubernetesConfig
+		cfg       config.MachineConfig
 	}{
-		{"dns", "docker", false, config.KubernetesConfig{DNSDomain: "1.1.1.1"}},
+		{"dns", "docker", false, config.MachineConfig{KubernetesConfig: config.KubernetesConfig{DNSDomain: "1.1.1.1"}}},
 	}
 	for _, version := range versions {
 		for _, tc := range tests {
@ -119,9 +119,14 @@ func TestGenerateKubeadmYAMLDNS(t *testing.T) {
 			tname := tc.name + "_" + version
 			t.Run(tname, func(t *testing.T) {
 				cfg := tc.cfg
-				cfg.NodeIP = "1.1.1.1"
-				cfg.NodeName = "mk"
-				cfg.KubernetesVersion = version + ".0"
+				cfg.Nodes = []config.Node{
+					config.Node{
+						IP:           "1.1.1.1",
+						Name:         "mk",
+						ControlPlane: true,
+					},
+				}
+				cfg.KubernetesConfig.KubernetesVersion = version + ".0"

 				got, err := GenerateKubeadmYAML(cfg, runtime)
 				if err != nil && !tc.shouldErr {
@ -166,17 +171,17 @@ func TestGenerateKubeadmYAML(t *testing.T) {
 		name      string
 		runtime   string
 		shouldErr bool
-		cfg       config.KubernetesConfig
+		cfg       config.MachineConfig
 	}{
-		{"default", "docker", false, config.KubernetesConfig{}},
-		{"containerd", "containerd", false, config.KubernetesConfig{}},
-		{"crio", "crio", false, config.KubernetesConfig{}},
-		{"options", "docker", false, config.KubernetesConfig{ExtraOptions: extraOpts}},
-		{"crio-options-gates", "crio", false, config.KubernetesConfig{ExtraOptions: extraOpts, FeatureGates: "a=b"}},
-		{"unknown-component", "docker", true, config.KubernetesConfig{ExtraOptions: config.ExtraOptionSlice{config.ExtraOption{Component: "not-a-real-component", Key: "killswitch", Value: "true"}}}},
-		{"containerd-api-port", "containerd", false, config.KubernetesConfig{NodePort: 12345}},
-		{"containerd-pod-network-cidr", "containerd", false, config.KubernetesConfig{ExtraOptions: extraOptsPodCidr}},
-		{"image-repository", "docker", false, config.KubernetesConfig{ImageRepository: "test/repo"}},
+		{"default", "docker", false, config.MachineConfig{}},
+		{"containerd", "containerd", false, config.MachineConfig{}},
+		{"crio", "crio", false, config.MachineConfig{}},
+		{"options", "docker", false, config.MachineConfig{KubernetesConfig: config.KubernetesConfig{ExtraOptions: extraOpts}}},
+		{"crio-options-gates", "crio", false, config.MachineConfig{KubernetesConfig: config.KubernetesConfig{ExtraOptions: extraOpts, FeatureGates: "a=b"}}},
+		{"unknown-component", "docker", true, config.MachineConfig{KubernetesConfig: config.KubernetesConfig{ExtraOptions: config.ExtraOptionSlice{config.ExtraOption{Component: "not-a-real-component", Key: "killswitch", Value: "true"}}}}},
+		{"containerd-api-port", "containerd", false, config.MachineConfig{Nodes: []config.Node{config.Node{Port: 12345}}}},
+		{"containerd-pod-network-cidr", "containerd", false, config.MachineConfig{KubernetesConfig: config.KubernetesConfig{ExtraOptions: extraOptsPodCidr}}},
+		{"image-repository", "docker", false, config.MachineConfig{KubernetesConfig: config.KubernetesConfig{ImageRepository: "test/repo"}}},
 	}
 	for _, version := range versions {
 		for _, tc := range tests {
@ -187,9 +192,21 @@ func TestGenerateKubeadmYAML(t *testing.T) {
 			tname := tc.name + "_" + version
 			t.Run(tname, func(t *testing.T) {
 				cfg := tc.cfg
-				cfg.NodeIP = "1.1.1.1"
-				cfg.NodeName = "mk"
-				cfg.KubernetesVersion = version + ".0"
+
+				if len(cfg.Nodes) > 0 {
+					cfg.Nodes[0].IP = "1.1.1.1"
+					cfg.Nodes[0].Name = "mk"
+					cfg.Nodes[0].ControlPlane = true
+				} else {
+					cfg.Nodes = []config.Node{
+						config.Node{
+							IP:           "1.1.1.1",
+							Name:         "mk",
+							ControlPlane: true,
+						},
+					}
+				}
+				cfg.KubernetesConfig.KubernetesVersion = version + ".0"

 				got, err := GenerateKubeadmYAML(cfg, runtime)
 				if err != nil && !tc.shouldErr {
--- a/pkg/minikube/bootstrapper/bsutil/kubelet.go
+++ b/pkg/minikube/bootstrapper/bsutil/kubelet.go
@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */

-// bsutil package will eventually be renamed to kubeadm package after getting rid of older one
+// Package bsutil will eventually be renamed to kubeadm package after getting rid of older one
 package bsutil

 import (
@ -22,7 +22,7 @@ import (
 	"path"

 	"github.com/pkg/errors"
-	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/template"
+	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/ktmpl"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/cruntime"
@ -30,7 +30,8 @@ import (

 // NewKubeletConfig generates a new systemd unit containing a configured kubelet
 // based on the options present in the KubernetesConfig.
-func NewKubeletConfig(k8s config.KubernetesConfig, r cruntime.Manager) ([]byte, error) {
+func NewKubeletConfig(mc config.MachineConfig, r cruntime.Manager) ([]byte, error) {
+	k8s := mc.KubernetesConfig
 	version, err := ParseKubernetesVersion(k8s.KubernetesVersion)
 	if err != nil {
 		return nil, errors.Wrap(err, "parsing kubernetes version")
@ -41,14 +42,23 @@ func NewKubeletConfig(k8s config.KubernetesConfig, r cruntime.Manager) ([]byte,
 		return nil, errors.Wrap(err, "generating extra configuration for kubelet")
 	}

+	cgroupDriver, err := r.CGroupDriver()
+	if err == nil {
+		extraOpts["cgroup-driver"] = cgroupDriver
+	}
+
 	for k, v := range r.KubeletOptions() {
 		extraOpts[k] = v
 	}
 	if k8s.NetworkPlugin != "" {
 		extraOpts["network-plugin"] = k8s.NetworkPlugin
 	}
+	cp, err := config.PrimaryControlPlane(mc)
+	if err != nil {
+		return nil, errors.Wrap(err, "getting master node")
+	}
 	if _, ok := extraOpts["node-ip"]; !ok {
-		extraOpts["node-ip"] = k8s.NodeIP
+		extraOpts["node-ip"] = cp.IP
 	}

 	pauseImage := images.Pause(k8s.ImageRepository)
@ -76,7 +86,7 @@ func NewKubeletConfig(k8s config.KubernetesConfig, r cruntime.Manager) ([]byte,
 		ContainerRuntime: k8s.ContainerRuntime,
 		KubeletPath:      path.Join(binRoot(k8s.KubernetesVersion), "kubelet"),
 	}
-	if err := template.KubeletSystemdTemplate.Execute(&b, opts); err != nil {
+	if err := ktmpl.KubeletSystemdTemplate.Execute(&b, opts); err != nil {
 		return nil, err
 	}

@ -87,7 +97,7 @@ func NewKubeletConfig(k8s config.KubernetesConfig, r cruntime.Manager) ([]byte,
 func NewKubeletService(cfg config.KubernetesConfig) ([]byte, error) {
 	var b bytes.Buffer
 	opts := struct{ KubeletPath string }{KubeletPath: path.Join(binRoot(cfg.KubernetesVersion), "kubelet")}
-	if err := template.KubeletServiceTemplate.Execute(&b, opts); err != nil {
+	if err := ktmpl.KubeletServiceTemplate.Execute(&b, opts); err != nil {
 		return nil, errors.Wrap(err, "template execute")
 	}
 	return b.Bytes(), nil
--- a/pkg/minikube/bootstrapper/bsutil/kubelet_test.go
+++ b/pkg/minikube/bootstrapper/bsutil/kubelet_test.go
@ -14,13 +14,14 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */

-// bsutil package will eventually be renamed to kubeadm package after getting rid of older one
+// Package bsutil will eventually be renamed to kubeadm package after getting rid of older one
 package bsutil

 import (
 	"testing"

 	"github.com/pmezard/go-difflib/difflib"
+	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/cruntime"
@ -29,17 +30,24 @@ import (
 func TestGenerateKubeletConfig(t *testing.T) {
 	tests := []struct {
 		description string
-		cfg         config.KubernetesConfig
+		cfg         config.MachineConfig
 		expected    string
 		shouldErr   bool
 	}{
 		{
 			description: "old docker",
-			cfg: config.KubernetesConfig{
-				NodeIP:            "192.168.1.100",
-				KubernetesVersion: constants.OldestKubernetesVersion,
-				NodeName:          "minikube",
-				ContainerRuntime:  "docker",
+			cfg: config.MachineConfig{
+				KubernetesConfig: config.KubernetesConfig{
+					KubernetesVersion: constants.OldestKubernetesVersion,
+					ContainerRuntime:  "docker",
+				},
+				Nodes: []config.Node{
+					config.Node{
+						IP:           "192.168.1.100",
+						Name:         "minikube",
+						ControlPlane: true,
+					},
+				},
 			},
 			expected: `[Unit]
 Wants=docker.socket
@ -53,52 +61,41 @@ ExecStart=/var/lib/minikube/binaries/v1.11.10/kubelet --allow-privileged=true --
 		},
 		{
 			description: "newest cri runtime",
-			cfg: config.KubernetesConfig{
-				NodeIP:            "192.168.1.100",
-				KubernetesVersion: constants.NewestKubernetesVersion,
-				NodeName:          "minikube",
-				ContainerRuntime:  "cri-o",
+			cfg: config.MachineConfig{
+				KubernetesConfig: config.KubernetesConfig{
+					KubernetesVersion: constants.NewestKubernetesVersion,
+					ContainerRuntime:  "cri-o",
+				},
+				Nodes: []config.Node{
+					config.Node{
+						IP:           "192.168.1.100",
+						Name:         "minikube",
+						ControlPlane: true,
+					},
+				},
 			},
 			expected: `[Unit]
 Wants=crio.service

 [Service]
 ExecStart=
-ExecStart=/var/lib/minikube/binaries/v1.17.0/kubelet --authorization-mode=Webhook --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --cgroup-driver=cgroupfs --client-ca-file=/var/lib/minikube/certs/ca.crt --cluster-dns=10.96.0.10 --cluster-domain=cluster.local --config=/var/lib/kubelet/config.yaml --container-runtime=remote --container-runtime-endpoint=/var/run/crio/crio.sock --fail-swap-on=false --hostname-override=minikube --image-service-endpoint=/var/run/crio/crio.sock --kubeconfig=/etc/kubernetes/kubelet.conf --node-ip=192.168.1.100 --pod-manifest-path=/etc/kubernetes/manifests --runtime-request-timeout=15m
+ExecStart=/var/lib/minikube/binaries/v1.17.2/kubelet --authorization-mode=Webhook --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --cgroup-driver=cgroupfs --client-ca-file=/var/lib/minikube/certs/ca.crt --cluster-dns=10.96.0.10 --cluster-domain=cluster.local --config=/var/lib/kubelet/config.yaml --container-runtime=remote --container-runtime-endpoint=/var/run/crio/crio.sock --fail-swap-on=false --hostname-override=minikube --image-service-endpoint=/var/run/crio/crio.sock --kubeconfig=/etc/kubernetes/kubelet.conf --node-ip=192.168.1.100 --pod-manifest-path=/etc/kubernetes/manifests --runtime-request-timeout=15m

 [Install]
 `,
 		},
 		{
 			description: "default containerd runtime",
-			cfg: config.KubernetesConfig{
-				NodeIP:            "192.168.1.100",
-				KubernetesVersion: constants.DefaultKubernetesVersion,
-				NodeName:          "minikube",
-				ContainerRuntime:  "containerd",
-			},
-			expected: `[Unit]
-Wants=containerd.service
-
-[Service]
-ExecStart=
-ExecStart=/var/lib/minikube/binaries/v1.17.0/kubelet --authorization-mode=Webhook --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --cgroup-driver=cgroupfs --client-ca-file=/var/lib/minikube/certs/ca.crt --cluster-dns=10.96.0.10 --cluster-domain=cluster.local --config=/var/lib/kubelet/config.yaml --container-runtime=remote --container-runtime-endpoint=unix:///run/containerd/containerd.sock --fail-swap-on=false --hostname-override=minikube --image-service-endpoint=unix:///run/containerd/containerd.sock --kubeconfig=/etc/kubernetes/kubelet.conf --node-ip=192.168.1.100 --pod-manifest-path=/etc/kubernetes/manifests --runtime-request-timeout=15m
-
-[Install]
-`,
-		},
-		{
-			description: "default containerd runtime",
-			cfg: config.KubernetesConfig{
-				NodeIP:            "192.168.1.100",
-				KubernetesVersion: constants.DefaultKubernetesVersion,
-				NodeName:          "minikube",
-				ContainerRuntime:  "containerd",
-				ExtraOptions: config.ExtraOptionSlice{
-					config.ExtraOption{
-						Component: Kubelet,
-						Key:       "node-ip",
-						Value:     "192.168.1.200",
+			cfg: config.MachineConfig{
+				KubernetesConfig: config.KubernetesConfig{
+					KubernetesVersion: constants.DefaultKubernetesVersion,
+					ContainerRuntime:  "containerd",
+				},
+				Nodes: []config.Node{
+					config.Node{
+						IP:           "192.168.1.100",
+						Name:         "minikube",
+						ControlPlane: true,
 					},
 				},
 			},
@ -107,26 +104,65 @@ Wants=containerd.service

 [Service]
 ExecStart=
-ExecStart=/var/lib/minikube/binaries/v1.17.0/kubelet --authorization-mode=Webhook --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --cgroup-driver=cgroupfs --client-ca-file=/var/lib/minikube/certs/ca.crt --cluster-dns=10.96.0.10 --cluster-domain=cluster.local --config=/var/lib/kubelet/config.yaml --container-runtime=remote --container-runtime-endpoint=unix:///run/containerd/containerd.sock --fail-swap-on=false --hostname-override=minikube --image-service-endpoint=unix:///run/containerd/containerd.sock --kubeconfig=/etc/kubernetes/kubelet.conf --node-ip=192.168.1.200 --pod-manifest-path=/etc/kubernetes/manifests --runtime-request-timeout=15m
+ExecStart=/var/lib/minikube/binaries/v1.17.2/kubelet --authorization-mode=Webhook --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --cgroup-driver=cgroupfs --client-ca-file=/var/lib/minikube/certs/ca.crt --cluster-dns=10.96.0.10 --cluster-domain=cluster.local --config=/var/lib/kubelet/config.yaml --container-runtime=remote --container-runtime-endpoint=unix:///run/containerd/containerd.sock --fail-swap-on=false --hostname-override=minikube --image-service-endpoint=unix:///run/containerd/containerd.sock --kubeconfig=/etc/kubernetes/kubelet.conf --node-ip=192.168.1.100 --pod-manifest-path=/etc/kubernetes/manifests --runtime-request-timeout=15m
+
+[Install]
+`,
+		},
+		{
+			description: "default containerd runtime with IP override",
+			cfg: config.MachineConfig{
+				KubernetesConfig: config.KubernetesConfig{
+					KubernetesVersion: constants.DefaultKubernetesVersion,
+					ContainerRuntime:  "containerd",
+					ExtraOptions: config.ExtraOptionSlice{
+						config.ExtraOption{
+							Component: Kubelet,
+							Key:       "node-ip",
+							Value:     "192.168.1.200",
+						},
+					},
+				},
+				Nodes: []config.Node{
+					config.Node{
+						IP:           "192.168.1.100",
+						Name:         "minikube",
+						ControlPlane: true,
+					},
+				},
+			},
+			expected: `[Unit]
+Wants=containerd.service
+
+[Service]
+ExecStart=
+ExecStart=/var/lib/minikube/binaries/v1.17.2/kubelet --authorization-mode=Webhook --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --cgroup-driver=cgroupfs --client-ca-file=/var/lib/minikube/certs/ca.crt --cluster-dns=10.96.0.10 --cluster-domain=cluster.local --config=/var/lib/kubelet/config.yaml --container-runtime=remote --container-runtime-endpoint=unix:///run/containerd/containerd.sock --fail-swap-on=false --hostname-override=minikube --image-service-endpoint=unix:///run/containerd/containerd.sock --kubeconfig=/etc/kubernetes/kubelet.conf --node-ip=192.168.1.200 --pod-manifest-path=/etc/kubernetes/manifests --runtime-request-timeout=15m

 [Install]
 `,
 		},
 		{
 			description: "docker with custom image repository",
-			cfg: config.KubernetesConfig{
-				NodeIP:            "192.168.1.100",
-				KubernetesVersion: constants.DefaultKubernetesVersion,
-				NodeName:          "minikube",
-				ContainerRuntime:  "docker",
-				ImageRepository:   "docker-proxy-image.io/google_containers",
+			cfg: config.MachineConfig{
+				KubernetesConfig: config.KubernetesConfig{
+					KubernetesVersion: constants.DefaultKubernetesVersion,
+					ContainerRuntime:  "docker",
+					ImageRepository:   "docker-proxy-image.io/google_containers",
+				},
+				Nodes: []config.Node{
+					config.Node{
+						IP:           "192.168.1.100",
+						Name:         "minikube",
+						ControlPlane: true,
+					},
+				},
 			},
 			expected: `[Unit]
 Wants=docker.socket

 [Service]
 ExecStart=
-ExecStart=/var/lib/minikube/binaries/v1.17.0/kubelet --authorization-mode=Webhook --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --cgroup-driver=cgroupfs --client-ca-file=/var/lib/minikube/certs/ca.crt --cluster-dns=10.96.0.10 --cluster-domain=cluster.local --config=/var/lib/kubelet/config.yaml --container-runtime=docker --fail-swap-on=false --hostname-override=minikube --kubeconfig=/etc/kubernetes/kubelet.conf --node-ip=192.168.1.100 --pod-infra-container-image=docker-proxy-image.io/google_containers/pause:3.1 --pod-manifest-path=/etc/kubernetes/manifests
+ExecStart=/var/lib/minikube/binaries/v1.17.2/kubelet --authorization-mode=Webhook --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --cgroup-driver=cgroupfs --client-ca-file=/var/lib/minikube/certs/ca.crt --cluster-dns=10.96.0.10 --cluster-domain=cluster.local --config=/var/lib/kubelet/config.yaml --container-runtime=docker --fail-swap-on=false --hostname-override=minikube --kubeconfig=/etc/kubernetes/kubelet.conf --node-ip=192.168.1.100 --pod-infra-container-image=docker-proxy-image.io/google_containers/pause:3.1 --pod-manifest-path=/etc/kubernetes/manifests

 [Install]
 `,
@ -135,7 +171,8 @@ ExecStart=/var/lib/minikube/binaries/v1.17.0/kubelet --authorization-mode=Webhoo

 	for _, tc := range tests {
 		t.Run(tc.description, func(t *testing.T) {
-			runtime, err := cruntime.New(cruntime.Config{Type: tc.cfg.ContainerRuntime})
+			runtime, err := cruntime.New(cruntime.Config{Type: tc.cfg.KubernetesConfig.ContainerRuntime,
+				Runner: command.NewFakeCommandRunner()})
 			if err != nil {
 				t.Fatalf("runtime: %v", err)
 			}
--- a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
+++ b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
@ -0,0 +1,141 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package kverify verifies a running kubernetes cluster is healthy
+package kverify
+
+import (
+	"crypto/tls"
+	"fmt"
+	"net"
+	"net/http"
+	"os/exec"
+	"time"
+
+	"github.com/docker/machine/libmachine/state"
+	"github.com/golang/glog"
+	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
+	kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	"k8s.io/minikube/pkg/minikube/command"
+)
+
+// APIServerProcess waits for api server to be healthy returns error if it doesn't
+func APIServerProcess(runner command.Runner, start time.Time, timeout time.Duration) error {
+	glog.Infof("waiting for apiserver process to appear ...")
+	err := wait.PollImmediate(time.Second*1, timeout, func() (bool, error) {
+		if time.Since(start) > timeout {
+			return false, fmt.Errorf("cluster wait timed out during process check")
+		}
+		rr, ierr := runner.RunCmd(exec.Command("sudo", "pgrep", "kube-apiserver"))
+		if ierr != nil {
+			glog.Warningf("pgrep apiserver: %v cmd: %s", ierr, rr.Command())
+			return false, nil
+		}
+		return true, nil
+	})
+	if err != nil {
+		return fmt.Errorf("apiserver process never appeared")
+	}
+	glog.Infof("duration metric: took %s to wait for apiserver process to appear ...", time.Since(start))
+	return nil
+}
+
+// SystemPods verifies essential pods for running kurnetes is running
+func SystemPods(client *kubernetes.Clientset, start time.Time, timeout time.Duration) error {
+	glog.Info("waiting for kube-system pods to appear ...")
+	pStart := time.Now()
+	podStart := time.Time{}
+	podList := func() (bool, error) {
+		if time.Since(start) > timeout {
+			return false, fmt.Errorf("cluster wait timed out during pod check")
+		}
+		// Wait for any system pod, as waiting for apiserver may block until etcd
+		pods, err := client.CoreV1().Pods("kube-system").List(meta.ListOptions{})
+		if len(pods.Items) < 2 {
+			podStart = time.Time{}
+			return false, nil
+		}
+		if err != nil {
+			podStart = time.Time{}
+			return false, nil
+		}
+		if podStart.IsZero() {
+			podStart = time.Now()
+		}
+
+		glog.Infof("%d kube-system pods found since %s", len(pods.Items), podStart)
+		if time.Since(podStart) > 2*kconst.APICallRetryInterval {
+			glog.Infof("stability requirement met, returning")
+			return true, nil
+		}
+		return false, nil
+	}
+	if err := wait.PollImmediate(kconst.APICallRetryInterval, kconst.DefaultControlPlaneTimeout, podList); err != nil {
+		return fmt.Errorf("apiserver never returned a pod list")
+	}
+	glog.Infof("duration metric: took %s to wait for pod list to return data ...", time.Since(pStart))
+	return nil
+}
+
+// APIServerIsRunning waits for api server status to be running
+func APIServerIsRunning(start time.Time, ip string, port int, timeout time.Duration) error {
+	glog.Infof("waiting for apiserver healthz status ...")
+	hStart := time.Now()
+	healthz := func() (bool, error) {
+		if time.Since(start) > timeout {
+			return false, fmt.Errorf("cluster wait timed out during healthz check")
+		}
+
+		status, err := APIServerStatus(net.ParseIP(ip), port)
+		if err != nil {
+			glog.Warningf("status: %v", err)
+			return false, nil
+		}
+		if status != "Running" {
+			return false, nil
+		}
+		return true, nil
+	}
+
+	if err := wait.PollImmediate(kconst.APICallRetryInterval, kconst.DefaultControlPlaneTimeout, healthz); err != nil {
+		return fmt.Errorf("apiserver healthz never reported healthy")
+	}
+	glog.Infof("duration metric: took %s to wait for apiserver healthz status ...", time.Since(hStart))
+	return nil
+}
+
+// APIServerStatus hits the /healthz endpoint and returns libmachine style state.State
+func APIServerStatus(ip net.IP, apiserverPort int) (string, error) {
+	url := fmt.Sprintf("https://%s/healthz", net.JoinHostPort(ip.String(), fmt.Sprint(apiserverPort)))
+	// To avoid: x509: certificate signed by unknown authority
+	tr := &http.Transport{
+		Proxy:           nil, // To avoid connectiv issue if http(s)_proxy is set.
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	}
+	client := &http.Client{Transport: tr}
+	resp, err := client.Get(url)
+	// Connection refused, usually.
+	if err != nil {
+		return state.Stopped.String(), nil
+	}
+	if resp.StatusCode != http.StatusOK {
+		glog.Warningf("%s response: %v %+v", url, err, resp)
+		return state.Error.String(), nil
+	}
+	return state.Running.String(), nil
+}
--- a/pkg/minikube/bootstrapper/kubeadm/rbac.go
+++ b/pkg/minikube/bootstrapper/kubeadm/rbac.go
@ -14,10 +14,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */

-package kubeadm
+package bsutil

 import (
+	"fmt"
 	"net"
+	"os/exec"
+	"strings"
 	"time"

 	"github.com/golang/glog"
@ -25,6 +28,7 @@ import (
 	rbac "k8s.io/api/rbac/v1beta1"
 	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/util/retry"
 )

@ -32,9 +36,9 @@ const (
 	rbacName = "minikube-rbac"
 )

-// elevateKubeSystemPrivileges gives the kube-system service account
+// ElevateKubeSystemPrivileges gives the kube-system service account
 // cluster admin privileges to work with RBAC.
-func elevateKubeSystemPrivileges(client kubernetes.Interface) error {
+func ElevateKubeSystemPrivileges(client kubernetes.Interface) error {
 	start := time.Now()
 	clusterRoleBinding := &rbac.ClusterRoleBinding{
 		ObjectMeta: meta.ObjectMeta{
@ -67,3 +71,31 @@ func elevateKubeSystemPrivileges(client kubernetes.Interface) error {
 	glog.Infof("duration metric: took %s to wait for elevateKubeSystemPrivileges.", time.Since(start))
 	return nil
 }
+
+// AdjustResourceLimits makes fine adjustments to pod resources that aren't possible via kubeadm config.
+func AdjustResourceLimits(c command.Runner) error {
+	rr, err := c.RunCmd(exec.Command("/bin/bash", "-c", "cat /proc/$(pgrep kube-apiserver)/oom_adj"))
+	if err != nil {
+		return errors.Wrapf(err, "oom_adj check cmd %s. ", rr.Command())
+	}
+	glog.Infof("apiserver oom_adj: %s", rr.Stdout.String())
+	// oom_adj is already a negative number
+	if strings.HasPrefix(rr.Stdout.String(), "-") {
+		return nil
+	}
+	glog.Infof("adjusting apiserver oom_adj to -10")
+
+	// Prevent the apiserver from OOM'ing before other pods, as it is our gateway into the cluster.
+	// It'd be preferable to do this via Kubernetes, but kubeadm doesn't have a way to set pod QoS.
+	if _, err = c.RunCmd(exec.Command("/bin/bash", "-c", "echo -10 | sudo tee /proc/$(pgrep kube-apiserver)/oom_adj")); err != nil {
+		return errors.Wrap(err, fmt.Sprintf("oom_adj adjust"))
+	}
+	return nil
+}
+
+// ExistingConfig checks if there are config files from possible previous kubernets cluster
+func ExistingConfig(c command.Runner) error {
+	args := append([]string{"ls"}, expectedRemoteArtifacts...)
+	_, err := c.RunCmd(exec.Command("sudo", args...))
+	return err
+}
--- a/pkg/minikube/bootstrapper/bsutil/template/template.go
+++ b/pkg/minikube/bootstrapper/bsutil/template/template.go
@ -1,171 +0,0 @@
-/*
-Copyright 2016 The Kubernetes Authors All rights reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package template
-
-import (
-	"fmt"
-	"sort"
-	"text/template"
-)
-
-// KubeAdmConfigTmplV1Alpha1 is for Kubernetes v1.11
-var KubeAdmConfigTmplV1Alpha1 = template.Must(template.New("configTmpl-v1alpha1").Funcs(template.FuncMap{
-	"printMapInOrder": printMapInOrder,
-}).Parse(`apiVersion: kubeadm.k8s.io/v1alpha1
-kind: MasterConfiguration
-{{if .NoTaintMaster}}noTaintMaster: true{{end}}
-api:
-  advertiseAddress: {{.AdvertiseAddress}}
-  bindPort: {{.APIServerPort}}
-  controlPlaneEndpoint: localhost
-kubernetesVersion: {{.KubernetesVersion}}
-certificatesDir: {{.CertDir}}
-networking:
-  serviceSubnet: {{.ServiceCIDR}}
-etcd:
-  dataDir: {{.EtcdDataDir}}
-nodeName: {{.NodeName}}
-{{if .ImageRepository}}imageRepository: {{.ImageRepository}}
-{{end}}{{if .CRISocket}}criSocket: {{.CRISocket}}
-{{end}}{{range .ExtraArgs}}{{.Component}}ExtraArgs:{{range $i, $val := printMapInOrder .Options ": " }}
-  {{$val}}{{end}}
-{{end}}{{if .FeatureArgs}}featureGates: {{range $i, $val := .FeatureArgs}}
-  {{$i}}: {{$val}}{{end}}
-{{end}}`))
-
-// KubeAdmConfigTmplV1Alpha3 is for Kubernetes v1.12
-var KubeAdmConfigTmplV1Alpha3 = template.Must(template.New("configTmpl-v1alpha3").Funcs(template.FuncMap{
-	"printMapInOrder": printMapInOrder,
-}).Parse(`apiVersion: kubeadm.k8s.io/v1alpha3
-kind: InitConfiguration
-apiEndpoint:
-  advertiseAddress: {{.AdvertiseAddress}}
-  bindPort: {{.APIServerPort}}
-bootstrapTokens:
-  - groups:
-      - system:bootstrappers:kubeadm:default-node-token
-    ttl: 24h0m0s
-    usages:
-      - signing
-      - authentication
-nodeRegistration:
-  criSocket: {{if .CRISocket}}{{.CRISocket}}{{else}}/var/run/dockershim.sock{{end}}
-  name: {{.NodeName}}
-  taints: []
---
-apiVersion: kubeadm.k8s.io/v1alpha3
-kind: ClusterConfiguration
-{{if .ImageRepository}}imageRepository: {{.ImageRepository}}
-{{end}}{{range .ExtraArgs}}{{.Component}}ExtraArgs:{{range $i, $val := printMapInOrder .Options ": " }}
-  {{$val}}{{end}}
-{{end -}}
-{{if .FeatureArgs}}featureGates: {{range $i, $val := .FeatureArgs}}
-  {{$i}}: {{$val}}{{end}}
-{{end -}}
-certificatesDir: {{.CertDir}}
-clusterName: kubernetes
-controlPlaneEndpoint: localhost:{{.APIServerPort}}
-etcd:
-  local:
-    dataDir: {{.EtcdDataDir}}
-kubernetesVersion: {{.KubernetesVersion}}
-networking:
-  dnsDomain: {{if .DNSDomain}}{{.DNSDomain}}{{else}}cluster.local{{end}}
-  podSubnet: {{if .PodSubnet}}{{.PodSubnet}}{{else}}""{{end}}
-  serviceSubnet: {{.ServiceCIDR}}
---
-apiVersion: kubelet.config.k8s.io/v1beta1
-kind: KubeletConfiguration
-evictionHard:
-  nodefs.available: "0%"
-  nodefs.inodesFree: "0%"
-  imagefs.available: "0%"
-`))
-
-// KubeAdmConfigTmplV1Beta1 is for Kubernetes v1.13+
-var KubeAdmConfigTmplV1Beta1 = template.Must(template.New("configTmpl-v1beta1").Funcs(template.FuncMap{
-	"printMapInOrder": printMapInOrder,
-}).Parse(`apiVersion: kubeadm.k8s.io/v1beta1
-kind: InitConfiguration
-localAPIEndpoint:
-  advertiseAddress: {{.AdvertiseAddress}}
-  bindPort: {{.APIServerPort}}
-bootstrapTokens:
-  - groups:
-      - system:bootstrappers:kubeadm:default-node-token
-    ttl: 24h0m0s
-    usages:
-      - signing
-      - authentication
-nodeRegistration:
-  criSocket: {{if .CRISocket}}{{.CRISocket}}{{else}}/var/run/dockershim.sock{{end}}
-  name: {{.NodeName}}
-  taints: []
---
-apiVersion: kubeadm.k8s.io/v1beta1
-kind: ClusterConfiguration
-{{ if .ImageRepository}}imageRepository: {{.ImageRepository}}
-{{end}}{{range .ExtraArgs}}{{.Component}}:
-  extraArgs:
-{{- range $i, $val := printMapInOrder .Options ": " }}
-    {{$val}}
-{{- end}}
-{{end -}}
-{{if .FeatureArgs}}featureGates:
-{{range $i, $val := .FeatureArgs}}{{$i}}: {{$val}}
-{{end -}}{{end -}}
-certificatesDir: {{.CertDir}}
-clusterName: kubernetes
-controlPlaneEndpoint: localhost:{{.APIServerPort}}
-dns:
-  type: CoreDNS
-etcd:
-  local:
-    dataDir: {{.EtcdDataDir}}
-kubernetesVersion: {{.KubernetesVersion}}
-networking:
-  dnsDomain: {{if .DNSDomain}}{{.DNSDomain}}{{else}}cluster.local{{end}}
-  podSubnet: ""
-  serviceSubnet: {{.ServiceCIDR}}
---
-apiVersion: kubelet.config.k8s.io/v1beta1
-kind: KubeletConfiguration
-imageGCHighThresholdPercent: 100
-evictionHard:
-  nodefs.available: "0%"
-  nodefs.inodesFree: "0%"
-  imagefs.available: "0%"
-`))
-
-// printMapInOrder sorts the keys and prints the map in order, combining key
-// value pairs with the separator character
-//
-// Note: this is not necessary, but makes testing easy
-func printMapInOrder(m map[string]string, sep string) []string {
-	if m == nil {
-		return nil
-	}
-	keys := []string{}
-	for k := range m {
-		keys = append(keys, k)
-	}
-	sort.Strings(keys)
-	for i, k := range keys {
-		keys[i] = fmt.Sprintf("%s%s\"%s\"", k, sep, m[k])
-	}
-	return keys
-}
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd-api-port.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd-api-port.yaml
@ -12,6 +12,7 @@ networking:
 etcd:
  dataDir: /var/lib/minikube/etcd
 nodeName: mk
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 criSocket: /run/containerd/containerd.sock
 apiServerExtraArgs:
  enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd-pod-network-cidr.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd-pod-network-cidr.yaml
@ -12,6 +12,7 @@ networking:
 etcd:
  dataDir: /var/lib/minikube/etcd
 nodeName: mk
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 criSocket: /run/containerd/containerd.sock
 apiServerExtraArgs:
  enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd.yaml
@ -12,6 +12,7 @@ networking:
 etcd:
  dataDir: /var/lib/minikube/etcd
 nodeName: mk
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 criSocket: /run/containerd/containerd.sock
 apiServerExtraArgs:
  enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/crio-options-gates.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/crio-options-gates.yaml
@ -12,6 +12,7 @@ networking:
 etcd:
  dataDir: /var/lib/minikube/etcd
 nodeName: mk
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 criSocket: /var/run/crio/crio.sock
 apiServerExtraArgs:
  enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/crio.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/crio.yaml
@ -12,6 +12,7 @@ networking:
 etcd:
  dataDir: /var/lib/minikube/etcd
 nodeName: mk
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 criSocket: /var/run/crio/crio.sock
 apiServerExtraArgs:
  enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/default.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/default.yaml
@ -12,5 +12,6 @@ networking:
 etcd:
  dataDir: /var/lib/minikube/etcd
 nodeName: mk
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 apiServerExtraArgs:
  enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/image-repository.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/image-repository.yaml
@ -12,6 +12,7 @@ networking:
 etcd:
  dataDir: /var/lib/minikube/etcd
 nodeName: mk
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 imageRepository: test/repo
 apiServerExtraArgs:
  enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/options.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/options.yaml
@ -12,6 +12,7 @@ networking:
 etcd:
  dataDir: /var/lib/minikube/etcd
 nodeName: mk
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 apiServerExtraArgs:
  enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
  fail-no-swap: "true"
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd-api-port.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd-api-port.yaml
@ -21,6 +21,7 @@ apiServerExtraArgs:
  enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:12345
 etcd:
  local:
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd-pod-network-cidr.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd-pod-network-cidr.yaml
@ -21,6 +21,7 @@ apiServerExtraArgs:
  enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
  local:
@ -28,7 +29,7 @@ etcd:
 kubernetesVersion: v1.12.0
 networking:
  dnsDomain: cluster.local
-  podSubnet: 192.168.32.0/20
+  podSubnet: "192.168.32.0/20"
  serviceSubnet: 10.96.0.0/12
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd.yaml
@ -21,6 +21,7 @@ apiServerExtraArgs:
  enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
  local:
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/crio-options-gates.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/crio-options-gates.yaml
@ -29,6 +29,7 @@ schedulerExtraArgs:
  scheduler-name: "mini-scheduler"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
  local:
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/crio.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/crio.yaml
@ -21,6 +21,7 @@ apiServerExtraArgs:
  enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
  local:
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/default.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/default.yaml
@ -21,6 +21,7 @@ apiServerExtraArgs:
  enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
  local:
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/dns.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/dns.yaml
@ -21,6 +21,7 @@ apiServerExtraArgs:
  enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
  local:
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/image-repository.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/image-repository.yaml
@ -22,6 +22,7 @@ apiServerExtraArgs:
  enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
  local:
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/options.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/options.yaml
@ -26,6 +26,7 @@ schedulerExtraArgs:
  scheduler-name: "mini-scheduler"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
  local:
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd-api-port.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd-api-port.yaml
@ -21,6 +21,7 @@ apiServerExtraArgs:
  enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:12345
 etcd:
  local:
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd-pod-network-cidr.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd-pod-network-cidr.yaml
@ -21,6 +21,7 @@ apiServerExtraArgs:
  enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
  local:
@ -28,7 +29,7 @@ etcd:
 kubernetesVersion: v1.13.0
 networking:
  dnsDomain: cluster.local
-  podSubnet: 192.168.32.0/20
+  podSubnet: "192.168.32.0/20"
  serviceSubnet: 10.96.0.0/12
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd.yaml
@ -21,6 +21,7 @@ apiServerExtraArgs:
  enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
  local:
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/crio-options-gates.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/crio-options-gates.yaml
@ -29,6 +29,7 @@ schedulerExtraArgs:
  scheduler-name: "mini-scheduler"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
  local:
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/crio.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/crio.yaml
@ -21,6 +21,7 @@ apiServerExtraArgs:
  enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
  local:
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/default.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/default.yaml
@ -21,6 +21,7 @@ apiServerExtraArgs:
  enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
  local:
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/dns.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/dns.yaml
@ -21,6 +21,7 @@ apiServerExtraArgs:
  enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
  local:
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/image-repository.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/image-repository.yaml
@ -22,6 +22,7 @@ apiServerExtraArgs:
  enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
  local:
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/options.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/options.yaml
@ -26,6 +26,7 @@ schedulerExtraArgs:
  scheduler-name: "mini-scheduler"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
  local:
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd-api-port.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd-api-port.yaml
@ -22,12 +22,16 @@ apiServer:
    enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:12345
 dns:
  type: CoreDNS
 etcd:
  local:
    dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.14.0
 networking:
  dnsDomain: cluster.local
@ -41,3 +45,7 @@ evictionHard:
  nodefs.available: "0%"
  nodefs.inodesFree: "0%"
  imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd-pod-network-cidr.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd-pod-network-cidr.yaml
@ -22,16 +22,20 @@ apiServer:
    enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 dns:
  type: CoreDNS
 etcd:
  local:
    dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.14.0
 networking:
  dnsDomain: cluster.local
-  podSubnet: ""
+  podSubnet: "192.168.32.0/20"
  serviceSubnet: 10.96.0.0/12
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
@ -41,3 +45,7 @@ evictionHard:
  nodefs.available: "0%"
  nodefs.inodesFree: "0%"
  imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd.yaml
@ -22,12 +22,16 @@ apiServer:
    enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 dns:
  type: CoreDNS
 etcd:
  local:
    dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.14.0
 networking:
  dnsDomain: cluster.local
@ -41,3 +45,7 @@ evictionHard:
  nodefs.available: "0%"
  nodefs.inodesFree: "0%"
  imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/crio-options-gates.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/crio-options-gates.yaml
@ -32,12 +32,16 @@ scheduler:
    scheduler-name: "mini-scheduler"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 dns:
  type: CoreDNS
 etcd:
  local:
    dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.14.0
 networking:
  dnsDomain: cluster.local
@ -51,3 +55,7 @@ evictionHard:
  nodefs.available: "0%"
  nodefs.inodesFree: "0%"
  imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/crio.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/crio.yaml
@ -22,12 +22,16 @@ apiServer:
    enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 dns:
  type: CoreDNS
 etcd:
  local:
    dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.14.0
 networking:
  dnsDomain: cluster.local
@ -41,3 +45,7 @@ evictionHard:
  nodefs.available: "0%"
  nodefs.inodesFree: "0%"
  imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/default.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/default.yaml
@ -22,12 +22,16 @@ apiServer:
    enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 dns:
  type: CoreDNS
 etcd:
  local:
    dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.14.0
 networking:
  dnsDomain: cluster.local
@ -41,3 +45,7 @@ evictionHard:
  nodefs.available: "0%"
  nodefs.inodesFree: "0%"
  imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/dns.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/dns.yaml
@ -22,12 +22,16 @@ apiServer:
    enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 dns:
  type: CoreDNS
 etcd:
  local:
    dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.14.0
 networking:
  dnsDomain: 1.1.1.1
@ -41,3 +45,7 @@ evictionHard:
  nodefs.available: "0%"
  nodefs.inodesFree: "0%"
  imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/image-repository.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/image-repository.yaml
@ -23,12 +23,16 @@ apiServer:
    enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 dns:
  type: CoreDNS
 etcd:
  local:
    dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.14.0
 networking:
  dnsDomain: cluster.local
@ -42,3 +46,7 @@ evictionHard:
  nodefs.available: "0%"
  nodefs.inodesFree: "0%"
  imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/options.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/options.yaml
@ -29,12 +29,16 @@ scheduler:
    scheduler-name: "mini-scheduler"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 dns:
  type: CoreDNS
 etcd:
  local:
    dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.14.0
 networking:
  dnsDomain: cluster.local
@ -48,3 +52,7 @@ evictionHard:
  nodefs.available: "0%"
  nodefs.inodesFree: "0%"
  imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd-api-port.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd-api-port.yaml
@ -22,12 +22,16 @@ apiServer:
    enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:12345
 dns:
  type: CoreDNS
 etcd:
  local:
    dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.15.0
 networking:
  dnsDomain: cluster.local
@ -41,3 +45,7 @@ evictionHard:
  nodefs.available: "0%"
  nodefs.inodesFree: "0%"
  imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd-pod-network-cidr.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd-pod-network-cidr.yaml
@ -22,16 +22,20 @@ apiServer:
    enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 dns:
  type: CoreDNS
 etcd:
  local:
    dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.15.0
 networking:
  dnsDomain: cluster.local
-  podSubnet: ""
+  podSubnet: "192.168.32.0/20"
  serviceSubnet: 10.96.0.0/12
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
@ -41,3 +45,7 @@ evictionHard:
  nodefs.available: "0%"
  nodefs.inodesFree: "0%"
  imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
--- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd.yaml
+++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd.yaml
@ -22,12 +22,16 @@ apiServer:
    enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 dns:
  type: CoreDNS
 etcd:
  local:
    dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      listen-metrics-urls: http://127.0.0.1:2381,http://1.1.1.1:2381
 kubernetesVersion: v1.15.0
 networking:
  dnsDomain: cluster.local
@ -41,3 +45,7 @@ evictionHard:
  nodefs.available: "0%"
  nodefs.inodesFree: "0%"
  imagefs.available: "0%"
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+metricsBindAddress: 1.1.1.1:10249
--- a/Show More
+++ b/Show More