Rebased on master

CHANGELOG.md

@@ -1,5 +1,29 @@
 # Release Notes
 
+
+## Version 1.6.2  - 2019-12-19
+
+* Offline: always transfer image if lookup fails, always download drivers [#6111](https://github.com/kubernetes/minikube/pull/6111)
+* Update ingress-dns addon [#6091](https://github.com/kubernetes/minikube/pull/6091)
+* Fix update-context to use KUBECONFIG when the env is set [#6090](https://github.com/kubernetes/minikube/pull/6090)
+* Fixed IPv6 format for SSH [#6110](https://github.com/kubernetes/minikube/pull/6110)
+* Add hyperkit version check whether user's hyperkit is newer [#5833](https://github.com/kubernetes/minikube/pull/5833)
+* start: Remove create/delete retry loop [#6129](https://github.com/kubernetes/minikube/pull/6129)
+* Change error text to encourage better issue reports [#6121](https://github.com/kubernetes/minikube/pull/6121)
+
+Huge thank you for this release towards our contributors: 
+- Anukul Sangwan
+- Aresforchina
+- Curtis Carter
+- Kenta Iso
+- Medya Ghazizadeh
+- Sharif Elgamal
+- Thomas Strömberg
+- Zhou Hao
+- priyawadhwa
+- tstromberg
+
+
 ## Version 1.6.1  - 2019-12-11
 
 A special bugfix release to fix a Windows regression:

Makefile

@@ -15,7 +15,7 @@
 # Bump these on release - and please check ISO_VERSION for correctness.
 VERSION_MAJOR ?= 1
 VERSION_MINOR ?= 6
-VERSION_BUILD ?= 1
+VERSION_BUILD ?= 2
 RAW_VERSION=$(VERSION_MAJOR).$(VERSION_MINOR).${VERSION_BUILD}
 VERSION ?= v$(RAW_VERSION)
 
@@ -29,7 +29,7 @@ RPM_VERSION ?= $(DEB_VERSION)
 GO_VERSION ?= 1.13.4
 
 INSTALL_SIZE ?= $(shell du out/minikube-windows-amd64.exe | cut -f1)
-BUILDROOT_BRANCH ?= 2019.02.7
+BUILDROOT_BRANCH ?= 2019.02.8
 REGISTRY?=gcr.io/k8s-minikube
 
 # Get git commit id
@@ -49,7 +49,7 @@ MINIKUBE_BUCKET ?= minikube/releases
 MINIKUBE_UPLOAD_LOCATION := gs://${MINIKUBE_BUCKET}
 MINIKUBE_RELEASES_URL=https://github.com/kubernetes/minikube/releases/download
 
-KERNEL_VERSION ?= 4.19.81
+KERNEL_VERSION ?= 4.19.88
 # latest from https://github.com/golangci/golangci-lint/releases
 GOLINT_VERSION ?= v1.21.0
 # Limit number of default jobs, to avoid the CI builds running out of memory
@@ -60,7 +60,8 @@ GOLINT_GOGC ?= 100
 GOLINT_OPTIONS = --timeout 4m \
 	  --build-tags "${MINIKUBE_INTEGRATION_BUILD_TAGS}" \
 	  --enable goimports,gocritic,golint,gocyclo,misspell,nakedret,stylecheck,unconvert,unparam,dogsled \
-	  --exclude 'variable on range scope.*in function literal|ifElseChain'
+	  --exclude 'variable on range scope.*in function literal|ifElseChain' \
+	  --skip-files "pkg/minikube/translate/translations.go|pkg/minikube/assets/assets.go"
 
 # Major version of gvisor image. Increment when there are breaking changes.
 GVISOR_IMAGE_VERSION ?= 2
@@ -472,7 +473,7 @@ $(ISO_BUILD_IMAGE): deploy/iso/minikube-iso/Dockerfile
 	@echo "$(@) successfully built"
 
 out/storage-provisioner:
-	GOOS=linux go build -o $@ -ldflags=$(PROVISIONER_LDFLAGS) cmd/storage-provisioner/main.go
+	CGO_ENABLED=0 GOOS=linux go build -o $@ -ldflags=$(PROVISIONER_LDFLAGS) cmd/storage-provisioner/main.go
 
 .PHONY: storage-provisioner-image
 storage-provisioner-image: out/storage-provisioner ## Build storage-provisioner docker image
@@ -482,6 +483,13 @@ else
 	docker build -t $(REGISTRY)/storage-provisioner-$(GOARCH):$(STORAGE_PROVISIONER_TAG) -f deploy/storage-provisioner/Dockerfile-$(GOARCH) .
 endif
 
+.PHONY: kic-base-image
+kic-base-image: ## builds the base image used for kic.
+	docker rmi -f $(REGISTRY)/kicbase:v0.0.1-snapshot || true
+	docker build -f ./hack/images/kicbase.Dockerfile -t $(REGISTRY)/kicbase:v0.0.1-snapshot  --build-arg COMMIT_SHA=${VERSION}-$(COMMIT)  .
+
+
+
 .PHONY: push-storage-provisioner-image
 push-storage-provisioner-image: storage-provisioner-image ## Push storage-provisioner docker image using gcloud
 ifeq ($(GOARCH),amd64)
@@ -590,6 +598,7 @@ out/mkcmp:
 out/performance-monitor:
 	GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o $@ cmd/performance/monitor/monitor.go
 
+
 .PHONY: help
 help:
 	@printf "\033[1mAvailable targets for minikube ${VERSION}\033[21m\n"

README.md

@@ -23,7 +23,7 @@ minikube runs the latest stable release of Kubernetes, with support for standard
 * Multi-cluster - using `minikube start -p <name>`
 * NodePorts - using `minikube service`
 * [Persistent Volumes](https://minikube.sigs.k8s.io/docs/reference/persistent_volumes/)
-* Ingress
+* [Ingress](https://kubernetes.io/docs/tasks/access-application-cluster/ingress-minikube/)
 * [Dashboard](https://minikube.sigs.k8s.io/docs/tasks/dashboard/) - `minikube dashboard`
 * [Container runtimes](https://minikube.sigs.k8s.io/docs/reference/runtimes/) - `start --container-runtime`
 * [Configure apiserver and kubelet options](https://minikube.sigs.k8s.io/docs/reference/configuration/kubernetes/) via command-line flags

cmd/minikube/cmd/cache.go

@@ -22,6 +22,7 @@ import (
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/exit"
+	"k8s.io/minikube/pkg/minikube/image"
 	"k8s.io/minikube/pkg/minikube/localpath"
 	"k8s.io/minikube/pkg/minikube/machine"
 )
@@ -64,12 +65,25 @@ var deleteCacheCmd = &cobra.Command{
 			exit.WithError("Failed to delete images from config", err)
 		}
 		// Delete images from cache/images directory
-		if err := machine.DeleteFromImageCacheDir(args); err != nil {
+		if err := image.DeleteFromCacheDir(args); err != nil {
 			exit.WithError("Failed to delete images", err)
 		}
 	},
 }
 
+// reloadCacheCmd represents the cache reload command
+var reloadCacheCmd = &cobra.Command{
+	Use:   "reload",
+	Short: "reload cached images.",
+	Long:  "reloads images previously added using the 'cache add' subcommand",
+	Run: func(cmd *cobra.Command, args []string) {
+		err := cacheAndLoadImagesInConfig()
+		if err != nil {
+			exit.WithError("Failed to reload cached images", err)
+		}
+	},
+}
+
 func imagesInConfigFile() ([]string, error) {
 	configFile, err := config.ReadConfig(localpath.ConfigFile)
 	if err != nil {
@@ -85,8 +99,9 @@ func imagesInConfigFile() ([]string, error) {
 	return []string{}, nil
 }
 
-// CacheImagesInConfigFile caches the images currently in the config file (minikube start)
-func CacheImagesInConfigFile() error {
+// saveImagesToTarFromConfig saves images to tar in cache which specified in config file.
+// currently only used by download-only option
+func saveImagesToTarFromConfig() error {
 	images, err := imagesInConfigFile()
 	if err != nil {
 		return err
@@ -94,11 +109,12 @@ func CacheImagesInConfigFile() error {
 	if len(images) == 0 {
 		return nil
 	}
-	return machine.CacheImages(images, constants.ImageCacheDir)
+	return image.SaveToDir(images, constants.ImageCacheDir)
 }
 
-// loadCachedImagesInConfigFile loads the images currently in the config file (minikube start)
-func loadCachedImagesInConfigFile() error {
+// cacheAndLoadImagesInConfig loads the images currently in the config file
+// called by 'start' and 'cache reload' commands.
+func cacheAndLoadImagesInConfig() error {
 	images, err := imagesInConfigFile()
 	if err != nil {
 		return err
@@ -112,4 +128,5 @@ func loadCachedImagesInConfigFile() error {
 func init() {
 	cacheCmd.AddCommand(addCacheCmd)
 	cacheCmd.AddCommand(deleteCacheCmd)
+	cacheCmd.AddCommand(reloadCacheCmd)
 }

cmd/minikube/cmd/config/addons_list.go

@@ -22,17 +22,17 @@ import (
 	"os"
 	"sort"
 	"strings"
-	"text/template"
 
+	"github.com/golang/glog"
+	"github.com/olekukonko/tablewriter"
 	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
 	"k8s.io/minikube/pkg/minikube/assets"
+	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/exit"
 	"k8s.io/minikube/pkg/minikube/out"
 )
 
-const defaultAddonListFormat = "- {{.AddonName}}: {{.AddonStatus}}\n"
-
-var addonListFormat string
 var addonListOutput string
 
 // AddonListTemplate represents the addon list template
@@ -50,10 +50,6 @@ var addonsListCmd = &cobra.Command{
 			exit.UsageT("usage: minikube addons list")
 		}
 
-		if addonListOutput != "list" && addonListFormat != defaultAddonListFormat {
-			exit.UsageT("Cannot use both --output and --format options")
-		}
-
 		switch strings.ToLower(addonListOutput) {
 		case "list":
 			printAddonsList()
@@ -66,14 +62,6 @@ var addonsListCmd = &cobra.Command{
 }
 
 func init() {
-	addonsListCmd.Flags().StringVarP(
-		&addonListFormat,
-		"format",
-		"f",
-		defaultAddonListFormat,
-		`Go template format string for the addon list output.  The format for Go templates can be found here: https://golang.org/pkg/text/template/
-For the list of accessible variables for the template, see the struct values here: https://godoc.org/k8s.io/minikube/cmd/minikube/cmd/config#AddonListTemplate`)
-
 	addonsListCmd.Flags().StringVarP(
 		&addonListOutput,
 		"output",
@@ -84,6 +72,13 @@ For the list of accessible variables for the template, see the struct values her
 	AddonsCmd.AddCommand(addonsListCmd)
 }
 
+var iconFromStatus = func(addonStatus bool) string {
+	if addonStatus {
+		return "✅"
+	}
+	return "   " // because emoji indentation is different
+}
+
 var stringFromStatus = func(addonStatus bool) string {
 	if addonStatus {
 		return "enabled"
@@ -97,6 +92,13 @@ var printAddonsList = func() {
 		addonNames = append(addonNames, addonName)
 	}
 	sort.Strings(addonNames)
+	var tData [][]string
+	table := tablewriter.NewWriter(os.Stdout)
+	table.SetHeader([]string{"Addon Name", "Profile", "Status"})
+	table.SetAutoFormatHeaders(true)
+	table.SetBorders(tablewriter.Border{Left: true, Top: true, Right: true, Bottom: true})
+	table.SetCenterSeparator("|")
+	pName := viper.GetString(config.MachineProfile)
 
 	for _, addonName := range addonNames {
 		addonBundle := assets.Addons[addonName]
@@ -104,20 +106,25 @@ var printAddonsList = func() {
 		if err != nil {
 			exit.WithError("Error getting addons status", err)
 		}
-		tmpl, err := template.New("list").Parse(addonListFormat)
-		if err != nil {
-			exit.WithError("Error creating list template", err)
-		}
-		listTmplt := AddonListTemplate{addonName, stringFromStatus(addonStatus)}
-		err = tmpl.Execute(os.Stdout, listTmplt)
-		if err != nil {
-			exit.WithError("Error executing list template", err)
-		}
+		tData = append(tData, []string{addonName, pName, fmt.Sprintf("%s %s", stringFromStatus(addonStatus), iconFromStatus(addonStatus))})
 	}
+
+	table.AppendBulk(tData)
+	table.Render()
+
+	v, _, err := config.ListProfiles()
+	if err != nil {
+		glog.Infof("error getting list of porfiles: %v", err)
+	}
+	if len(v) > 1 {
+		out.T(out.Tip, "To see addons list for other profiles use: `minikube addons -p name list`")
+	}
+
 }
 
 var printAddonsJSON = func() {
 	addonNames := make([]string, 0, len(assets.Addons))
+	pName := viper.GetString(config.MachineProfile)
 	for addonName := range assets.Addons {
 		addonNames = append(addonNames, addonName)
 	}
@@ -134,7 +141,8 @@ var printAddonsJSON = func() {
 		}
 
 		addonsMap[addonName] = map[string]interface{}{
-			"Status": stringFromStatus(addonStatus),
+			"Status":  stringFromStatus(addonStatus),
+			"Profile": pName,
 		}
 	}
 	jsonString, _ := json.Marshal(addonsMap)

cmd/minikube/cmd/config/config.go

@@ -142,111 +142,8 @@ var settings = []Setting{
 		set:  SetBool,
 	},
 	{
-		name:        "dashboard",
-		set:         SetBool,
-		validations: []setFn{IsValidAddon},
-		callbacks:   []setFn{EnableOrDisableAddon},
-	},
-	{
-		name:        "addon-manager",
-		set:         SetBool,
-		validations: []setFn{IsValidAddon},
-		callbacks:   []setFn{EnableOrDisableAddon},
-	},
-	{
-		name:        "default-storageclass",
-		set:         SetBool,
-		validations: []setFn{IsValidAddon},
-		callbacks:   []setFn{EnableOrDisableStorageClasses},
-	},
-	{
-		name:        "efk",
-		set:         SetBool,
-		validations: []setFn{IsValidAddon},
-		callbacks:   []setFn{EnableOrDisableAddon},
-	},
-	{
-		name:        "ingress",
-		set:         SetBool,
-		validations: []setFn{IsValidAddon},
-		callbacks:   []setFn{EnableOrDisableAddon},
-	},
-	{
-		name:        "insecure-registry",
-		set:         SetBool,
-		validations: []setFn{IsValidAddon},
-		callbacks:   []setFn{EnableOrDisableAddon},
-	},
-	{
-		name:        "registry",
-		set:         SetBool,
-		validations: []setFn{IsValidAddon},
-		callbacks:   []setFn{EnableOrDisableAddon},
-	},
-	{
-		name:        "registry-creds",
-		set:         SetBool,
-		validations: []setFn{IsValidAddon},
-		callbacks:   []setFn{EnableOrDisableAddon},
-	},
-	{
-		name:        "freshpod",
-		set:         SetBool,
-		validations: []setFn{IsValidAddon},
-		callbacks:   []setFn{EnableOrDisableAddon},
-	},
-	{
-		name:        "storage-provisioner",
-		set:         SetBool,
-		validations: []setFn{IsValidAddon},
-		callbacks:   []setFn{EnableOrDisableAddon},
-	},
-	{
-		name:        "storage-provisioner-gluster",
-		set:         SetBool,
-		validations: []setFn{IsValidAddon},
-		callbacks:   []setFn{EnableOrDisableStorageClasses},
-	},
-	{
-		name:        "metrics-server",
-		set:         SetBool,
-		validations: []setFn{IsValidAddon},
-		callbacks:   []setFn{EnableOrDisableAddon},
-	},
-	{
-		name:        "nvidia-driver-installer",
-		set:         SetBool,
-		validations: []setFn{IsValidAddon},
-		callbacks:   []setFn{EnableOrDisableAddon},
-	},
-	{
-		name:        "nvidia-gpu-device-plugin",
-		set:         SetBool,
-		validations: []setFn{IsValidAddon},
-		callbacks:   []setFn{EnableOrDisableAddon},
-	},
-	{
-		name:        "logviewer",
-		set:         SetBool,
-		validations: []setFn{IsValidAddon},
-	},
-	{
-		name:        "gvisor",
-		set:         SetBool,
-		validations: []setFn{IsValidAddon, IsContainerdRuntime},
-		callbacks:   []setFn{EnableOrDisableAddon},
-	},
-	{
-		name:        "helm-tiller",
-		set:         SetBool,
-		validations: []setFn{IsValidAddon},
-		callbacks:   []setFn{EnableOrDisableAddon},
-	},
-	{
-		name:        "ingress-dns",
-		set:         SetBool,
-		validations: []setFn{IsValidAddon},
-		callbacks:   []setFn{EnableOrDisableAddon},
+		name: "insecure-registry",
+		set:  SetString,
 	},
 	{
 		name: "hyperv-virtual-switch",

cmd/minikube/cmd/config/disable.go

@@ -18,6 +18,9 @@ package config
 
 import (
 	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"k8s.io/minikube/pkg/addons"
+	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/exit"
 	"k8s.io/minikube/pkg/minikube/out"
 )
@@ -32,7 +35,7 @@ var addonsDisableCmd = &cobra.Command{
 		}
 
 		addon := args[0]
-		err := Set(addon, "false")
+		err := addons.Set(addon, "false", viper.GetString(config.MachineProfile))
 		if err != nil {
 			exit.WithError("disable failed", err)
 		}

cmd/minikube/cmd/config/disable_test.go

@@ -1,39 +0,0 @@
-/*
-Copyright 2016 The Kubernetes Authors All rights reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package config
-
-import (
-	"testing"
-
-	"gotest.tools/assert"
-	pkgConfig "k8s.io/minikube/pkg/minikube/config"
-	"k8s.io/minikube/pkg/minikube/localpath"
-)
-
-func TestDisableUnknownAddon(t *testing.T) {
-	if err := Set("InvalidAddon", "false"); err == nil {
-		t.Fatalf("Disable did not return error for unknown addon")
-	}
-}
-
-func TestDisableAddon(t *testing.T) {
-	if err := Set("dashboard", "false"); err != nil {
-		t.Fatalf("Disable returned unexpected error: " + err.Error())
-	}
-	config, _ := pkgConfig.ReadConfig(localpath.ConfigFile)
-	assert.Equal(t, config["dashboard"], false)
-}

cmd/minikube/cmd/config/enable.go

@@ -18,6 +18,9 @@ package config
 
 import (
 	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"k8s.io/minikube/pkg/addons"
+	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/exit"
 	"k8s.io/minikube/pkg/minikube/out"
 )
@@ -32,7 +35,7 @@ var addonsEnableCmd = &cobra.Command{
 		}
 
 		addon := args[0]
-		err := Set(addon, "true")
+		err := addons.Set(addon, "true", viper.GetString(config.MachineProfile))
 		if err != nil {
 			exit.WithError("enable failed", err)
 		}

cmd/minikube/cmd/config/get_test.go

@@ -0,0 +1,41 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package config
+
+import "testing"
+
+func TestGetNotFound(t *testing.T) {
+	_, err := Get("nonexistent")
+	if err == nil {
+		t.Fatalf("Get did not return error for unknown property")
+	}
+}
+
+func TestGetOK(t *testing.T) {
+	name := "vm-driver"
+	err := Set(name, "virtualbox")
+	if err != nil {
+		t.Fatalf("Set returned error for property %s", name)
+	}
+	val, err := Get(name)
+	if err != nil {
+		t.Fatalf("Get returned error for property %s", name)
+	}
+	if val != "virtualbox" {
+		t.Fatalf("Get returned %s, expected virtualbox", val)
+	}
+}

cmd/minikube/cmd/config/profile_list.go

@@ -59,7 +59,6 @@ var profileListCmd = &cobra.Command{
 var printProfilesTable = func() {
 
 	var validData [][]string
-
 	table := tablewriter.NewWriter(os.Stdout)
 	table.SetHeader([]string{"Profile", "VM Driver", "NodeIP", "Node Port", "Kubernetes Version", "Status"})
 	table.SetAutoFormatHeaders(false)

cmd/minikube/cmd/config/set_test.go

@@ -24,3 +24,24 @@ func TestNotFound(t *testing.T) {
 		t.Fatalf("Set did not return error for unknown property")
 	}
 }
+
+func TestSetNotAllowed(t *testing.T) {
+	err := Set("vm-driver", "123456")
+	if err == nil || err.Error() != "[driver \"123456\" is not supported]" {
+		t.Fatalf("Set did not return error for unallowed value")
+	}
+}
+
+func TestSetOK(t *testing.T) {
+	err := Set("vm-driver", "virtualbox")
+	if err != nil {
+		t.Fatalf("Set returned error for valid property value")
+	}
+	val, err := Get("vm-driver")
+	if err != nil {
+		t.Fatalf("Get returned error for valid property")
+	}
+	if val != "virtualbox" {
+		t.Fatalf("Get returned %s, expected \"virtualbox\"", val)
+	}
+}

cmd/minikube/cmd/config/util.go

@@ -18,26 +18,13 @@ package config
 
 import (
 	"fmt"
-	"os"
 	"strconv"
 	"strings"
 
-	"github.com/pkg/errors"
-	"github.com/spf13/viper"
-	"k8s.io/minikube/pkg/minikube/addons"
-	"k8s.io/minikube/pkg/minikube/assets"
-	"k8s.io/minikube/pkg/minikube/cluster"
-	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/minikube/config"
-	"k8s.io/minikube/pkg/minikube/exit"
-	"k8s.io/minikube/pkg/minikube/machine"
 	"k8s.io/minikube/pkg/minikube/out"
-	"k8s.io/minikube/pkg/minikube/storageclass"
 )
 
-// defaultStorageClassProvisioner is the name of the default storage class provisioner
-const defaultStorageClassProvisioner = "standard"
-
 // Runs all the validation or callback functions and collects errors
 func run(name string, value string, fns []setFn) error {
 	var errors []error
@@ -107,159 +94,6 @@ func SetBool(m config.MinikubeConfig, name string, val string) error {
 	return nil
 }
 
-// EnableOrDisableAddon updates addon status executing any commands necessary
-func EnableOrDisableAddon(name string, val string) error {
-	enable, err := strconv.ParseBool(val)
-	if err != nil {
-		return errors.Wrapf(err, "parsing bool: %s", name)
-	}
-	addon := assets.Addons[name]
-
-	// check addon status before enabling/disabling it
-	alreadySet, err := isAddonAlreadySet(addon, enable)
-	if err != nil {
-		out.ErrT(out.Conflict, "{{.error}}", out.V{"error": err})
-		return err
-	}
-	//if addon is already enabled or disabled, do nothing
-	if alreadySet {
-		return nil
-	}
-
-	// TODO(r2d4): config package should not reference API, pull this out
-	api, err := machine.NewAPIClient()
-	if err != nil {
-		return errors.Wrap(err, "machine client")
-	}
-	defer api.Close()
-
-	//if minikube is not running, we return and simply update the value in the addon
-	//config and rewrite the file
-	if !cluster.IsMinikubeRunning(api) {
-		return nil
-	}
-
-	cfg, err := config.Load(viper.GetString(config.MachineProfile))
-	if err != nil && !os.IsNotExist(err) {
-		exit.WithCodeT(exit.Data, "Unable to load config: {{.error}}", out.V{"error": err})
-	}
-
-	host, err := cluster.CheckIfHostExistsAndLoad(api, cfg.Name)
-	if err != nil {
-		return errors.Wrap(err, "getting host")
-	}
-
-	cmd, err := machine.CommandRunner(host)
-	if err != nil {
-		return errors.Wrap(err, "command runner")
-	}
-
-	data := assets.GenerateTemplateData(cfg.KubernetesConfig)
-	return enableOrDisableAddonInternal(addon, cmd, data, enable)
-}
-
-func isAddonAlreadySet(addon *assets.Addon, enable bool) (bool, error) {
-	addonStatus, err := addon.IsEnabled()
-
-	if err != nil {
-		return false, errors.Wrap(err, "get the addon status")
-	}
-
-	if addonStatus && enable {
-		return true, nil
-	} else if !addonStatus && !enable {
-		return true, nil
-	}
-
-	return false, nil
-}
-
-func isAddonManagerEnabled() (bool, error) {
-	addonManager := assets.Addons["addon-manager"]
-	return addonManager.IsEnabled()
-}
-
-func enableOrDisableAddonInternal(addon *assets.Addon, cmd command.Runner, data interface{}, enable bool) error {
-	var err error
-
-	if enable {
-		for _, addon := range addon.Assets {
-			var addonFile assets.CopyableFile
-			if addon.IsTemplate() {
-				addonFile, err = addon.Evaluate(data)
-				if err != nil {
-					return errors.Wrapf(err, "evaluate bundled addon %s asset", addon.GetAssetName())
-				}
-
-			} else {
-				addonFile = addon
-			}
-			if err := cmd.Copy(addonFile); err != nil {
-				return errors.Wrapf(err, "enabling addon %s", addon.AssetName)
-			}
-		}
-	} else {
-		for _, addon := range addon.Assets {
-			var addonFile assets.CopyableFile
-			if addon.IsTemplate() {
-				addonFile, err = addon.Evaluate(data)
-				if err != nil {
-					return errors.Wrapf(err, "evaluate bundled addon %s asset", addon.GetAssetName())
-				}
-
-			} else {
-				addonFile = addon
-			}
-			if err := cmd.Remove(addonFile); err != nil {
-				return errors.Wrapf(err, "disabling addon %s", addon.AssetName)
-			}
-		}
-	}
-	// If addon manager is enabled, return as it will handle whatever changes have been made.
-	// If not, reconcile addons ourselves.
-	enabled, err := isAddonManagerEnabled()
-	if err != nil {
-		return errors.Wrapf(err, "checking if addon manager is enabled")
-	}
-	if enabled {
-		return nil
-	}
-	return addons.Reconcile(cmd)
-}
-
-// EnableOrDisableStorageClasses enables or disables storage classes
-func EnableOrDisableStorageClasses(name, val string) error {
-	enable, err := strconv.ParseBool(val)
-	if err != nil {
-		return errors.Wrap(err, "Error parsing boolean")
-	}
-
-	class := defaultStorageClassProvisioner
-	if name == "storage-provisioner-gluster" {
-		class = "glusterfile"
-	}
-	storagev1, err := storageclass.GetStoragev1()
-	if err != nil {
-		return errors.Wrapf(err, "Error getting storagev1 interface %v ", err)
-	}
-
-	if enable {
-		// Only StorageClass for 'name' should be marked as default
-		err = storageclass.SetDefaultStorageClass(storagev1, class)
-		if err != nil {
-			return errors.Wrapf(err, "Error making %s the default storage class", class)
-		}
-	} else {
-		// Unset the StorageClass as default
-		err := storageclass.DisableDefaultStorageClass(storagev1, class)
-		if err != nil {
-			return errors.Wrapf(err, "Error disabling %s as the default storage class", class)
-		}
-	}
-
-	return EnableOrDisableAddon(name, val)
-}
-
 // ErrValidateProfile Error to validate profile
 type ErrValidateProfile struct {
 	Name string

cmd/minikube/cmd/config/util_test.go

@@ -20,7 +20,6 @@ import (
 	"fmt"
 	"testing"
 
-	"k8s.io/minikube/pkg/minikube/assets"
 	pkgConfig "k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/driver"
 )
@@ -83,37 +82,6 @@ func TestSetBool(t *testing.T) {
 	}
 }
 
-func TestIsAddonAlreadySet(t *testing.T) {
-	testCases := []struct {
-		addonName string
-	}{
-		{
-			addonName: "ingress",
-		},
-
-		{
-			addonName: "registry",
-		},
-	}
-
-	for _, test := range testCases {
-		addon := assets.Addons[test.addonName]
-		addonStatus, _ := addon.IsEnabled()
-
-		alreadySet, err := isAddonAlreadySet(addon, addonStatus)
-		if !alreadySet {
-			if addonStatus {
-				t.Errorf("Did not get expected status, \n\n expected %+v already enabled", test.addonName)
-			} else {
-				t.Errorf("Did not get expected status, \n\n expected %+v already disabled", test.addonName)
-			}
-		}
-		if err != nil {
-			t.Errorf("Got unexpected error: %+v", err)
-		}
-	}
-}
-
 func TestValidateProfile(t *testing.T) {
 	testCases := []struct {
 		profileName string

cmd/minikube/cmd/config/validations.go

@@ -25,25 +25,11 @@ import (
 	"strings"
 
 	units "github.com/docker/go-units"
-	"github.com/pkg/errors"
-	"github.com/spf13/viper"
-	"k8s.io/minikube/pkg/minikube/assets"
-	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/cruntime"
 	"k8s.io/minikube/pkg/minikube/driver"
 	"k8s.io/minikube/pkg/minikube/out"
 )
 
-// containerdOnlyMsg is the message shown when a containerd-only addon is enabled
-const containerdOnlyAddonMsg = `
-This addon can only be enabled with the containerd runtime backend. To enable this backend, please first stop minikube with:
-
-minikube stop
-
-and then start minikube again with the following flags:
-
-minikube start --container-runtime=containerd --docker-opt containerd=/var/run/containerd/containerd.sock`
-
 // IsValidDriver checks if a driver is supported
 func IsValidDriver(string, name string) error {
 	if driver.Supported(name) {
@@ -140,14 +126,6 @@ func IsValidPath(name string, path string) error {
 	return nil
 }
 
-// IsValidAddon checks if a string is a valid addon
-func IsValidAddon(name string, val string) error {
-	if _, ok := assets.Addons[name]; ok {
-		return nil
-	}
-	return errors.Errorf("Cannot enable/disable invalid addon %s", name)
-}
-
 // IsValidRuntime checks if a string is a valid runtime
 func IsValidRuntime(name string, runtime string) error {
 	_, err := cruntime.New(cruntime.Config{Type: runtime})
@@ -156,20 +134,3 @@ func IsValidRuntime(name string, runtime string) error {
 	}
 	return nil
 }
-
-// IsContainerdRuntime is a validator which returns an error if the current runtime is not containerd
-func IsContainerdRuntime(_, _ string) error {
-	config, err := config.Load(viper.GetString(config.MachineProfile))
-	if err != nil {
-		return fmt.Errorf("config.Load: %v", err)
-	}
-	r, err := cruntime.New(cruntime.Config{Type: config.KubernetesConfig.ContainerRuntime})
-	if err != nil {
-		return err
-	}
-	_, ok := r.(*cruntime.Containerd)
-	if !ok {
-		return fmt.Errorf(containerdOnlyAddonMsg)
-	}
-	return nil
-}

cmd/minikube/cmd/dashboard.go

@@ -33,7 +33,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	configcmd "k8s.io/minikube/cmd/minikube/cmd/config"
+	pkgaddons "k8s.io/minikube/pkg/addons"
 	"k8s.io/minikube/pkg/minikube/assets"
 	"k8s.io/minikube/pkg/minikube/cluster"
 	"k8s.io/minikube/pkg/minikube/config"
@@ -106,7 +106,7 @@ var dashboardCmd = &cobra.Command{
 			// Send status messages to stderr for folks re-using this output.
 			out.ErrT(out.Enabling, "Enabling dashboard ...")
 			// Enable the dashboard add-on
-			err = configcmd.Set("dashboard", "true")
+			err = pkgaddons.Set("dashboard", "true", viper.GetString(config.MachineProfile))
 			if err != nil {
 				exit.WithError("Unable to enable dashboard", err)
 			}

cmd/minikube/cmd/options.go

@@ -0,0 +1,68 @@
+/*
+Copyright 2019 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cmd
+
+import (
+	"bytes"
+	"fmt"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+
+	"k8s.io/minikube/pkg/minikube/out"
+)
+
+// optionsCmd represents the options command
+var optionsCmd = &cobra.Command{
+	Use:   "options",
+	Short: "Show a list of global command-line options (applies to all commands).",
+	Long:  "Show a list of global command-line options (applies to all commands).",
+	Run:   runOptions,
+}
+
+// runOptions handles the executes the flow of "minikube options"
+func runOptions(cmd *cobra.Command, args []string) {
+	out.String("The following options can be passed to any command:\n\n")
+	for _, flagName := range viperWhiteList {
+		f := pflag.Lookup(flagName)
+		out.String(flagUsage(f))
+	}
+}
+
+func flagUsage(flag *pflag.Flag) string {
+	x := new(bytes.Buffer)
+
+	if flag.Hidden {
+		return ""
+	}
+
+	format := "--%s=%s: %s\n"
+
+	if flag.Value.Type() == "string" {
+		format = "--%s='%s': %s\n"
+	}
+
+	if len(flag.Shorthand) > 0 {
+		format = "  -%s, " + format
+	} else {
+		format = "   %s   " + format
+	}
+
+	fmt.Fprintf(x, format, flag.Shorthand, flag.Name, flag.DefValue, flag.Usage)
+
+	return x.String()
+}

cmd/minikube/cmd/root.go

@@ -32,6 +32,7 @@ import (
 	"k8s.io/kubectl/pkg/util/templates"
 	configCmd "k8s.io/minikube/cmd/minikube/cmd/config"
 	"k8s.io/minikube/pkg/minikube/bootstrapper"
+	"k8s.io/minikube/pkg/minikube/bootstrapper/kicbs"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/kubeadm"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
@@ -53,9 +54,9 @@ var dirs = [...]string{
 }
 
 var viperWhiteList = []string{
-	"v",
 	"alsologtostderr",
 	"log_dir",
+	"v",
 }
 
 // RootCmd represents the base command when called without any subcommands
@@ -213,6 +214,7 @@ func init() {
 				logsCmd,
 				updateCheckCmd,
 				versionCmd,
+				optionsCmd,
 			},
 		},
 	}
@@ -267,11 +269,17 @@ func getClusterBootstrapper(api libmachine.API, bootstrapperName string) (bootst
 	var b bootstrapper.Bootstrapper
 	var err error
 	switch bootstrapperName {
-	case bootstrapper.BootstrapperTypeKubeadm:
-		b, err = kubeadm.NewKubeadmBootstrapper(api)
+	case bootstrapper.Kubeadm:
+		b, err = kubeadm.NewBootstrapper(api)
 		if err != nil {
-			return nil, errors.Wrap(err, "getting kubeadm bootstrapper")
+			return nil, errors.Wrap(err, "getting a new kubeadm bootstrapper")
 		}
+	case bootstrapper.KIC:
+		b, err = kicbs.NewBootstrapper(api)
+		if err != nil {
+			return nil, errors.Wrap(err, "getting a new kic bootstrapper")
+		}
+
 	default:
 		return nil, fmt.Errorf("unknown bootstrapper: %s", bootstrapperName)
 	}

cmd/minikube/cmd/start.go

@@ -46,9 +46,10 @@ import (
 	"github.com/spf13/viper"
 	"golang.org/x/sync/errgroup"
 	cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config"
+	pkgaddons "k8s.io/minikube/pkg/addons"
 	"k8s.io/minikube/pkg/minikube/bootstrapper"
+	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
-	"k8s.io/minikube/pkg/minikube/bootstrapper/kubeadm"
 	"k8s.io/minikube/pkg/minikube/cluster"
 	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/minikube/config"
@@ -115,6 +116,7 @@ const (
 	hostDNSResolver       = "host-dns-resolver"
 	waitUntilHealthy      = "wait"
 	force                 = "force"
+	dryRun                = "dry-run"
 	interactive           = "interactive"
 	waitTimeout           = "wait-timeout"
 	nativeSSH             = "native-ssh"
@@ -157,6 +159,7 @@ func initMinikubeFlags() {
 
 	startCmd.Flags().Bool(force, false, "Force minikube to perform possibly dangerous operations")
 	startCmd.Flags().Bool(interactive, true, "Allow user prompts for more information")
+	startCmd.Flags().Bool(dryRun, false, "dry-run mode. Validates configuration, but does does not mutate system state")
 
 	startCmd.Flags().Int(cpus, 2, "Number of CPUs allocated to the minikube VM.")
 	startCmd.Flags().String(memory, defaultMemorySize, "Amount of RAM allocated to the minikube VM (format: <number>[<unit>], where unit = b, k, m or g).")
@@ -186,7 +189,7 @@ func initKubernetesFlags() {
 		`A set of key=value pairs that describe configuration that may be passed to different components.
 		The key should be '.' separated, and the first part before the dot is the component to apply the configuration to.
 		Valid components are: kubelet, kubeadm, apiserver, controller-manager, etcd, proxy, scheduler
-		Valid kubeadm parameters: `+fmt.Sprintf("%s, %s", strings.Join(kubeadm.KubeadmExtraArgsWhitelist[kubeadm.KubeadmCmdParam], ", "), strings.Join(kubeadm.KubeadmExtraArgsWhitelist[kubeadm.KubeadmConfigParam], ",")))
+		Valid kubeadm parameters: `+fmt.Sprintf("%s, %s", strings.Join(bsutil.KubeadmExtraArgsWhitelist[bsutil.KubeadmCmdParam], ", "), strings.Join(bsutil.KubeadmExtraArgsWhitelist[bsutil.KubeadmConfigParam], ",")))
 	startCmd.Flags().String(featureGates, "", "A set of key=value pairs that describe feature gates for alpha/experimental features.")
 	startCmd.Flags().String(dnsDomain, constants.ClusterDNSDomain, "The cluster dns domain name used in the kubernetes cluster")
 	startCmd.Flags().Int(apiServerPort, constants.APIServerPort, "The apiserver listening port")
@@ -231,7 +234,7 @@ func initNetworkingFlags() {
 	startCmd.Flags().StringSliceVar(&registryMirror, "registry-mirror", nil, "Registry mirrors to pass to the Docker daemon")
 	startCmd.Flags().String(imageRepository, "", "Alternative image repository to pull docker images from. This can be used when you have limited access to gcr.io. Set it to \"auto\" to let minikube decide one for you. For Chinese mainland users, you may use local gcr.io mirrors such as registry.cn-hangzhou.aliyuncs.com/google_containers")
 	startCmd.Flags().String(imageMirrorCountry, "", "Country code of the image mirror to be used. Leave empty to use the global one. For Chinese mainland users, set it to cn.")
-	startCmd.Flags().String(serviceCIDR, pkgutil.DefaultServiceCIDR, "The CIDR to be used for service cluster IPs.")
+	startCmd.Flags().String(serviceCIDR, constants.DefaultServiceCIDR, "The CIDR to be used for service cluster IPs.")
 	startCmd.Flags().StringArrayVar(&dockerEnv, "docker-env", nil, "Environment variables to pass to the Docker daemon. (format: key=value)")
 	startCmd.Flags().StringArrayVar(&dockerOpt, "docker-opt", nil, "Specify arbitrary flags to pass to the Docker daemon. (format: key=value)")
 }
@@ -267,7 +270,11 @@ func platform() string {
 
 	// This environment is exotic, let's output a bit more.
 	if vrole == "guest" || runtime.GOARCH != "amd64" {
-		s.WriteString(fmt.Sprintf(" (%s/%s)", vsys, runtime.GOARCH))
+		if vsys != "" {
+			s.WriteString(fmt.Sprintf(" (%s/%s)", vsys, runtime.GOARCH))
+		} else {
+			s.WriteString(fmt.Sprintf(" (%s)", runtime.GOARCH))
+		}
 	}
 	return s.String()
 }
@@ -304,8 +311,8 @@ func runStart(cmd *cobra.Command, args []string) {
 	validateFlags(cmd, driverName)
 	validateUser(driverName)
 
-	// No need to install a driver in download-only mode
-	if !viper.GetBool(downloadOnly) {
+	// Download & update the driver, even in --download-only mode
+	if !viper.GetBool(dryRun) {
 		updateDriver(driverName)
 	}
 
@@ -315,12 +322,14 @@ func runStart(cmd *cobra.Command, args []string) {
 		exit.WithError("Failed to generate config", err)
 	}
 
-	if !driver.BareMetal(driverName) {
-		if err := cluster.CacheISO(config); err != nil {
-			exit.WithError("Failed to cache ISO", err)
-		}
+	// This is about as far as we can go without overwriting config files
+	if viper.GetBool(dryRun) {
+		out.T(out.DryRun, `dry-run validation complete!`)
+		return
 	}
 
+	cacheISO(&config, driverName)
+
 	if viper.GetBool(nativeSSH) {
 		ssh.SetDefaultClient(ssh.Native)
 	} else {
@@ -329,7 +338,7 @@ func runStart(cmd *cobra.Command, args []string) {
 
 	// Now that the ISO is downloaded, pull images in the background while the VM boots.
 	var cacheGroup errgroup.Group
-	beginCacheImages(&cacheGroup, config.KubernetesConfig.ImageRepository, k8sVersion)
+	beginCacheRequiredImages(&cacheGroup, config.KubernetesConfig.ImageRepository, k8sVersion)
 
 	// Abstraction leakage alert: startHost requires the config to be saved, to satistfy pkg/provision/buildroot.
 	// Hence, saveConfig must be called before startHost, and again afterwards when we know the IP.
@@ -344,7 +353,7 @@ func runStart(cmd *cobra.Command, args []string) {
 	// configure the runtime (docker, containerd, crio)
 	cr := configureRuntimes(mRunner, driverName, config.KubernetesConfig)
 	showVersionInfo(k8sVersion, cr)
-	waitCacheImages(&cacheGroup)
+	waitCacheRequiredImages(&cacheGroup)
 
 	// Must be written before bootstrap, otherwise health checks may flake due to stale IP
 	kubeconfig, err := setupKubeconfig(host, &config, config.Name)
@@ -362,7 +371,7 @@ func runStart(cmd *cobra.Command, args []string) {
 	// enable addons with start command
 	enableAddons()
 
-	if err = loadCachedImagesInConfigFile(); err != nil {
+	if err = cacheAndLoadImagesInConfig(); err != nil {
 		out.T(out.FailureType, "Unable to load cached images from config file.")
 	}
 
@@ -391,9 +400,17 @@ func updateDriver(driverName string) {
 	}
 }
 
+func cacheISO(config *cfg.MachineConfig, driverName string) {
+	if !driver.BareMetal(driverName) && !driver.IsKIC(driverName) {
+		if err := cluster.CacheISO(*config); err != nil {
+			exit.WithError("Failed to cache ISO", err)
+		}
+	}
+}
+
 func enableAddons() {
 	for _, a := range addonList {
-		err := cmdcfg.Set(a, "true")
+		err := pkgaddons.Set(a, "true", viper.GetString(config.MachineProfile))
 		if err != nil {
 			exit.WithError("addon enable failed", err)
 		}
@@ -427,16 +444,22 @@ func displayEnviron(env []string) {
 }
 
 func setupKubeconfig(h *host.Host, c *cfg.MachineConfig, clusterName string) (*kubeconfig.Settings, error) {
-	addr, err := h.Driver.GetURL()
-	if err != nil {
-		exit.WithError("Failed to get driver URL", err)
+	addr := ""
+	var err error
+	if driver.IsKIC(h.DriverName) {
+		addr = fmt.Sprintf("https://%s", net.JoinHostPort("127.0.0.1", fmt.Sprint(c.KubernetesConfig.NodePort)))
+	} else {
+		addr, err = h.Driver.GetURL()
+		if err != nil {
+			exit.WithError("Failed to get driver URL", err)
+		}
+		addr = strings.Replace(addr, "tcp://", "https://", -1)
+		addr = strings.Replace(addr, ":2376", ":"+strconv.Itoa(c.KubernetesConfig.NodePort), -1)
 	}
-	addr = strings.Replace(addr, "tcp://", "https://", -1)
-	addr = strings.Replace(addr, ":2376", ":"+strconv.Itoa(c.KubernetesConfig.NodePort), -1)
+
 	if c.KubernetesConfig.APIServerName != constants.APIServerName {
 		addr = strings.Replace(addr, c.KubernetesConfig.NodeIP, c.KubernetesConfig.APIServerName, -1)
 	}
-
 	kcs := &kubeconfig.Settings{
 		ClusterName:          clusterName,
 		ClusterServerAddress: addr,
@@ -462,9 +485,9 @@ func handleDownloadOnly(cacheGroup *errgroup.Group, k8sVersion string) {
 	if err := doCacheBinaries(k8sVersion); err != nil {
 		exit.WithError("Failed to cache binaries", err)
 	}
-	waitCacheImages(cacheGroup)
-	if err := CacheImagesInConfigFile(); err != nil {
-		exit.WithError("Failed to cache images", err)
+	waitCacheRequiredImages(cacheGroup)
+	if err := saveImagesToTarFromConfig(); err != nil {
+		exit.WithError("Failed to cache images to tar", err)
 	}
 	out.T(out.Check, "Download complete!")
 	os.Exit(0)
@@ -483,6 +506,7 @@ func startMachine(config *cfg.MachineConfig) (runner command.Runner, preExists b
 	}
 
 	ip := validateNetwork(host, runner)
+
 	// Bypass proxy for minikube's vm host ip
 	err = proxy.ExcludeIP(ip)
 	if err != nil {
@@ -555,6 +579,8 @@ func showKubectlInfo(kcs *kubeconfig.Settings, k8sVersion string, machineName st
 func selectDriver(existing *cfg.MachineConfig) string {
 	name := viper.GetString("vm-driver")
 	glog.Infof("selectDriver: flag=%q, old=%v", name, existing)
+
+	driver.SetLibvirtURI(viper.GetString(kvmQemuURI))
 	options := driver.Choices()
 	pick, alts := driver.Choose(name, options)
 
@@ -705,7 +731,7 @@ func minikubeCmd() string {
 	return "minikube"
 }
 
-// validerUser validates minikube is run by the recommended user (privileged or regular)
+// validateUser validates minikube is run by the recommended user (privileged or regular)
 func validateUser(drvName string) {
 	u, err := user.Current()
 	if err != nil {
@@ -739,23 +765,50 @@ func validateUser(drvName string) {
 	}
 }
 
-// validateFlags validates the supplied flags against known bad combinations
-func validateFlags(cmd *cobra.Command, drvName string) {
+// validateDiskSize validates the disk size matches the minimum recommended
+func validateDiskSize() {
 	diskSizeMB := pkgutil.CalculateSizeInMB(viper.GetString(humanReadableDiskSize))
 	if diskSizeMB < pkgutil.CalculateSizeInMB(minimumDiskSize) && !viper.GetBool(force) {
 		exit.WithCodeT(exit.Config, "Requested disk size {{.requested_size}} is less than minimum of {{.minimum_size}}", out.V{"requested_size": diskSizeMB, "minimum_size": pkgutil.CalculateSizeInMB(minimumDiskSize)})
 	}
+}
 
+// validateMemorySize validates the memory size matches the minimum recommended
+func validateMemorySize() {
 	memorySizeMB := pkgutil.CalculateSizeInMB(viper.GetString(memory))
 	if memorySizeMB < pkgutil.CalculateSizeInMB(minimumMemorySize) && !viper.GetBool(force) {
-		exit.UsageT("Requested memory allocation {{.requested_size}} is less than the minimum allowed of {{.minimum_size}}", out.V{"requested_size": memorySizeMB, "minimum_size": pkgutil.CalculateSizeInMB(minimumMemorySize)})
+		exit.WithCodeT(exit.Config, "Requested memory allocation {{.requested_size}} is less than the minimum allowed of {{.minimum_size}}", out.V{"requested_size": memorySizeMB, "minimum_size": pkgutil.CalculateSizeInMB(minimumMemorySize)})
 	}
 	if memorySizeMB < pkgutil.CalculateSizeInMB(defaultMemorySize) && !viper.GetBool(force) {
 		out.T(out.Notice, "Requested memory allocation ({{.memory}}MB) is less than the default memory allocation of {{.default_memorysize}}MB. Beware that minikube might not work correctly or crash unexpectedly.",
 			out.V{"memory": memorySizeMB, "default_memorysize": pkgutil.CalculateSizeInMB(defaultMemorySize)})
 	}
+}
 
+// validateCPUCount validates the cpu count matches the minimum recommended
+func validateCPUCount(local bool) {
 	var cpuCount int
+	if local {
+		// Uses the gopsutil cpu package to count the number of physical cpu cores
+		ci, err := cpu.Counts(false)
+		if err != nil {
+			glog.Warningf("Unable to get CPU info: %v", err)
+		} else {
+			cpuCount = ci
+		}
+	} else {
+		cpuCount = viper.GetInt(cpus)
+	}
+	if cpuCount < minimumCPUS && !viper.GetBool(force) {
+		exit.UsageT("Requested cpu count {{.requested_cpus}} is less than the minimum allowed of {{.minimum_cpus}}", out.V{"requested_cpus": cpuCount, "minimum_cpus": minimumCPUS})
+	}
+}
+
+// validateFlags validates the supplied flags against known bad combinations
+func validateFlags(cmd *cobra.Command, drvName string) {
+	validateDiskSize()
+	validateMemorySize()
+
 	if driver.BareMetal(drvName) {
 		if viper.GetString(cfg.MachineProfile) != constants.DefaultMachineName {
 			exit.WithCodeT(exit.Config, "The 'none' driver does not support multiple profiles: https://minikube.sigs.k8s.io/docs/reference/drivers/none/")
@@ -772,25 +825,14 @@ func validateFlags(cmd *cobra.Command, drvName string) {
 		if runtime != "docker" {
 			out.WarningT("Using the '{{.runtime}}' runtime with the 'none' driver is an untested configuration!", out.V{"runtime": runtime})
 		}
+	}
 
-		// Uses the gopsutil cpu package to count the number of physical cpu cores
-		ci, err := cpu.Counts(false)
-		if err != nil {
-			glog.Warningf("Unable to get CPU info: %v", err)
-		} else {
-			cpuCount = ci
-		}
-	} else {
-		cpuCount = viper.GetInt(cpus)
-	}
-	if cpuCount < minimumCPUS && !viper.GetBool(force) {
-		exit.UsageT("Requested cpu count {{.requested_cpus}} is less than the minimum allowed of {{.minimum_cpus}}", out.V{"requested_cpus": cpuCount, "minimum_cpus": minimumCPUS})
-	}
+	validateCPUCount(driver.BareMetal(drvName))
 
 	// check that kubeadm extra args contain only whitelisted parameters
-	for param := range extraOptions.AsMap().Get(kubeadm.Kubeadm) {
-		if !cfg.ContainsParam(kubeadm.KubeadmExtraArgsWhitelist[kubeadm.KubeadmCmdParam], param) &&
-			!cfg.ContainsParam(kubeadm.KubeadmExtraArgsWhitelist[kubeadm.KubeadmConfigParam], param) {
+	for param := range extraOptions.AsMap().Get(bsutil.Kubeadm) {
+		if !cfg.ContainsParam(bsutil.KubeadmExtraArgsWhitelist[bsutil.KubeadmCmdParam], param) &&
+			!cfg.ContainsParam(bsutil.KubeadmExtraArgsWhitelist[bsutil.KubeadmConfigParam], param) {
 			exit.UsageT("Sorry, the kubeadm.{{.parameter_name}} parameter is currently not supported by --extra-config", out.V{"parameter_name": param})
 		}
 	}
@@ -821,8 +863,8 @@ func doCacheBinaries(k8sVersion string) error {
 	return machine.CacheBinariesForBootstrapper(k8sVersion, viper.GetString(cmdcfg.Bootstrapper))
 }
 
-// beginCacheImages caches Docker images in the background
-func beginCacheImages(g *errgroup.Group, imageRepository string, k8sVersion string) {
+// beginCacheRequiredImages caches images required for kubernetes version in the background
+func beginCacheRequiredImages(g *errgroup.Group, imageRepository string, k8sVersion string) {
 	if !viper.GetBool(cacheImages) {
 		return
 	}
@@ -832,8 +874,8 @@ func beginCacheImages(g *errgroup.Group, imageRepository string, k8sVersion stri
 	})
 }
 
-// waitCacheImages blocks until the image cache jobs complete
-func waitCacheImages(g *errgroup.Group) {
+// waitCacheRequiredImages blocks until the required images are all cached.
+func waitCacheRequiredImages(g *errgroup.Group) {
 	if !viper.GetBool(cacheImages) {
 		return
 	}
@@ -960,16 +1002,35 @@ func setDockerProxy() {
 }
 
 // autoSetDriverOptions sets the options needed for specific vm-driver automatically.
-func autoSetDriverOptions(cmd *cobra.Command, drvName string) error {
+func autoSetDriverOptions(cmd *cobra.Command, drvName string) (err error) {
+	err = nil
 	hints := driver.FlagDefaults(drvName)
-	if !cmd.Flags().Changed("extra-config") && hints.ExtraOptions != "" {
-		return extraOptions.Set(hints.ExtraOptions)
+	if !cmd.Flags().Changed("extra-config") && len(hints.ExtraOptions) > 0 {
+		for _, eo := range hints.ExtraOptions {
+			glog.Infof("auto setting extra-config to %q.", eo)
+			err = extraOptions.Set(eo)
+			if err != nil {
+				err = errors.Wrapf(err, "setting extra option %s", eo)
+			}
+		}
 	}
 
 	if !cmd.Flags().Changed(cacheImages) {
 		viper.Set(cacheImages, hints.CacheImages)
 	}
-	return nil
+
+	if !cmd.Flags().Changed(containerRuntime) && hints.ContainerRuntime != "" {
+		viper.Set(containerRuntime, hints.ContainerRuntime)
+		glog.Infof("auto set %s to %q.", containerRuntime, hints.ContainerRuntime)
+	}
+
+	if !cmd.Flags().Changed(cmdcfg.Bootstrapper) && hints.Bootstrapper != "" {
+		viper.Set(cmdcfg.Bootstrapper, hints.Bootstrapper)
+		glog.Infof("auto set %s to %q.", cmdcfg.Bootstrapper, hints.Bootstrapper)
+
+	}
+
+	return err
 }
 
 // prepareNone prepares the user and host for the joy of the "none" driver
@@ -1008,20 +1069,9 @@ func startHost(api libmachine.API, mc cfg.MachineConfig) (*host.Host, bool) {
 		exit.WithError("Failed to check if machine exists", err)
 	}
 
-	var host *host.Host
-	start := func() (err error) {
-		host, err = cluster.StartHost(api, mc)
-		if err != nil {
-			out.T(out.Resetting, "Retriable failure: {{.error}}", out.V{"error": err})
-			if derr := cluster.DeleteHost(api, mc.Name); derr != nil {
-				glog.Warningf("DeleteHost: %v", derr)
-			}
-		}
-		return err
-	}
-
-	if err = retry.Expo(start, 5*time.Second, 3*time.Minute, 3); err != nil {
-		exit.WithError("Unable to start VM", err)
+	host, err := cluster.StartHost(api, mc)
+	if err != nil {
+		exit.WithError("Unable to start VM. Please investigate and run 'minikube delete' if possible", err)
 	}
 	return host, exists
 }
@@ -1051,7 +1101,7 @@ func validateNetwork(h *host.Host, r command.Runner) string {
 		}
 	}
 
-	if !driver.BareMetal(h.Driver.DriverName()) {
+	if !driver.BareMetal(h.Driver.DriverName()) && !driver.IsKIC(h.Driver.DriverName()) {
 		trySSH(h, ip)
 	}
 
@@ -1061,7 +1111,7 @@ func validateNetwork(h *host.Host, r command.Runner) string {
 }
 
 func trySSH(h *host.Host, ip string) {
-	sshAddr := fmt.Sprintf("%s:22", ip)
+	sshAddr := net.JoinHostPort(ip, "22")
 
 	dial := func() (err error) {
 		d := net.Dialer{Timeout: 3 * time.Second}
@@ -1125,9 +1175,10 @@ func getKubernetesVersion(old *cfg.MachineConfig) (string, bool) {
 	isUpgrade := false
 
 	if paramVersion == "" { // if the user did not specify any version then ...
-		if old != nil { // .. use the old version from config
+		if old != nil { // .. use the old version from config (if any)
 			paramVersion = old.KubernetesConfig.KubernetesVersion
-		} else { // .. otherwise use the default version
+		}
+		if paramVersion == "" { // .. otherwise use the default version
 			paramVersion = constants.DefaultKubernetesVersion
 		}
 	}

cmd/minikube/cmd/update-context.go

@@ -17,6 +17,8 @@ limitations under the License.
 package cmd
 
 import (
+	"os"
+
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"k8s.io/minikube/pkg/minikube/cluster"
@@ -45,7 +47,13 @@ var updateContextCmd = &cobra.Command{
 		if err != nil {
 			exit.WithError("Error host driver ip status", err)
 		}
-		updated, err := kubeconfig.UpdateIP(ip, machineName, constants.KubeconfigPath)
+		var updated bool
+		kubeConfigPath := os.Getenv("KUBECONFIG")
+		if kubeConfigPath == "" {
+			updated, err = kubeconfig.UpdateIP(ip, machineName, constants.KubeconfigPath)
+		} else {
+			updated, err = kubeconfig.UpdateIP(ip, machineName, kubeConfigPath)
+		}
 		if err != nil {
 			exit.WithError("update config", err)
 		}

deploy/addons/addon-manager.yaml.tmpl

@@ -33,6 +33,8 @@ spec:
       value: "5"
     - name: ADDON_MANAGER_LEADER_ELECTION
       value: "false"
+    - name: KUBECTL_EXTRA_PRUNE_WHITELIST
+      value: install.istio.io/v1alpha2/IstioControlPlane
     imagePullPolicy: IfNotPresent
     resources:
       requests:

deploy/addons/ingress-dns/ingress-dns-pod.yaml

@@ -79,7 +79,7 @@ spec:
   hostNetwork: true
   containers:
     - name: minikube-ingress-dns
-      image: "cryptexlabs/minikube-ingress-dns:0.2.0"
+      image: "cryptexlabs/minikube-ingress-dns:0.2.1"
       imagePullPolicy: IfNotPresent
       ports:
         - containerPort: 53

deploy/addons/istio-provisioner/istio-operator.yaml.tmpl

@@ -0,0 +1,271 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: istio-operator
+  labels:
+    kubernetes.io/minikube-addons: istio
+    addonmanager.kubernetes.io/mode: EnsureExists 
+...
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: istiocontrolplanes.install.istio.io
+  labels:
+    kubernetes.io/minikube-addons: istio
+    addonmanager.kubernetes.io/mode: EnsureExists 
+spec:
+  group: install.istio.io
+  names:
+    kind: IstioControlPlane
+    listKind: IstioControlPlaneList
+    plural: istiocontrolplanes
+    singular: istiocontrolplane
+    shortNames:
+    - icp
+  scope: Namespaced
+  subresources:
+    status: {}
+  validation:
+    openAPIV3Schema:
+      properties:
+        apiVersion:
+          description: 'APIVersion defines the versioned schema of this representation
+            of an object. Servers should convert recognized schemas to the latest
+            internal value, and may reject unrecognized values.
+            More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#resources'
+          type: string
+        kind:
+          description: 'Kind is a string value representing the REST resource this
+            object represents. Servers may infer this from the endpoint the client
+            submits requests to. Cannot be updated. In CamelCase.
+            More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+          type: string
+        spec:
+          description: 'Specification of the desired state of the istio control plane resource.
+            More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status'
+          type: object
+        status:
+          description: 'Status describes each of istio control plane component status at the current time.
+            0 means NONE, 1 means UPDATING, 2 means HEALTHY, 3 means ERROR, 4 means RECONCILING.
+            More info: https://github.com/istio/operator/blob/master/pkg/apis/istio/v1alpha2/v1alpha2.pb.html &
+            https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status'
+          type: object
+  versions:
+  - name: v1alpha2
+    served: true
+    storage: true
+...
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  namespace: istio-operator
+  name: istio-operator
+  labels:
+    kubernetes.io/minikube-addons: istio
+    addonmanager.kubernetes.io/mode: EnsureExists 
+...
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  name: istio-operator
+  labels:
+    kubernetes.io/minikube-addons: istio
+    addonmanager.kubernetes.io/mode: EnsureExists 
+rules:
+# istio groups
+- apiGroups:
+  - authentication.istio.io
+  resources:
+  - '*'
+  verbs:
+  - '*'
+- apiGroups:
+  - config.istio.io
+  resources:
+  - '*'
+  verbs:
+  - '*'
+- apiGroups:
+  - install.istio.io
+  resources:
+  - '*'
+  verbs:
+  - '*'
+- apiGroups:
+  - networking.istio.io
+  resources:
+  - '*'
+  verbs:
+  - '*'
+- apiGroups:
+  - rbac.istio.io
+  resources:
+  - '*'
+  verbs:
+  - '*'
+- apiGroups:
+  - security.istio.io
+  resources:
+  - '*'
+  verbs:
+  - '*'
+# k8s groups
+- apiGroups:
+  - admissionregistration.k8s.io
+  resources:
+  - mutatingwebhookconfigurations
+  - validatingwebhookconfigurations
+  verbs:
+  - '*'
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions.apiextensions.k8s.io
+  - customresourcedefinitions
+  verbs:
+  - '*'
+- apiGroups:
+  - apps
+  - extensions
+  resources:
+  - daemonsets
+  - deployments
+  - deployments/finalizers
+  - ingresses
+  - replicasets
+  - statefulsets
+  verbs:
+  - '*'
+- apiGroups:
+  - autoscaling
+  resources:
+  - horizontalpodautoscalers
+  verbs:
+  - '*'
+- apiGroups:
+  - monitoring.coreos.com
+  resources:
+  - servicemonitors
+  verbs:
+  - get
+  - create
+- apiGroups:
+  - policy
+  resources:
+  - poddisruptionbudgets
+  verbs:
+  - '*'
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - clusterrolebindings
+  - clusterroles
+  - roles
+  - rolebindings
+  verbs:
+  - '*'
+- apiGroups:
+  - ""
+  resources:
+  - configmaps  
+  - endpoints
+  - events
+  - namespaces
+  - pods
+  - persistentvolumeclaims
+  - secrets
+  - services
+  - serviceaccounts  
+  verbs:
+  - '*'
+...
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: istio-operator
+  labels:
+    kubernetes.io/minikube-addons: istio
+    addonmanager.kubernetes.io/mode: EnsureExists 
+subjects:
+- kind: ServiceAccount
+  name: istio-operator
+  namespace: istio-operator
+roleRef:
+  kind: ClusterRole
+  name: istio-operator
+  apiGroup: rbac.authorization.k8s.io
+...
+---
+apiVersion: v1
+kind: Service
+metadata:
+  namespace: istio-operator
+  labels:
+    name: istio-operator
+    kubernetes.io/minikube-addons: istio
+    addonmanager.kubernetes.io/mode: EnsureExists 
+  name: istio-operator-metrics
+spec:
+  ports:
+  - name: http-metrics
+    port: 8383
+    targetPort: 8383
+  selector:
+    name: istio-operator
+...
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  namespace: istio-operator
+  name: istio-operator
+  labels:
+    kubernetes.io/minikube-addons: istio
+    addonmanager.kubernetes.io/mode: Reconcile 
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      name: istio-operator
+  template:
+    metadata:
+      labels:
+        name: istio-operator
+        kubernetes.io/minikube-addons: istio
+        addonmanager.kubernetes.io/mode: EnsureExists 
+    spec:
+      serviceAccountName: istio-operator
+      containers:
+        - name: istio-operator
+          image: docker.io/istio/operator:1.4.0
+          command:
+          - istio-operator
+          - server
+          imagePullPolicy: Always
+          resources:
+            limits:
+              cpu: 200m
+              memory: 256Mi
+            requests:
+              cpu: 50m
+              memory: 128Mi
+          env:
+            - name: WATCH_NAMESPACE
+              value: ""
+            - name: LEADER_ELECTION_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: OPERATOR_NAME
+              value: "istio-operator"
+...

deploy/addons/istio/README.md

@@ -0,0 +1,26 @@
+## istio Addon
+[istio](https://istio.io/docs/setup/getting-started/) - Cloud platforms provide a wealth of benefits for the organizations that use them.
+
+### Enabling istio
+Propose to startup minikube with at least 8192 MB of memory and 4 CPUs to enable istio.
+To enable this addon, simply run:
+
+```shell script
+minikube addons enable istio
+```
+
+In a minute or so istio default components will be installed into your cluster. You could run `kubectl get po -n istio-system` to see the progress for istio installation.
+
+### Testing installation
+
+```shell script
+kubectl get po -n istio-system
+```
+
+If everything went well you shouldn't get any errors about istio being installed in your cluster. If you haven't deployed any releases `kubectl get po -n istio-system` won't return anything.
+
+### Deprecation of istio
+To disable this addon, simply run:
+```shell script
+minikube addons disable istio
+```

deploy/addons/istio/istio-default-profile.yaml.tmpl

@@ -0,0 +1,10 @@
+apiVersion: install.istio.io/v1alpha2
+kind: IstioControlPlane
+metadata:
+  namespace: istio-operator
+  name: example-istiocontrolplane
+  labels:
+    kubernetes.io/minikube-addons: istio
+    addonmanager.kubernetes.io/mode: Reconcile 
+spec:
+  profile: default

deploy/addons/layouts/istio/single.html

@@ -0,0 +1,5 @@
+{{ define "main" }}
+   <div style="padding-top:20px">
+      {{ .Render "content" }}
+   </div>
+{{ end }}

deploy/iso/minikube-iso/board/coreos/minikube/patches/systemd/240/rdrand-workaround-on-amd.patch

@@ -0,0 +1,36 @@
+Index: systemd-240/src/basic/random-util.c
+===================================================================
+--- systemd-240.orig/src/basic/random-util.c
++++ systemd-240/src/basic/random-util.c
+@@ -37,6 +37,7 @@ int rdrand(unsigned long *ret) {
+ 
+ #if defined(__i386__) || defined(__x86_64__)
+         static int have_rdrand = -1;
++        unsigned long v;
+         unsigned char err;
+ 
+         if (have_rdrand < 0) {
+@@ -56,9 +57,22 @@ int rdrand(unsigned long *ret) {
+ 
+         asm volatile("rdrand %0;"
+                      "setc %1"
+-                     : "=r" (*ret),
++                     : "=r" (v),
+                        "=qm" (err));
+ 
++       /* Apparently on some AMD CPUs RDRAND will sometimes (after a suspend/resume cycle?) report success
++        * via the carry flag but nonetheless return the same fixed value -1 in all cases. This appears to be
++        * a bad bug in the CPU or firmware. Let's deal with that and work-around this by explicitly checking
++        * for this special value (and also 0, just to be sure) and filtering it out. This is a work-around
++        * only however and something AMD really should fix properly. The Linux kernel should probably work
++        * around this issue by turning off RDRAND altogether on those CPUs. See:
++        * https://github.com/systemd/systemd/issues/11810 */
++       if (v == 0 || v == ULONG_MAX)
++               return log_debug_errno(SYNTHETIC_ERRNO(EUCLEAN),
++                                      "RDRAND returned suspicious value %lx, assuming bad hardware RNG, not using value.", v);
++
++       *ret = v;
++
+ #if HAS_FEATURE_MEMORY_SANITIZER
+         __msan_unpoison(&err, sizeof(err));
+ #endif

deploy/iso/minikube-iso/board/coreos/minikube/rootfs-overlay/etc/sysctl.d/disable-ipv6.conf

@@ -0,0 +1,2 @@
+net.ipv6.conf.all.disable_ipv6=1
+net.ipv6.conf.default.disable_ipv6=1

deploy/iso/minikube-iso/configs/minikube_defconfig

@@ -17,6 +17,7 @@ BR2_SYSTEM_BIN_SH_BASH=y
 # BR2_TARGET_GENERIC_GETTY is not set
 BR2_ROOTFS_USERS_TABLES="$(BR2_EXTERNAL_MINIKUBE_PATH)/board/coreos/minikube/users"
 BR2_ROOTFS_OVERLAY="$(BR2_EXTERNAL_MINIKUBE_PATH)/board/coreos/minikube/rootfs-overlay"
+BR2_GLOBAL_PATCH_DIR="$(BR2_EXTERNAL_MINIKUBE_PATH)/board/coreos/minikube/patches"
 BR2_LINUX_KERNEL=y
 BR2_LINUX_KERNEL_USE_CUSTOM_CONFIG=y
 BR2_LINUX_KERNEL_CUSTOM_CONFIG_FILE="$(BR2_EXTERNAL_MINIKUBE_PATH)/board/coreos/minikube/linux_defconfig"

deploy/iso/minikube-iso/package/crio-bin/crio-bin.hash

@@ -9,3 +9,4 @@ sha256 f7041a92e2d3a4c341be8df58f1076ba57ecb5daa02b6c65e652530c5f242739 v1.15.0.
 sha256 6218a99877da9b9895e0088944731f5384803c15628d4b3c6b40ba1ddd39e052 v1.15.1.tar.gz
 sha256 70d4c746fe207422c78420dc4239768f485eea639a38c993c02872ec6305dd1d v1.15.2.tar.gz
 sha256 05f9614c4d5970b4662499b84c270b0ab953596ee863dcd09c9dc7a2d2f09789 v1.16.0.tar.gz
+sha256 57e1ee990ef2d5af8b32c33a21b4998682608e3556dcf1d3349666f55e7d95b9 v1.16.1.tar.gz

deploy/iso/minikube-iso/package/crio-bin/crio-bin.mk

@@ -4,8 +4,8 @@
 #
 ################################################################################
 
-CRIO_BIN_VERSION = v1.16.0
-CRIO_BIN_COMMIT = fa99ff4ae2aa45115bf3c6bb33db07191db2518e
+CRIO_BIN_VERSION = v1.16.1
+CRIO_BIN_COMMIT = bf8fcf34c942ba973a4f694094b46f914c779c0a
 CRIO_BIN_SITE = https://github.com/cri-o/cri-o/archive
 CRIO_BIN_SOURCE = $(CRIO_BIN_VERSION).tar.gz
 CRIO_BIN_DEPENDENCIES = host-go libgpgme

deploy/iso/minikube-iso/package/hyperv-daemons/hyperv-daemons.mk

@@ -4,7 +4,7 @@
 #
 ################################################################################
 
-HYPERV_DAEMONS_VERSION = 4.19.81
+HYPERV_DAEMONS_VERSION = 4.19.88
 HYPERV_DAEMONS_SITE = https://www.kernel.org/pub/linux/kernel/v4.x
 HYPERV_DAEMONS_SOURCE = linux-$(HYPERV_DAEMONS_VERSION).tar.xz
 

deploy/iso/minikube-iso/package/podman/podman.hash

@@ -10,3 +10,4 @@ sha256 17fdf68e85106d0848e89825e191198a4079bb6d9ca6dd3e415e5c010192db9e v1.4.0.t
 sha256 45eb7bccd81a1431b0c7a0697829c0bcc397048595d143fd91179b31d22a3c63 v1.4.1.tar.gz
 sha256 2e027c1b935f3a03f27ef7f17823ccf334607a17d033d4ce53a90b98294e7f68 v1.4.4.tar.gz
 sha256 61b44b739c485125f179044f7aa7dc58c820f771bce4ce495fa555a38dc68b57 v1.6.3.tar.gz
+sha256 6e59821320b435543bc7554e73faa66d5956e4ad3f7e7f4ea03bebd6726758e9 v1.6.4.tar.gz

deploy/iso/minikube-iso/package/podman/podman.mk

@@ -1,5 +1,5 @@
-PODMAN_VERSION = v1.6.3
-PODMAN_COMMIT = 9d087f6a766259ba53b224944f1b7b778035c370
+PODMAN_VERSION = v1.6.4
+PODMAN_COMMIT = 5cc92849f7fc9dd734ca2fd8f3ae8830b9a7eb26
 PODMAN_SITE = https://github.com/containers/libpod/archive
 PODMAN_SOURCE = $(PODMAN_VERSION).tar.gz
 PODMAN_LICENSE = Apache-2.0
@@ -24,7 +24,7 @@ endef
 
 define PODMAN_BUILD_CMDS
 	mkdir -p $(@D)/bin
-	$(PODMAN_BIN_ENV) $(MAKE) $(TARGET_CONFIGURE_OPTS) -C $(@D) GIT_COMMIT=$(PODMAN_COMMIT) PREFIX=/usr podman
+	$(PODMAN_BIN_ENV) CIRRUS_TAG=$(PODMAN_VERSION) $(MAKE) $(TARGET_CONFIGURE_OPTS) -C $(@D) GIT_COMMIT=$(PODMAN_COMMIT) PREFIX=/usr podman
 endef
 
 define PODMAN_INSTALL_TARGET_CMDS

deploy/iso/minikube-iso/package/runc-master/runc-master.mk

@@ -13,15 +13,15 @@ RUNC_MASTER_LICENSE_FILES = LICENSE
 
 RUNC_MASTER_DEPENDENCIES = host-go
 
-RUNC_MASTER_GOPATH = "$(@D)/Godeps/_workspace"
+RUNC_MASTER_GOPATH = $(@D)/_output
 RUNC_MASTER_MAKE_ENV = $(HOST_GO_TARGET_ENV) \
 	CGO_ENABLED=1 \
-	GOBIN="$(@D)/bin" \
+	GO111MODULE=off \
 	GOPATH="$(RUNC_MASTER_GOPATH)" \
-	PATH=$(BR_PATH)
+	GOBIN="$(RUNC_MASTER_GOPATH)/bin" \
+	PATH=$(RUNC_MASTER_GOPATH)/bin:$(BR_PATH)
 
-RUNC_MASTER_GLDFLAGS = \
-	-buildmode=pie -X main.gitCommit=$(RUNC_MASTER_VERSION)
+RUNC_MASTER_COMPILE_SRC = $(RUNC_MASTER_GOPATH)/src/github.com/opencontainers/runc
 
 ifeq ($(BR2_PACKAGE_LIBSECCOMP),y)
 RUNC_MASTER_GOTAGS += seccomp
@@ -34,13 +34,11 @@ define RUNC_MASTER_CONFIGURE_CMDS
 endef
 
 define RUNC_MASTER_BUILD_CMDS
-	cd $(@D) && $(RUNC_MASTER_MAKE_ENV) $(HOST_DIR)/usr/bin/go \
-		build -v -o $(@D)/bin/runc \
-		-tags "$(RUNC_MASTER_GOTAGS)" -ldflags "$(RUNC_MASTER_GLDFLAGS)" github.com/opencontainers/runc
+	PWD=$(RUNC_MASTER_COMPILE_SRC) $(RUNC_MASTER_MAKE_ENV) $(MAKE) $(TARGET_CONFIGURE_OPTS) -C $(@D) BUILDTAGS="$(RUNC_MASTER_GOTAGS)" COMMIT_NO=$(RUNC_MASTER_VERSION) PREFIX=/usr
 endef
 
 define RUNC_MASTER_INSTALL_TARGET_CMDS
-	$(INSTALL) -D -m 0755 $(@D)/bin/runc $(TARGET_DIR)/usr/bin/runc
+	$(INSTALL) -D -m 0755 $(@D)/runc $(TARGET_DIR)/usr/bin/runc
 endef
 
 $(eval $(generic-package))

deploy/minikube/releases.json

@@ -1,4 +1,12 @@
 [
+  {
+      "name": "v1.6.2",
+      "checksums": {
+          "darwin": "5ea5168a80597ee6221bf50a524429a24a37f0c0f36725e6b297dc5a7a6a2105",
+          "linux": "eabd027438953d29a4b0f7b810c801919cc13bef3ebe7aff08c9534ac2b091ab",
+          "windows": "79d66c874cfe3497656e9ba191680cc95abd92d2f722b10de38f00b76ef82393"
+      }
+  },
   {
       "name": "v1.6.1",
       "checksums": {

go.mod

@@ -17,15 +17,17 @@ require (
 	github.com/cheggaaa/pb/v3 v3.0.1
 	github.com/cloudfoundry-attic/jibber_jabber v0.0.0-20151120183258-bcc4c8345a21
 	github.com/cloudfoundry/jibber_jabber v0.0.0-20151120183258-bcc4c8345a21 // indirect
-	github.com/docker/docker v1.13.1 // indirect
+	github.com/docker/docker v1.13.1
 	github.com/docker/go-units v0.3.3
 	github.com/docker/machine v0.7.1-0.20190718054102-a555e4f7a8f5 // version is 0.7.1 to pin to a555e4f7a8f5
 	github.com/elazarl/goproxy v0.0.0-20190421051319-9d40249d3c2f
 	github.com/elazarl/goproxy/ext v0.0.0-20190421051319-9d40249d3c2f // indirect
+	github.com/evanphx/json-patch v4.5.0+incompatible // indirect
 	github.com/go-ole/go-ole v1.2.4 // indirect
 	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
 	github.com/google/go-cmp v0.3.0
+	github.com/googleapis/gnostic v0.3.0 // indirect
 	github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce // indirect
 	github.com/hashicorp/go-getter v1.4.0
 	github.com/hashicorp/go-multierror v0.0.0-20160811015721-8c5f0ad93604 // indirect
@@ -69,17 +71,19 @@ require (
 	github.com/xeipuuv/gojsonschema v0.0.0-20160623135812-c539bca196be
 	github.com/zchee/go-vmnet v0.0.0-20161021174912-97ebf9174097
 	golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d
-	golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8
+	golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4
 	golang.org/x/sync v0.0.0-20190423024810-112230192c58
 	golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb
 	golang.org/x/text v0.3.2
 	gopkg.in/airbrake/gobrake.v2 v2.0.9 // indirect
+	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
 	gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 // indirect
 	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
 	gotest.tools v2.2.0+incompatible
 	k8s.io/api v0.0.0
 	k8s.io/apimachinery v0.0.0
-	k8s.io/client-go v0.0.0
+	k8s.io/client-go v11.0.0+incompatible
+	k8s.io/klog v0.3.3 // indirect
 	k8s.io/kubectl v0.0.0-00010101000000-000000000000
 	sigs.k8s.io/sig-storage-lib-external-provisioner v4.0.0+incompatible
 )

go.sum

@@ -151,6 +151,8 @@ github.com/euank/go-kmsg-parser v2.0.0+incompatible/go.mod h1:MhmAMZ8V4CYH4ybgdR
 github.com/evanphx/json-patch v0.0.0-20190203023257-5858425f7550/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch v4.2.0+incompatible h1:fUDGZCv/7iAN7u0puUVhvKCcsR6vRfwrJatElLBEf0I=
 github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M=
+github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
 github.com/fatih/camelcase v0.0.0-20160318181535-f6a740d52f96/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
 github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
@@ -243,6 +245,8 @@ github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k=
 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
+github.com/googleapis/gnostic v0.3.0 h1:CcQijm0XKekKjP/YCz28LXVSpgguuB+nCxaSjCe09y0=
+github.com/googleapis/gnostic v0.3.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
 github.com/gophercloud/gophercloud v0.0.0-20190126172459-c818fa66e4c8/go.mod h1:3WdhXV3rUYy9p6AUW8d94kr+HS62Y4VL9mBnFxsD8q4=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
@@ -337,10 +341,12 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGi
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/fs v0.0.0-20131111012553-2788f0dbd169/go.mod h1:glhvuHOU9Hy7/8PwwdtnarXqLagOX0b/TbZx2zLMqEg=
 github.com/kr/pretty v0.0.0-20140812000539-f31442d60e51/go.mod h1:Bvhd+E3laJ0AVkG0c9rmtZcnhV0HQ3+c3YxxqTvc/gA=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.0.0-20130911015532-6807e777504f/go.mod h1:sjUstKUATFIcff4qlB53Kml0wQPtJVc/3fWrmuUmcfA=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/libopenstorage/openstorage v0.0.0-20170906232338-093a0c388875/go.mod h1:Sp1sIObHjat1BeXhfMqLZ14wnOzEhNx2YQedreMcUyc=
 github.com/libvirt/libvirt-go v3.4.0+incompatible h1:Cpyalgj1x8JIeTlL6SDYZBo7j8nY3+5XHqmi8DaunCk=
@@ -574,8 +580,8 @@ golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 h1:1wopBVtVdWnn03fZelqdXTqk7U7zPQCb+T4rbU9ZEoU=
-golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc=
+golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -705,6 +711,8 @@ gopkg.in/airbrake/gobrake.v2 v2.0.9 h1:7z2uVWwn7oVeeugY1DtlPAy5H+KYgB1KeKTnqjNat
 gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/cheggaaa/pb.v1 v1.0.27 h1:kJdccidYzt3CaHD1crCFTS1hxyhSi059NhOFUf03YFo=
 gopkg.in/cheggaaa/pb.v1 v1.0.27/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
 gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
@@ -739,6 +747,8 @@ k8s.io/gengo v0.0.0-20190116091435-f8a0810f38af/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8
 k8s.io/heapster v1.2.0-beta.1/go.mod h1:h1uhptVXMwC8xtZBYsPXKVi8fpdlYkTs6k949KozGrM=
 k8s.io/klog v0.3.1 h1:RVgyDHY/kFKtLqh67NvEWIgkMneNoIrdkN0CxDSQc68=
 k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
+k8s.io/klog v0.3.3 h1:niceAagH1tzskmaie/icWd7ci1wbG7Bf2c6YGcQv+3c=
+k8s.io/klog v0.3.3/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
 k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30 h1:TRb4wNWoBVrH9plmkp2q86FIDppkbrEXdXlxU3a3BMI=
 k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc=
 k8s.io/kubernetes v1.15.2 h1:RO9EuRw5vlN3oa/lnmPxmywOoJRtg9o40KcklHXNIAQ=

hack/images/kicbase.Dockerfile

@@ -0,0 +1,18 @@
+ARG COMMIT_SHA
+FROM kindest/node:v1.16.2
+USER root
+RUN apt-get update && apt-get install -y \
+  sudo \
+  dnsutils \
+  && apt-get clean -y 
+RUN rm -rf \
+    /var/cache/debconf/* \
+    /var/lib/apt/lists/* \
+    /var/log/* \
+    /tmp/* \
+    /var/tmp/* \
+    /usr/share/doc/* \
+    /usr/share/man/* \
+    /usr/share/local/* \
+    /kind/bin/kubeadm /kind/bin/kubelet /kind/systemd /kind/images /kind/manifests
+RUN echo "kic! Build: ${COMMIT_SHA} Time :$(date)" > "/kic.txt"

hack/jenkins/common.sh

@@ -26,6 +26,9 @@
 
 readonly TEST_ROOT="${HOME}/minikube-integration"
 readonly TEST_HOME="${TEST_ROOT}/${OS_ARCH}-${VM_DRIVER}-${MINIKUBE_LOCATION}-$$-${COMMIT}"
+export GOPATH="$HOME/go"
+export PATH=$PATH:"/usr/local/bin/:/usr/local/go/bin/:$GOPATH/bin"
+
 echo ">> Starting at $(date)"
 echo ""
 echo "arch:      ${OS_ARCH}"
@@ -39,6 +42,8 @@ echo "uptime:    $(uptime)"
 # Setting KUBECONFIG prevents the version ceck from erroring out due to permission issues
 echo "kubectl:   $(env KUBECONFIG=${TEST_HOME} kubectl version --client --short=true)"
 echo "docker:    $(docker version --format '{{ .Client.Version }}')"
+echo "go:        $(go version || true)"
+
 
 case "${VM_DRIVER}" in
   kvm2)
@@ -258,16 +263,27 @@ if [[ "${LOAD}" -gt 2 ]]; then
   uptime
 fi
 
+readonly TEST_OUT="${TEST_HOME}/testout.txt"
+readonly JSON_OUT="${TEST_HOME}/test.json"
+readonly HTML_OUT="${TEST_HOME}/test.html"
+
 e2e_start_time="$(date -u +%s)"
 echo ""
 echo ">> Starting ${E2E_BIN} at $(date)"
 set -x
+
+if test -f "${TEST_OUT}"; then
+  rm "${TEST_OUT}" || true # clean up previous runs of same build
+fi
+touch "${TEST_OUT}"
 ${SUDO_PREFIX}${E2E_BIN} \
   -minikube-start-args="--vm-driver=${VM_DRIVER} ${EXTRA_START_ARGS}" \
   -expected-default-driver="${EXPECTED_DEFAULT_DRIVER}" \
-  -test.timeout=70m \
+  -test.timeout=70m -test.v \
   ${EXTRA_TEST_ARGS} \
-  -binary="${MINIKUBE_BIN}" && result=$? || result=$?
+  -binary="${MINIKUBE_BIN}" 2>&1 | tee "${TEST_OUT}"
+
+result=${PIPESTATUS[0]} # capture the exit code of the first cmd in pipe.
 set +x
 echo ">> ${E2E_BIN} exited with ${result} at $(date)"
 echo ""
@@ -286,18 +302,60 @@ elapsed=$(($e2e_end_time-$e2e_start_time))
 min=$(($elapsed/60))
 sec=$(tail -c 3 <<< $((${elapsed}00/60)))
 elapsed=$min.$sec
+
+JOB_GCS_BUCKET="minikube-builds/logs/${MINIKUBE_LOCATION}/${JOB_NAME}"
+echo ">> Copying ${TEST_OUT} to gs://${JOB_GCS_BUCKET}out.txt"
+gsutil -qm cp "${TEST_OUT}" "gs://${JOB_GCS_BUCKET}out.txt"
+
+
+echo ">> Attmpting to convert test logs to json"
+if test -f "${JSON_OUT}"; then
+  rm "${JSON_OUT}" || true # clean up previous runs of same build
+fi
+
+touch "${JSON_OUT}"
+
+# Generate JSON output
+echo ">> Running go test2json"
+go tool test2json -t < "${TEST_OUT}" > "${JSON_OUT}" || true
+echo ">> Installing gopogh"
+cd /tmp
+GO111MODULE="on" go get -u github.com/medyagh/gopogh@v0.0.17 || true
+cd -
+echo ">> Running gopogh"
+if test -f "${HTML_OUT}"; then
+    rm "${HTML_OUT}" || true # clean up previous runs of same build
+fi
+
+touch "${HTML_OUT}"
+pessimistic_status=$(gopogh -in "${JSON_OUT}" -out "${HTML_OUT}" -name "${JOB_NAME}" -pr "${MINIKUBE_LOCATION}" -repo github.com/kubernetes/minikube/  -details "${COMMIT}") || true
 description="completed with ${status} in ${elapsed} minute(s)."
+if [ "$status" = "failure" ]; then
+  description="completed with ${pessimistic_status} in ${elapsed} minute(s)."
+fi
 echo $description
 
+echo ">> uploading ${JSON_OUT}"
+gsutil -qm cp "${JSON_OUT}" "gs://${JOB_GCS_BUCKET}.json" || true
+echo ">> uploading ${HTML_OUT}"
+gsutil -qm cp "${HTML_OUT}" "gs://${JOB_GCS_BUCKET}.html" || true
+
+public_log_url="https://storage.googleapis.com/${JOB_GCS_BUCKET}.txt"
+if grep -q html "$HTML_OUT"; then
+  public_log_url="https://storage.googleapis.com/${JOB_GCS_BUCKET}.html"
+fi
 
 echo ">> Cleaning up after ourselves ..."
 ${SUDO_PREFIX}${MINIKUBE_BIN} tunnel --cleanup || true
-${SUDO_PREFIX}${MINIKUBE_BIN} delete >/dev/null 2>/dev/null || true
+${SUDO_PREFIX}${MINIKUBE_BIN} delete --all >/dev/null 2>/dev/null || true
 cleanup_stale_routes || true
 
 ${SUDO_PREFIX} rm -Rf "${MINIKUBE_HOME}" || true
 ${SUDO_PREFIX} rm -f "${KUBECONFIG}" || true
-rmdir "${TEST_HOME}"
+${SUDO_PREFIX} rm -f "${TEST_OUT}" || true
+${SUDO_PREFIX} rm -f "${JSON_OUT}" || true
+${SUDO_PREFIX} rm -f "${HTML_OUT}" || true
+rmdir "${TEST_HOME}" || true
 echo ">> ${TEST_HOME} completed at $(date)"
 
 if [[ "${MINIKUBE_LOCATION}" == "master" ]]; then
@@ -341,5 +399,5 @@ function retry_github_status() {
 
 
 
-retry_github_status "${COMMIT}" "${JOB_NAME}" "${status}" "${access_token}" "https://storage.googleapis.com/minikube-builds/logs/${MINIKUBE_LOCATION}/${JOB_NAME}.txt" "${description}"
+retry_github_status "${COMMIT}" "${JOB_NAME}" "${status}" "${access_token}" "${public_log_url}" "${description}"
 exit $result

hack/jenkins/linux_integration_tests_kvm.sh

@@ -36,5 +36,4 @@ EXTRA_TEST_ARGS="-gvisor"
 mkdir -p cron && gsutil -qm rsync "gs://minikube-builds/${MINIKUBE_LOCATION}/cron" cron || echo "FAILED TO GET CRON FILES"
 sudo install cron/cleanup_and_reboot_Linux.sh /etc/cron.hourly/cleanup_and_reboot || echo "FAILED TO INSTALL CLEANUP"
 
-# Download files and set permissions
 source ./common.sh

hack/jenkins/linux_integration_tests_none.sh

@@ -54,5 +54,4 @@ systemctl is-active --quiet kubelet \
 mkdir -p cron && gsutil -m rsync "gs://minikube-builds/${MINIKUBE_LOCATION}/cron" cron || echo "FAILED TO GET CRON FILES"
 sudo install cron/cleanup_and_reboot_Linux.sh /etc/cron.hourly/cleanup_and_reboot || echo "FAILED TO INSTALL CLEANUP"
 
-# Download files and set permissions
 source ./common.sh

hack/jenkins/linux_integration_tests_virtualbox.sh

@@ -33,6 +33,4 @@ EXPECTED_DEFAULT_DRIVER="kvm2"
 mkdir -p cron && gsutil -qm rsync "gs://minikube-builds/${MINIKUBE_LOCATION}/cron" cron || echo "FAILED TO GET CRON FILES"
 sudo install cron/cleanup_and_reboot_Linux.sh /etc/cron.hourly/cleanup_and_reboot || echo "FAILED TO INSTALL CLEANUP"
 
-
-# Download files and set permissions
 source ./common.sh

hack/jenkins/minikube_set_pending.sh

@@ -78,6 +78,6 @@ function retry_github_status() {
 
 for j in ${jobs[@]}; do
   retry_github_status "${ghprbActualCommit}" "${j}" "pending" "${access_token}" \
-    "https://storage.googleapis.com/minikube-builds/logs/${ghprbPullId}/${j}.txt"
+    "https://storage.googleapis.com/minikube-builds/logs/${ghprbPullId}/${j}.pending"
 done
 

hack/jenkins/osx_integration_tests_hyperkit.sh

@@ -39,5 +39,4 @@ install cron/cleanup_and_reboot_Darwin.sh $HOME/cleanup_and_reboot.sh || echo "F
 echo "*/30 * * * * $HOME/cleanup_and_reboot.sh" | crontab
 crontab -l
 
-# Download files and set permissions
 source common.sh

hack/jenkins/osx_integration_tests_virtualbox.sh

@@ -39,5 +39,4 @@ install cron/cleanup_and_reboot_Darwin.sh $HOME/cleanup_and_reboot.sh  || echo "
 echo "*/30 * * * * $HOME/cleanup_and_reboot.sh" | crontab
 crontab -l
 
-# Download files and set permissions
 source common.sh

hack/jenkins/release_build_and_upload.sh

@@ -55,7 +55,11 @@ gsutil -m cp out/* "gs://$BUCKET/releases/$TAGNAME/"
 # Update "latest" release for non-beta/non-alpha builds
 if ! [[ ${VERSION_BUILD} =~ ^[0-9]+$ ]]; then
   echo "NOTE: ${VERSION} appears to be a non-standard release, not updating /releases/latest"
-else
-  echo "Updating latest bucket for ${VERSION} release"
-  gsutil cp -r "gs://${BUCKET}/releases/${TAGNAME}/*" "gs://${BUCKET}/releases/latest/"
-fi
+  exit 0
+fi
+
+echo "Updating Docker images ..."
+make push-gvisor-addon-image push-storage-provisioner-image
+
+echo "Updating latest bucket for ${VERSION} release ..."
+gsutil cp -r "gs://${BUCKET}/releases/${TAGNAME}/*" "gs://${BUCKET}/releases/latest/"

hack/jenkins/windows_integration_test_hyperv.ps1

@@ -29,4 +29,4 @@ $env:target_url="https://storage.googleapis.com/minikube-builds/logs/$env:MINIKU
 $json = "{`"state`": `"$env:status`", `"description`": `"Jenkins`", `"target_url`": `"$env:target_url`", `"context`": `"Hyper-V_Windows`"}"
 Invoke-WebRequest -Uri "https://api.github.com/repos/kubernetes/minikube/statuses/$env:COMMIT`?access_token=$env:access_token" -Body $json -ContentType "application/json" -Method Post -usebasicparsing
 
-Exit $env:result
+Exit $env:result

hack/jenkins/windows_integration_test_virtualbox.ps1

@@ -29,4 +29,4 @@ $env:target_url="https://storage.googleapis.com/minikube-builds/logs/$env:MINIKU
 $json = "{`"state`": `"$env:status`", `"description`": `"Jenkins`", `"target_url`": `"$env:target_url`", `"context`": `"VirtualBox_Windows`"}"
 Invoke-WebRequest -Uri "https://api.github.com/repos/kubernetes/minikube/statuses/$env:COMMIT`?access_token=$env:access_token" -Body $json -ContentType "application/json" -Method Post -usebasicparsing
 
-Exit $env:result
+Exit $env:result

pkg/addons/addons.go

@@ -0,0 +1,233 @@
+/*
+Copyright 2019 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package addons
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+
+	"github.com/pkg/errors"
+	"github.com/spf13/viper"
+	"k8s.io/minikube/pkg/minikube/assets"
+	"k8s.io/minikube/pkg/minikube/cluster"
+	"k8s.io/minikube/pkg/minikube/command"
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/exit"
+	"k8s.io/minikube/pkg/minikube/machine"
+	"k8s.io/minikube/pkg/minikube/out"
+	"k8s.io/minikube/pkg/minikube/storageclass"
+	pkgutil "k8s.io/minikube/pkg/util"
+)
+
+// defaultStorageClassProvisioner is the name of the default storage class provisioner
+const defaultStorageClassProvisioner = "standard"
+
+// Set sets a value
+func Set(name, value, profile string) error {
+	a, valid := isAddonValid(name)
+	if !valid {
+		return errors.Errorf("%s is not a valid addon", name)
+	}
+
+	// Run any additional validations for this property
+	if err := run(name, value, profile, a.validations); err != nil {
+		return errors.Wrap(err, "running validations")
+	}
+
+	// Set the value
+	c, err := config.Load(profile)
+	if err != nil {
+		return errors.Wrap(err, "loading profile")
+	}
+
+	if err := a.set(c, name, value); err != nil {
+		return errors.Wrap(err, "setting new value of addon")
+	}
+
+	// Run any callbacks for this property
+	if err := run(name, value, profile, a.callbacks); err != nil {
+		return errors.Wrap(err, "running callbacks")
+	}
+
+	// Write the value
+	return config.Write(profile, c)
+}
+
+// Runs all the validation or callback functions and collects errors
+func run(name, value, profile string, fns []setFn) error {
+	var errors []error
+	for _, fn := range fns {
+		err := fn(name, value, profile)
+		if err != nil {
+			errors = append(errors, err)
+		}
+	}
+	if len(errors) > 0 {
+		return fmt.Errorf("%v", errors)
+	}
+	return nil
+}
+
+// SetBool sets a bool value
+func SetBool(m *config.MachineConfig, name string, val string) error {
+	b, err := strconv.ParseBool(val)
+	if err != nil {
+		return err
+	}
+	if m.Addons == nil {
+		m.Addons = map[string]bool{}
+	}
+	m.Addons[name] = b
+	return nil
+}
+
+// enableOrDisableAddon updates addon status executing any commands necessary
+func enableOrDisableAddon(name, val, profile string) error {
+	enable, err := strconv.ParseBool(val)
+	if err != nil {
+		return errors.Wrapf(err, "parsing bool: %s", name)
+	}
+	addon := assets.Addons[name]
+
+	// check addon status before enabling/disabling it
+	alreadySet, err := isAddonAlreadySet(addon, enable)
+	if err != nil {
+		out.ErrT(out.Conflict, "{{.error}}", out.V{"error": err})
+		return err
+	}
+	//if addon is already enabled or disabled, do nothing
+	if alreadySet {
+		return nil
+	}
+
+	if name == "istio" && enable {
+		minMem := 8192
+		minCpus := 4
+		memorySizeMB := pkgutil.CalculateSizeInMB(viper.GetString("memory"))
+		cpuCount := viper.GetInt("cpus")
+		if memorySizeMB < minMem || cpuCount < minCpus {
+			out.WarningT("Enable istio needs {{.minMem}} MB of memory and {{.minCpus}} CPUs.", out.V{"minMem": minMem, "minCpus": minCpus})
+		}
+	}
+
+	// TODO(r2d4): config package should not reference API, pull this out
+	api, err := machine.NewAPIClient()
+	if err != nil {
+		return errors.Wrap(err, "machine client")
+	}
+	defer api.Close()
+
+	//if minikube is not running, we return and simply update the value in the addon
+	//config and rewrite the file
+	if !cluster.IsMinikubeRunning(api) {
+		return nil
+	}
+
+	cfg, err := config.Load(profile)
+	if err != nil && !os.IsNotExist(err) {
+		exit.WithCodeT(exit.Data, "Unable to load config: {{.error}}", out.V{"error": err})
+	}
+
+	host, err := cluster.CheckIfHostExistsAndLoad(api, cfg.Name)
+	if err != nil {
+		return errors.Wrap(err, "getting host")
+	}
+
+	cmd, err := machine.CommandRunner(host)
+	if err != nil {
+		return errors.Wrap(err, "command runner")
+	}
+
+	data := assets.GenerateTemplateData(cfg.KubernetesConfig)
+	return enableOrDisableAddonInternal(addon, cmd, data, enable)
+}
+
+func isAddonAlreadySet(addon *assets.Addon, enable bool) (bool, error) {
+	addonStatus, err := addon.IsEnabled()
+
+	if err != nil {
+		return false, errors.Wrap(err, "get the addon status")
+	}
+
+	if addonStatus && enable {
+		return true, nil
+	} else if !addonStatus && !enable {
+		return true, nil
+	}
+
+	return false, nil
+}
+
+func enableOrDisableAddonInternal(addon *assets.Addon, cmd command.Runner, data interface{}, enable bool) error {
+	var err error
+
+	updateFile := cmd.Copy
+	if !enable {
+		updateFile = cmd.Remove
+	}
+
+	for _, addon := range addon.Assets {
+		var addonFile assets.CopyableFile
+		if addon.IsTemplate() {
+			addonFile, err = addon.Evaluate(data)
+			if err != nil {
+				return errors.Wrapf(err, "evaluate bundled addon %s asset", addon.GetAssetName())
+			}
+
+		} else {
+			addonFile = addon
+		}
+		if err := updateFile(addonFile); err != nil {
+			return errors.Wrapf(err, "updating addon %s", addon.AssetName)
+		}
+	}
+	return nil
+}
+
+// enableOrDisableStorageClasses enables or disables storage classes
+func enableOrDisableStorageClasses(name, val, profile string) error {
+	enable, err := strconv.ParseBool(val)
+	if err != nil {
+		return errors.Wrap(err, "Error parsing boolean")
+	}
+
+	class := defaultStorageClassProvisioner
+	if name == "storage-provisioner-gluster" {
+		class = "glusterfile"
+	}
+	storagev1, err := storageclass.GetStoragev1()
+	if err != nil {
+		return errors.Wrapf(err, "Error getting storagev1 interface %v ", err)
+	}
+
+	if enable {
+		// Only StorageClass for 'name' should be marked as default
+		err = storageclass.SetDefaultStorageClass(storagev1, class)
+		if err != nil {
+			return errors.Wrapf(err, "Error making %s the default storage class", class)
+		}
+	} else {
+		// Unset the StorageClass as default
+		err := storageclass.DisableDefaultStorageClass(storagev1, class)
+		if err != nil {
+			return errors.Wrapf(err, "Error disabling %s as the default storage class", class)
+		}
+	}
+
+	return enableOrDisableAddon(name, val, profile)
+}

pkg/addons/addons_test.go

@@ -0,0 +1,109 @@
+/*
+Copyright 2019 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package addons
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"gotest.tools/assert"
+	"k8s.io/minikube/pkg/minikube/assets"
+	"k8s.io/minikube/pkg/minikube/config"
+)
+
+func TestIsAddonAlreadySet(t *testing.T) {
+	testCases := []struct {
+		addonName string
+	}{
+		{
+			addonName: "ingress",
+		},
+
+		{
+			addonName: "registry",
+		},
+	}
+
+	for _, test := range testCases {
+		addon := assets.Addons[test.addonName]
+		addonStatus, _ := addon.IsEnabled()
+
+		alreadySet, err := isAddonAlreadySet(addon, addonStatus)
+		if !alreadySet {
+			if addonStatus {
+				t.Errorf("Did not get expected status, \n\n expected %+v already enabled", test.addonName)
+			} else {
+				t.Errorf("Did not get expected status, \n\n expected %+v already disabled", test.addonName)
+			}
+		}
+		if err != nil {
+			t.Errorf("Got unexpected error: %+v", err)
+		}
+	}
+}
+
+func TestDisableUnknownAddon(t *testing.T) {
+	tmpProfile := "temp-minikube-profile"
+	if err := Set("InvalidAddon", "false", tmpProfile); err == nil {
+		t.Fatalf("Disable did not return error for unknown addon")
+	}
+}
+
+func TestEnableUnknownAddon(t *testing.T) {
+	tmpProfile := "temp-minikube-profile"
+	if err := Set("InvalidAddon", "true", tmpProfile); err == nil {
+		t.Fatalf("Enable did not return error for unknown addon")
+	}
+}
+
+func TestEnableAndDisableAddon(t *testing.T) {
+	tests := []struct {
+		name   string
+		enable bool
+	}{
+		{
+			name:   "test enable",
+			enable: true,
+		}, {
+			name:   "test disable",
+			enable: false,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			tmpProfile := "temp-minikube-profile"
+			if err := os.MkdirAll(config.ProfileFolderPath(tmpProfile), 0777); err != nil {
+				t.Fatalf("error creating temporary directory")
+			}
+			defer os.RemoveAll(config.ProfileFolderPath(tmpProfile))
+
+			if err := config.DefaultLoader.WriteConfigToFile(tmpProfile, &config.MachineConfig{}); err != nil {
+				t.Fatalf("error creating temporary profile config: %v", err)
+			}
+			if err := Set("dashboard", fmt.Sprintf("%t", test.enable), tmpProfile); err != nil {
+				t.Fatalf("Disable returned unexpected error: " + err.Error())
+			}
+			c, err := config.DefaultLoader.LoadConfigFromFile(tmpProfile)
+			if err != nil {
+				t.Fatalf("error loading config: %v", err)
+			}
+			assert.Equal(t, c.Addons["dashboard"], test.enable)
+		})
+	}
+}

pkg/addons/config.go

@@ -0,0 +1,130 @@
+/*
+Copyright 2019 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package addons
+
+import "k8s.io/minikube/pkg/minikube/config"
+
+type setFn func(string, string, string) error
+
+// Addon represents an addon
+type Addon struct {
+	name        string
+	set         func(*config.MachineConfig, string, string) error
+	validations []setFn
+	callbacks   []setFn
+}
+
+// Addons is a list of all addons
+var Addons = []*Addon{
+	{
+		name:      "addon-manager",
+		set:       SetBool,
+		callbacks: []setFn{enableOrDisableAddon},
+	},
+	{
+		name:      "dashboard",
+		set:       SetBool,
+		callbacks: []setFn{enableOrDisableAddon},
+	},
+
+	{
+		name:      "default-storageclass",
+		set:       SetBool,
+		callbacks: []setFn{enableOrDisableStorageClasses},
+	},
+	{
+		name:      "efk",
+		set:       SetBool,
+		callbacks: []setFn{enableOrDisableAddon},
+	},
+	{
+		name:      "freshpod",
+		set:       SetBool,
+		callbacks: []setFn{enableOrDisableAddon},
+	},
+	{
+		name:        "gvisor",
+		set:         SetBool,
+		validations: []setFn{IsContainerdRuntime},
+		callbacks:   []setFn{enableOrDisableAddon},
+	},
+	{
+		name:      "helm-tiller",
+		set:       SetBool,
+		callbacks: []setFn{enableOrDisableAddon},
+	},
+	{
+		name:      "ingress",
+		set:       SetBool,
+		callbacks: []setFn{enableOrDisableAddon},
+	},
+	{
+		name:      "ingress-dns",
+		set:       SetBool,
+		callbacks: []setFn{enableOrDisableAddon},
+	},
+	{
+		name:      "istio-provisioner",
+		set:       SetBool,
+		callbacks: []setFn{enableOrDisableAddon},
+	},
+	{
+		name:      "istio",
+		set:       SetBool,
+		callbacks: []setFn{enableOrDisableAddon},
+	},
+	{
+		name: "logviewer",
+		set:  SetBool,
+	},
+	{
+		name:      "metrics-server",
+		set:       SetBool,
+		callbacks: []setFn{enableOrDisableAddon},
+	},
+	{
+		name:      "nvidia-driver-installer",
+		set:       SetBool,
+		callbacks: []setFn{enableOrDisableAddon},
+	},
+	{
+		name:      "nvidia-gpu-device-plugin",
+		set:       SetBool,
+		callbacks: []setFn{enableOrDisableAddon},
+	},
+
+	{
+		name:      "registry",
+		set:       SetBool,
+		callbacks: []setFn{enableOrDisableAddon},
+	},
+	{
+		name:      "registry-creds",
+		set:       SetBool,
+		callbacks: []setFn{enableOrDisableAddon},
+	},
+	{
+		name:      "storage-provisioner",
+		set:       SetBool,
+		callbacks: []setFn{enableOrDisableAddon},
+	},
+	{
+		name:      "storage-provisioner-gluster",
+		set:       SetBool,
+		callbacks: []setFn{enableOrDisableStorageClasses},
+	},
+}

pkg/addons/validations.go

@@ -0,0 +1,62 @@
+/*
+Copyright 2019 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package addons
+
+import (
+	"fmt"
+
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/cruntime"
+)
+
+// containerdOnlyMsg is the message shown when a containerd-only addon is enabled
+const containerdOnlyAddonMsg = `
+This addon can only be enabled with the containerd runtime backend. To enable this backend, please first stop minikube with:
+
+minikube stop
+
+and then start minikube again with the following flags:
+
+minikube start --container-runtime=containerd --docker-opt containerd=/var/run/containerd/containerd.sock`
+
+// IsContainerdRuntime is a validator which returns an error if the current runtime is not containerd
+func IsContainerdRuntime(_, _, profile string) error {
+	config, err := config.Load(profile)
+	if err != nil {
+		return fmt.Errorf("config.Load: %v", err)
+	}
+	r, err := cruntime.New(cruntime.Config{Type: config.KubernetesConfig.ContainerRuntime})
+	if err != nil {
+		return err
+	}
+	_, ok := r.(*cruntime.Containerd)
+	if !ok {
+		return fmt.Errorf(containerdOnlyAddonMsg)
+	}
+	return nil
+}
+
+// isAddonValid returns the addon, true if it is valid
+// otherwise returns nil, false
+func isAddonValid(name string) (*Addon, bool) {
+	for _, a := range Addons {
+		if a.name == name {
+			return a, true
+		}
+	}
+	return nil, false
+}

pkg/addons/validations_test.go

@@ -0,0 +1,45 @@
+/*
+Copyright 2019 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package addons
+
+import "testing"
+
+func TestIsAddonValid(t *testing.T) {
+	tests := []struct {
+		description string
+		name        string
+		isValid     bool
+	}{
+		{
+			description: "valid addon",
+			name:        "gvisor",
+			isValid:     true,
+		}, {
+			description: "invalid addon",
+			name:        "invalid",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.description, func(t *testing.T) {
+			_, valid := isAddonValid(test.name)
+			if test.isValid != valid {
+				t.Fatalf("expected: %t\nactual:%t\n", test.isValid, valid)
+			}
+		})
+	}
+}

pkg/drivers/kic/kic.go

@@ -0,0 +1,260 @@
+/*
+Copyright 2019 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kic
+
+import (
+	"fmt"
+	"os/exec"
+	"strconv"
+	"strings"
+
+	"github.com/docker/machine/libmachine/drivers"
+	"github.com/docker/machine/libmachine/state"
+	"github.com/pkg/errors"
+	pkgdrivers "k8s.io/minikube/pkg/drivers"
+	"k8s.io/minikube/pkg/drivers/kic/node"
+	"k8s.io/minikube/pkg/drivers/kic/oci"
+	"k8s.io/minikube/pkg/minikube/command"
+	"k8s.io/minikube/pkg/minikube/constants"
+)
+
+// DefaultPodCIDR is The CIDR to be used for pods inside the node.
+const DefaultPodCIDR = "10.244.0.0/16"
+
+// DefaultBindIPV4 is The default IP the container will bind to.
+const DefaultBindIPV4 = "127.0.0.1"
+
+// BaseImage is the base image is used to spin up kic containers
+const BaseImage = "gcr.io/k8s-minikube/kicbase:v0.0.1@sha256:c4ad2938877d2ae0d5b7248a5e7182ff58c0603165c3bedfe9d503e2d380a0db"
+
+// Driver represents a kic driver https://minikube.sigs.k8s.io/docs/reference/drivers/kic/
+type Driver struct {
+	*drivers.BaseDriver
+	*pkgdrivers.CommonDriver
+	URL        string
+	exec       command.Runner
+	NodeConfig Config
+	OCIBinary  string // docker,podman
+}
+
+// Config is configuration for the kic driver used by registry
+type Config struct {
+	MachineName  string            // maps to the container name being created
+	CPU          int               // Number of CPU cores assigned to the container
+	Memory       int               // max memory in MB
+	StorePath    string            // libmachine store path
+	OCIBinary    string            // oci tool to use (docker, podman,...)
+	ImageDigest  string            // image name with sha to use for the node
+	HostBindPort int               // port to connect to forward from container to user's machine
+	Mounts       []oci.Mount       // mounts
+	PortMappings []oci.PortMapping // container port mappings
+	Envs         map[string]string // key,value of environment variables passed to the node
+}
+
+// NewDriver returns a fully configured Kic driver
+func NewDriver(c Config) *Driver {
+	d := &Driver{
+		BaseDriver: &drivers.BaseDriver{
+			MachineName: c.MachineName,
+			StorePath:   c.StorePath,
+		},
+		exec:       command.NewKICRunner(c.MachineName, c.OCIBinary),
+		NodeConfig: c,
+		OCIBinary:  c.OCIBinary,
+	}
+	return d
+}
+
+// Create a host using the driver's config
+func (d *Driver) Create() error {
+	params := node.CreateConfig{
+		Name:         d.NodeConfig.MachineName,
+		Image:        d.NodeConfig.ImageDigest,
+		ClusterLabel: node.ClusterLabelKey + "=" + d.MachineName,
+		CPUs:         strconv.Itoa(d.NodeConfig.CPU),
+		Memory:       strconv.Itoa(d.NodeConfig.Memory) + "mb",
+		Envs:         d.NodeConfig.Envs,
+		ExtraArgs:    []string{"--expose", fmt.Sprintf("%d", d.NodeConfig.HostBindPort)},
+		OCIBinary:    d.NodeConfig.OCIBinary,
+	}
+
+	// control plane specific options
+	params.PortMappings = append(params.PortMappings, oci.PortMapping{
+		ListenAddress: "127.0.0.1",
+		HostPort:      int32(d.NodeConfig.HostBindPort),
+		ContainerPort: constants.APIServerPort,
+	})
+
+	_, err := node.CreateNode(params)
+	if err != nil {
+		return errors.Wrap(err, "create kic node")
+	}
+	return nil
+}
+
+// DriverName returns the name of the driver
+func (d *Driver) DriverName() string {
+	if d.NodeConfig.OCIBinary == "podman" {
+		return "podman"
+	}
+	return "docker"
+}
+
+// GetIP returns an IP or hostname that this host is available at
+func (d *Driver) GetIP() (string, error) {
+	node, err := node.Find(d.OCIBinary, d.MachineName, d.exec)
+	if err != nil {
+		return "", fmt.Errorf("ip not found for nil node")
+	}
+	ip, _, err := node.IP()
+	return ip, err
+}
+
+// GetSSHHostname returns hostname for use with ssh
+func (d *Driver) GetSSHHostname() (string, error) {
+	return "", fmt.Errorf("driver does not have SSHHostName")
+}
+
+// GetSSHPort returns port for use with ssh
+func (d *Driver) GetSSHPort() (int, error) {
+	return 0, fmt.Errorf("driver does not support GetSSHPort")
+}
+
+// GetURL returns ip of the container running kic control-panel
+func (d *Driver) GetURL() (string, error) {
+	return d.GetIP()
+}
+
+// GetState returns the state that the host is in (running, stopped, etc)
+func (d *Driver) GetState() (state.State, error) {
+	cmd := exec.Command(d.NodeConfig.OCIBinary, "inspect", "-f", "{{.State.Status}}", d.MachineName)
+	out, err := cmd.CombinedOutput()
+	o := strings.Trim(string(out), "\n")
+	if err != nil {
+		return state.Error, errors.Wrapf(err, "error stop node %s", d.MachineName)
+	}
+	switch o {
+	case "running":
+		return state.Running, nil
+	case "exited":
+		return state.Stopped, nil
+	case "paused":
+		return state.Paused, nil
+	case "restarting":
+		return state.Starting, nil
+	case "dead":
+		return state.Error, nil
+	default:
+		return state.None, fmt.Errorf("unknown state")
+	}
+}
+
+// Kill stops a host forcefully, including any containers that we are managing.
+func (d *Driver) Kill() error {
+	cmd := exec.Command(d.NodeConfig.OCIBinary, "kill", d.MachineName)
+	if err := cmd.Run(); err != nil {
+		return errors.Wrapf(err, "killing kic node %s", d.MachineName)
+	}
+	return nil
+}
+
+// Remove will delete the Kic Node Container
+func (d *Driver) Remove() error {
+	if _, err := d.nodeID(d.MachineName); err != nil {
+		return errors.Wrapf(err, "not found node %s", d.MachineName)
+	}
+	cmd := exec.Command(d.NodeConfig.OCIBinary, "rm", "-f", "-v", d.MachineName)
+	if err := cmd.Run(); err != nil {
+		return errors.Wrapf(err, "error removing node %s", d.MachineName)
+	}
+	return nil
+}
+
+// Restart a host
+func (d *Driver) Restart() error {
+	s, err := d.GetState()
+	if err != nil {
+		return errors.Wrap(err, "get kic state")
+	}
+	switch s {
+	case state.Paused:
+		return d.Unpause()
+	case state.Stopped:
+		return d.Start()
+	case state.Running, state.Error:
+		if err = d.Stop(); err != nil {
+			return fmt.Errorf("restarting a kic stop phase %v", err)
+		}
+		if err = d.Start(); err != nil {
+			return fmt.Errorf("restarting a kic start phase %v", err)
+		}
+		return nil
+	}
+
+	return fmt.Errorf("restarted not implemented for kic state %s yet", s)
+}
+
+// Unpause a kic container
+func (d *Driver) Unpause() error {
+	cmd := exec.Command(d.NodeConfig.OCIBinary, "unpause", d.MachineName)
+	if err := cmd.Run(); err != nil {
+		return errors.Wrapf(err, "unpausing %s", d.MachineName)
+	}
+	return nil
+}
+
+// Start a _stopped_ kic container
+// not meant to be used for Create().
+func (d *Driver) Start() error {
+	s, err := d.GetState()
+	if err != nil {
+		return errors.Wrap(err, "get kic state")
+	}
+	if s == state.Stopped {
+		cmd := exec.Command(d.NodeConfig.OCIBinary, "start", d.MachineName)
+		if err := cmd.Run(); err != nil {
+			return errors.Wrapf(err, "starting a stopped kic node %s", d.MachineName)
+		}
+		return nil
+	}
+	// TODO:medyagh maybe make it idempotent
+	return fmt.Errorf("cant start a not-stopped (%s) kic node", s)
+}
+
+// Stop a host gracefully, including any containers that we are managing.
+func (d *Driver) Stop() error {
+	cmd := exec.Command(d.NodeConfig.OCIBinary, "stop", d.MachineName)
+	if err := cmd.Run(); err != nil {
+		return errors.Wrapf(err, "stopping %s", d.MachineName)
+	}
+	return nil
+}
+
+// RunSSHCommandFromDriver implements direct ssh control to the driver
+func (d *Driver) RunSSHCommandFromDriver() error {
+	return fmt.Errorf("driver does not support RunSSHCommandFromDriver commands")
+}
+
+// looks up for a container node by name, will return error if not found.
+func (d *Driver) nodeID(nameOrID string) (string, error) {
+	cmd := exec.Command(d.NodeConfig.OCIBinary, "inspect", "-f", "{{.Id}}", nameOrID)
+	id, err := cmd.CombinedOutput()
+	if err != nil {
+		id = []byte{}
+	}
+	return string(id), err
+}

pkg/drivers/kic/node/node.go

@@ -0,0 +1,195 @@
+/*
+Copyright 2019 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package node
+
+import (
+	"fmt"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"k8s.io/minikube/pkg/drivers/kic/oci"
+	"k8s.io/minikube/pkg/minikube/assets"
+	"k8s.io/minikube/pkg/minikube/command"
+
+	"github.com/pkg/errors"
+)
+
+const (
+	// Docker default bridge network is named "bridge" (https://docs.docker.com/network/bridge/#use-the-default-bridge-network)
+	DefaultNetwork  = "bridge"
+	ClusterLabelKey = "io.x-k8s.kic.cluster" // ClusterLabelKey is applied to each node docker container for identification
+	NodeRoleKey     = "io.k8s.sigs.kic.role"
+)
+
+// Node represents a handle to a kic node
+// This struct must be created by one of: CreateControlPlane
+type Node struct {
+	id        string         // container id
+	name      string         // container name
+	r         command.Runner // Runner
+	ociBinary string
+}
+
+type CreateConfig struct {
+	Name         string            // used for container name and hostname
+	Image        string            // container image to use to create the node.
+	ClusterLabel string            // label the containers we create using minikube so we can clean up
+	Role         string            // currently only role supported is control-plane
+	Mounts       []oci.Mount       // volume mounts
+	PortMappings []oci.PortMapping // ports to map to container from host
+	CPUs         string            // number of cpu cores assign to container
+	Memory       string            // memory (mbs) to assign to the container
+	Envs         map[string]string // environment variables to pass to the container
+	ExtraArgs    []string          // a list of any extra option to pass to oci binary during creation time, for example --expose 8080...
+	OCIBinary    string            // docker or podman
+}
+
+// CreateNode creates a new container node
+func CreateNode(p CreateConfig) (*Node, error) {
+	cmder := command.NewKICRunner(p.Name, p.OCIBinary)
+	runArgs := []string{
+		fmt.Sprintf("--cpus=%s", p.CPUs),
+		fmt.Sprintf("--memory=%s", p.Memory),
+		"-d", // run the container detached
+		"-t", // allocate a tty for entrypoint logs
+		// running containers in a container requires privileged
+		// NOTE: we could try to replicate this with --cap-add, and use less
+		// privileges, but this flag also changes some mounts that are necessary
+		// including some ones docker would otherwise do by default.
+		// for now this is what we want. in the future we may revisit this.
+		"--privileged",
+		"--security-opt", "seccomp=unconfined", // also ignore seccomp
+		"--tmpfs", "/tmp", // various things depend on working /tmp
+		"--tmpfs", "/run", // systemd wants a writable /run
+		// logs,pods be stroed on  filesystem vs inside container,
+		"--volume", "/var",
+		// some k8s things want /lib/modules
+		"-v", "/lib/modules:/lib/modules:ro",
+		"--hostname", p.Name, // make hostname match container name
+		"--name", p.Name, // ... and set the container name
+		// label the node with the cluster ID
+		"--label", p.ClusterLabel,
+		// label the node with the role ID
+		"--label", fmt.Sprintf("%s=%s", NodeRoleKey, p.Role),
+	}
+
+	for key, val := range p.Envs {
+		runArgs = append(runArgs, "-e", fmt.Sprintf("%s=%s", key, val))
+	}
+
+	// adds node specific args
+	runArgs = append(runArgs, p.ExtraArgs...)
+
+	if oci.UsernsRemap(p.OCIBinary) {
+		// We need this argument in order to make this command work
+		// in systems that have userns-remap enabled on the docker daemon
+		runArgs = append(runArgs, "--userns=host")
+	}
+
+	_, err := oci.CreateContainer(p.OCIBinary,
+		p.Image,
+		oci.WithRunArgs(runArgs...),
+		oci.WithMounts(p.Mounts),
+		oci.WithPortMappings(p.PortMappings),
+	)
+
+	if err != nil {
+		return nil, errors.Wrap(err, "oci create ")
+	}
+
+	// we should return a handle so the caller can clean it up
+	node, err := Find(p.OCIBinary, p.Name, cmder)
+	if err != nil {
+		return node, errors.Wrap(err, "find node")
+	}
+
+	return node, nil
+}
+
+// Find finds a node
+func Find(ociBinary string, name string, cmder command.Runner) (*Node, error) {
+	n, err := oci.Inspect(ociBinary, name, "{{.Id}}")
+	if err != nil {
+		return nil, fmt.Errorf("can't find node %v", err)
+	}
+	return &Node{
+		ociBinary: ociBinary,
+		id:        n[0],
+		name:      name,
+		r:         cmder,
+	}, nil
+}
+
+// WriteFile writes content to dest on the node
+func (n *Node) WriteFile(dest, content string, perm string) error {
+	// create destination directory
+	cmd := exec.Command("mkdir", "-p", filepath.Dir(dest))
+	rr, err := n.r.RunCmd(cmd)
+	if err != nil {
+		return errors.Wrapf(err, "failed to create directory %s cmd: %v output:%q", cmd.Args, dest, rr.Output())
+	}
+
+	cmd = exec.Command("cp", "/dev/stdin", dest)
+	cmd.Stdin = strings.NewReader(content)
+
+	if rr, err := n.r.RunCmd(cmd); err != nil {
+		return errors.Wrapf(err, "failed to run: cp /dev/stdin %s cmd: %v output:%q", dest, cmd.Args, rr.Output())
+	}
+
+	cmd = exec.Command("chmod", perm, dest)
+	_, err = n.r.RunCmd(cmd)
+	if err != nil {
+		return errors.Wrapf(err, "failed to run: chmod %s %s", perm, dest)
+	}
+	return nil
+}
+
+// IP returns the IP address of the node
+func (n *Node) IP() (ipv4 string, ipv6 string, err error) {
+	// retrieve the IP address of the node using docker inspect
+	lines, err := oci.Inspect(n.ociBinary, n.name, "{{range .NetworkSettings.Networks}}{{.IPAddress}},{{.GlobalIPv6Address}}{{end}}")
+	if err != nil {
+		return "", "", errors.Wrap(err, "node ips")
+	}
+	if len(lines) != 1 {
+		return "", "", errors.Errorf("file should only be one line, got %d lines", len(lines))
+	}
+	ips := strings.Split(lines[0], ",")
+	if len(ips) != 2 {
+		return "", "", errors.Errorf("container addresses should have 2 values, got %d values: %+v", len(ips), ips)
+	}
+	return ips[0], ips[1], nil
+}
+
+// Copy copies a local asset into the node
+func (n *Node) Copy(ociBinary string, asset assets.CopyableFile) error {
+	if err := oci.Copy(ociBinary, n.name, asset); err != nil {
+		return errors.Wrap(err, "failed to copy file/folder")
+	}
+
+	cmd := exec.Command("chmod", asset.GetPermissions(), asset.GetTargetName())
+	if _, err := n.r.RunCmd(cmd); err != nil {
+		return errors.Wrap(err, "failed to chmod file permissions")
+	}
+	return nil
+}
+
+// Remove removes the node
+func (n *Node) Remove() error {
+	return oci.Remove(n.ociBinary, n.name)
+}

pkg/drivers/kic/oci/oci.go

@@ -0,0 +1,396 @@
+/*
+Copyright 2019 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package oci
+
+import (
+	"os"
+
+	"github.com/docker/machine/libmachine/state"
+	"k8s.io/minikube/pkg/minikube/assets"
+
+	"bufio"
+	"bytes"
+
+	"github.com/pkg/errors"
+
+	"fmt"
+	"net"
+	"os/exec"
+	"strings"
+	"time"
+
+	"github.com/cenkalti/backoff"
+)
+
+// Stop stops a container
+func Stop(ociBinary, ociID string) error {
+	cmd := exec.Command(ociBinary, "stop", ociID)
+	err := cmd.Run()
+	if err != nil {
+		return errors.Wrapf(err, "error stop node %s", ociID)
+	}
+
+	return nil
+}
+
+// Status returns the status of the container
+func Status(ociBinary string, ociID string) (state.State, error) {
+	cmd := exec.Command(ociBinary, "inspect", "-f", "{{.State.Status}}", ociID)
+	out, err := cmd.CombinedOutput()
+	o := strings.Trim(string(out), "\n")
+	s := state.Error
+	switch o {
+	case "running":
+		s = state.Running
+	case "exited":
+		s = state.Stopped
+	case "paused":
+		s = state.Paused
+	case "restaring":
+		s = state.Starting
+	}
+
+	if err != nil {
+		return state.Error, errors.Wrapf(err, "error getting node %s status", ociID)
+	}
+	return s, nil
+}
+
+// SystemStatus checks if the oci container engine is running
+func SystemStatus(ociBinary string, ociID string) (state.State, error) {
+	_, err := exec.LookPath(ociBinary)
+	if err != nil {
+		return state.Error, err
+	}
+
+	err = exec.Command("docker", "info").Run()
+	if err != nil {
+		return state.Error, err
+	}
+
+	return state.Running, nil
+}
+
+// Remove removes a container
+func Remove(ociBinary string, ociID string) error {
+	// TODO: force remove should be an option
+	cmd := exec.Command(ociBinary, "rm", "-f", "-v", ociID)
+	if err := cmd.Run(); err != nil {
+		return errors.Wrapf(err, "error removing node %s", ociID)
+	}
+
+	return nil
+}
+
+// Pause pauses a container
+func Pause(ociBinary string, ociID string) error {
+	cmd := exec.Command(ociBinary, "pause", ociID)
+	if err := cmd.Run(); err != nil {
+		return errors.Wrapf(err, "error pausing node %s", ociID)
+	}
+
+	return nil
+}
+
+// Inspect return low-level information on containers
+func Inspect(ociBinary string, containerNameOrID, format string) ([]string, error) {
+	cmd := exec.Command(ociBinary, "inspect",
+		"-f", format,
+		containerNameOrID) // ... against the "node" container
+	var buff bytes.Buffer
+	cmd.Stdout = &buff
+	cmd.Stderr = &buff
+	err := cmd.Run()
+	scanner := bufio.NewScanner(&buff)
+	var lines []string
+	for scanner.Scan() {
+		lines = append(lines, scanner.Text())
+	}
+	return lines, err
+}
+
+// NetworkInspect displays detailed information on one or more networks
+func NetworkInspect(networkNames []string, format string) ([]string, error) {
+	cmd := exec.Command("docker", "network", "inspect",
+		"-f", format,
+		strings.Join(networkNames, " "))
+	var buff bytes.Buffer
+	cmd.Stdout = &buff
+	cmd.Stderr = &buff
+	err := cmd.Run()
+	scanner := bufio.NewScanner(&buff)
+	var lines []string
+	for scanner.Scan() {
+		lines = append(lines, scanner.Text())
+	}
+	return lines, err
+}
+
+// GetSubnets returns a slice of subnets for a specified network name
+// For example the command : docker network inspect -f '{{range (index (index . "IPAM") "Config")}}{{index . "Subnet"}} {{end}}' bridge
+// returns 172.17.0.0/16
+func GetSubnets(networkName string) ([]string, error) {
+	format := `{{range (index (index . "IPAM") "Config")}}{{index . "Subnet"}} {{end}}`
+	lines, err := NetworkInspect([]string{networkName}, format)
+	if err != nil {
+		return nil, err
+	}
+	return strings.Split(lines[0], " "), nil
+}
+
+// ImageInspect return low-level information on containers images
+func ImageInspect(containerNameOrID, format string) ([]string, error) {
+	cmd := exec.Command("docker", "image", "inspect",
+		"-f", format,
+		containerNameOrID,
+	)
+	var buff bytes.Buffer
+	cmd.Stdout = &buff
+	cmd.Stderr = &buff
+	err := cmd.Run()
+	scanner := bufio.NewScanner(&buff)
+	var lines []string
+	for scanner.Scan() {
+		lines = append(lines, scanner.Text())
+	}
+	return lines, err
+}
+
+// ImageID return the Id of the container image
+func ImageID(containerNameOrID string) (string, error) {
+	lines, err := ImageInspect(containerNameOrID, "{{ .Id }}")
+	if err != nil {
+		return "", err
+	}
+	if len(lines) != 1 {
+		return "", fmt.Errorf("docker image ID should only be one line, got %d lines", len(lines))
+	}
+	return lines[0], nil
+}
+
+/*
+This is adapated from:
+https://github.com/kubernetes/kubernetes/blob/07a5488b2a8f67add543da72e8819407d8314204/pkg/kubelet/dockershim/helpers.go#L115-L155
+*/
+// generateMountBindings converts the mount list to a list of strings that
+// can be understood by docker
+// '<HostPath>:<ContainerPath>[:options]', where 'options'
+// is a comma-separated list of the following strings:
+// 'ro', if the path is read only
+// 'Z', if the volume requires SELinux relabeling
+func generateMountBindings(mounts ...Mount) []string {
+	result := make([]string, 0, len(mounts))
+	for _, m := range mounts {
+		bind := fmt.Sprintf("%s:%s", m.HostPath, m.ContainerPath)
+		var attrs []string
+		if m.Readonly {
+			attrs = append(attrs, "ro")
+		}
+		// Only request relabeling if the pod provides an SELinux context. If the pod
+		// does not provide an SELinux context relabeling will label the volume with
+		// the container's randomly allocated MCS label. This would restrict access
+		// to the volume to the container which mounts it first.
+		if m.SelinuxRelabel {
+			attrs = append(attrs, "Z")
+		}
+		switch m.Propagation {
+		case MountPropagationNone:
+			// noop, private is default
+		case MountPropagationBidirectional:
+			attrs = append(attrs, "rshared")
+		case MountPropagationHostToContainer:
+			attrs = append(attrs, "rslave")
+		default:
+			// Falls back to "private"
+		}
+
+		if len(attrs) > 0 {
+			bind = fmt.Sprintf("%s:%s", bind, strings.Join(attrs, ","))
+		}
+		// our specific modification is the following line: make this a docker flag
+		bind = fmt.Sprintf("--volume=%s", bind)
+		result = append(result, bind)
+	}
+	return result
+}
+
+// PullIfNotPresent pulls docker image if not present back off exponentially
+func PullIfNotPresent(ociBinary string, image string, forceUpdate bool, maxWait time.Duration) error {
+	cmd := exec.Command(ociBinary, "inspect", "--type=image", image)
+	err := cmd.Run()
+	if err == nil && !forceUpdate {
+		return nil // if presents locally and not force
+	}
+	b := backoff.NewExponentialBackOff()
+	b.MaxElapsedTime = maxWait
+	f := func() error {
+		return pull(ociBinary, image)
+	}
+	return backoff.Retry(f, b)
+}
+
+// Pull pulls an image, retrying up to retries times
+func pull(ociBinary string, image string) error {
+	cmd := exec.Command(ociBinary, "pull", image)
+	err := cmd.Run()
+	if err != nil {
+		return fmt.Errorf("error pull image %s : %v", image, err)
+	}
+	return err
+}
+
+// UsernsRemap checks if userns-remap is enabled in dockerd
+func UsernsRemap(ociBinary string) bool {
+	cmd := exec.Command(ociBinary, "info", "--format", "'{{json .SecurityOptions}}'")
+	var buff bytes.Buffer
+	cmd.Stdout = &buff
+	cmd.Stderr = &buff
+	err := cmd.Run()
+	scanner := bufio.NewScanner(&buff)
+	var lines []string
+	for scanner.Scan() {
+		lines = append(lines, scanner.Text())
+	}
+	if err != nil {
+		return false
+	}
+	if len(lines) > 0 {
+		if strings.Contains(lines[0], "name=userns") {
+			return true
+		}
+	}
+	return false
+}
+
+func generatePortMappings(portMappings ...PortMapping) []string {
+	result := make([]string, 0, len(portMappings))
+	for _, pm := range portMappings {
+		var hostPortBinding string
+		if pm.ListenAddress != "" {
+			hostPortBinding = net.JoinHostPort(pm.ListenAddress, fmt.Sprintf("%d", pm.HostPort))
+		} else {
+			hostPortBinding = fmt.Sprintf("%d", pm.HostPort)
+		}
+		publish := fmt.Sprintf("--publish=%s:%d", hostPortBinding, pm.ContainerPort)
+		result = append(result, publish)
+	}
+	return result
+}
+
+// Save saves an image archive "docker/podman save"
+func Save(ociBinary string, image, dest string) error {
+	cmd := exec.Command(ociBinary, "save", "-o", dest, image)
+	var buff bytes.Buffer
+	cmd.Stdout = &buff
+	cmd.Stderr = &buff
+	err := cmd.Run()
+	scanner := bufio.NewScanner(&buff)
+	var lines []string
+	for scanner.Scan() {
+		lines = append(lines, scanner.Text())
+	}
+	if err != nil {
+		return errors.Wrapf(err, "saving image to tar failed, output %s", lines[0])
+	}
+	return nil
+}
+
+// CreateOpt is an option for Create
+type CreateOpt func(*createOpts) *createOpts
+
+// actual options struct
+type createOpts struct {
+	RunArgs       []string
+	ContainerArgs []string
+	Mounts        []Mount
+	PortMappings  []PortMapping
+}
+
+// CreateContainer creates a container with "docker/podman run"
+func CreateContainer(ociBinary string, image string, opts ...CreateOpt) ([]string, error) {
+	o := &createOpts{}
+	for _, opt := range opts {
+		o = opt(o)
+	}
+	// convert mounts to container run args
+	runArgs := o.RunArgs
+	for _, mount := range o.Mounts {
+		runArgs = append(runArgs, generateMountBindings(mount)...)
+	}
+	for _, portMapping := range o.PortMappings {
+		runArgs = append(runArgs, generatePortMappings(portMapping)...)
+	}
+	// construct the actual docker run argv
+	args := []string{"run"}
+	args = append(args, runArgs...)
+	args = append(args, image)
+	args = append(args, o.ContainerArgs...)
+	cmd := exec.Command(ociBinary, args...)
+	var buff bytes.Buffer
+	cmd.Stdout = &buff
+	cmd.Stderr = &buff
+	err := cmd.Run()
+	scanner := bufio.NewScanner(&buff)
+	var output []string
+	for scanner.Scan() {
+		output = append(output, scanner.Text())
+	}
+
+	if err != nil {
+		return output, errors.Wrapf(err, "args: %v  output: %s ", args, output)
+	}
+	return output, nil
+}
+
+// WithRunArgs sets the args for docker run
+// as in the args portion of `docker run args... image containerArgs...`
+func WithRunArgs(args ...string) CreateOpt {
+	return func(r *createOpts) *createOpts {
+		r.RunArgs = args
+		return r
+	}
+}
+
+// WithMounts sets the container mounts
+func WithMounts(mounts []Mount) CreateOpt {
+	return func(r *createOpts) *createOpts {
+		r.Mounts = mounts
+		return r
+	}
+}
+
+// WithPortMappings sets the container port mappings to the host
+func WithPortMappings(portMappings []PortMapping) CreateOpt {
+	return func(r *createOpts) *createOpts {
+		r.PortMappings = portMappings
+		return r
+	}
+}
+
+// Copy copies a local asset into the container
+func Copy(ociBinary string, ociID string, asset assets.CopyableFile) error {
+	if _, err := os.Stat(asset.GetAssetName()); os.IsNotExist(err) {
+		return errors.Wrapf(err, "error source %s does not exist", asset.GetAssetName())
+	}
+	destination := fmt.Sprintf("%s:%s", ociID, asset.GetTargetDir())
+	cmd := exec.Command(ociBinary, "cp", asset.GetAssetName(), destination)
+	err := cmd.Run()
+	if err != nil {
+		return errors.Wrapf(err, "error copying %s into node", asset.GetAssetName())
+	}
+	return nil
+}

pkg/drivers/kic/oci/types.go

@@ -0,0 +1,101 @@
+/*
+Copyright 2019 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package oci
+
+const (
+	Docker = "docker"
+	Podman = "podman"
+)
+
+/*
+These types are from
+https://github.com/kubernetes/kubernetes/blob/063e7ff358fdc8b0916e6f39beedc0d025734cb1/pkg/kubelet/apis/cri/runtime/v1alpha2/api.pb.go#L183
+*/
+
+// Mount specifies a host volume to mount into a container.
+// This is a close copy of the upstream cri Mount type
+// see: k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2
+// It additionally serializes the "propagation" field with the string enum
+// names on disk as opposed to the int32 values, and the serlialzed field names
+// have been made closer to core/v1 VolumeMount field names
+// In yaml this looks like:
+//  containerPath: /foo
+//  hostPath: /bar
+//  readOnly: true
+//  selinuxRelabel: false
+//  propagation: None
+// Propagation may be one of: None, HostToContainer, Bidirectional
+type Mount struct {
+	// Path of the mount within the container.
+	ContainerPath string `protobuf:"bytes,1,opt,name=container_path,json=containerPath,proto3" json:"containerPath,omitempty"`
+	// Path of the mount on the host. If the hostPath doesn't exist, then runtimes
+	// should report error. If the hostpath is a symbolic link, runtimes should
+	// follow the symlink and mount the real destination to container.
+	HostPath string `protobuf:"bytes,2,opt,name=host_path,json=hostPath,proto3" json:"hostPath,omitempty"`
+	// If set, the mount is read-only.
+	Readonly bool `protobuf:"varint,3,opt,name=readonly,proto3,json=readOnly,proto3" json:"readOnly,omitempty"`
+	// If set, the mount needs SELinux relabeling.
+	SelinuxRelabel bool `protobuf:"varint,4,opt,name=selinux_relabel,json=selinuxRelabel,proto3" json:"selinuxRelabel,omitempty"`
+	// Requested propagation mode.
+	Propagation MountPropagation `protobuf:"varint,5,opt,name=propagation,proto3,enum=runtime.v1alpha2.MountPropagation" json:"propagation,omitempty"`
+}
+
+// PortMapping specifies a host port mapped into a container port.
+// In yaml this looks like:
+//  containerPort: 80
+//  hostPort: 8000
+//  listenAddress: 127.0.0.1
+type PortMapping struct {
+	// Port within the container.
+	ContainerPort int32 `protobuf:"varint,1,opt,name=container_port,json=containerPort,proto3" json:"containerPort,omitempty"`
+	// Port on the host.
+	HostPort      int32  `protobuf:"varint,2,opt,name=host_path,json=hostPort,proto3" json:"hostPort,omitempty"`
+	ListenAddress string `protobuf:"bytes,3,opt,name=listenAddress,json=hostPort,proto3" json:"listenAddress,omitempty"`
+}
+
+// MountPropagation represents an "enum" for mount propagation options,
+// see also Mount.
+type MountPropagation int32
+
+const (
+	// MountPropagationNone specifies that no mount propagation
+	// ("private" in Linux terminology).
+	MountPropagationNone MountPropagation = 0
+	// MountPropagationHostToContainer specifies that mounts get propagated
+	// from the host to the container ("rslave" in Linux).
+	MountPropagationHostToContainer MountPropagation = 1
+	// MountPropagationBidirectional specifies that mounts get propagated from
+	// the host to the container and from the container to the host
+	// ("rshared" in Linux).
+	MountPropagationBidirectional MountPropagation = 2
+)
+
+// MountPropagationValueToName is a map of valid MountPropogation values to
+// their string names
+var MountPropagationValueToName = map[MountPropagation]string{
+	MountPropagationNone:            "None",
+	MountPropagationHostToContainer: "HostToContainer",
+	MountPropagationBidirectional:   "Bidirectional",
+}
+
+// MountPropagationNameToValue is a map of valid MountPropogation names to
+// their values
+var MountPropagationNameToValue = map[string]MountPropagation{
+	"None":            MountPropagationNone,
+	"HostToContainer": MountPropagationHostToContainer,
+	"Bidirectional":   MountPropagationBidirectional,
+}

pkg/minikube/assets/addons.go

@@ -22,9 +22,9 @@ import (
 	"path"
 	"path/filepath"
 	"runtime"
-	"strconv"
 
 	"github.com/pkg/errors"
+	"github.com/spf13/viper"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/localpath"
@@ -54,15 +54,13 @@ func (a *Addon) Name() string {
 	return a.addonName
 }
 
-// IsEnabled checks if an Addon is enabled
+// IsEnabled checks if an Addon is enabled for the current profile
 func (a *Addon) IsEnabled() (bool, error) {
-	addonStatusText, err := config.Get(a.addonName)
+	c, err := config.Load(viper.GetString(config.MachineProfile))
 	if err == nil {
-		addonStatus, err := strconv.ParseBool(addonStatusText)
-		if err != nil {
-			return false, err
+		if status, ok := c.Addons[a.Name()]; ok {
+			return status, nil
 		}
-		return addonStatus, nil
 	}
 	return a.enabled, nil
 }
@@ -190,6 +188,22 @@ var Addons = map[string]*Addon{
 			"0640",
 			true),
 	}, false, "ingress"),
+	"istio-provisioner": NewAddon([]*BinAsset{
+		MustBinAsset(
+			"deploy/addons/istio-provisioner/istio-operator.yaml.tmpl",
+			vmpath.GuestAddonsDir,
+			"istio-operator.yaml",
+			"0640",
+			true),
+	}, true, "istio-provisioner"),
+	"istio": NewAddon([]*BinAsset{
+		MustBinAsset(
+			"deploy/addons/istio/istio-default-profile.yaml.tmpl",
+			vmpath.GuestAddonsDir,
+			"istio-default-profile.yaml",
+			"0640",
+			false),
+	}, false, "istio"),
 	"metrics-server": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/metrics-server/metrics-apiservice.yaml.tmpl",

pkg/minikube/bootstrapper/bootstrapper.go

@@ -49,26 +49,17 @@ type Bootstrapper interface {
 }
 
 const (
-	// BootstrapperTypeKubeadm is the kubeadm bootstrapper type
-	BootstrapperTypeKubeadm = "kubeadm"
+	// Kubeadm is the kubeadm bootstrapper type
+	Kubeadm = "kubeadm"
+	KIC     = "kic"
 )
 
 // GetCachedBinaryList returns the list of binaries
 func GetCachedBinaryList(bootstrapper string) []string {
-	switch bootstrapper {
-	case BootstrapperTypeKubeadm:
-		return constants.KubernetesReleaseBinaries
-	default:
-		return []string{}
-	}
+	return constants.KubernetesReleaseBinaries
 }
 
 // GetCachedImageList returns the list of images for a version
 func GetCachedImageList(imageRepository string, version string, bootstrapper string) ([]string, error) {
-	switch bootstrapper {
-	case BootstrapperTypeKubeadm:
-		return images.Kubeadm(imageRepository, version)
-	default:
-		return []string{}, nil
-	}
+	return images.Kubeadm(imageRepository, version)
 }

pkg/minikube/bootstrapper/bsutil/binaries.go

@@ -0,0 +1,57 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package bsutil package will eventually be renamed to kubeadm package after getting rid of older one
+package bsutil
+
+import (
+	"path"
+	"runtime"
+
+	"github.com/pkg/errors"
+	"golang.org/x/sync/errgroup"
+	"k8s.io/minikube/pkg/minikube/command"
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/constants"
+	"k8s.io/minikube/pkg/minikube/machine"
+	"k8s.io/minikube/pkg/minikube/vmpath"
+)
+
+// TransferBinaries transfers all required Kubernetes binaries
+func TransferBinaries(cfg config.KubernetesConfig, c command.Runner) error {
+	var g errgroup.Group
+	for _, name := range constants.KubernetesReleaseBinaries {
+		name := name
+		g.Go(func() error {
+			src, err := machine.CacheBinary(name, cfg.KubernetesVersion, "linux", runtime.GOARCH)
+			if err != nil {
+				return errors.Wrapf(err, "downloading %s", name)
+			}
+
+			dst := path.Join(binRoot(cfg.KubernetesVersion), name)
+			if err := machine.CopyBinary(c, src, dst); err != nil {
+				return errors.Wrapf(err, "copybinary %s -> %s", src, dst)
+			}
+			return nil
+		})
+	}
+	return g.Wait()
+}
+
+// binRoot returns the persistent path binaries are stored in
+func binRoot(version string) string {
+	return path.Join(vmpath.GuestPersistentDir, "binaries", version)
+}

pkg/minikube/bootstrapper/bsutil/extraconfig.go

@@ -0,0 +1,193 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package bsutil will eventually be renamed to kubeadm package after getting rid of older one
+package bsutil
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/blang/semver"
+	"github.com/golang/glog"
+	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/minikube/config"
+)
+
+const (
+	KubeadmCmdParam    = iota
+	KubeadmConfigParam = iota
+)
+
+// componentExtraArgs holds extra args for a component
+type componentExtraArgs struct {
+	Component string
+	Options   map[string]string
+}
+
+// mapping of component to the section name in kubeadm.
+var componentToKubeadmConfigKey = map[string]string{
+	Apiserver:         "apiServer",
+	ControllerManager: "controllerManager",
+	Scheduler:         "scheduler",
+	Kubeadm:           "kubeadm",
+	// The Kubelet is not configured in kubeadm, only in systemd.
+	Kubelet: "",
+}
+
+// KubeadmExtraArgsWhitelist is a whitelist of supported kubeadm params that can be supplied to kubeadm through
+// minikube's ExtraArgs parameter. The list is split into two parts - params that can be supplied as flags on the
+// command line and params that have to be inserted into the kubeadm config file. This is because of a kubeadm
+// constraint which allows only certain params to be provided from the command line when the --config parameter
+// is specified
+var KubeadmExtraArgsWhitelist = map[int][]string{
+	KubeadmCmdParam: {
+		"ignore-preflight-errors",
+		"dry-run",
+		"kubeconfig",
+		"kubeconfig-dir",
+		"node-name",
+		"cri-socket",
+		"experimental-upload-certs",
+		"certificate-key",
+		"rootfs",
+	},
+	KubeadmConfigParam: {
+		"pod-network-cidr",
+	},
+}
+
+// CreateFlagsFromExtraArgs converts kubeadm extra args into flags to be supplied from the command linne
+func CreateFlagsFromExtraArgs(extraOptions config.ExtraOptionSlice) string {
+	kubeadmExtraOpts := extraOptions.AsMap().Get(Kubeadm)
+
+	// kubeadm allows only a small set of parameters to be supplied from the command line when the --config param
+	// is specified, here we remove those that are not allowed
+	for opt := range kubeadmExtraOpts {
+		if !config.ContainsParam(KubeadmExtraArgsWhitelist[KubeadmCmdParam], opt) {
+			// kubeadmExtraOpts is a copy so safe to delete
+			delete(kubeadmExtraOpts, opt)
+		}
+	}
+	return convertToFlags(kubeadmExtraOpts)
+}
+
+// extraConfigForComponent generates a map of flagname-value pairs for a k8s
+// component.
+func extraConfigForComponent(component string, opts config.ExtraOptionSlice, version semver.Version) (map[string]string, error) {
+	versionedOpts, err := defaultOptionsForComponentAndVersion(component, version)
+	if err != nil {
+		return nil, errors.Wrapf(err, "setting version specific options for %s", component)
+	}
+
+	for _, opt := range opts {
+		if opt.Component == component {
+			if val, ok := versionedOpts[opt.Key]; ok {
+				glog.Infof("Overwriting default %s=%s with user provided %s=%s for component %s", opt.Key, val, opt.Key, opt.Value, component)
+			}
+			versionedOpts[opt.Key] = opt.Value
+		}
+	}
+
+	return versionedOpts, nil
+}
+
+// defaultOptionsForComponentAndVersion returns the default option for a component and version
+func defaultOptionsForComponentAndVersion(component string, version semver.Version) (map[string]string, error) {
+	versionedOpts := map[string]string{}
+	for _, opts := range versionSpecificOpts {
+		if opts.Option.Component == component {
+			if versionIsBetween(version, opts.GreaterThanOrEqual, opts.LessThanOrEqual) {
+				if val, ok := versionedOpts[opts.Option.Key]; ok {
+					return nil, fmt.Errorf("flag %s=%q already set %s=%q", opts.Option.Key, opts.Option.Value, opts.Option.Key, val)
+				}
+				versionedOpts[opts.Option.Key] = opts.Option.Value
+			}
+		}
+	}
+	return versionedOpts, nil
+}
+
+// newComponentExtraArgs creates a new ComponentExtraArgs
+func newComponentExtraArgs(opts config.ExtraOptionSlice, version semver.Version, featureGates string) ([]componentExtraArgs, error) {
+	var kubeadmExtraArgs []componentExtraArgs
+	for _, extraOpt := range opts {
+		if _, ok := componentToKubeadmConfigKey[extraOpt.Component]; !ok {
+			return nil, fmt.Errorf("unknown component %q. valid components are: %v", componentToKubeadmConfigKey, componentToKubeadmConfigKey)
+		}
+	}
+
+	keys := []string{}
+	for k := range componentToKubeadmConfigKey {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	for _, component := range keys {
+		kubeadmComponentKey := componentToKubeadmConfigKey[component]
+		if kubeadmComponentKey == "" {
+			continue
+		}
+		extraConfig, err := extraConfigForComponent(component, opts, version)
+		if err != nil {
+			return nil, errors.Wrapf(err, "getting kubeadm extra args for %s", component)
+		}
+		if featureGates != "" {
+			extraConfig["feature-gates"] = featureGates
+		}
+		if len(extraConfig) > 0 {
+			kubeadmExtraArgs = append(kubeadmExtraArgs, componentExtraArgs{
+				Component: kubeadmComponentKey,
+				Options:   extraConfig,
+			})
+		}
+	}
+
+	return kubeadmExtraArgs, nil
+}
+
+// createExtraComponentConfig generates a map of component to extra args for all of the components except kubeadm
+func createExtraComponentConfig(extraOptions config.ExtraOptionSlice, version semver.Version, componentFeatureArgs string) ([]componentExtraArgs, error) {
+	extraArgsSlice, err := newComponentExtraArgs(extraOptions, version, componentFeatureArgs)
+	if err != nil {
+		return nil, err
+	}
+
+	// kubeadm extra args should not be included in the kubeadm config in the extra args section (instead, they must
+	// be inserted explicitly in the appropriate places or supplied from the command line); here we remove all of the
+	// kubeadm extra args from the slice
+	for i, extraArgs := range extraArgsSlice {
+		if extraArgs.Component == Kubeadm {
+			extraArgsSlice = append(extraArgsSlice[:i], extraArgsSlice[i+1:]...)
+			break
+		}
+	}
+	return extraArgsSlice, nil
+}
+
+func convertToFlags(opts map[string]string) string {
+	var flags []string
+	var keys []string
+	for k := range opts {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		flags = append(flags, fmt.Sprintf("--%s=%s", k, opts[k]))
+	}
+	return strings.Join(flags, " ")
+}

pkg/minikube/bootstrapper/bsutil/featuregates.go

@@ -0,0 +1,69 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package bsutil will eventually be renamed to kubeadm package after getting rid of older one
+package bsutil
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/pkg/errors"
+	"k8s.io/kubernetes/cmd/kubeadm/app/features"
+)
+
+// supportedFG indicates whether a feature name is supported by the bootstrapper
+func supportedFG(featureName string) bool {
+	for k := range features.InitFeatureGates {
+		if featureName == k {
+			return true
+		}
+	}
+	return false
+}
+
+// parseFeatureArgs parses feature args into extra args
+func parseFeatureArgs(featureGates string) (map[string]bool, string, error) {
+	kubeadmFeatureArgs := map[string]bool{}
+	componentFeatureArgs := ""
+	for _, s := range strings.Split(featureGates, ",") {
+		if len(s) == 0 {
+			continue
+		}
+
+		fg := strings.SplitN(s, "=", 2)
+		if len(fg) != 2 {
+			return nil, "", fmt.Errorf("missing value for key \"%v\"", s)
+		}
+
+		k := strings.TrimSpace(fg[0])
+		v := strings.TrimSpace(fg[1])
+
+		if !supportedFG(k) {
+			componentFeatureArgs = fmt.Sprintf("%s%s,", componentFeatureArgs, s)
+			continue
+		}
+
+		boolValue, err := strconv.ParseBool(v)
+		if err != nil {
+			return nil, "", errors.Wrapf(err, "failed to convert bool value \"%v\"", v)
+		}
+		kubeadmFeatureArgs[k] = boolValue
+	}
+	componentFeatureArgs = strings.TrimRight(componentFeatureArgs, ",")
+	return kubeadmFeatureArgs, componentFeatureArgs, nil
+}

pkg/minikube/bootstrapper/bsutil/featuregates_test.go

@@ -0,0 +1,94 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package bsutil will eventually be renamed to kubeadm package after getting rid of older one
+package bsutil
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestParseFeatureArgs(t *testing.T) {
+	tests := []struct {
+		description                  string
+		featureGates                 string
+		expectedKubeadmFeatureArgs   map[string]bool
+		expectedComponentFeatureArgs string
+	}{
+		{
+			description:  "CoreDNS enabled",
+			featureGates: "CoreDNS=true",
+			expectedKubeadmFeatureArgs: map[string]bool{
+				"CoreDNS": true,
+			},
+			expectedComponentFeatureArgs: "",
+		},
+		{
+			description:  "CoreDNS disabled",
+			featureGates: "CoreDNS=false",
+			expectedKubeadmFeatureArgs: map[string]bool{
+				"CoreDNS": false,
+			},
+			expectedComponentFeatureArgs: "",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.description, func(t *testing.T) {
+			kubeadm, component, err := parseFeatureArgs(test.featureGates)
+
+			if err != nil {
+				t.Fatalf("Error parsing feature args: %v", err)
+			}
+
+			if !reflect.DeepEqual(kubeadm, test.expectedKubeadmFeatureArgs) {
+				t.Errorf("Kubeadm Actual: %v, Expected: %v", kubeadm, test.expectedKubeadmFeatureArgs)
+			}
+
+			if !reflect.DeepEqual(component, test.expectedComponentFeatureArgs) {
+				t.Errorf("Component Actual: %v, Expected: %v", component, test.expectedComponentFeatureArgs)
+			}
+		})
+	}
+}
+
+func TestSupport(t *testing.T) {
+
+	tests := []struct {
+		name     string
+		expected bool
+	}{
+		{
+			name:     "CoreDNS",
+			expected: true,
+		},
+		{
+			name:     "Life is Beautiful !",
+			expected: false,
+		},
+		{
+			name:     "",
+			expected: false,
+		},
+	}
+	for _, tc := range tests {
+		if supportedFG(tc.name) != tc.expected {
+			t.Errorf("expected supportedFG(%s) to be %t ! ", tc.name, tc.expected)
+		}
+	}
+
+}

pkg/minikube/bootstrapper/bsutil/files.go

@@ -0,0 +1,84 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package bsutil will eventually be renamed to kubeadm package after getting rid of older one
+package bsutil
+
+import (
+	"path"
+
+	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/minikube/assets"
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/vmpath"
+)
+
+// KubeadmYamlPath is the path to the kubeadm configuration
+var KubeadmYamlPath = path.Join(vmpath.GuestEphemeralDir, "kubeadm.yaml")
+
+const (
+	DefaultCNIConfigPath = "/etc/cni/net.d/k8s.conf"
+	KubeletServiceFile   = "/lib/systemd/system/kubelet.service"
+	// enum to differentiate kubeadm command line parameters from kubeadm config file parameters (see the
+	// KubeadmExtraArgsWhitelist variable for more info)
+	KubeletSystemdConfFile = "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf"
+)
+
+// ConfigFileAssets returns configuration file assets
+func ConfigFileAssets(cfg config.KubernetesConfig, kubeadm []byte, kubelet []byte, kubeletSvc []byte, defaultCNIConfig []byte) []assets.CopyableFile {
+	fs := []assets.CopyableFile{
+		assets.NewMemoryAssetTarget(kubeadm, KubeadmYamlPath, "0640"),
+		assets.NewMemoryAssetTarget(kubelet, KubeletSystemdConfFile, "0644"),
+		assets.NewMemoryAssetTarget(kubeletSvc, KubeletServiceFile, "0644"),
+	}
+	// Copy the default CNI config (k8s.conf), so that kubelet can successfully
+	// start a Pod in the case a user hasn't manually installed any CNI plugin
+	// and minikube was started with "--extra-config=kubelet.network-plugin=cni".
+	if defaultCNIConfig != nil {
+		fs = append(fs, assets.NewMemoryAssetTarget(defaultCNIConfig, DefaultCNIConfigPath, "0644"))
+	}
+	return fs
+}
+
+// AddAddons adds addons to list of files
+func AddAddons(files *[]assets.CopyableFile, data interface{}) error {
+	// add addons to file list
+	// custom addons
+	if err := assets.AddMinikubeDirAssets(files); err != nil {
+		return errors.Wrap(err, "adding minikube dir assets")
+	}
+	// bundled addons
+	for _, addonBundle := range assets.Addons {
+		if isEnabled, err := addonBundle.IsEnabled(); err == nil && isEnabled {
+			for _, addon := range addonBundle.Assets {
+				if addon.IsTemplate() {
+					addonFile, err := addon.Evaluate(data)
+					if err != nil {
+						return errors.Wrapf(err, "evaluate bundled addon %s asset", addon.GetAssetName())
+					}
+
+					*files = append(*files, addonFile)
+				} else {
+					*files = append(*files, addon)
+				}
+			}
+		} else if err != nil {
+			return nil
+		}
+	}
+
+	return nil
+}

pkg/minikube/bootstrapper/bsutil/ktmpl/kubelet.go

@@ -0,0 +1,46 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ktmpl
+
+import "text/template"
+
+// KubeletSystemdTemplate hosts the override kubelet flags, written to kubeletSystemdConfFile
+var KubeletSystemdTemplate = template.Must(template.New("kubeletSystemdTemplate").Parse(`[Unit]
+{{if or (eq .ContainerRuntime "cri-o") (eq .ContainerRuntime "cri")}}Wants=crio.service{{else if eq .ContainerRuntime "containerd"}}Wants=containerd.service{{else}}Wants=docker.socket{{end}}
+
+[Service]
+ExecStart=
+ExecStart={{.KubeletPath}}{{if .ExtraOptions}} {{.ExtraOptions}}{{end}}
+
+[Install]
+`))
+
+// KubeletServiceTemplate is the base kubelet systemd template, written to kubeletServiceFile
+var KubeletServiceTemplate = template.Must(template.New("kubeletServiceTemplate").Parse(`[Unit]
+Description=kubelet: The Kubernetes Node Agent
+Documentation=http://kubernetes.io/docs/
+
+[Service]
+ExecStart={{.KubeletPath}}
+Restart=always
+StartLimitInterval=0
+# Tuned for local dev: faster than upstream default (10s), but slower than systemd default (100ms)
+RestartSec=600ms
+
+[Install]
+WantedBy=multi-user.target
+`))

pkg/minikube/bootstrapper/bsutil/ktmpl/template.go

@@ -14,26 +14,28 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package config
+package ktmpl
 
 import (
-	"testing"
-
-	"gotest.tools/assert"
-	pkgConfig "k8s.io/minikube/pkg/minikube/config"
-	"k8s.io/minikube/pkg/minikube/localpath"
+	"fmt"
+	"sort"
 )
 
-func TestEnableUnknownAddon(t *testing.T) {
-	if err := Set("InvalidAddon", "false"); err == nil {
-		t.Fatalf("Enable did not return error for unknown addon")
+// printMapInOrder sorts the keys and prints the map in order, combining key
+// value pairs with the separator character
+//
+// Note: this is not necessary, but makes testing easy
+func printMapInOrder(m map[string]string, sep string) []string {
+	if m == nil {
+		return nil
 	}
-}
-
-func TestEnableAddon(t *testing.T) {
-	if err := Set("ingress", "true"); err != nil {
-		t.Fatalf("Enable returned unexpected error: " + err.Error())
+	keys := []string{}
+	for k := range m {
+		keys = append(keys, k)
 	}
-	config, _ := pkgConfig.ReadConfig(localpath.ConfigFile)
-	assert.Equal(t, config["ingress"], true)
+	sort.Strings(keys)
+	for i, k := range keys {
+		keys[i] = fmt.Sprintf("%s%s\"%s\"", k, sep, m[k])
+	}
+	return keys
 }

pkg/minikube/bootstrapper/bsutil/ktmpl/template_test.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package kubeadm
+package ktmpl
 
 import (
 	"reflect"

pkg/minikube/bootstrapper/bsutil/ktmpl/v1alpha1.go

@@ -0,0 +1,45 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ktmpl
+
+import "text/template"
+
+// V1Alpha1 is for Kubernetes v1.11
+var V1Alpha1 = template.Must(template.New("configTmpl-v1alpha1").Funcs(template.FuncMap{
+	"printMapInOrder": printMapInOrder,
+}).Parse(`apiVersion: kubeadm.k8s.io/v1alpha1
+kind: MasterConfiguration
+{{if .NoTaintMaster}}noTaintMaster: true{{end}}
+api:
+  advertiseAddress: {{.AdvertiseAddress}}
+  bindPort: {{.APIServerPort}}
+  controlPlaneEndpoint: localhost
+kubernetesVersion: {{.KubernetesVersion}}
+certificatesDir: {{.CertDir}}
+networking:
+  serviceSubnet: {{.ServiceCIDR}}
+etcd:
+  dataDir: {{.EtcdDataDir}}
+nodeName: {{.NodeName}}
+apiServerCertSANs: ["127.0.0.1", "localhost", "{{.AdvertiseAddress}}"]
+{{if .ImageRepository}}imageRepository: {{.ImageRepository}}
+{{end}}{{if .CRISocket}}criSocket: {{.CRISocket}}
+{{end}}{{range .ExtraArgs}}{{.Component}}ExtraArgs:{{range $i, $val := printMapInOrder .Options ": " }}
+  {{$val}}{{end}}
+{{end}}{{if .FeatureArgs}}featureGates: {{range $i, $val := .FeatureArgs}}
+  {{$i}}: {{$val}}{{end}}
+{{end}}`))

pkg/minikube/bootstrapper/bsutil/ktmpl/v1alpha3.go

@@ -0,0 +1,69 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ktmpl
+
+import "text/template"
+
+// V1Alpha3 is for Kubernetes v1.12
+var V1Alpha3 = template.Must(template.New("configTmpl-v1alpha3").Funcs(template.FuncMap{
+	"printMapInOrder": printMapInOrder,
+}).Parse(`apiVersion: kubeadm.k8s.io/v1alpha3
+kind: InitConfiguration
+apiEndpoint:
+  advertiseAddress: {{.AdvertiseAddress}}
+  bindPort: {{.APIServerPort}}
+bootstrapTokens:
+  - groups:
+      - system:bootstrappers:kubeadm:default-node-token
+    ttl: 24h0m0s
+    usages:
+      - signing
+      - authentication
+nodeRegistration:
+  criSocket: {{if .CRISocket}}{{.CRISocket}}{{else}}/var/run/dockershim.sock{{end}}
+  name: {{.NodeName}}
+  taints: []
+---
+apiVersion: kubeadm.k8s.io/v1alpha3
+kind: ClusterConfiguration
+{{if .ImageRepository}}imageRepository: {{.ImageRepository}}
+{{end}}{{range .ExtraArgs}}{{.Component}}ExtraArgs:{{range $i, $val := printMapInOrder .Options ": " }}
+  {{$val}}{{end}}
+{{end -}}
+{{if .FeatureArgs}}featureGates: {{range $i, $val := .FeatureArgs}}
+  {{$i}}: {{$val}}{{end}}
+{{end -}}
+certificatesDir: {{.CertDir}}
+clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "{{.AdvertiseAddress}}"]
+controlPlaneEndpoint: localhost:{{.APIServerPort}}
+etcd:
+  local:
+    dataDir: {{.EtcdDataDir}}
+kubernetesVersion: {{.KubernetesVersion}}
+networking:
+  dnsDomain: {{if .DNSDomain}}{{.DNSDomain}}{{else}}cluster.local{{end}}
+  podSubnet: "{{ .PodSubnet }}"
+  serviceSubnet: {{.ServiceCIDR}}
+---
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+evictionHard:
+  nodefs.available: "0%"
+  nodefs.inodesFree: "0%"
+  imagefs.available: "0%"
+`))

pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta1.go

@@ -0,0 +1,76 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ktmpl
+
+import "text/template"
+
+// V1Beta1 is kubeadm config template for Kubernetes v1.13+
+var V1Beta1 = template.Must(template.New("configTmpl-v1beta1").Funcs(template.FuncMap{
+	"printMapInOrder": printMapInOrder,
+}).Parse(`apiVersion: kubeadm.k8s.io/v1beta1
+kind: InitConfiguration
+localAPIEndpoint:
+  advertiseAddress: {{.AdvertiseAddress}}
+  bindPort: {{.APIServerPort}}
+bootstrapTokens:
+  - groups:
+      - system:bootstrappers:kubeadm:default-node-token
+    ttl: 24h0m0s
+    usages:
+      - signing
+      - authentication
+nodeRegistration:
+  criSocket: {{if .CRISocket}}{{.CRISocket}}{{else}}/var/run/dockershim.sock{{end}}
+  name: {{.NodeName}}
+  taints: []
+---
+apiVersion: kubeadm.k8s.io/v1beta1
+kind: ClusterConfiguration
+{{ if .ImageRepository}}imageRepository: {{.ImageRepository}}
+{{end}}{{range .ExtraArgs}}{{.Component}}:
+  extraArgs:
+{{- range $i, $val := printMapInOrder .Options ": " }}
+    {{$val}}
+{{- end}}
+{{end -}}
+{{if .FeatureArgs}}featureGates:
+{{range $i, $val := .FeatureArgs}}{{$i}}: {{$val}}
+{{end -}}{{end -}}
+certificatesDir: {{.CertDir}}
+clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "{{.AdvertiseAddress}}"]
+controlPlaneEndpoint: localhost:{{.APIServerPort}}
+dns:
+  type: CoreDNS
+etcd:
+  local:
+    dataDir: {{.EtcdDataDir}}
+kubernetesVersion: {{.KubernetesVersion}}
+networking:
+  dnsDomain: {{if .DNSDomain}}{{.DNSDomain}}{{else}}cluster.local{{end}}
+  podSubnet: "{{.PodSubnet }}"
+  serviceSubnet: {{.ServiceCIDR}}
+---
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+imageGCHighThresholdPercent: 100
+evictionHard:
+  nodefs.available: "0%"
+  nodefs.inodesFree: "0%"
+  imagefs.available: "0%"
+`))

pkg/minikube/bootstrapper/bsutil/kubeadm.go

@@ -0,0 +1,132 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package bsutil will eventually be renamed to kubeadm package after getting rid of older one
+package bsutil
+
+import (
+	"bytes"
+	"fmt"
+	"path"
+
+	"github.com/blang/semver"
+	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/ktmpl"
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/constants"
+	"k8s.io/minikube/pkg/minikube/cruntime"
+	"k8s.io/minikube/pkg/minikube/vmpath"
+)
+
+// Container runtimes
+const remoteContainerRuntime = "remote"
+
+// GenerateKubeadmYAML generates the kubeadm.yaml file
+func GenerateKubeadmYAML(k8s config.KubernetesConfig, r cruntime.Manager) ([]byte, error) {
+	version, err := ParseKubernetesVersion(k8s.KubernetesVersion)
+	if err != nil {
+		return nil, errors.Wrap(err, "parsing kubernetes version")
+	}
+
+	// parses a map of the feature gates for kubeadm and component
+	kubeadmFeatureArgs, componentFeatureArgs, err := parseFeatureArgs(k8s.FeatureGates)
+	if err != nil {
+		return nil, errors.Wrap(err, "parses feature gate config for kubeadm and component")
+	}
+
+	extraComponentConfig, err := createExtraComponentConfig(k8s.ExtraOptions, version, componentFeatureArgs)
+	if err != nil {
+		return nil, errors.Wrap(err, "generating extra component config for kubeadm")
+	}
+
+	// In case of no port assigned, use default
+	nodePort := k8s.NodePort
+	if nodePort <= 0 {
+		nodePort = constants.APIServerPort
+	}
+
+	opts := struct {
+		CertDir           string
+		ServiceCIDR       string
+		PodSubnet         string
+		AdvertiseAddress  string
+		APIServerPort     int
+		KubernetesVersion string
+		EtcdDataDir       string
+		NodeName          string
+		DNSDomain         string
+		CRISocket         string
+		ImageRepository   string
+		ExtraArgs         []componentExtraArgs
+		FeatureArgs       map[string]bool
+		NoTaintMaster     bool
+	}{
+		CertDir:           vmpath.GuestCertsDir,
+		ServiceCIDR:       constants.DefaultServiceCIDR,
+		PodSubnet:         k8s.ExtraOptions.Get("pod-network-cidr", Kubeadm),
+		AdvertiseAddress:  k8s.NodeIP,
+		APIServerPort:     nodePort,
+		KubernetesVersion: k8s.KubernetesVersion,
+		EtcdDataDir:       EtcdDataDir(),
+		NodeName:          k8s.NodeName,
+		CRISocket:         r.SocketPath(),
+		ImageRepository:   k8s.ImageRepository,
+		ExtraArgs:         extraComponentConfig,
+		FeatureArgs:       kubeadmFeatureArgs,
+		NoTaintMaster:     false, // That does not work with k8s 1.12+
+		DNSDomain:         k8s.DNSDomain,
+	}
+
+	if k8s.ServiceCIDR != "" {
+		opts.ServiceCIDR = k8s.ServiceCIDR
+	}
+
+	opts.NoTaintMaster = true
+	b := bytes.Buffer{}
+	configTmpl := ktmpl.V1Alpha1
+	if version.GTE(semver.MustParse("1.12.0")) {
+		configTmpl = ktmpl.V1Alpha3
+	}
+	// v1beta1 works in v1.13, but isn't required until v1.14.
+	if version.GTE(semver.MustParse("1.14.0-alpha.0")) {
+		configTmpl = ktmpl.V1Beta1
+	}
+	if err := configTmpl.Execute(&b, opts); err != nil {
+		return nil, err
+	}
+
+	return b.Bytes(), nil
+}
+
+// These are the components that can be configured
+// through the "extra-config"
+const (
+	Kubelet           = "kubelet"
+	Kubeadm           = "kubeadm"
+	Apiserver         = "apiserver"
+	Scheduler         = "scheduler"
+	ControllerManager = "controller-manager"
+)
+
+// InvokeKubeadm returns the invocation command for Kubeadm
+func InvokeKubeadm(version string) string {
+	return fmt.Sprintf("sudo env PATH=%s:$PATH kubeadm", binRoot(version))
+}
+
+// EtcdDataDir is where etcd data is stored.
+func EtcdDataDir() string {
+	return path.Join(vmpath.GuestPersistentDir, "etcd")
+}

pkg/minikube/bootstrapper/bsutil/kubeadm_test.go

@@ -0,0 +1,224 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package bsutil
+
+import (
+	"fmt"
+	"io/ioutil"
+	"strings"
+	"testing"
+
+	"github.com/pmezard/go-difflib/difflib"
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/constants"
+	"k8s.io/minikube/pkg/minikube/cruntime"
+)
+
+func getExtraOpts() []config.ExtraOption {
+	return config.ExtraOptionSlice{
+		config.ExtraOption{
+			Component: Apiserver,
+			Key:       "fail-no-swap",
+			Value:     "true",
+		},
+		config.ExtraOption{
+			Component: ControllerManager,
+			Key:       "kube-api-burst",
+			Value:     "32",
+		},
+		config.ExtraOption{
+			Component: Scheduler,
+			Key:       "scheduler-name",
+			Value:     "mini-scheduler",
+		},
+		config.ExtraOption{
+			Component: Kubeadm,
+			Key:       "ignore-preflight-errors",
+			Value:     "true",
+		},
+		config.ExtraOption{
+			Component: Kubeadm,
+			Key:       "dry-run",
+			Value:     "true",
+		},
+	}
+}
+
+func getExtraOptsPodCidr() []config.ExtraOption {
+	return config.ExtraOptionSlice{
+		config.ExtraOption{
+			Component: Kubeadm,
+			Key:       "pod-network-cidr",
+			Value:     "192.168.32.0/20",
+		},
+	}
+}
+
+func recentReleases() ([]string, error) {
+	// test the 6 most recent releases
+	versions := []string{"v1.17", "v1.16", "v1.15", "v1.14", "v1.13", "v1.12", "v1.11"}
+	foundNewest := false
+	foundDefault := false
+
+	for _, v := range versions {
+		if strings.HasPrefix(constants.NewestKubernetesVersion, v) {
+			foundNewest = true
+		}
+		if strings.HasPrefix(constants.DefaultKubernetesVersion, v) {
+			foundDefault = true
+		}
+	}
+
+	if !foundNewest {
+		return nil, fmt.Errorf("No tests exist yet for newest minor version: %s", constants.NewestKubernetesVersion)
+	}
+
+	if !foundDefault {
+		return nil, fmt.Errorf("No tests exist yet for default minor version: %s", constants.DefaultKubernetesVersion)
+	}
+
+	return versions, nil
+}
+
+/**
+Need a separate test function to test the DNS server IP
+as v1.11 yaml file is very different compared to v1.12+.
+This test case has only 1 thing to test and that is the
+nnetworking/dnsDomain value
+*/
+func TestGenerateKubeadmYAMLDNS(t *testing.T) {
+	versions := []string{"v1.17", "v1.16", "v1.15", "v1.14", "v1.13", "v1.12"}
+	tests := []struct {
+		name      string
+		runtime   string
+		shouldErr bool
+		cfg       config.KubernetesConfig
+	}{
+		{"dns", "docker", false, config.KubernetesConfig{DNSDomain: "1.1.1.1"}},
+	}
+	for _, version := range versions {
+		for _, tc := range tests {
+			runtime, err := cruntime.New(cruntime.Config{Type: tc.runtime})
+			if err != nil {
+				t.Fatalf("runtime: %v", err)
+			}
+			tname := tc.name + "_" + version
+			t.Run(tname, func(t *testing.T) {
+				cfg := tc.cfg
+				cfg.NodeIP = "1.1.1.1"
+				cfg.NodeName = "mk"
+				cfg.KubernetesVersion = version + ".0"
+
+				got, err := GenerateKubeadmYAML(cfg, runtime)
+				if err != nil && !tc.shouldErr {
+					t.Fatalf("got unexpected error generating config: %v", err)
+				}
+				if err == nil && tc.shouldErr {
+					t.Fatalf("expected error but got none, config: %s", got)
+				}
+				if tc.shouldErr {
+					return
+				}
+				expected, err := ioutil.ReadFile(fmt.Sprintf("testdata/%s/%s.yaml", version, tc.name))
+				if err != nil {
+					t.Fatalf("unable to read testdata: %v", err)
+				}
+				diff, err := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{
+					A:        difflib.SplitLines(string(expected)),
+					B:        difflib.SplitLines(string(got)),
+					FromFile: "Expected",
+					ToFile:   "Got",
+					Context:  1,
+				})
+				if err != nil {
+					t.Fatalf("diff error: %v", err)
+				}
+				if diff != "" {
+					t.Errorf("unexpected diff:\n%s\n===== [RAW OUTPUT] =====\n%s", diff, got)
+				}
+			})
+		}
+	}
+}
+
+func TestGenerateKubeadmYAML(t *testing.T) {
+	extraOpts := getExtraOpts()
+	extraOptsPodCidr := getExtraOptsPodCidr()
+	versions, err := recentReleases()
+	if err != nil {
+		t.Errorf("versions: %v", err)
+	}
+	tests := []struct {
+		name      string
+		runtime   string
+		shouldErr bool
+		cfg       config.KubernetesConfig
+	}{
+		{"default", "docker", false, config.KubernetesConfig{}},
+		{"containerd", "containerd", false, config.KubernetesConfig{}},
+		{"crio", "crio", false, config.KubernetesConfig{}},
+		{"options", "docker", false, config.KubernetesConfig{ExtraOptions: extraOpts}},
+		{"crio-options-gates", "crio", false, config.KubernetesConfig{ExtraOptions: extraOpts, FeatureGates: "a=b"}},
+		{"unknown-component", "docker", true, config.KubernetesConfig{ExtraOptions: config.ExtraOptionSlice{config.ExtraOption{Component: "not-a-real-component", Key: "killswitch", Value: "true"}}}},
+		{"containerd-api-port", "containerd", false, config.KubernetesConfig{NodePort: 12345}},
+		{"containerd-pod-network-cidr", "containerd", false, config.KubernetesConfig{ExtraOptions: extraOptsPodCidr}},
+		{"image-repository", "docker", false, config.KubernetesConfig{ImageRepository: "test/repo"}},
+	}
+	for _, version := range versions {
+		for _, tc := range tests {
+			runtime, err := cruntime.New(cruntime.Config{Type: tc.runtime})
+			if err != nil {
+				t.Fatalf("runtime: %v", err)
+			}
+			tname := tc.name + "_" + version
+			t.Run(tname, func(t *testing.T) {
+				cfg := tc.cfg
+				cfg.NodeIP = "1.1.1.1"
+				cfg.NodeName = "mk"
+				cfg.KubernetesVersion = version + ".0"
+
+				got, err := GenerateKubeadmYAML(cfg, runtime)
+				if err != nil && !tc.shouldErr {
+					t.Fatalf("got unexpected error generating config: %v", err)
+				}
+				if err == nil && tc.shouldErr {
+					t.Fatalf("expected error but got none, config: %s", got)
+				}
+				if tc.shouldErr {
+					return
+				}
+				expected, err := ioutil.ReadFile(fmt.Sprintf("testdata/%s/%s.yaml", version, tc.name))
+				if err != nil {
+					t.Fatalf("unable to read testdata: %v", err)
+				}
+				diff, err := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{
+					A:        difflib.SplitLines(string(expected)),
+					B:        difflib.SplitLines(string(got)),
+					FromFile: "Expected",
+					ToFile:   "Got",
+					Context:  1,
+				})
+				if err != nil {
+					t.Fatalf("diff error: %v", err)
+				}
+				if diff != "" {
+					t.Errorf("unexpected diff:\n%s\n===== [RAW OUTPUT] =====\n%s", diff, got)
+				}
+			})
+		}
+	}
+}

pkg/minikube/bootstrapper/bsutil/kubelet.go

@@ -0,0 +1,94 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package bsutil will eventually be renamed to kubeadm package after getting rid of older one
+package bsutil
+
+import (
+	"bytes"
+	"path"
+
+	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/ktmpl"
+	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/cruntime"
+)
+
+// NewKubeletConfig generates a new systemd unit containing a configured kubelet
+// based on the options present in the KubernetesConfig.
+func NewKubeletConfig(k8s config.KubernetesConfig, r cruntime.Manager) ([]byte, error) {
+	version, err := ParseKubernetesVersion(k8s.KubernetesVersion)
+	if err != nil {
+		return nil, errors.Wrap(err, "parsing kubernetes version")
+	}
+
+	extraOpts, err := extraConfigForComponent(Kubelet, k8s.ExtraOptions, version)
+	if err != nil {
+		return nil, errors.Wrap(err, "generating extra configuration for kubelet")
+	}
+
+	for k, v := range r.KubeletOptions() {
+		extraOpts[k] = v
+	}
+	if k8s.NetworkPlugin != "" {
+		extraOpts["network-plugin"] = k8s.NetworkPlugin
+	}
+	if _, ok := extraOpts["node-ip"]; !ok {
+		extraOpts["node-ip"] = k8s.NodeIP
+	}
+
+	pauseImage := images.Pause(k8s.ImageRepository)
+	if _, ok := extraOpts["pod-infra-container-image"]; !ok && k8s.ImageRepository != "" && pauseImage != "" && k8s.ContainerRuntime != remoteContainerRuntime {
+		extraOpts["pod-infra-container-image"] = pauseImage
+	}
+
+	// parses a map of the feature gates for kubelet
+	_, kubeletFeatureArgs, err := parseFeatureArgs(k8s.FeatureGates)
+	if err != nil {
+		return nil, errors.Wrap(err, "parses feature gate config for kubelet")
+	}
+
+	if kubeletFeatureArgs != "" {
+		extraOpts["feature-gates"] = kubeletFeatureArgs
+	}
+
+	b := bytes.Buffer{}
+	opts := struct {
+		ExtraOptions     string
+		ContainerRuntime string
+		KubeletPath      string
+	}{
+		ExtraOptions:     convertToFlags(extraOpts),
+		ContainerRuntime: k8s.ContainerRuntime,
+		KubeletPath:      path.Join(binRoot(k8s.KubernetesVersion), "kubelet"),
+	}
+	if err := ktmpl.KubeletSystemdTemplate.Execute(&b, opts); err != nil {
+		return nil, err
+	}
+
+	return b.Bytes(), nil
+}
+
+// NewKubeletService returns a generated systemd unit file for the kubelet
+func NewKubeletService(cfg config.KubernetesConfig) ([]byte, error) {
+	var b bytes.Buffer
+	opts := struct{ KubeletPath string }{KubeletPath: path.Join(binRoot(cfg.KubernetesVersion), "kubelet")}
+	if err := ktmpl.KubeletServiceTemplate.Execute(&b, opts); err != nil {
+		return nil, errors.Wrap(err, "template execute")
+	}
+	return b.Bytes(), nil
+}

pkg/minikube/bootstrapper/bsutil/kubelet_test.go

@@ -14,12 +14,10 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package kubeadm
+// Package bsutil will eventually be renamed to kubeadm package after getting rid of older one
+package bsutil
 
 import (
-	"fmt"
-	"io/ioutil"
-	"strings"
 	"testing"
 
 	"github.com/pmezard/go-difflib/difflib"
@@ -168,198 +166,3 @@ ExecStart=/var/lib/minikube/binaries/v1.17.0/kubelet --authorization-mode=Webhoo
 		})
 	}
 }
-
-func getExtraOpts() []config.ExtraOption {
-	return config.ExtraOptionSlice{
-		config.ExtraOption{
-			Component: Apiserver,
-			Key:       "fail-no-swap",
-			Value:     "true",
-		},
-		config.ExtraOption{
-			Component: ControllerManager,
-			Key:       "kube-api-burst",
-			Value:     "32",
-		},
-		config.ExtraOption{
-			Component: Scheduler,
-			Key:       "scheduler-name",
-			Value:     "mini-scheduler",
-		},
-		config.ExtraOption{
-			Component: Kubeadm,
-			Key:       "ignore-preflight-errors",
-			Value:     "true",
-		},
-		config.ExtraOption{
-			Component: Kubeadm,
-			Key:       "dry-run",
-			Value:     "true",
-		},
-	}
-}
-
-func getExtraOptsPodCidr() []config.ExtraOption {
-	return config.ExtraOptionSlice{
-		config.ExtraOption{
-			Component: Kubeadm,
-			Key:       "pod-network-cidr",
-			Value:     "192.168.32.0/20",
-		},
-	}
-}
-
-func recentReleases() ([]string, error) {
-	// test the 6 most recent releases
-	versions := []string{"v1.17", "v1.16", "v1.15", "v1.14", "v1.13", "v1.12", "v1.11"}
-	foundNewest := false
-	foundDefault := false
-
-	for _, v := range versions {
-		if strings.HasPrefix(constants.NewestKubernetesVersion, v) {
-			foundNewest = true
-		}
-		if strings.HasPrefix(constants.DefaultKubernetesVersion, v) {
-			foundDefault = true
-		}
-	}
-
-	if !foundNewest {
-		return nil, fmt.Errorf("No tests exist yet for newest minor version: %s", constants.NewestKubernetesVersion)
-	}
-
-	if !foundDefault {
-		return nil, fmt.Errorf("No tests exist yet for default minor version: %s", constants.DefaultKubernetesVersion)
-	}
-
-	return versions, nil
-}
-
-/**
-Need a separate test function to test the DNS server IP
-as v1.11 yaml file is very different compared to v1.12+.
-This test case has only 1 thing to test and that is the
-nnetworking/dnsDomain value
-*/
-func TestGenerateConfigDNS(t *testing.T) {
-	versions := []string{"v1.16", "v1.15", "v1.14", "v1.13", "v1.12"}
-	tests := []struct {
-		name      string
-		runtime   string
-		shouldErr bool
-		cfg       config.KubernetesConfig
-	}{
-		{"dns", "docker", false, config.KubernetesConfig{DNSDomain: "1.1.1.1"}},
-	}
-	for _, version := range versions {
-		for _, tc := range tests {
-			runtime, err := cruntime.New(cruntime.Config{Type: tc.runtime})
-			if err != nil {
-				t.Fatalf("runtime: %v", err)
-			}
-			tname := tc.name + "_" + version
-			t.Run(tname, func(t *testing.T) {
-				cfg := tc.cfg
-				cfg.NodeIP = "1.1.1.1"
-				cfg.NodeName = "mk"
-				cfg.KubernetesVersion = version + ".0"
-
-				got, err := generateConfig(cfg, runtime)
-				if err != nil && !tc.shouldErr {
-					t.Fatalf("got unexpected error generating config: %v", err)
-				}
-				if err == nil && tc.shouldErr {
-					t.Fatalf("expected error but got none, config: %s", got)
-				}
-				if tc.shouldErr {
-					return
-				}
-				expected, err := ioutil.ReadFile(fmt.Sprintf("testdata/%s/%s.yaml", version, tc.name))
-				if err != nil {
-					t.Fatalf("unable to read testdata: %v", err)
-				}
-				diff, err := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{
-					A:        difflib.SplitLines(string(expected)),
-					B:        difflib.SplitLines(string(got)),
-					FromFile: "Expected",
-					ToFile:   "Got",
-					Context:  1,
-				})
-				if err != nil {
-					t.Fatalf("diff error: %v", err)
-				}
-				if diff != "" {
-					t.Errorf("unexpected diff:\n%s\n===== [RAW OUTPUT] =====\n%s", diff, got)
-				}
-			})
-		}
-	}
-}
-
-func TestGenerateConfig(t *testing.T) {
-	extraOpts := getExtraOpts()
-	extraOptsPodCidr := getExtraOptsPodCidr()
-	versions, err := recentReleases()
-	if err != nil {
-		t.Errorf("versions: %v", err)
-	}
-	tests := []struct {
-		name      string
-		runtime   string
-		shouldErr bool
-		cfg       config.KubernetesConfig
-	}{
-		{"default", "docker", false, config.KubernetesConfig{}},
-		{"containerd", "containerd", false, config.KubernetesConfig{}},
-		{"crio", "crio", false, config.KubernetesConfig{}},
-		{"options", "docker", false, config.KubernetesConfig{ExtraOptions: extraOpts}},
-		{"crio-options-gates", "crio", false, config.KubernetesConfig{ExtraOptions: extraOpts, FeatureGates: "a=b"}},
-		{"unknown-component", "docker", true, config.KubernetesConfig{ExtraOptions: config.ExtraOptionSlice{config.ExtraOption{Component: "not-a-real-component", Key: "killswitch", Value: "true"}}}},
-		{"containerd-api-port", "containerd", false, config.KubernetesConfig{NodePort: 12345}},
-		{"containerd-pod-network-cidr", "containerd", false, config.KubernetesConfig{ExtraOptions: extraOptsPodCidr}},
-		{"image-repository", "docker", false, config.KubernetesConfig{ImageRepository: "test/repo"}},
-	}
-	for _, version := range versions {
-		for _, tc := range tests {
-			runtime, err := cruntime.New(cruntime.Config{Type: tc.runtime})
-			if err != nil {
-				t.Fatalf("runtime: %v", err)
-			}
-			tname := tc.name + "_" + version
-			t.Run(tname, func(t *testing.T) {
-				cfg := tc.cfg
-				cfg.NodeIP = "1.1.1.1"
-				cfg.NodeName = "mk"
-				cfg.KubernetesVersion = version + ".0"
-
-				got, err := generateConfig(cfg, runtime)
-				if err != nil && !tc.shouldErr {
-					t.Fatalf("got unexpected error generating config: %v", err)
-				}
-				if err == nil && tc.shouldErr {
-					t.Fatalf("expected error but got none, config: %s", got)
-				}
-				if tc.shouldErr {
-					return
-				}
-				expected, err := ioutil.ReadFile(fmt.Sprintf("testdata/%s/%s.yaml", version, tc.name))
-				if err != nil {
-					t.Fatalf("unable to read testdata: %v", err)
-				}
-				diff, err := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{
-					A:        difflib.SplitLines(string(expected)),
-					B:        difflib.SplitLines(string(got)),
-					FromFile: "Expected",
-					ToFile:   "Got",
-					Context:  1,
-				})
-				if err != nil {
-					t.Fatalf("diff error: %v", err)
-				}
-				if diff != "" {
-					t.Errorf("unexpected diff:\n%s\n===== [RAW OUTPUT] =====\n%s", diff, got)
-				}
-			})
-		}
-	}
-}

pkg/minikube/bootstrapper/bsutil/kverify/kverify.go

@@ -0,0 +1,141 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package kverify verifies a running kubernetes cluster is healthy
+package kverify
+
+import (
+	"crypto/tls"
+	"fmt"
+	"net"
+	"net/http"
+	"os/exec"
+	"time"
+
+	"github.com/docker/machine/libmachine/state"
+	"github.com/golang/glog"
+	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
+	kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	"k8s.io/minikube/pkg/minikube/command"
+)
+
+// APIServerProcess waits for api server to be healthy returns error if it doesn't
+func APIServerProcess(runner command.Runner, start time.Time, timeout time.Duration) error {
+	glog.Infof("waiting for apiserver process to appear ...")
+	err := wait.PollImmediate(time.Second*1, timeout, func() (bool, error) {
+		if time.Since(start) > timeout {
+			return false, fmt.Errorf("cluster wait timed out during process check")
+		}
+		rr, ierr := runner.RunCmd(exec.Command("sudo", "pgrep", "kube-apiserver"))
+		if ierr != nil {
+			glog.Warningf("pgrep apiserver: %v cmd: %s", ierr, rr.Command())
+			return false, nil
+		}
+		return true, nil
+	})
+	if err != nil {
+		return fmt.Errorf("apiserver process never appeared")
+	}
+	glog.Infof("duration metric: took %s to wait for apiserver process to appear ...", time.Since(start))
+	return nil
+}
+
+// SystemPods verifies essential pods for running kurnetes is running
+func SystemPods(client *kubernetes.Clientset, start time.Time, ip string, port int, timeout time.Duration) error {
+	glog.Infof("waiting for kube-system pods to appear %s...", net.JoinHostPort(ip, fmt.Sprint(port)))
+	pStart := time.Now()
+	podStart := time.Time{}
+	podList := func() (bool, error) {
+		if time.Since(start) > timeout {
+			return false, fmt.Errorf("cluster wait timed out during pod check")
+		}
+		// Wait for any system pod, as waiting for apiserver may block until etcd
+		pods, err := client.CoreV1().Pods("kube-system").List(meta.ListOptions{})
+		if len(pods.Items) < 2 {
+			podStart = time.Time{}
+			return false, nil
+		}
+		if err != nil {
+			podStart = time.Time{}
+			return false, nil
+		}
+		if podStart.IsZero() {
+			podStart = time.Now()
+		}
+
+		glog.Infof("%d kube-system pods found since %s", len(pods.Items), podStart)
+		if time.Since(podStart) > 2*kconst.APICallRetryInterval {
+			glog.Infof("stability requirement met, returning")
+			return true, nil
+		}
+		return false, nil
+	}
+	if err := wait.PollImmediate(kconst.APICallRetryInterval, kconst.DefaultControlPlaneTimeout, podList); err != nil {
+		return fmt.Errorf("apiserver never returned a pod list")
+	}
+	glog.Infof("duration metric: took %s to wait for pod list to return data ...", time.Since(pStart))
+	return nil
+}
+
+// APIServerIsRunning waits for api server status to be running
+func APIServerIsRunning(start time.Time, ip string, port int, timeout time.Duration) error {
+	glog.Infof("waiting for apiserver healthz status ...")
+	hStart := time.Now()
+	healthz := func() (bool, error) {
+		if time.Since(start) > timeout {
+			return false, fmt.Errorf("cluster wait timed out during healthz check")
+		}
+
+		status, err := APIServerStatus(net.ParseIP(ip), port)
+		if err != nil {
+			glog.Warningf("status: %v", err)
+			return false, nil
+		}
+		if status != "Running" {
+			return false, nil
+		}
+		return true, nil
+	}
+
+	if err := wait.PollImmediate(kconst.APICallRetryInterval, kconst.DefaultControlPlaneTimeout, healthz); err != nil {
+		return fmt.Errorf("apiserver healthz never reported healthy")
+	}
+	glog.Infof("duration metric: took %s to wait for apiserver healthz status ...", time.Since(hStart))
+	return nil
+}
+
+// APIServerStatus hits the /healthz endpoint and returns libmachine style state.State
+func APIServerStatus(ip net.IP, apiserverPort int) (string, error) {
+	url := fmt.Sprintf("https://%s/healthz", net.JoinHostPort(ip.String(), fmt.Sprint(apiserverPort)))
+	// To avoid: x509: certificate signed by unknown authority
+	tr := &http.Transport{
+		Proxy:           nil, // To avoid connectiv issue if http(s)_proxy is set.
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	}
+	client := &http.Client{Transport: tr}
+	resp, err := client.Get(url)
+	// Connection refused, usually.
+	if err != nil {
+		return state.Stopped.String(), nil
+	}
+	if resp.StatusCode != http.StatusOK {
+		glog.Warningf("%s response: %v %+v", url, err, resp)
+		return state.Error.String(), nil
+	}
+	return state.Running.String(), nil
+}

pkg/minikube/bootstrapper/bsutil/rbac.go

@@ -14,10 +14,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package kubeadm
+package bsutil
 
 import (
+	"fmt"
 	"net"
+	"os/exec"
+	"strings"
 	"time"
 
 	"github.com/golang/glog"
@@ -25,6 +28,7 @@ import (
 	rbac "k8s.io/api/rbac/v1beta1"
 	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/util/retry"
 )
 
@@ -32,9 +36,9 @@ const (
 	rbacName = "minikube-rbac"
 )
 
-// elevateKubeSystemPrivileges gives the kube-system service account
+// ElevateKubeSystemPrivileges gives the kube-system service account
 // cluster admin privileges to work with RBAC.
-func elevateKubeSystemPrivileges(client kubernetes.Interface) error {
+func ElevateKubeSystemPrivileges(client kubernetes.Interface) error {
 	start := time.Now()
 	clusterRoleBinding := &rbac.ClusterRoleBinding{
 		ObjectMeta: meta.ObjectMeta{
@@ -67,3 +71,31 @@ func elevateKubeSystemPrivileges(client kubernetes.Interface) error {
 	glog.Infof("duration metric: took %s to wait for elevateKubeSystemPrivileges.", time.Since(start))
 	return nil
 }
+
+// AdjustResourceLimits makes fine adjustments to pod resources that aren't possible via kubeadm config.
+func AdjustResourceLimits(c command.Runner) error {
+	rr, err := c.RunCmd(exec.Command("/bin/bash", "-c", "cat /proc/$(pgrep kube-apiserver)/oom_adj"))
+	if err != nil {
+		return errors.Wrapf(err, "oom_adj check cmd %s. ", rr.Command())
+	}
+	glog.Infof("apiserver oom_adj: %s", rr.Stdout.String())
+	// oom_adj is already a negative number
+	if strings.HasPrefix(rr.Stdout.String(), "-") {
+		return nil
+	}
+	glog.Infof("adjusting apiserver oom_adj to -10")
+
+	// Prevent the apiserver from OOM'ing before other pods, as it is our gateway into the cluster.
+	// It'd be preferable to do this via Kubernetes, but kubeadm doesn't have a way to set pod QoS.
+	if _, err = c.RunCmd(exec.Command("/bin/bash", "-c", "echo -10 | sudo tee /proc/$(pgrep kube-apiserver)/oom_adj")); err != nil {
+		return errors.Wrap(err, fmt.Sprintf("oom_adj adjust"))
+	}
+	return nil
+}
+
+// ExistingConfig checks if there are config files from possible previous kubernets cluster
+func ExistingConfig(c command.Runner) error {
+	args := append([]string{"ls"}, expectedRemoteArtifacts...)
+	_, err := c.RunCmd(exec.Command("sudo", args...))
+	return err
+}

pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd-api-port.yaml

@@ -12,6 +12,7 @@ networking:
 etcd:
   dataDir: /var/lib/minikube/etcd
 nodeName: mk
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 criSocket: /run/containerd/containerd.sock
 apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"

pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd-pod-network-cidr.yaml

@@ -12,6 +12,7 @@ networking:
 etcd:
   dataDir: /var/lib/minikube/etcd
 nodeName: mk
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 criSocket: /run/containerd/containerd.sock
 apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"

pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd.yaml

@@ -12,6 +12,7 @@ networking:
 etcd:
   dataDir: /var/lib/minikube/etcd
 nodeName: mk
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 criSocket: /run/containerd/containerd.sock
 apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"

pkg/minikube/bootstrapper/bsutil/testdata/v1.11/crio-options-gates.yaml

@@ -12,6 +12,7 @@ networking:
 etcd:
   dataDir: /var/lib/minikube/etcd
 nodeName: mk
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 criSocket: /var/run/crio/crio.sock
 apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"

pkg/minikube/bootstrapper/bsutil/testdata/v1.11/crio.yaml

@@ -12,6 +12,7 @@ networking:
 etcd:
   dataDir: /var/lib/minikube/etcd
 nodeName: mk
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 criSocket: /var/run/crio/crio.sock
 apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"

pkg/minikube/bootstrapper/bsutil/testdata/v1.11/default.yaml

@@ -12,5 +12,6 @@ networking:
 etcd:
   dataDir: /var/lib/minikube/etcd
 nodeName: mk
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"

pkg/minikube/bootstrapper/bsutil/testdata/v1.11/image-repository.yaml

@@ -12,6 +12,7 @@ networking:
 etcd:
   dataDir: /var/lib/minikube/etcd
 nodeName: mk
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 imageRepository: test/repo
 apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"

pkg/minikube/bootstrapper/bsutil/testdata/v1.11/options.yaml

@@ -12,6 +12,7 @@ networking:
 etcd:
   dataDir: /var/lib/minikube/etcd
 nodeName: mk
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
   fail-no-swap: "true"

pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd-api-port.yaml

@@ -21,6 +21,7 @@ apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:12345
 etcd:
   local:

pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd-pod-network-cidr.yaml

@@ -21,6 +21,7 @@ apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
   local:
@@ -28,7 +29,7 @@ etcd:
 kubernetesVersion: v1.12.0
 networking:
   dnsDomain: cluster.local
-  podSubnet: 192.168.32.0/20
+  podSubnet: "192.168.32.0/20"
   serviceSubnet: 10.96.0.0/12
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1

pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd.yaml

@@ -21,6 +21,7 @@ apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
   local:

pkg/minikube/bootstrapper/bsutil/testdata/v1.12/crio-options-gates.yaml

@@ -29,6 +29,7 @@ schedulerExtraArgs:
   scheduler-name: "mini-scheduler"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
   local:

pkg/minikube/bootstrapper/bsutil/testdata/v1.12/crio.yaml

@@ -21,6 +21,7 @@ apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
   local:

pkg/minikube/bootstrapper/bsutil/testdata/v1.12/default.yaml

@@ -21,6 +21,7 @@ apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
   local:

pkg/minikube/bootstrapper/bsutil/testdata/v1.12/dns.yaml

@@ -21,6 +21,7 @@ apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
   local:

pkg/minikube/bootstrapper/bsutil/testdata/v1.12/image-repository.yaml

@@ -22,6 +22,7 @@ apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
   local:

pkg/minikube/bootstrapper/bsutil/testdata/v1.12/options.yaml

@@ -26,6 +26,7 @@ schedulerExtraArgs:
   scheduler-name: "mini-scheduler"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
   local:

pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd-api-port.yaml

@@ -21,6 +21,7 @@ apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:12345
 etcd:
   local:

pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd-pod-network-cidr.yaml

@@ -21,6 +21,7 @@ apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
   local:
@@ -28,7 +29,7 @@ etcd:
 kubernetesVersion: v1.13.0
 networking:
   dnsDomain: cluster.local
-  podSubnet: 192.168.32.0/20
+  podSubnet: "192.168.32.0/20"
   serviceSubnet: 10.96.0.0/12
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1

pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd.yaml

@@ -21,6 +21,7 @@ apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
   local:

pkg/minikube/bootstrapper/bsutil/testdata/v1.13/crio-options-gates.yaml

@@ -29,6 +29,7 @@ schedulerExtraArgs:
   scheduler-name: "mini-scheduler"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
   local:

pkg/minikube/bootstrapper/bsutil/testdata/v1.13/crio.yaml

@@ -21,6 +21,7 @@ apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
   local:

pkg/minikube/bootstrapper/bsutil/testdata/v1.13/default.yaml

@@ -21,6 +21,7 @@ apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
   local: