Merge branch 'master' of github.com:kubernetes/minikube into m2

.github/workflows/main.yml

@@ -1,5 +1,7 @@
 name: CI
 on: [pull_request]
+env:
+  GOPROXY: https://proxy.golang.org
 jobs:
   # Runs before all other jobs
   # builds the minikube binaries
@@ -11,7 +13,9 @@ jobs:
     runs-on: ubuntu-18.04
     steps:
     - uses: actions/checkout@v2
-    - name: build binaries
+    - name: Download Dependencies
+      run : go mod download
+    - name: Build Binaries
       run : |
         make minikube-linux-amd64
         make e2e-linux-amd64
@@ -36,11 +40,13 @@ jobs:
     runs-on: ubuntu-18.04
     steps:
     - uses: actions/checkout@v2
-    - name: install libvirt
+    - name: Install libvirt
       run : |
         sudo apt-get update
         sudo apt-get install -y libvirt-dev
-    - name: lint
+    - name: Download Dependencies
+      run : go mod download
+    - name: Lint
       env:
         TESTSUITE: lintall
       run : make test
@@ -53,11 +59,13 @@ jobs:
     runs-on: ubuntu-18.04
     steps:
     - uses: actions/checkout@v2
-    - name: install libvirt
+    - name: Install libvirt
       run : |
         sudo apt-get update
         sudo apt-get install -y libvirt-dev
-    - name: unit test
+    - name: Download Dependencies
+      run : go mod download
+    - name: Unit Test
       env:
         TESTSUITE: unittest
       run :
@@ -74,16 +82,22 @@ jobs:
       SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643
     runs-on: ubuntu-16.04
     steps:
+    - name: Docker Info
+      shell: bash
+      run: |
+        docker info || true
+        docker version || true
+        docker ps || true
     - name: Install gopogh
       shell: bash
       run: |
         curl -LO https://github.com/medyagh/gopogh/releases/download/v0.1.16/gopogh-linux-amd64
         sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
-    - name: Download binaries
+    - name: Download Binaries
       uses: actions/download-artifact@v1
       with:
         name: minikube_binaries
-    - name: Run integration test
+    - name: Run Integration Test
       continue-on-error: true
       # bash {0} to allow test to continue to next step. in case of
       shell: bash {0}
@@ -101,7 +115,7 @@ jobs:
         sec=$((${TIME_ELAPSED}%60))
         TIME_ELAPSED="${min} min $sec seconds "
         echo ::set-env name=TIME_ELAPSED::${TIME_ELAPSED}
-    - name: Generate html report
+    - name: Generate HTML Report
       shell: bash
       run: |
         cd minikube_binaries
@@ -136,16 +150,22 @@ jobs:
       SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643
     needs: [build_minikube]
     steps:
+    - name: Docker Info
+      shell: bash
+      run: |
+        docker info || true
+        docker version || true
+        docker ps || true
     - name: Install gopogh
       shell: bash
       run: |
         curl -LO https://github.com/medyagh/gopogh/releases/download/v0.1.16/gopogh-linux-amd64
         sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
-    - name: Download binaries
+    - name: Download Binaries
       uses: actions/download-artifact@v1
       with:
         name: minikube_binaries
-    - name: Run integration test
+    - name: Run Integration Test
       continue-on-error: true
       # bash {0} to allow test to continue to next step. in case of
       shell: bash {0}
@@ -156,14 +176,14 @@ jobs:
         chmod a+x e2e-*
         chmod a+x minikube-*
         START_TIME=$(date -u +%s)
-        KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--vm-driver=docker -test.timeout=70m -test.v -timeout-multiplier=3 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
+        KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--driver=docker -test.timeout=70m -test.v -timeout-multiplier=3 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
         END_TIME=$(date -u +%s)
         TIME_ELAPSED=$(($END_TIME-$START_TIME))
         min=$((${TIME_ELAPSED}/60))
         sec=$((${TIME_ELAPSED}%60))
         TIME_ELAPSED="${min} min $sec seconds "
         echo ::set-env name=TIME_ELAPSED::${TIME_ELAPSED}
-    - name: Generate html report
+    - name: Generate HTML Report
       shell: bash
       run: |
         cd minikube_binaries
@@ -203,11 +223,11 @@ jobs:
       run: |
         curl -LO https://github.com/medyagh/gopogh/releases/download/v0.1.16/gopogh-linux-amd64
         sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
-    - name: Download binaries
+    - name: Download Binaries
       uses: actions/download-artifact@v1
       with:
         name: minikube_binaries
-    - name: Run integration test
+    - name: Run Integration Test
       continue-on-error: true
       # bash {0} to allow test to continue to next step. in case of
       shell: bash {0}
@@ -218,14 +238,14 @@ jobs:
         chmod a+x e2e-*
         chmod a+x minikube-*
         START_TIME=$(date -u +%s)
-        KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome sudo -E ./e2e-linux-amd64 -minikube-start-args=--vm-driver=none -test.timeout=70m -test.v -timeout-multiplier=3 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
+        KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome sudo -E ./e2e-linux-amd64 -minikube-start-args=--driver=none -test.timeout=70m -test.v -timeout-multiplier=3 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
         END_TIME=$(date -u +%s)
         TIME_ELAPSED=$(($END_TIME-$START_TIME))
         min=$((${TIME_ELAPSED}/60))
         sec=$((${TIME_ELAPSED}%60))
         TIME_ELAPSED="${min} min $sec seconds "
         echo ::set-env name=TIME_ELAPSED::${TIME_ELAPSED}
-    - name: Generate html report
+    - name: Generate HTML Report
       shell: bash
       run: |
         cd minikube_binaries
@@ -265,11 +285,11 @@ jobs:
       run: |
         curl -LO https://github.com/medyagh/gopogh/releases/download/v0.1.16/gopogh-linux-amd64
         sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
-    - name: Download binaries
+    - name: Download Binaries
       uses: actions/download-artifact@v1
       with:
         name: minikube_binaries
-    - name: Run integration test
+    - name: Run Integration Test
       continue-on-error: true
       # bash {0} to allow test to continue to next step. in case of
       shell: bash {0}
@@ -280,14 +300,14 @@ jobs:
         chmod a+x e2e-*
         chmod a+x minikube-*
         START_TIME=$(date -u +%s)
-        KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome sudo -E ./e2e-linux-amd64 -minikube-start-args=--vm-driver=none -test.timeout=70m -test.v -timeout-multiplier=3 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
+        KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome sudo -E ./e2e-linux-amd64 -minikube-start-args=--driver=none -test.timeout=70m -test.v -timeout-multiplier=3 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
         END_TIME=$(date -u +%s)
         TIME_ELAPSED=$(($END_TIME-$START_TIME))
         min=$((${TIME_ELAPSED}/60))
         sec=$((${TIME_ELAPSED}%60))
         TIME_ELAPSED="${min} min $sec seconds "
         echo ::set-env name=TIME_ELAPSED::${TIME_ELAPSED}
-    - name: Generate html report
+    - name: Generate HTML Report
       shell: bash
       run: |
         cd minikube_binaries
@@ -342,7 +362,7 @@ jobs:
         uses: actions/download-artifact@v1
         with:
           name: minikube_binaries
-      - name: Run integration test
+      - name: Run Integration Test
         continue-on-error: true
         # bash {0} to allow test to continue to next step. in case of
         shell: bash {0}
@@ -353,14 +373,14 @@ jobs:
           chmod a+x e2e-*
           chmod a+x minikube-*
           START_TIME=$(date -u +%s)
-          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome sudo -E ./e2e-linux-amd64 -minikube-start-args=--vm-driver=podman -test.timeout=70m -test.v -timeout-multiplier=3 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
+          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome sudo -E ./e2e-linux-amd64 -minikube-start-args=--driver=podman -test.timeout=70m -test.v -timeout-multiplier=3 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
           END_TIME=$(date -u +%s)
           TIME_ELAPSED=$(($END_TIME-$START_TIME))
           min=$((${TIME_ELAPSED}/60))
           sec=$((${TIME_ELAPSED}%60))
           TIME_ELAPSED="${min} min $sec seconds "
           echo ::set-env name=TIME_ELAPSED::${TIME_ELAPSED}
-      - name: Generate html report
+      - name: Generate HTML Report
         shell: bash
         run: |
           cd minikube_binaries
@@ -393,7 +413,7 @@ jobs:
     needs: [docker_ubuntu_16_04,docker_ubuntu_18_04,none_ubuntu16_04,none_ubuntu18_04,podman_ubuntu_18_04]
     runs-on: ubuntu-18.04
     steps:
-    - name: download results docker_ubuntu_16_04
+    - name: Download Results docker_ubuntu_16_04
       uses: actions/download-artifact@v1
       with:
         name: docker_ubuntu_16_04
@@ -403,7 +423,7 @@ jobs:
       run: |
         mkdir -p all_reports
         cp -r docker_ubuntu_16_04 ./all_reports/
-    - name: download results docker_ubuntu_18_04
+    - name: Download Results docker_ubuntu_18_04
       uses: actions/download-artifact@v1
       with:
         name: docker_ubuntu_18_04
@@ -413,7 +433,7 @@ jobs:
       run: |
         mkdir -p all_reports
         cp -r docker_ubuntu_18_04 ./all_reports/
-    - name: download results none_ubuntu16_04
+    - name: Download Results none_ubuntu16_04
       uses: actions/download-artifact@v1
       with:
         name: none_ubuntu16_04
@@ -423,21 +443,21 @@ jobs:
       run: |
         mkdir -p all_reports
         cp -r none_ubuntu16_04 ./all_reports/
-    - name: download results none_ubuntu18_04
+    - name: Download Results none_ubuntu18_04
       uses: actions/download-artifact@v1
       with:
         name: none_ubuntu18_04
-    - name: cp none_ubuntu18_04 to all_report
+    - name: Copy none_ubuntu18_04 to all_report
       continue-on-error: true
       shell: bash {0}
       run: |
         mkdir -p all_reports
         cp -r none_ubuntu18_04 ./all_reports/
-    - name: download results podman_ubuntu_18_04
+    - name: Download Results podman_ubuntu_18_04
       uses: actions/download-artifact@v1
       with:
         name: podman_ubuntu_18_04
-    - name: cp podman_ubuntu_18_04 to all_report
+    - name: Copy podman_ubuntu_18_04 to all_report
       continue-on-error: true
       shell: bash {0}
       run: |

.travis.yml

@@ -3,7 +3,7 @@
 os: linux
 language: go
 go:
-  - 1.13.4
+  - 1.13.6
 env:
   global:
     - GOPROXY=https://proxy.golang.org
@@ -11,7 +11,7 @@ matrix:
   include:
     - language: go
       name: Code Lint
-      go: 1.13.4
+      go: 1.13.6
       env:
         - TESTSUITE=lintall
       before_install:
@@ -20,7 +20,7 @@ matrix:
 
     - language: go
       name: Unit Test
-      go: 1.13.4
+      go: 1.13.6
       env:
         - TESTSUITE=unittest
       before_install:
@@ -29,7 +29,7 @@ matrix:
 
     - language: go
       name: Build
-      go: 1.13.4
+      go: 1.13.6
       script: make
 after_success:
   - bash <(curl -s https://codecov.io/bash)

Makefile

@@ -19,8 +19,10 @@ VERSION_BUILD ?= 3
 RAW_VERSION=$(VERSION_MAJOR).$(VERSION_MINOR).${VERSION_BUILD}
 VERSION ?= v$(RAW_VERSION)
 
-KUBERNETES_VERSION ?= $(shell egrep "^var DefaultKubernetesVersion" pkg/minikube/constants/constants.go | cut -d \" -f2)
+KUBERNETES_VERSION ?= $(shell egrep "DefaultKubernetesVersion =" pkg/minikube/constants/constants.go | cut -d \" -f2)
 KIC_VERSION ?= $(shell egrep "Version =" pkg/drivers/kic/types.go | cut -d \" -f2)
+PRELOADED_TARBALL_VERSION ?= $(shell egrep "PreloadVersion =" pkg/minikube/download/preload.go | cut -d \" -f2)
+PRELOADED_VOLUMES_GCS_BUCKET ?= $(shell egrep "PreloadBucket =" pkg/minikube/download/preload.go | cut -d \" -f2)
 
 # Default to .0 for higher cache hit rates, as build increments typically don't require new ISO versions
 ISO_VERSION ?= v$(VERSION_MAJOR).$(VERSION_MINOR).3
@@ -29,7 +31,7 @@ DEB_VERSION ?= $(subst -,~,$(RAW_VERSION))
 RPM_VERSION ?= $(DEB_VERSION)
 
 # used by hack/jenkins/release_build_and_upload.sh and KVM_BUILD_IMAGE, see also BUILD_IMAGE below
-GO_VERSION ?= 1.13.4
+GO_VERSION ?= 1.13.6
 
 INSTALL_SIZE ?= $(shell du out/minikube-windows-amd64.exe | cut -f1)
 BUILDROOT_BRANCH ?= 2019.02.9
@@ -243,7 +245,7 @@ test-pkg/%: pkg/minikube/assets/assets.go pkg/minikube/translate/translations.go
 	go test -v -test.timeout=60m ./$* --tags="$(MINIKUBE_BUILD_TAGS)"
 
 .PHONY: all
-all: cross drivers e2e-cross out/gvisor-addon ## Build all different minikube components
+all: cross drivers e2e-cross exotic out/gvisor-addon ## Build all different minikube components
 
 .PHONY: drivers
 drivers: docker-machine-driver-hyperkit docker-machine-driver-kvm2 ## Build Hyperkit and KVM2 drivers
@@ -260,7 +262,7 @@ integration: out/minikube ## Trigger minikube integration test
 
 .PHONY: integration-none-driver
 integration-none-driver: e2e-linux-$(GOARCH) out/minikube-linux-$(GOARCH)  ## Trigger minikube none driver test
-	sudo -E out/e2e-linux-$(GOARCH) -testdata-dir "test/integration/testdata" -minikube-start-args="--vm-driver=none" -test.v -test.timeout=60m -binary=out/minikube-linux-amd64 $(TEST_ARGS)
+	sudo -E out/e2e-linux-$(GOARCH) -testdata-dir "test/integration/testdata" -minikube-start-args="--driver=none" -test.v -test.timeout=60m -binary=out/minikube-linux-amd64 $(TEST_ARGS)
 
 .PHONY: integration-versioned
 integration-versioned: out/minikube ## Trigger minikube integration testing
@@ -302,7 +304,10 @@ endif
 	@sed -i -e 's/Json/JSON/' $@ && rm -f ./-e
 
 .PHONY: cross
-cross: minikube-linux-amd64 minikube-linux-arm64 minikube-darwin-amd64 minikube-windows-amd64.exe ## Build minikube for all platform
+cross: minikube-linux-amd64 minikube-darwin-amd64 minikube-windows-amd64.exe ## Build minikube for all platform
+
+.PHONY: exotic
+exotic: out/minikube-linux-arm out/minikube-linux-arm64 out/minikube-linux-ppc64le out/minikube-linux-s390x ## Build minikube for non-amd64 linux
 
 .PHONY: windows
 windows: minikube-windows-amd64.exe ## Build minikube for Windows 64bit
@@ -318,7 +323,8 @@ e2e-cross: e2e-linux-amd64 e2e-darwin-amd64 e2e-windows-amd64.exe ## End-to-end
 
 .PHONY: checksum
 checksum: ## Generate checksums
-	for f in out/minikube.iso out/minikube-linux-amd64 minikube-linux-arm64 \
+	for f in out/minikube.iso out/minikube-linux-amd64 out/minikube-linux-arm \
+		 out/minikube-linux-arm64 out/minikube-linux-ppc64le out/minikube-linux-s390x \
 		 out/minikube-darwin-amd64 out/minikube-windows-amd64.exe \
 		 out/docker-machine-driver-kvm2 out/docker-machine-driver-hyperkit; do \
 		if [ -f "$${f}" ]; then \
@@ -364,8 +370,14 @@ out/linters/golangci-lint-$(GOLINT_VERSION):
 
 # this one is meant for local use
 .PHONY: lint
+ifeq ($(MINIKUBE_BUILD_IN_DOCKER),y)
+lint: pkg/minikube/assets/assets.go pkg/minikube/translate/translations.go
+	docker run --rm -v $(pwd):/app -w /app golangci/golangci-lint:$(GOLINT_VERSION) \
+	golangci-lint run ${GOLINT_OPTIONS} --skip-dirs "cmd/drivers/kvm|cmd/drivers/hyperkit|pkg/drivers/kvm|pkg/drivers/hyperkit" ./...
+else
 lint: pkg/minikube/assets/assets.go pkg/minikube/translate/translations.go out/linters/golangci-lint-$(GOLINT_VERSION) ## Run lint
 	./out/linters/golangci-lint-$(GOLINT_VERSION) run ${GOLINT_OPTIONS} ./...
+endif
 
 # lint-ci is slower version of lint and is meant to be used in ci (travis) to avoid out of memory leaks.
 .PHONY: lint-ci
@@ -433,15 +445,13 @@ out/repodata/repomd.xml: out/minikube-$(RPM_VERSION).rpm
 
 .SECONDEXPANSION:
 TAR_TARGETS_linux-amd64   := out/minikube-linux-amd64 out/docker-machine-driver-kvm2
-TAR_TARGETS_linux-arm64   := out/minikube-linux-arm64
 TAR_TARGETS_darwin-amd64  := out/minikube-darwin-amd64 out/docker-machine-driver-hyperkit
 TAR_TARGETS_windows-amd64 := out/minikube-windows-amd64.exe
 out/minikube-%.tar.gz: $$(TAR_TARGETS_$$*)
 	tar -cvzf $@ $^
 
 .PHONY: cross-tars
-cross-tars: out/minikube-linux-amd64.tar.gz out/minikube-linux-arm64.tar.gz \ ## Cross-compile minikube
+cross-tars: out/minikube-linux-amd64.tar.gz out/minikube-windows-amd64.tar.gz out/minikube-darwin-amd64.tar.gz ## Cross-compile minikube
-	    out/minikube-windows-amd64.tar.gz out/minikube-darwin-amd64.tar.gz
 	-cd out && $(SHA512SUM) *.tar.gz > SHA512SUM
 
 out/minikube-installer.exe: out/minikube-windows-amd64.exe
@@ -515,15 +525,14 @@ kic-base-image: ## builds the base image used for kic.
 	docker rmi -f $(REGISTRY)/kicbase:$(KIC_VERSION)-snapshot || true
 	docker build -f ./hack/images/kicbase.Dockerfile -t $(REGISTRY)/kicbase:$(KIC_VERSION)-snapshot  --build-arg COMMIT_SHA=${VERSION}-$(COMMIT) --target base .
 
-
+.PHONY: upload-preloaded-images-tar
-.PHONY: kic-preloaded-base-image
+upload-preloaded-images-tar: generate-preloaded-images-tar # Upload the preloaded images tar to the GCS bucket. Specify a specific kubernetes version to build via `KUBERNETES_VERSION=vx.y.z make upload-preloaded-images-tar`.
-kic-preloaded-base-image: generate-preloaded-images-tar ## builds the base image used for kic.
+	gsutil cp out/preloaded-images-k8s-${PRELOADED_TARBALL_VERSION}-${KUBERNETES_VERSION}-docker-overlay2.tar.lz4 gs://${PRELOADED_VOLUMES_GCS_BUCKET}
-	docker rmi -f $(REGISTRY)/kicbase:$(KIC_VERSION)-k8s-${KUBERNETES_VERSION} || true
+	gsutil acl ch -u AllUsers:R gs://${PRELOADED_VOLUMES_GCS_BUCKET}/preloaded-images-k8s-${PRELOADED_TARBALL_VERSION}-${KUBERNETES_VERSION}-docker-overlay2.tar.lz4 
-	docker build -f ./hack/images/kicbase.Dockerfile -t $(REGISTRY)/kicbase:$(KIC_VERSION)-k8s-${KUBERNETES_VERSION}  --build-arg COMMIT_SHA=${VERSION}-$(COMMIT) --build-arg KUBERNETES_VERSION=${KUBERNETES_VERSION} .
 
 .PHONY: generate-preloaded-images-tar
-generate-preloaded-images-tar: out/minikube
+generate-preloaded-images-tar:
-	go run ./hack/preload-images/preload_images.go -kubernetes-version ${KUBERNETES_VERSION} 
+	go run ./hack/preload-images/preload_images.go -kubernetes-version ${KUBERNETES_VERSION} -preloaded-tarball-version ${PRELOADED_TARBALL_VERSION}
 
 
 .PHONY: push-storage-provisioner-image

cmd/minikube/cmd/config/addons_list.go

@@ -98,7 +98,7 @@ var printAddonsList = func() {
 	table.SetAutoFormatHeaders(true)
 	table.SetBorders(tablewriter.Border{Left: true, Top: true, Right: true, Bottom: true})
 	table.SetCenterSeparator("|")
-	pName := viper.GetString(config.MachineProfile)
+	pName := viper.GetString(config.ProfileName)
 
 	for _, addonName := range addonNames {
 		addonBundle := assets.Addons[addonName]
@@ -123,7 +123,7 @@ var printAddonsList = func() {
 
 var printAddonsJSON = func() {
 	addonNames := make([]string, 0, len(assets.Addons))
-	pName := viper.GetString(config.MachineProfile)
+	pName := viper.GetString(config.ProfileName)
 	for addonName := range assets.Addons {
 		addonNames = append(addonNames, addonName)
 	}

cmd/minikube/cmd/config/config.go

@@ -42,6 +42,12 @@ type Setting struct {
 // These are all the settings that are configurable
 // and their validation and callback fn run on Set
 var settings = []Setting{
+	{
+		name:        "driver",
+		set:         SetString,
+		validations: []setFn{IsValidDriver},
+		callbacks:   []setFn{RequiresRestartMsg},
+	},
 	{
 		name:        "vm-driver",
 		set:         SetString,
@@ -126,7 +132,7 @@ var settings = []Setting{
 		set:  SetBool,
 	},
 	{
-		name: config.MachineProfile,
+		name: config.ProfileName,
 		set:  SetString,
 	},
 	{
@@ -172,7 +178,7 @@ var settings = []Setting{
 var ConfigCmd = &cobra.Command{
 	Use:   "config SUBCOMMAND [flags]",
 	Short: "Modify minikube config",
-	Long: `config modifies minikube config files using subcommands like "minikube config set vm-driver kvm"
+	Long: `config modifies minikube config files using subcommands like "minikube config set driver kvm"
 Configurable fields: ` + "\n\n" + configurableFields(),
 	Run: func(cmd *cobra.Command, args []string) {
 		if err := cmd.Help(); err != nil {

cmd/minikube/cmd/config/disable.go

@@ -35,7 +35,7 @@ var addonsDisableCmd = &cobra.Command{
 		}
 
 		addon := args[0]
-		err := addons.Set(addon, "false", viper.GetString(config.MachineProfile))
+		err := addons.Set(addon, "false", viper.GetString(config.ProfileName))
 		if err != nil {
 			exit.WithError("disable failed", err)
 		}

cmd/minikube/cmd/config/enable.go

@@ -34,7 +34,7 @@ var addonsEnableCmd = &cobra.Command{
 			exit.UsageT("usage: minikube addons enable ADDON_NAME")
 		}
 		addon := args[0]
-		err := addons.Set(addon, "true", viper.GetString(config.MachineProfile))
+		err := addons.Set(addon, "true", viper.GetString(config.ProfileName))
 		if err != nil {
 			exit.WithError("enable failed", err)
 		}

cmd/minikube/cmd/config/get_test.go

@@ -30,7 +30,7 @@ func TestGetNotFound(t *testing.T) {
 
 func TestGetOK(t *testing.T) {
 	createTestConfig(t)
-	name := "vm-driver"
+	name := "driver"
 	err := Set(name, "virtualbox")
 	if err != nil {
 		t.Fatalf("Set returned error for property %s, %+v", name, err)

cmd/minikube/cmd/config/open.go

@@ -26,7 +26,9 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"k8s.io/minikube/pkg/minikube/assets"
+	"k8s.io/minikube/pkg/minikube/config"
 	pkg_config "k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/driver"
 	"k8s.io/minikube/pkg/minikube/exit"
 	"k8s.io/minikube/pkg/minikube/machine"
 	"k8s.io/minikube/pkg/minikube/out"
@@ -67,8 +69,16 @@ var addonsOpenCmd = &cobra.Command{
 		}
 		defer api.Close()
 
-		profileName := viper.GetString(pkg_config.MachineProfile)
+		profileName := viper.GetString(pkg_config.ProfileName)
-		if !machine.IsHostRunning(api, profileName) {
+		cc, err := config.Load(profileName)
+		if err != nil {
+			exit.WithError("Error getting cluster", err)
+		}
+		cp, err := config.PrimaryControlPlane(*cc)
+		if err != nil {
+			exit.WithError("Error getting control plane", err)
+		}
+		if !machine.IsHostRunning(api, driver.MachineName(*cc, cp)) {
 			os.Exit(1)
 		}
 		addon, ok := assets.Addons[addonName] // validate addon input

cmd/minikube/cmd/config/profile.go

@@ -35,7 +35,7 @@ var ProfileCmd = &cobra.Command{
 	Long:  "profile sets the current minikube profile, or gets the current profile if no arguments are provided.  This is used to run and manage multiple minikube instance.  You can return to the default minikube profile by running `minikube profile default`",
 	Run: func(cmd *cobra.Command, args []string) {
 		if len(args) == 0 {
-			profile := viper.GetString(pkgConfig.MachineProfile)
+			profile := viper.GetString(pkgConfig.ProfileName)
 			out.T(out.Empty, profile)
 			os.Exit(0)
 		}
@@ -65,10 +65,11 @@ var ProfileCmd = &cobra.Command{
 		}
 
 		if !pkgConfig.ProfileExists(profile) {
-			out.FailureT("if you want to create a profile you can by this command: minikube start -p {{.profile_name}}", out.V{"profile_name": profile})
+			out.ErrT(out.Tip, `if you want to create a profile you can by this command: minikube start -p {{.profile_name}}`, out.V{"profile_name": profile})
+			os.Exit(0)
 		}
 
-		err := Set(pkgConfig.MachineProfile, profile)
+		err := Set(pkgConfig.ProfileName, profile)
 		if err != nil {
 			exit.WithError("Setting profile failed", err)
 		}

cmd/minikube/cmd/config/profile_list.go

@@ -24,6 +24,7 @@ import (
 	"strings"
 
 	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/driver"
 	"k8s.io/minikube/pkg/minikube/exit"
 	"k8s.io/minikube/pkg/minikube/machine"
 	"k8s.io/minikube/pkg/minikube/out"
@@ -75,14 +76,13 @@ var printProfilesTable = func() {
 	defer api.Close()
 
 	for _, p := range validProfiles {
-		p.Status, err = machine.GetHostStatus(api, p.Name)
-		if err != nil {
-			glog.Warningf("error getting host status for %s: %v", p.Name, err)
-		}
 		cp, err := config.PrimaryControlPlane(*p.Config)
 		if err != nil {
-			glog.Errorf("%q has no control plane: %v", p.Name, err)
+			exit.WithError("error getting primary control plane", err)
-			// Print the data we know about anyways
+		}
+		p.Status, err = machine.GetHostStatus(api, driver.MachineName(*p.Config, cp))
+		if err != nil {
+			glog.Warningf("error getting host status for %s: %v", p.Name, err)
 		}
 		validData = append(validData, []string{p.Name, p.Config.Driver, p.Config.KubernetesConfig.ContainerRuntime, cp.IP, strconv.Itoa(cp.Port), p.Config.KubernetesConfig.KubernetesVersion, p.Status})
 	}
@@ -103,7 +103,7 @@ var printProfilesTable = func() {
 	}
 
 	if err != nil {
-		exit.WithCodeT(exit.Config, fmt.Sprintf("error loading profiles: %v", err))
+		glog.Warningf("error loading profiles: %v", err)
 	}
 
 }
@@ -117,7 +117,11 @@ var printProfilesJSON = func() {
 
 	validProfiles, invalidProfiles, err := config.ListProfiles()
 	for _, v := range validProfiles {
-		status, err := machine.GetHostStatus(api, v.Name)
+		cp, err := config.PrimaryControlPlane(*v.Config)
+		if err != nil {
+			exit.WithError("error getting primary control plane", err)
+		}
+		status, err := machine.GetHostStatus(api, driver.MachineName(*v.Config, cp))
 		if err != nil {
 			glog.Warningf("error getting host status for %s: %v", v.Name, err)
 		}

cmd/minikube/cmd/config/set_test.go

@@ -34,25 +34,25 @@ func TestNotFound(t *testing.T) {
 
 func TestSetNotAllowed(t *testing.T) {
 	createTestConfig(t)
-	err := Set("vm-driver", "123456")
+	err := Set("driver", "123456")
-	if err == nil || err.Error() != "run validations for \"vm-driver\" with value of \"123456\": [driver \"123456\" is not supported]" {
+	if err == nil || err.Error() != "run validations for \"driver\" with value of \"123456\": [driver \"123456\" is not supported]" {
 		t.Fatalf("Set did not return error for unallowed value: %+v", err)
 	}
 }
 
 func TestSetOK(t *testing.T) {
 	createTestConfig(t)
-	err := Set("vm-driver", "virtualbox")
+	err := Set("driver", "virtualbox")
 	defer func() {
-		err = Unset("vm-driver")
+		err = Unset("driver")
 		if err != nil {
-			t.Errorf("failed to unset vm-driver: %+v", err)
+			t.Errorf("failed to unset driver: %+v", err)
 		}
 	}()
 	if err != nil {
 		t.Fatalf("Set returned error for valid property value: %+v", err)
 	}
-	val, err := Get("vm-driver")
+	val, err := Get("driver")
 	if err != nil {
 		t.Fatalf("Get returned error for valid property: %+v", err)
 	}

cmd/minikube/cmd/config/util_test.go

@@ -25,7 +25,7 @@ import (
 )
 
 var minikubeConfig = pkgConfig.MinikubeConfig{
-	"vm-driver":            driver.KVM2,
+	"driver":               driver.KVM2,
 	"cpus":                 12,
 	"show-libmachine-logs": true,
 }
@@ -38,17 +38,17 @@ func TestFindSettingNotFound(t *testing.T) {
 }
 
 func TestFindSetting(t *testing.T) {
-	s, err := findSetting("vm-driver")
+	s, err := findSetting("driver")
 	if err != nil {
-		t.Fatalf("Couldn't find setting, vm-driver: %v", err)
+		t.Fatalf("Couldn't find setting, driver: %v", err)
 	}
-	if s.name != "vm-driver" {
+	if s.name != "driver" {
-		t.Fatalf("Found wrong setting, expected vm-driver, got %s", s.name)
+		t.Fatalf("Found wrong setting, expected driver, got %s", s.name)
 	}
 }
 
 func TestSetString(t *testing.T) {
-	err := SetString(minikubeConfig, "vm-driver", driver.VirtualBox)
+	err := SetString(minikubeConfig, "driver", driver.VirtualBox)
 	if err != nil {
 		t.Fatalf("Couldn't set string: %v", err)
 	}

cmd/minikube/cmd/config/validations_test.go

@@ -51,7 +51,7 @@ func TestDriver(t *testing.T) {
 		},
 	}
 
-	runValidations(t, tests, "vm-driver", IsValidDriver)
+	runValidations(t, tests, "driver", IsValidDriver)
 
 }
 

cmd/minikube/cmd/dashboard.go

@@ -35,7 +35,9 @@ import (
 	"github.com/spf13/viper"
 	pkgaddons "k8s.io/minikube/pkg/addons"
 	"k8s.io/minikube/pkg/minikube/assets"
+	"k8s.io/minikube/pkg/minikube/config"
 	pkg_config "k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/driver"
 	"k8s.io/minikube/pkg/minikube/exit"
 	"k8s.io/minikube/pkg/minikube/machine"
 	"k8s.io/minikube/pkg/minikube/out"
@@ -57,7 +59,7 @@ var dashboardCmd = &cobra.Command{
 	Short: "Access the kubernetes dashboard running within the minikube cluster",
 	Long:  `Access the kubernetes dashboard running within the minikube cluster`,
 	Run: func(cmd *cobra.Command, args []string) {
-		profileName := viper.GetString(pkg_config.MachineProfile)
+		profileName := viper.GetString(pkg_config.ProfileName)
 		cc, err := pkg_config.Load(profileName)
 		if err != nil && !pkg_config.IsNotExist(err) {
 			exit.WithError("Error loading profile config", err)
@@ -80,7 +82,13 @@ var dashboardCmd = &cobra.Command{
 			exit.WithError("Error getting client", err)
 		}
 
-		if _, err = api.Load(cc.Name); err != nil {
+		cp, err := config.PrimaryControlPlane(*cc)
+		if err != nil {
+			exit.WithError("Error getting primary control plane", err)
+		}
+
+		machineName := driver.MachineName(*cc, cp)
+		if _, err = api.Load(machineName); err != nil {
 			switch err := errors.Cause(err).(type) {
 			case mcnerror.ErrHostDoesNotExist:
 				exit.WithCodeT(exit.Unavailable, "{{.name}} cluster does not exist", out.V{"name": cc.Name})
@@ -101,7 +109,7 @@ var dashboardCmd = &cobra.Command{
 			exit.WithCodeT(exit.NoInput, "kubectl not found in PATH, but is required for the dashboard. Installation guide: https://kubernetes.io/docs/tasks/tools/install-kubectl/")
 		}
 
-		if !machine.IsHostRunning(api, profileName) {
+		if !machine.IsHostRunning(api, machineName) {
 			os.Exit(1)
 		}
 
@@ -127,7 +135,7 @@ var dashboardCmd = &cobra.Command{
 		}
 
 		out.ErrT(out.Launch, "Launching proxy ...")
-		p, hostPort, err := kubectlProxy(kubectl, cc.Name)
+		p, hostPort, err := kubectlProxy(kubectl, machineName)
 		if err != nil {
 			exit.WithError("kubectl proxy", err)
 		}

cmd/minikube/cmd/delete.go

@@ -35,7 +35,6 @@ import (
 	"k8s.io/minikube/pkg/drivers/kic/oci"
 	"k8s.io/minikube/pkg/minikube/cluster"
 	"k8s.io/minikube/pkg/minikube/config"
-	pkg_config "k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/cruntime"
 	"k8s.io/minikube/pkg/minikube/driver"
@@ -94,9 +93,8 @@ func runDelete(cmd *cobra.Command, args []string) {
 	if len(args) > 0 {
 		exit.UsageT("Usage: minikube delete")
 	}
-	profileFlag := viper.GetString(config.MachineProfile)
 
-	validProfiles, invalidProfiles, err := pkg_config.ListProfiles()
+	validProfiles, invalidProfiles, err := config.ListProfiles()
 	if err != nil {
 		glog.Warningf("'error loading profiles in minikube home %q: %v", localpath.MiniPath(), err)
 	}
@@ -112,11 +110,8 @@ func runDelete(cmd *cobra.Command, args []string) {
 	}
 
 	if deleteAll {
-		if profileFlag != constants.DefaultMachineName {
-			exit.UsageT("usage: minikube delete --all")
-		}
 		delLabel := fmt.Sprintf("%s=%s", oci.CreatedByLabelKey, "true")
-		errs := oci.DeleteAllContainersByLabel(oci.Docker, delLabel)
+		errs := oci.DeleteContainersByLabel(oci.Docker, delLabel)
 		if len(errs) > 0 { // it will error if there is no container to delete
 			glog.Infof("error delete containers by label %q (might be okay): %+v", delLabel, err)
 		}
@@ -142,12 +137,13 @@ func runDelete(cmd *cobra.Command, args []string) {
 			exit.UsageT("usage: minikube delete")
 		}
 
-		profile, err := pkg_config.LoadProfile(profileFlag)
+		profileName := viper.GetString(config.ProfileName)
+		profile, err := config.LoadProfile(profileName)
 		if err != nil {
-			out.ErrT(out.Meh, `"{{.name}}" profile does not exist, trying anyways.`, out.V{"name": profileFlag})
+			out.ErrT(out.Meh, `"{{.name}}" profile does not exist, trying anyways.`, out.V{"name": profileName})
 		}
 
-		errs := DeleteProfiles([]*pkg_config.Profile{profile})
+		errs := DeleteProfiles([]*config.Profile{profile})
 		if len(errs) > 0 {
 			HandleDeletionErrors(errs)
 		}
@@ -168,7 +164,7 @@ func purgeMinikubeDirectory() {
 }
 
 // DeleteProfiles deletes one or more profiles
-func DeleteProfiles(profiles []*pkg_config.Profile) []error {
+func DeleteProfiles(profiles []*config.Profile) []error {
 	var errs []error
 	for _, profile := range profiles {
 		err := deleteProfile(profile)
@@ -189,11 +185,11 @@ func DeleteProfiles(profiles []*pkg_config.Profile) []error {
 	return errs
 }
 
-func deleteProfile(profile *pkg_config.Profile) error {
+func deleteProfile(profile *config.Profile) error {
-	viper.Set(pkg_config.MachineProfile, profile.Name)
+	viper.Set(config.ProfileName, profile.Name)
 
 	delLabel := fmt.Sprintf("%s=%s", oci.ProfileLabelKey, profile.Name)
-	errs := oci.DeleteAllContainersByLabel(oci.Docker, delLabel)
+	errs := oci.DeleteContainersByLabel(oci.Docker, delLabel)
 	if errs != nil { // it will error if there is no container to delete
 		glog.Infof("error deleting containers for %s (might be okay):\n%v", profile.Name, errs)
 	}
@@ -206,21 +202,20 @@ func deleteProfile(profile *pkg_config.Profile) error {
 	if len(errs) > 0 { // it will not error if there is nothing to delete
 		glog.Warningf("error pruning volume (might be okay):\n%v", errs)
 	}
-
 	api, err := machine.NewAPIClient()
 	if err != nil {
 		delErr := profileDeletionErr(profile.Name, fmt.Sprintf("error getting client %v", err))
 		return DeletionError{Err: delErr, Errtype: Fatal}
 	}
 	defer api.Close()
-	cc, err := pkg_config.Load(profile.Name)
+	cc, err := config.Load(profile.Name)
-	if err != nil && !pkg_config.IsNotExist(err) {
+	if err != nil && !config.IsNotExist(err) {
 		delErr := profileDeletionErr(profile.Name, fmt.Sprintf("error loading profile config: %v", err))
 		return DeletionError{Err: delErr, Errtype: MissingProfile}
 	}
 
 	if err == nil && driver.BareMetal(cc.Driver) {
-		if err := uninstallKubernetes(api, profile.Name, cc.KubernetesConfig, viper.GetString(cmdcfg.Bootstrapper), cc.Nodes[0].Name); err != nil {
+		if err := uninstallKubernetes(api, *cc, cc.Nodes[0], viper.GetString(cmdcfg.Bootstrapper)); err != nil {
 			deletionError, ok := err.(DeletionError)
 			if ok {
 				delErr := profileDeletionErr(profile.Name, fmt.Sprintf("%v", err))
@@ -237,10 +232,11 @@ func deleteProfile(profile *pkg_config.Profile) error {
 
 	if cc != nil {
 		for _, n := range cc.Nodes {
-			if err = machine.DeleteHost(api, driver.MachineName(profile.Name, n.Name)); err != nil {
+			machineName := driver.MachineName(*cc, n)
+			if err = machine.DeleteHost(api, machineName); err != nil {
 				switch errors.Cause(err).(type) {
 				case mcnerror.ErrHostDoesNotExist:
-					glog.Infof("%s cluster does not exist. Proceeding ahead with cleanup.", profile.Name)
+					glog.Infof("Host %s does not exist. Proceeding ahead with cleanup.", machineName)
 				default:
 					out.T(out.FailureType, "Failed to delete cluster: {{.error}}", out.V{"error": err})
 					out.T(out.Notice, `You may need to manually remove the "{{.name}}" VM from your hypervisor`, out.V{"name": profile.Name})
@@ -252,8 +248,8 @@ func deleteProfile(profile *pkg_config.Profile) error {
 	// In case DeleteHost didn't complete the job.
 	deleteProfileDirectory(profile.Name)
 
-	if err := pkg_config.DeleteProfile(profile.Name); err != nil {
+	if err := config.DeleteProfile(profile.Name); err != nil {
-		if pkg_config.IsNotExist(err) {
+		if config.IsNotExist(err) {
 			delErr := profileDeletionErr(profile.Name, fmt.Sprintf("\"%s\" profile does not exist", profile.Name))
 			return DeletionError{Err: delErr, Errtype: MissingProfile}
 		}
@@ -273,17 +269,17 @@ func deleteContext(machineName string) error {
 		return DeletionError{Err: fmt.Errorf("update config: %v", err), Errtype: Fatal}
 	}
 
-	if err := cmdcfg.Unset(pkg_config.MachineProfile); err != nil {
+	if err := cmdcfg.Unset(config.ProfileName); err != nil {
 		return DeletionError{Err: fmt.Errorf("unset minikube profile: %v", err), Errtype: Fatal}
 	}
 	return nil
 }
 
-func deleteInvalidProfile(profile *pkg_config.Profile) []error {
+func deleteInvalidProfile(profile *config.Profile) []error {
 	out.T(out.DeletingHost, "Trying to delete invalid profile {{.profile}}", out.V{"profile": profile.Name})
 
 	var errs []error
-	pathToProfile := pkg_config.ProfileFolderPath(profile.Name, localpath.MiniPath())
+	pathToProfile := config.ProfileFolderPath(profile.Name, localpath.MiniPath())
 	if _, err := os.Stat(pathToProfile); !os.IsNotExist(err) {
 		err := os.RemoveAll(pathToProfile)
 		if err != nil {
@@ -305,14 +301,14 @@ func profileDeletionErr(profileName string, additionalInfo string) error {
 	return fmt.Errorf("error deleting profile \"%s\": %s", profileName, additionalInfo)
 }
 
-func uninstallKubernetes(api libmachine.API, profile string, kc pkg_config.KubernetesConfig, bsName string, nodeName string) error {
+func uninstallKubernetes(api libmachine.API, cc config.ClusterConfig, n config.Node, bsName string) error {
-	out.T(out.Resetting, "Uninstalling Kubernetes {{.kubernetes_version}} using {{.bootstrapper_name}} ...", out.V{"kubernetes_version": kc.KubernetesVersion, "bootstrapper_name": bsName})
+	out.T(out.Resetting, "Uninstalling Kubernetes {{.kubernetes_version}} using {{.bootstrapper_name}} ...", out.V{"kubernetes_version": cc.KubernetesConfig.KubernetesVersion, "bootstrapper_name": bsName})
-	clusterBootstrapper, err := cluster.Bootstrapper(api, bsName, profile, nodeName)
+	clusterBootstrapper, err := cluster.Bootstrapper(api, bsName, cc, n)
 	if err != nil {
 		return DeletionError{Err: fmt.Errorf("unable to get bootstrapper: %v", err), Errtype: Fatal}
 	}
 
-	host, err := machine.CheckIfHostExistsAndLoad(api, profile)
+	host, err := machine.CheckIfHostExistsAndLoad(api, driver.MachineName(cc, n))
 	if err != nil {
 		exit.WithError("Error getting host", err)
 	}
@@ -321,7 +317,7 @@ func uninstallKubernetes(api libmachine.API, profile string, kc pkg_config.Kuber
 		exit.WithError("Failed to get command runner", err)
 	}
 
-	cr, err := cruntime.New(cruntime.Config{Type: kc.ContainerRuntime, Runner: r})
+	cr, err := cruntime.New(cruntime.Config{Type: cc.KubernetesConfig.ContainerRuntime, Runner: r})
 	if err != nil {
 		exit.WithError("Failed runtime", err)
 	}
@@ -332,7 +328,7 @@ func uninstallKubernetes(api libmachine.API, profile string, kc pkg_config.Kuber
 		glog.Errorf("unpause failed: %v", err)
 	}
 
-	if err = clusterBootstrapper.DeleteCluster(kc); err != nil {
+	if err = clusterBootstrapper.DeleteCluster(cc.KubernetesConfig); err != nil {
 		return DeletionError{Err: fmt.Errorf("failed to delete cluster: %v", err), Errtype: Fatal}
 	}
 	return nil

cmd/minikube/cmd/delete_test.go

@@ -141,7 +141,7 @@ func TestDeleteProfile(t *testing.T) {
 				t.Errorf("machines mismatch (-want +got):\n%s", diff)
 			}
 
-			viper.Set(config.MachineProfile, "")
+			viper.Set(config.ProfileName, "")
 		})
 	}
 }
@@ -211,5 +211,5 @@ func TestDeleteAllProfiles(t *testing.T) {
 		t.Errorf("Did not delete all machines, remaining: %v", afterMachines)
 	}
 
-	viper.Set(config.MachineProfile, "")
+	viper.Set(config.ProfileName, "")
 }

cmd/minikube/cmd/docker-env.go

@@ -143,79 +143,82 @@ var dockerEnvCmd = &cobra.Command{
 		}
 		defer api.Close()
 
-		profile := viper.GetString(config.MachineProfile)
+		profile := viper.GetString(config.ProfileName)
 		cc, err := config.Load(profile)
 		if err != nil {
 			exit.WithError("Error getting config", err)
 		}
-		host, err := machine.CheckIfHostExistsAndLoad(api, cc.Name)
+		for _, n := range cc.Nodes {
-		if err != nil {
+			machineName := driver.MachineName(*cc, n)
-			exit.WithError("Error getting host", err)
+			host, err := machine.CheckIfHostExistsAndLoad(api, machineName)
-		}
-		if host.Driver.DriverName() == driver.None {
-			exit.UsageT(`'none' driver does not support 'minikube docker-env' command`)
-		}
-
-		hostSt, err := machine.GetHostStatus(api, cc.Name)
-		if err != nil {
-			exit.WithError("Error getting host status", err)
-		}
-		if hostSt != state.Running.String() {
-			exit.WithCodeT(exit.Unavailable, `'{{.profile}}' is not running`, out.V{"profile": profile})
-		}
-		ok, err := isDockerActive(host.Driver)
-		if err != nil {
-			exit.WithError("Error getting service status", err)
-		}
-
-		if !ok {
-			exit.WithCodeT(exit.Unavailable, `The docker service within '{{.profile}}' is not active`, out.V{"profile": profile})
-		}
-
-		hostIP, err := host.Driver.GetIP()
-		if err != nil {
-			exit.WithError("Error getting host IP", err)
-		}
-
-		sh := shell.EnvConfig{
-			Shell: shell.ForceShell,
-		}
-
-		port := constants.DockerDaemonPort
-		if driver.IsKIC(host.DriverName) { // for kic we need to find what port docker/podman chose for us
-			hostIP = oci.DefaultBindIPV4
-			port, err = oci.HostPortBinding(host.DriverName, profile, port)
 			if err != nil {
-				exit.WithCodeT(exit.Failure, "Error getting port binding for '{{.driver_name}} driver: {{.error}}", out.V{"driver_name": host.DriverName, "error": err})
+				exit.WithError("Error getting host", err)
+			}
+			if host.Driver.DriverName() == driver.None {
+				exit.UsageT(`'none' driver does not support 'minikube docker-env' command`)
 			}
-		}
 
-		ec := DockerEnvConfig{
+			hostSt, err := machine.GetHostStatus(api, machineName)
-			EnvConfig: sh,
-			profile:   profile,
-			driver:    host.DriverName,
-			hostIP:    hostIP,
-			port:      port,
-			certsDir:  localpath.MakeMiniPath("certs"),
-			noProxy:   noProxy,
-		}
-
-		if ec.Shell == "" {
-			ec.Shell, err = shell.Detect()
 			if err != nil {
-				exit.WithError("Error detecting shell", err)
+				exit.WithError("Error getting host status", err)
 			}
-		}
+			if hostSt != state.Running.String() {
-
+				exit.WithCodeT(exit.Unavailable, `'{{.profile}}' is not running`, out.V{"profile": profile})
-		if dockerUnset {
+			}
-			if err := dockerUnsetScript(ec, os.Stdout); err != nil {
+			ok, err := isDockerActive(host.Driver)
-				exit.WithError("Error generating unset output", err)
+			if err != nil {
+				exit.WithError("Error getting service status", err)
 			}
-			return
-		}
 
-		if err := dockerSetScript(ec, os.Stdout); err != nil {
+			if !ok {
-			exit.WithError("Error generating set output", err)
+				exit.WithCodeT(exit.Unavailable, `The docker service within '{{.profile}}' is not active`, out.V{"profile": profile})
+			}
+
+			hostIP, err := host.Driver.GetIP()
+			if err != nil {
+				exit.WithError("Error getting host IP", err)
+			}
+
+			sh := shell.EnvConfig{
+				Shell: shell.ForceShell,
+			}
+
+			port := constants.DockerDaemonPort
+			if driver.IsKIC(host.DriverName) { // for kic we need to find what port docker/podman chose for us
+				hostIP = oci.DefaultBindIPV4
+				port, err = oci.HostPortBinding(host.DriverName, profile, port)
+				if err != nil {
+					exit.WithCodeT(exit.Failure, "Error getting port binding for '{{.driver_name}} driver: {{.error}}", out.V{"driver_name": host.DriverName, "error": err})
+				}
+			}
+
+			ec := DockerEnvConfig{
+				EnvConfig: sh,
+				profile:   profile,
+				driver:    host.DriverName,
+				hostIP:    hostIP,
+				port:      port,
+				certsDir:  localpath.MakeMiniPath("certs"),
+				noProxy:   noProxy,
+			}
+
+			if ec.Shell == "" {
+				ec.Shell, err = shell.Detect()
+				if err != nil {
+					exit.WithError("Error detecting shell", err)
+				}
+			}
+
+			if dockerUnset {
+				if err := dockerUnsetScript(ec, os.Stdout); err != nil {
+					exit.WithError("Error generating unset output", err)
+				}
+				return
+			}
+
+			if err := dockerSetScript(ec, os.Stdout); err != nil {
+				exit.WithError("Error generating set output", err)
+			}
 		}
 	},
 }

cmd/minikube/cmd/ip.go

@@ -40,24 +40,26 @@ var ipCmd = &cobra.Command{
 		}
 		defer api.Close()
 
-		cc, err := config.Load(viper.GetString(config.MachineProfile))
+		cc, err := config.Load(viper.GetString(config.ProfileName))
 		if err != nil {
 			exit.WithError("Error getting config", err)
 		}
-		machineName := driver.MachineName(cc.Name, cc.Nodes[0].Name)
+		for _, n := range cc.Nodes {
-		host, err := api.Load(machineName)
+			machineName := driver.MachineName(*cc, n)
-		if err != nil {
+			host, err := api.Load(machineName)
-			switch err := errors.Cause(err).(type) {
+			if err != nil {
-			case mcnerror.ErrHostDoesNotExist:
+				switch err := errors.Cause(err).(type) {
-				exit.WithCodeT(exit.NoInput, `"{{.profile_name}}" host does not exist, unable to show an IP`, out.V{"profile_name": machineName})
+				case mcnerror.ErrHostDoesNotExist:
-			default:
+					exit.WithCodeT(exit.NoInput, `"{{.profile_name}}" host does not exist, unable to show an IP`, out.V{"profile_name": cc.Name})
-				exit.WithError("Error getting host", err)
+				default:
+					exit.WithError("Error getting host", err)
+				}
 			}
+			ip, err := host.Driver.GetIP()
+			if err != nil {
+				exit.WithError("Error getting IP", err)
+			}
+			out.Ln(ip)
 		}
-		ip, err := host.Driver.GetIP()
-		if err != nil {
-			exit.WithError("Error getting IP", err)
-		}
-		out.Ln(ip)
 	},
 }

cmd/minikube/cmd/kubectl.go

@@ -49,7 +49,7 @@ minikube kubectl -- get pods --namespace kube-system`,
 		}
 		defer api.Close()
 
-		cc, err := config.Load(viper.GetString(config.MachineProfile))
+		cc, err := config.Load(viper.GetString(config.ProfileName))
 		if err != nil && !config.IsNotExist(err) {
 			out.ErrLn("Error loading profile config: %v", err)
 		}

cmd/minikube/cmd/logs.go

@@ -23,9 +23,11 @@ import (
 	"k8s.io/minikube/pkg/minikube/cluster"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/cruntime"
+	"k8s.io/minikube/pkg/minikube/driver"
 	"k8s.io/minikube/pkg/minikube/exit"
 	"k8s.io/minikube/pkg/minikube/logs"
 	"k8s.io/minikube/pkg/minikube/machine"
+	"k8s.io/minikube/pkg/minikube/node"
 )
 
 const (
@@ -34,6 +36,7 @@ const (
 )
 
 var (
+	nodeName string
 	// followLogs triggers tail -f mode
 	followLogs bool
 	// numberOfLines is how many lines to output, set via -n
@@ -48,18 +51,33 @@ var logsCmd = &cobra.Command{
 	Short: "Gets the logs of the running instance, used for debugging minikube, not user code.",
 	Long:  `Gets the logs of the running instance, used for debugging minikube, not user code.`,
 	Run: func(cmd *cobra.Command, args []string) {
-		cfg, err := config.Load(viper.GetString(config.MachineProfile))
+		cfg, err := config.Load(viper.GetString(config.ProfileName))
 		if err != nil {
 			exit.WithError("Error getting config", err)
 		}
 
+		if nodeName == "" {
+			cp, err := config.PrimaryControlPlane(*cfg)
+			if err != nil {
+				exit.WithError("Error getting primary control plane", err)
+			}
+			nodeName = cp.Name
+		}
+
+		n, _, err := node.Retrieve(cfg, nodeName)
+		if err != nil {
+			exit.WithError("Error retrieving node", err)
+		}
+
+		machineName := driver.MachineName(*cfg, *n)
+
 		api, err := machine.NewAPIClient()
 		if err != nil {
 			exit.WithError("Error getting client", err)
 		}
 		defer api.Close()
 
-		h, err := api.Load(cfg.Name)
+		h, err := api.Load(machineName)
 		if err != nil {
 			exit.WithError("api load", err)
 		}
@@ -67,7 +85,7 @@ var logsCmd = &cobra.Command{
 		if err != nil {
 			exit.WithError("command runner", err)
 		}
-		bs, err := cluster.Bootstrapper(api, viper.GetString(cmdcfg.Bootstrapper), viper.GetString(config.MachineProfile), viper.GetString(config.MachineProfile))
+		bs, err := cluster.Bootstrapper(api, viper.GetString(cmdcfg.Bootstrapper), *cfg, *n)
 		if err != nil {
 			exit.WithError("Error getting cluster bootstrapper", err)
 		}
@@ -99,4 +117,5 @@ func init() {
 	logsCmd.Flags().BoolVarP(&followLogs, "follow", "f", false, "Show only the most recent journal entries, and continuously print new entries as they are appended to the journal.")
 	logsCmd.Flags().BoolVar(&showProblems, "problems", false, "Show only log entries which point to known problems")
 	logsCmd.Flags().IntVarP(&numberOfLines, "length", "n", 60, "Number of lines back to go within the log")
+	logsCmd.Flags().StringVar(&nodeName, "node", "", "The node to get logs from. Defaults to the primary control plane.")
 }

cmd/minikube/cmd/mount.go

@@ -104,11 +104,16 @@ var mountCmd = &cobra.Command{
 			exit.WithError("Error getting client", err)
 		}
 		defer api.Close()
-		cc, err := config.Load(viper.GetString(config.MachineProfile))
+		cc, err := config.Load(viper.GetString(config.ProfileName))
 		if err != nil {
 			exit.WithError("Error getting config", err)
 		}
-		host, err := api.Load(cc.Name)
+
+		cp, err := config.PrimaryControlPlane(*cc)
+		if err != nil {
+			exit.WithError("Error getting primary cp", err)
+		}
+		host, err := api.Load(driver.MachineName(*cc, cp))
 		if err != nil {
 			exit.WithError("Error loading api", err)
 		}

cmd/minikube/cmd/node_add.go

@@ -17,7 +17,7 @@ limitations under the License.
 package cmd
 
 import (
-	"strconv"
+	"fmt"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
@@ -29,39 +29,31 @@ import (
 )
 
 var (
-	nodeName string
+	cp     bool
-	cp       bool
+	worker bool
-	worker   bool
 )
 var nodeAddCmd = &cobra.Command{
 	Use:   "add",
 	Short: "Adds a node to the given cluster.",
 	Long:  "Adds a node to the given cluster config, and starts it.",
 	Run: func(cmd *cobra.Command, args []string) {
-		profile := viper.GetString(config.MachineProfile)
+		profile := viper.GetString(config.ProfileName)
-		mc, err := config.Load(profile)
+		cc, err := config.Load(profile)
 		if err != nil {
 			exit.WithError("Error getting config", err)
 		}
-		name := nodeName
+
-		if nodeName == "" {
+		//name := profile + strconv.Itoa(len(mc.Nodes)+1)
-			name = profile + strconv.Itoa(len(mc.Nodes)+1)
+		name := fmt.Sprintf("m%d", len(cc.Nodes)+1)
-		}
+
-		_, _, err = node.Retrieve(mc, name)
-		if err == nil {
-			exit.WithCodeT(100, "{{.nodeName}} already exists in cluster {{.cluster}}. Choose a different name.", out.V{"nodeName": name, "cluster": mc.Name})
-		}
 		out.T(out.Happy, "Adding node {{.name}} to cluster {{.cluster}}", out.V{"name": name, "cluster": profile})
 
-		// TODO: Deal with parameters better. Ideally we should be able to acceot any node-specific minikube start params here.
+		n, err := node.Add(cc, name, cp, worker, "", profile)
-		n := config.Node{
+		if err != nil {
-			Name:              name,
+			exit.WithError("Error adding node to cluster", err)
-			Worker:            worker,
-			ControlPlane:      cp,
-			KubernetesVersion: mc.KubernetesConfig.KubernetesVersion,
 		}
 
-		err = node.Add(mc, n)
+		_, err = node.Start(*cc, *n, false, nil)
 		if err != nil {
 			exit.WithError("Error adding node to cluster", err)
 		}
@@ -71,7 +63,6 @@ var nodeAddCmd = &cobra.Command{
 }
 
 func init() {
-	nodeAddCmd.Flags().StringVar(&nodeName, "name", "", "The name of the node to add.")
 	nodeAddCmd.Flags().BoolVar(&cp, "control-plane", false, "If true, the node added will also be a control plane in addition to a worker.")
 	nodeAddCmd.Flags().BoolVar(&worker, "worker", true, "If true, the added node will be marked for work. Defaults to true.")
 	//We should figure out which of these flags to actually import

cmd/minikube/cmd/node_delete.go

@@ -36,7 +36,7 @@ var nodeDeleteCmd = &cobra.Command{
 		}
 		name := args[0]
 
-		profile := viper.GetString(config.MachineProfile)
+		profile := viper.GetString(config.ProfileName)
 		out.T(out.DeletingHost, "Deleting node {{.name}} from cluster {{.cluster}}", out.V{"name": name, "cluster": profile})
 
 		cc, err := config.Load(profile)

cmd/minikube/cmd/node_start.go

@@ -50,7 +50,7 @@ var nodeStartCmd = &cobra.Command{
 			os.Exit(0)
 		}
 
-		cc, err := config.Load(viper.GetString(config.MachineProfile))
+		cc, err := config.Load(viper.GetString(config.ProfileName))
 		if err != nil {
 			exit.WithError("loading config", err)
 		}

cmd/minikube/cmd/node_stop.go

@@ -18,8 +18,12 @@ package cmd
 
 import (
 	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/driver"
 	"k8s.io/minikube/pkg/minikube/exit"
 	"k8s.io/minikube/pkg/minikube/machine"
+	"k8s.io/minikube/pkg/minikube/node"
 	"k8s.io/minikube/pkg/minikube/out"
 )
 
@@ -39,7 +43,19 @@ var nodeStopCmd = &cobra.Command{
 			exit.WithError("creating api client", err)
 		}
 
-		err = machine.StopHost(api, name)
+		cc, err := config.Load(viper.GetString(config.ProfileName))
+		if err != nil {
+			exit.WithError("getting config", err)
+		}
+
+		n, _, err := node.Retrieve(cc, name)
+		if err != nil {
+			exit.WithError("retrieving node", err)
+		}
+
+		machineName := driver.MachineName(*cc, *n)
+
+		err = machine.StopHost(api, machineName)
 		if err != nil {
 			out.FatalT("Failed to stop node {{.name}}", out.V{"name": name})
 		}

cmd/minikube/cmd/pause.go

@@ -27,6 +27,7 @@ import (
 	"k8s.io/minikube/pkg/minikube/cluster"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/cruntime"
+	"k8s.io/minikube/pkg/minikube/driver"
 	"k8s.io/minikube/pkg/minikube/exit"
 	"k8s.io/minikube/pkg/minikube/machine"
 	"k8s.io/minikube/pkg/minikube/out"
@@ -45,7 +46,7 @@ var pauseCmd = &cobra.Command{
 }
 
 func runPause(cmd *cobra.Command, args []string) {
-	cname := viper.GetString(config.MachineProfile)
+	cname := viper.GetString(config.ProfileName)
 	api, err := machine.NewAPIClient()
 	if err != nil {
 		exit.WithError("Error getting client", err)
@@ -63,37 +64,40 @@ func runPause(cmd *cobra.Command, args []string) {
 	}
 
 	glog.Infof("config: %+v", cc)
-	host, err := machine.CheckIfHostExistsAndLoad(api, cname)
-	if err != nil {
-		exit.WithError("Error getting host", err)
-	}
 
-	r, err := machine.CommandRunner(host)
+	for _, n := range cc.Nodes {
-	if err != nil {
+		host, err := machine.CheckIfHostExistsAndLoad(api, driver.MachineName(*cc, n))
-		exit.WithError("Failed to get command runner", err)
+		if err != nil {
-	}
+			exit.WithError("Error getting host", err)
+		}
 
-	cr, err := cruntime.New(cruntime.Config{Type: cc.KubernetesConfig.ContainerRuntime, Runner: r})
+		r, err := machine.CommandRunner(host)
-	if err != nil {
+		if err != nil {
-		exit.WithError("Failed runtime", err)
+			exit.WithError("Failed to get command runner", err)
-	}
+		}
 
-	glog.Infof("namespaces: %v keys: %v", namespaces, viper.AllSettings())
+		cr, err := cruntime.New(cruntime.Config{Type: cc.KubernetesConfig.ContainerRuntime, Runner: r})
-	if allNamespaces {
+		if err != nil {
-		namespaces = nil //all
+			exit.WithError("Failed runtime", err)
-	} else if len(namespaces) == 0 {
+		}
-		exit.WithCodeT(exit.BadUsage, "Use -A to specify all namespaces")
-	}
 
-	ids, err := cluster.Pause(cr, r, namespaces)
+		glog.Infof("namespaces: %v keys: %v", namespaces, viper.AllSettings())
-	if err != nil {
+		if allNamespaces {
-		exit.WithError("Pause", err)
+			namespaces = nil //all
-	}
+		} else if len(namespaces) == 0 {
+			exit.WithCodeT(exit.BadUsage, "Use -A to specify all namespaces")
+		}
 
-	if namespaces == nil {
+		ids, err := cluster.Pause(cr, r, namespaces)
-		out.T(out.Unpause, "Paused kubelet and {{.count}} containers", out.V{"count": len(ids)})
+		if err != nil {
-	} else {
+			exit.WithError("Pause", err)
-		out.T(out.Unpause, "Paused kubelet and {{.count}} containers in: {{.namespaces}}", out.V{"count": len(ids), "namespaces": strings.Join(namespaces, ", ")})
+		}
+
+		if namespaces == nil {
+			out.T(out.Unpause, "Paused kubelet and {{.count}} containers", out.V{"count": len(ids)})
+		} else {
+			out.T(out.Unpause, "Paused kubelet and {{.count}} containers in: {{.namespaces}}", out.V{"count": len(ids), "namespaces": strings.Join(namespaces, ", ")})
+		}
 	}
 }
 

cmd/minikube/cmd/podman-env.go

@@ -114,66 +114,69 @@ var podmanEnvCmd = &cobra.Command{
 		}
 		defer api.Close()
 
-		profile := viper.GetString(config.MachineProfile)
+		profile := viper.GetString(config.ProfileName)
 		cc, err := config.Load(profile)
 		if err != nil {
 			exit.WithError("Error getting config", err)
 		}
-		host, err := machine.CheckIfHostExistsAndLoad(api, cc.Name)
+		for _, n := range cc.Nodes {
-		if err != nil {
+			machineName := driver.MachineName(*cc, n)
-			exit.WithError("Error getting host", err)
+			host, err := machine.CheckIfHostExistsAndLoad(api, machineName)
-		}
-		if host.Driver.DriverName() == driver.None {
-			exit.UsageT(`'none' driver does not support 'minikube podman-env' command`)
-		}
-
-		hostSt, err := machine.GetHostStatus(api, cc.Name)
-		if err != nil {
-			exit.WithError("Error getting host status", err)
-		}
-		if hostSt != state.Running.String() {
-			exit.WithCodeT(exit.Unavailable, `'{{.profile}}' is not running`, out.V{"profile": profile})
-		}
-		ok, err := isPodmanAvailable(host)
-		if err != nil {
-			exit.WithError("Error getting service status", err)
-		}
-
-		if !ok {
-			exit.WithCodeT(exit.Unavailable, `The podman service within '{{.profile}}' is not active`, out.V{"profile": profile})
-		}
-
-		client, err := createExternalSSHClient(host.Driver)
-		if err != nil {
-			exit.WithError("Error getting ssh client", err)
-		}
-
-		sh := shell.EnvConfig{
-			Shell: shell.ForceShell,
-		}
-		ec := PodmanEnvConfig{
-			EnvConfig: sh,
-			profile:   profile,
-			driver:    host.DriverName,
-			client:    client,
-		}
-
-		if ec.Shell == "" {
-			ec.Shell, err = shell.Detect()
 			if err != nil {
-				exit.WithError("Error detecting shell", err)
+				exit.WithError("Error getting host", err)
 			}
-		}
+			if host.Driver.DriverName() == driver.None {
-
+				exit.UsageT(`'none' driver does not support 'minikube podman-env' command`)
-		if podmanUnset {
-			if err := podmanUnsetScript(ec, os.Stdout); err != nil {
-				exit.WithError("Error generating unset output", err)
 			}
-			return
-		}
 
-		if err := podmanSetScript(ec, os.Stdout); err != nil {
+			hostSt, err := machine.GetHostStatus(api, machineName)
-			exit.WithError("Error generating set output", err)
+			if err != nil {
+				exit.WithError("Error getting host status", err)
+			}
+			if hostSt != state.Running.String() {
+				exit.WithCodeT(exit.Unavailable, `'{{.profile}}' is not running`, out.V{"profile": profile})
+			}
+			ok, err := isPodmanAvailable(host)
+			if err != nil {
+				exit.WithError("Error getting service status", err)
+			}
+
+			if !ok {
+				exit.WithCodeT(exit.Unavailable, `The podman service within '{{.profile}}' is not active`, out.V{"profile": profile})
+			}
+
+			client, err := createExternalSSHClient(host.Driver)
+			if err != nil {
+				exit.WithError("Error getting ssh client", err)
+			}
+
+			sh := shell.EnvConfig{
+				Shell: shell.ForceShell,
+			}
+			ec := PodmanEnvConfig{
+				EnvConfig: sh,
+				profile:   profile,
+				driver:    host.DriverName,
+				client:    client,
+			}
+
+			if ec.Shell == "" {
+				ec.Shell, err = shell.Detect()
+				if err != nil {
+					exit.WithError("Error detecting shell", err)
+				}
+			}
+
+			if podmanUnset {
+				if err := podmanUnsetScript(ec, os.Stdout); err != nil {
+					exit.WithError("Error generating unset output", err)
+				}
+				return
+			}
+
+			if err := podmanSetScript(ec, os.Stdout); err != nil {
+				exit.WithError("Error generating set output", err)
+			}
 		}
 	},
 }

cmd/minikube/cmd/root.go

@@ -156,7 +156,7 @@ func setFlagsUsingViper() {
 
 func init() {
 	translate.DetermineLocale()
-	RootCmd.PersistentFlags().StringP(config.MachineProfile, "p", constants.DefaultMachineName, `The name of the minikube VM being used. This can be set to allow having multiple instances of minikube independently.`)
+	RootCmd.PersistentFlags().StringP(config.ProfileName, "p", constants.DefaultClusterName, `The name of the minikube VM being used. This can be set to allow having multiple instances of minikube independently.`)
 	RootCmd.PersistentFlags().StringP(configCmd.Bootstrapper, "b", "kubeadm", "The name of the cluster bootstrapper that will set up the kubernetes cluster.")
 
 	groups := templates.CommandGroups{

cmd/minikube/cmd/service.go

@@ -20,18 +20,29 @@ import (
 	"fmt"
 	"net/url"
 	"os"
+	"os/signal"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
 	"text/template"
+	"time"
 
 	"github.com/golang/glog"
 	"github.com/pkg/browser"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
+	"k8s.io/minikube/pkg/drivers/kic/oci"
+	"k8s.io/minikube/pkg/minikube/config"
 	pkg_config "k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/driver"
 	"k8s.io/minikube/pkg/minikube/exit"
+	"k8s.io/minikube/pkg/minikube/localpath"
 	"k8s.io/minikube/pkg/minikube/machine"
 	"k8s.io/minikube/pkg/minikube/out"
 	"k8s.io/minikube/pkg/minikube/service"
+	"k8s.io/minikube/pkg/minikube/tunnel/kic"
 )
 
 const defaultServiceFormatTemplate = "http://{{.IP}}:{{.Port}}"
@@ -72,33 +83,31 @@ var serviceCmd = &cobra.Command{
 		}
 		defer api.Close()
 
-		profileName := viper.GetString(pkg_config.MachineProfile)
+		profileName := viper.GetString(pkg_config.ProfileName)
-		if !machine.IsHostRunning(api, profileName) {
+		cfg, err := config.Load(profileName)
+		if err != nil {
+			exit.WithError("Error getting config", err)
+		}
+		cp, err := config.PrimaryControlPlane(*cfg)
+		if err != nil {
+			exit.WithError("Error getting control plane", err)
+		}
+		machineName := driver.MachineName(*cfg, cp)
+		if !machine.IsHostRunning(api, machineName) {
 			os.Exit(1)
 		}
 
+		if runtime.GOOS == "darwin" && cfg.Driver == oci.Docker {
+			startKicServiceTunnel(svc, cfg.Name)
+			return
+		}
+
 		urls, err := service.WaitForService(api, namespace, svc, serviceURLTemplate, serviceURLMode, https, wait, interval)
 		if err != nil {
 			exit.WithError("Error opening service", err)
 		}
 
-		for _, u := range urls {
+		openURLs(svc, urls)
-			_, err := url.Parse(u)
-			if err != nil {
-				glog.Warningf("failed to parse url %q: %v (will not open)", u, err)
-				out.String(fmt.Sprintf("%s\n", u))
-				continue
-			}
-
-			if serviceURLMode {
-				out.String(fmt.Sprintf("%s\n", u))
-				continue
-			}
-			out.T(out.Celebrate, "Opening service {{.namespace_name}}/{{.service_name}} in default browser...", out.V{"namespace_name": namespace, "service_name": svc})
-			if err := browser.OpenURL(u); err != nil {
-				exit.WithError(fmt.Sprintf("open url failed: %s", u), err)
-			}
-		}
 	},
 }
 
@@ -112,3 +121,63 @@ func init() {
 	serviceCmd.PersistentFlags().StringVar(&serviceURLFormat, "format", defaultServiceFormatTemplate, "Format to output service URL in. This format will be applied to each url individually and they will be printed one at a time.")
 
 }
+
+func startKicServiceTunnel(svc, configName string) {
+	ctrlC := make(chan os.Signal, 1)
+	signal.Notify(ctrlC, os.Interrupt)
+
+	clientset, err := service.K8s.GetClientset(1 * time.Second)
+	if err != nil {
+		exit.WithError("error creating clientset", err)
+	}
+
+	port, err := oci.HostPortBinding(oci.Docker, configName, 22)
+	if err != nil {
+		exit.WithError("error getting ssh port", err)
+	}
+	sshPort := strconv.Itoa(port)
+	sshKey := filepath.Join(localpath.MiniPath(), "machines", configName, "id_rsa")
+
+	serviceTunnel := kic.NewServiceTunnel(sshPort, sshKey, clientset.CoreV1())
+	urls, err := serviceTunnel.Start(svc, namespace)
+	if err != nil {
+		exit.WithError("error starting tunnel", err)
+	}
+
+	// wait for tunnel to come up
+	time.Sleep(1 * time.Second)
+
+	data := [][]string{{namespace, svc, "", strings.Join(urls, "\n")}}
+	service.PrintServiceList(os.Stdout, data)
+
+	openURLs(svc, urls)
+	out.T(out.WarningType, "Because you are using docker driver on Mac, the terminal needs to be open to run it.")
+
+	<-ctrlC
+
+	err = serviceTunnel.Stop()
+	if err != nil {
+		exit.WithError("error stopping tunnel", err)
+	}
+}
+
+func openURLs(svc string, urls []string) {
+	for _, u := range urls {
+		_, err := url.Parse(u)
+		if err != nil {
+			glog.Warningf("failed to parse url %q: %v (will not open)", u, err)
+			out.String(fmt.Sprintf("%s\n", u))
+			continue
+		}
+
+		if serviceURLMode {
+			out.String(fmt.Sprintf("%s\n", u))
+			continue
+		}
+
+		out.T(out.Celebrate, "Opening service {{.namespace_name}}/{{.service_name}} in default browser...", out.V{"namespace_name": namespace, "service_name": svc})
+		if err := browser.OpenURL(u); err != nil {
+			exit.WithError(fmt.Sprintf("open url failed: %s", u), err)
+		}
+	}
+}

cmd/minikube/cmd/service_list.go

@@ -18,10 +18,16 @@ package cmd
 
 import (
 	"os"
+	"runtime"
 	"strings"
 
 	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
 	core "k8s.io/api/core/v1"
+	"k8s.io/minikube/pkg/drivers/kic/oci"
+	"k8s.io/minikube/pkg/minikube/config"
+	pkg_config "k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/driver"
 	"k8s.io/minikube/pkg/minikube/exit"
 	"k8s.io/minikube/pkg/minikube/machine"
 	"k8s.io/minikube/pkg/minikube/out"
@@ -41,6 +47,18 @@ var serviceListCmd = &cobra.Command{
 			exit.WithError("Error getting client", err)
 		}
 		defer api.Close()
+		profileName := viper.GetString(pkg_config.ProfileName)
+		cfg, err := config.Load(profileName)
+		if err != nil {
+			exit.WithError("Error getting config", err)
+		}
+		cp, err := config.PrimaryControlPlane(*cfg)
+		if err != nil {
+			exit.WithError("Error getting primary control plane", err)
+		}
+		if !machine.IsHostRunning(api, driver.MachineName(*cfg, cp)) {
+			exit.WithCodeT(exit.Unavailable, "profile {{.name}} is not running.", out.V{"name": profileName})
+		}
 		serviceURLs, err := service.GetServiceURLs(api, serviceListNamespace, serviceURLTemplate)
 		if err != nil {
 			out.FatalT("Failed to get service URL: {{.error}}", out.V{"error": err})
@@ -53,9 +71,15 @@ var serviceListCmd = &cobra.Command{
 			if len(serviceURL.URLs) == 0 {
 				data = append(data, []string{serviceURL.Namespace, serviceURL.Name, "No node port"})
 			} else {
-				data = append(data, []string{serviceURL.Namespace, serviceURL.Name, strings.Join(serviceURL.URLs, "\n")})
+				serviceURLs := strings.Join(serviceURL.URLs, "\n")
-			}
 
+				// if we are running Docker on OSX we empty the internal service URLs
+				if runtime.GOOS == "darwin" && cfg.Driver == oci.Docker {
+					serviceURLs = ""
+				}
+
+				data = append(data, []string{serviceURL.Namespace, serviceURL.Name, "", serviceURLs})
+			}
 		}
 
 		service.PrintServiceList(os.Stdout, data)

cmd/minikube/cmd/ssh-key.go

@@ -33,7 +33,7 @@ var sshKeyCmd = &cobra.Command{
 	Short: "Retrieve the ssh identity key path of the specified cluster",
 	Long:  "Retrieve the ssh identity key path of the specified cluster.",
 	Run: func(cmd *cobra.Command, args []string) {
-		cc, err := config.Load(viper.GetString(config.MachineProfile))
+		cc, err := config.Load(viper.GetString(config.ProfileName))
 		if err != nil {
 			exit.WithError("Getting machine config failed", err)
 		}

cmd/minikube/cmd/ssh.go

@@ -27,9 +27,14 @@ import (
 	"k8s.io/minikube/pkg/minikube/driver"
 	"k8s.io/minikube/pkg/minikube/exit"
 	"k8s.io/minikube/pkg/minikube/machine"
+	"k8s.io/minikube/pkg/minikube/node"
 	"k8s.io/minikube/pkg/minikube/out"
 )
 
+var (
+	nativeSSHClient bool
+)
+
 // sshCmd represents the docker-ssh command
 var sshCmd = &cobra.Command{
 	Use:   "ssh",
@@ -41,32 +46,34 @@ var sshCmd = &cobra.Command{
 			exit.WithError("Error getting client", err)
 		}
 		defer api.Close()
-		cc, err := config.Load(viper.GetString(config.MachineProfile))
+		cc, err := config.Load(viper.GetString(config.ProfileName))
 		if err != nil {
 			exit.WithError("Error getting config", err)
 		}
-
+		var n *config.Node
 		if nodeName == "" {
 			cp, err := config.PrimaryControlPlane(*cc)
 			if err != nil {
 				exit.WithError("Getting primary control plane", err)
 			}
-			nodeName = cp.Name
+			n = &cp
+		} else {
+			n = node.Retrieve(cc, nodeName)
 		}
-		host, err := machine.CheckIfHostExistsAndLoad(api, driver.MachineName(cc.Name, nodeName))
+		host, err := machine.CheckIfHostExistsAndLoad(api, driver.MachineName(*cc, *n))
 		if err != nil {
 			exit.WithError("Error getting host", err)
 		}
 		if host.Driver.DriverName() == driver.None {
 			exit.UsageT("'none' driver does not support 'minikube ssh' command")
 		}
-		if viper.GetBool(nativeSSH) {
+		if nativeSSHClient {
 			ssh.SetDefaultClient(ssh.Native)
 		} else {
 			ssh.SetDefaultClient(ssh.External)
 		}
 
-		err = machine.CreateSSHShell(api, driver.MachineName(cc.Name, nodeName), args)
+		err = machine.CreateSSHShell(api, *cc, cp, args)
 		if err != nil {
 			// This is typically due to a non-zero exit code, so no need for flourish.
 			out.ErrLn("ssh: %v", err)

cmd/minikube/cmd/start.go

@@ -47,6 +47,7 @@ import (
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/cruntime"
+	"k8s.io/minikube/pkg/minikube/download"
 	"k8s.io/minikube/pkg/minikube/driver"
 	"k8s.io/minikube/pkg/minikube/exit"
 	"k8s.io/minikube/pkg/minikube/kubeconfig"
@@ -154,8 +155,8 @@ func initMinikubeFlags() {
 	startCmd.Flags().String(memory, defaultMemorySize, "Amount of RAM allocated to the minikube VM (format: <number>[<unit>], where unit = b, k, m or g).")
 	startCmd.Flags().String(humanReadableDiskSize, defaultDiskSize, "Disk size allocated to the minikube VM (format: <number>[<unit>], where unit = b, k, m or g).")
 	startCmd.Flags().Bool(downloadOnly, false, "If true, only download and cache files for later use - don't install or start anything.")
-	startCmd.Flags().Bool(cacheImages, true, "If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --vm-driver=none.")
+	startCmd.Flags().Bool(cacheImages, true, "If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --driver=none.")
-	startCmd.Flags().String(isoURL, constants.DefaultISOURL, "Location of the minikube iso.")
+	startCmd.Flags().StringSlice(isoURL, download.DefaultISOURLs(), "Locations to fetch the minikube ISO from.")
 	startCmd.Flags().Bool(keepContext, false, "This will keep the existing kubectl context and will create a minikube context.")
 	startCmd.Flags().Bool(embedCerts, false, "if true, will embed the certs in kubeconfig.")
 	startCmd.Flags().String(containerRuntime, "docker", "The container runtime to be used (docker, crio, containerd).")
@@ -191,7 +192,8 @@ func initKubernetesFlags() {
 
 // initDriverFlags inits the commandline flags for vm drivers
 func initDriverFlags() {
-	startCmd.Flags().String("vm-driver", "", fmt.Sprintf("Driver is one of: %v (defaults to auto-detect)", driver.DisplaySupportedDrivers()))
+	startCmd.Flags().String("driver", "", fmt.Sprintf("Driver is one of: %v (defaults to auto-detect)", driver.DisplaySupportedDrivers()))
+	startCmd.Flags().String("vm-driver", "", "DEPRECATED, use `driver` instead.")
 	startCmd.Flags().Bool(disableDriverMounts, false, "Disables the filesystem mounts provided by the hypervisors")
 
 	// kvm2
@@ -288,7 +290,7 @@ func runStart(cmd *cobra.Command, args []string) {
 		registryMirror = viper.GetStringSlice("registry_mirror")
 	}
 
-	existing, err := config.Load(viper.GetString(config.MachineProfile))
+	existing, err := config.Load(viper.GetString(config.ProfileName))
 	if err != nil && !config.IsNotExist(err) {
 		exit.WithCodeT(exit.Data, "Unable to load config: {{.error}}", out.V{"error": err})
 	}
@@ -322,7 +324,13 @@ func runStart(cmd *cobra.Command, args []string) {
 		return
 	}
 
-	cacheISO(&mc, driverName)
+	if !driver.BareMetal(driverName) && !driver.IsKIC(driverName) {
+		url, err := download.ISO(viper.GetStringSlice(isoURL))
+		if err != nil {
+			exit.WithError("Failed to cache ISO", err)
+		}
+		mc.MinikubeISO = url
+	}
 
 	if viper.GetBool(nativeSSH) {
 		ssh.SetDefaultClient(ssh.Native)
@@ -383,18 +391,10 @@ func updateDriver(driverName string) {
 	}
 }
 
-func cacheISO(cfg *config.ClusterConfig, driverName string) {
-	if !driver.BareMetal(driverName) && !driver.IsKIC(driverName) {
-		if err := cluster.CacheISO(*cfg); err != nil {
-			exit.WithError("Failed to cache ISO", err)
-		}
-	}
-}
-
 func displayVersion(version string) {
 	prefix := ""
-	if viper.GetString(config.MachineProfile) != constants.DefaultMachineName {
+	if viper.GetString(config.ProfileName) != constants.DefaultClusterName {
-		prefix = fmt.Sprintf("[%s] ", viper.GetString(config.MachineProfile))
+		prefix = fmt.Sprintf("[%s] ", viper.GetString(config.ProfileName))
 	}
 
 	versionState := out.Happy
@@ -465,12 +465,6 @@ func selectDriver(existing *config.ClusterConfig) registry.DriverState {
 	// Technically unrelated, but important to perform before detection
 	driver.SetLibvirtURI(viper.GetString(kvmQemuURI))
 
-	if viper.GetString("vm-driver") != "" {
-		ds := driver.Status(viper.GetString("vm-driver"))
-		out.T(out.Sparkle, `Using the {{.driver}} driver based on user configuration`, out.V{"driver": ds.String()})
-		return ds
-	}
-
 	// By default, the driver is whatever we used last time
 	if existing != nil && existing.Driver != "" {
 		ds := driver.Status(existing.Driver)
@@ -478,9 +472,23 @@ func selectDriver(existing *config.ClusterConfig) registry.DriverState {
 		return ds
 	}
 
+	// Default to looking at the new driver parameter
+	if viper.GetString("driver") != "" {
+		ds := driver.Status(viper.GetString("driver"))
+		out.T(out.Sparkle, `Using the {{.driver}} driver based on user configuration`, out.V{"driver": ds.String()})
+		return ds
+	}
+
+	// Fallback to old driver parameter
+	if viper.GetString("vm-driver") != "" {
+		ds := driver.Status(viper.GetString("vm-driver"))
+		out.T(out.Sparkle, `Using the {{.driver}} driver based on user configuration`, out.V{"driver": ds.String()})
+		return ds
+	}
+
 	pick, alts := driver.Suggest(driver.Choices())
 	if pick.Name == "" {
-		exit.WithCodeT(exit.Config, "Unable to determine a default driver to use. Try specifying --vm-driver, or see https://minikube.sigs.k8s.io/docs/start/")
+		exit.WithCodeT(exit.Config, "Unable to determine a default driver to use. Try specifying --driver, or see https://minikube.sigs.k8s.io/docs/start/")
 	}
 
 	if len(alts) > 1 {
@@ -536,9 +544,10 @@ func validateDriver(ds registry.DriverState, existing *config.ClusterConfig) {
 
 	cp, err := config.PrimaryControlPlane(*existing)
 	if err != nil {
-		glog.Warningf("selectDriver PrimaryControlPlane: %v", err)
+		exit.WithError("Error getting primary cp", err)
 	}
-	machineName := driver.MachineName(viper.GetString(config.MachineProfile), cp.Name)
+
+	machineName := driver.MachineName(*existing, cp)
 	h, err := api.Load(machineName)
 	if err != nil {
 		glog.Warningf("selectDriver api.Load: %v", err)
@@ -558,7 +567,7 @@ func validateDriver(ds registry.DriverState, existing *config.ClusterConfig) {
 
     * or *
 
-    2) Start the existing "{{.profile_name}}" cluster using: '{{.command}} start --vm-driver={{.old_driver}}'
+    2) Start the existing "{{.profile_name}}" cluster using: '{{.command}} start --driver={{.old_driver}}'
 	`, out.V{"command": minikubeCmd(), "old_driver": h.Driver.DriverName(), "profile_name": machineName})
 
 	exit.WithCodeT(exit.Config, "Exiting.")
@@ -617,8 +626,8 @@ func selectImageRepository(mirrorCountry string) (bool, string, error) {
 
 // Return a minikube command containing the current profile name
 func minikubeCmd() string {
-	if viper.GetString(config.MachineProfile) != constants.DefaultMachineName {
+	if viper.GetString(config.ProfileName) != constants.DefaultClusterName {
-		return fmt.Sprintf("minikube -p %s", config.MachineProfile)
+		return fmt.Sprintf("minikube -p %s", config.ProfileName)
 	}
 	return "minikube"
 }
@@ -634,7 +643,7 @@ func validateUser(drvName string) {
 	useForce := viper.GetBool(force)
 
 	if driver.NeedsRoot(drvName) && u.Uid != "0" && !useForce {
-		exit.WithCodeT(exit.Permissions, `The "{{.driver_name}}" driver requires root privileges. Please run minikube using 'sudo minikube --vm-driver={{.driver_name}}'.`, out.V{"driver_name": drvName})
+		exit.WithCodeT(exit.Permissions, `The "{{.driver_name}}" driver requires root privileges. Please run minikube using 'sudo minikube --driver={{.driver_name}}'.`, out.V{"driver_name": drvName})
 	}
 
 	if driver.NeedsRoot(drvName) || u.Uid != "0" {
@@ -642,13 +651,13 @@ func validateUser(drvName string) {
 	}
 
 	out.T(out.Stopped, `The "{{.driver_name}}" driver should not be used with root privileges.`, out.V{"driver_name": drvName})
-	out.T(out.Tip, "If you are running minikube within a VM, consider using --vm-driver=none:")
+	out.T(out.Tip, "If you are running minikube within a VM, consider using --driver=none:")
 	out.T(out.Documentation, "  https://minikube.sigs.k8s.io/docs/reference/drivers/none/")
 
 	if !useForce {
 		os.Exit(exit.Permissions)
 	}
-	_, err = config.Load(viper.GetString(config.MachineProfile))
+	_, err = config.Load(viper.GetString(config.ProfileName))
 	if err == nil || !config.IsNotExist(err) {
 		out.T(out.Tip, "Tip: To remove this root owned cluster, run: sudo {{.cmd}} delete", out.V{"cmd": minikubeCmd()})
 	}
@@ -711,7 +720,7 @@ func validateFlags(cmd *cobra.Command, drvName string) {
 	}
 
 	if driver.BareMetal(drvName) {
-		if viper.GetString(config.MachineProfile) != constants.DefaultMachineName {
+		if viper.GetString(config.ProfileName) != constants.DefaultClusterName {
 			exit.WithCodeT(exit.Config, "The '{{.name}} driver does not support multiple profiles: https://minikube.sigs.k8s.io/docs/reference/drivers/none/", out.V{"name": drvName})
 		}
 
@@ -799,7 +808,7 @@ func generateCfgFromFlags(cmd *cobra.Command, k8sVersion string, drvName string)
 
 	var kubeNodeName string
 	if drvName != driver.None {
-		kubeNodeName = viper.GetString(config.MachineProfile)
+		kubeNodeName = "m01"
 	}
 
 	// Create the initial node, which will necessarily be a control plane
@@ -812,10 +821,9 @@ func generateCfgFromFlags(cmd *cobra.Command, k8sVersion string, drvName string)
 	}
 
 	cfg := config.ClusterConfig{
-		Name:                    viper.GetString(config.MachineProfile),
+		Name:                    viper.GetString(config.ProfileName),
 		KeepContext:             viper.GetBool(keepContext),
 		EmbedCerts:              viper.GetBool(embedCerts),
-		MinikubeISO:             viper.GetString(isoURL),
 		Memory:                  pkgutil.CalculateSizeInMB(viper.GetString(memory)),
 		CPUs:                    viper.GetInt(cpus),
 		DiskSize:                pkgutil.CalculateSizeInMB(viper.GetString(humanReadableDiskSize)),
@@ -836,7 +844,6 @@ func generateCfgFromFlags(cmd *cobra.Command, k8sVersion string, drvName string)
 		KVMQemuURI:              viper.GetString(kvmQemuURI),
 		KVMGPU:                  viper.GetBool(kvmGPU),
 		KVMHidden:               viper.GetBool(kvmHidden),
-		Downloader:              pkgutil.DefaultDownloader{},
 		DisableDriverMounts:     viper.GetBool(disableDriverMounts),
 		UUID:                    viper.GetString(uuid),
 		NoVTXCheck:              viper.GetBool(noVTXCheck),
@@ -846,7 +853,7 @@ func generateCfgFromFlags(cmd *cobra.Command, k8sVersion string, drvName string)
 		NatNicType:              viper.GetString(natNicType),
 		KubernetesConfig: config.KubernetesConfig{
 			KubernetesVersion:      k8sVersion,
-			ClusterName:            viper.GetString(config.MachineProfile),
+			ClusterName:            viper.GetString(config.ProfileName),
 			APIServerName:          viper.GetString(apiServerName),
 			APIServerNames:         apiServerNames,
 			APIServerIPs:           apiServerIPs,
@@ -884,7 +891,7 @@ func setDockerProxy() {
 	}
 }
 
-// autoSetDriverOptions sets the options needed for specific vm-driver automatically.
+// autoSetDriverOptions sets the options needed for specific driver automatically.
 func autoSetDriverOptions(cmd *cobra.Command, drvName string) (err error) {
 	err = nil
 	hints := driver.FlagDefaults(drvName)
@@ -965,7 +972,7 @@ func getKubernetesVersion(old *config.ClusterConfig) string {
 	if nvs.LT(ovs) {
 		nv = version.VersionPrefix + ovs.String()
 		profileArg := ""
-		if old.Name != constants.DefaultMachineName {
+		if old.Name != constants.DefaultClusterName {
 			profileArg = fmt.Sprintf("-p %s", old.Name)
 		}
 		exit.WithCodeT(exit.Config, `Error: You have selected Kubernetes v{{.new}}, but the existing cluster for your profile is running Kubernetes v{{.old}}. Non-destructive downgrades are not supported, but you can proceed by performing one of the following options:

cmd/minikube/cmd/status.go

@@ -100,15 +100,14 @@ var statusCmd = &cobra.Command{
 		}
 		defer api.Close()
 
-		cluster := viper.GetString(config.MachineProfile)
+		cc, err := config.Load(viper.GetString(config.ProfileName))
-		cc, err := config.Load(cluster)
 		if err != nil {
 			exit.WithError("getting config", err)
 		}
 
 		var st *Status
 		for _, n := range cc.Nodes {
-			machineName := driver.MachineName(cluster, n.Name)
+			machineName := driver.MachineName(*cc, n)
 			st, err = status(api, machineName, n.ControlPlane)
 			if err != nil {
 				glog.Errorf("status error: %v", err)

cmd/minikube/cmd/stop.go

@@ -17,6 +17,7 @@ limitations under the License.
 package cmd
 
 import (
+	"github.com/docker/machine/libmachine"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"k8s.io/minikube/pkg/minikube/config"
@@ -39,7 +40,7 @@ itself, leaving all files intact. The cluster can be started again with the "sta
 
 // runStop handles the executes the flow of "minikube stop"
 func runStop(cmd *cobra.Command, args []string) {
-	profile := viper.GetString(pkg_config.MachineProfile)
+	profile := viper.GetString(pkg_config.ProfileName)
 	api, err := machine.NewAPIClient()
 	if err != nil {
 		exit.WithError("Error getting client", err)
@@ -48,21 +49,16 @@ func runStop(cmd *cobra.Command, args []string) {
 
 	cc, err := config.Load(profile)
 	if err != nil {
-		exit.WithError("Error retrieving config", err)
+		exit.WithError("Error getting cluster config", err)
 	}
 
-	// TODO replace this back with expo backoff
 	for _, n := range cc.Nodes {
-		err := machine.StopHost(api, driver.MachineName(profile, n.Name))
+		nonexistent := stop(api, *cc, n)
-		if err != nil {
-			exit.WithError("Unable to stop VM", err)
-		}
-		/*if err := retry.Expo(fn, 5*time.Second, 3*time.Minute, 5); err != nil {
-			exit.WithError("Unable to stop VM", err)
-		}*/
-	}
 
-	out.T(out.Stopped, `"{{.profile_name}}" stopped.`, out.V{"profile_name": profile})
+		if !nonexistent {
+			out.T(out.Stopped, `"{{.node_name}}" stopped.`, out.V{"node_name": n.Name})
+		}
+	}
 
 	if err := killMountProcess(); err != nil {
 		out.T(out.WarningType, "Unable to kill mount process: {{.error}}", out.V{"error": err})
@@ -73,3 +69,20 @@ func runStop(cmd *cobra.Command, args []string) {
 		exit.WithError("update config", err)
 	}
 }
+
+func stop(api libmachine.API, cluster config.ClusterConfig, n config.Node) bool {
+	nonexistent := false
+
+	// TODO replace this back with expo backoff
+	for _, n := range cluster.Nodes {
+		err := machine.StopHost(api, driver.MachineName(cluster, n))
+		if err != nil {
+			exit.WithError("Unable to stop VM", err)
+		}
+		/*if err := retry.Expo(fn, 5*time.Second, 3*time.Minute, 5); err != nil {
+			exit.WithError("Unable to stop VM", err)
+		}*/
+	}
+
+	return nonexistent
+}

cmd/minikube/cmd/tunnel.go

@@ -20,16 +20,23 @@ import (
 	"context"
 	"os"
 	"os/signal"
+	"path/filepath"
+	"runtime"
+	"strconv"
 	"time"
 
 	"github.com/golang/glog"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
+
+	"k8s.io/minikube/pkg/drivers/kic/oci"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/exit"
+	"k8s.io/minikube/pkg/minikube/localpath"
 	"k8s.io/minikube/pkg/minikube/machine"
 	"k8s.io/minikube/pkg/minikube/service"
 	"k8s.io/minikube/pkg/minikube/tunnel"
+	"k8s.io/minikube/pkg/minikube/tunnel/kic"
 )
 
 var cleanup bool
@@ -69,6 +76,11 @@ var tunnelCmd = &cobra.Command{
 			exit.WithError("error creating clientset", err)
 		}
 
+		cfg, err := config.Load(viper.GetString(config.ProfileName))
+		if err != nil {
+			exit.WithError("Error getting config", err)
+		}
+
 		ctrlC := make(chan os.Signal, 1)
 		signal.Notify(ctrlC, os.Interrupt)
 		ctx, cancel := context.WithCancel(context.Background())
@@ -77,10 +89,23 @@ var tunnelCmd = &cobra.Command{
 			cancel()
 		}()
 
-		cfg, err := config.Load(viper.GetString(config.MachineProfile))
+		if runtime.GOOS == "darwin" && cfg.Driver == oci.Docker {
-		if err != nil {
+			port, err := oci.HostPortBinding(oci.Docker, cfg.Name, 22)
-			exit.WithError("Error getting config", err)
+			if err != nil {
+				exit.WithError("error getting ssh port", err)
+			}
+			sshPort := strconv.Itoa(port)
+			sshKey := filepath.Join(localpath.MiniPath(), "machines", cfg.Name, "id_rsa")
+
+			kicSSHTunnel := kic.NewSSHTunnel(ctx, sshPort, sshKey, clientset.CoreV1())
+			err = kicSSHTunnel.Start()
+			if err != nil {
+				exit.WithError("error starting tunnel", err)
+			}
+
+			return
 		}
+
 		done, err := manager.StartTunnel(ctx, cfg.Name, api, config.DefaultLoader, clientset.CoreV1())
 		if err != nil {
 			exit.WithError("error starting tunnel", err)

cmd/minikube/cmd/unpause.go

@@ -27,6 +27,7 @@ import (
 	"k8s.io/minikube/pkg/minikube/cluster"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/cruntime"
+	"k8s.io/minikube/pkg/minikube/driver"
 	"k8s.io/minikube/pkg/minikube/exit"
 	"k8s.io/minikube/pkg/minikube/machine"
 	"k8s.io/minikube/pkg/minikube/out"
@@ -37,7 +38,7 @@ var unpauseCmd = &cobra.Command{
 	Use:   "unpause",
 	Short: "unpause Kubernetes",
 	Run: func(cmd *cobra.Command, args []string) {
-		cname := viper.GetString(config.MachineProfile)
+		cname := viper.GetString(config.ProfileName)
 		api, err := machine.NewAPIClient()
 		if err != nil {
 			exit.WithError("Error getting client", err)
@@ -54,39 +55,43 @@ var unpauseCmd = &cobra.Command{
 			os.Exit(1)
 		}
 		glog.Infof("config: %+v", cc)
-		host, err := machine.CheckIfHostExistsAndLoad(api, cname)
-		if err != nil {
-			exit.WithError("Error getting host", err)
-		}
 
-		r, err := machine.CommandRunner(host)
+		for _, n := range cc.Nodes {
-		if err != nil {
+			machineName := driver.MachineName(*cc, n)
-			exit.WithError("Failed to get command runner", err)
+			host, err := machine.CheckIfHostExistsAndLoad(api, machineName)
-		}
+			if err != nil {
-
+				exit.WithError("Error getting host", err)
-		cr, err := cruntime.New(cruntime.Config{Type: cc.KubernetesConfig.ContainerRuntime, Runner: r})
-		if err != nil {
-			exit.WithError("Failed runtime", err)
-		}
-
-		glog.Infof("namespaces: %v keys: %v", namespaces, viper.AllSettings())
-		if allNamespaces {
-			namespaces = nil //all
-		} else {
-			if len(namespaces) == 0 {
-				exit.WithCodeT(exit.BadUsage, "Use -A to specify all namespaces")
 			}
-		}
 
-		ids, err := cluster.Unpause(cr, r, namespaces)
+			r, err := machine.CommandRunner(host)
-		if err != nil {
+			if err != nil {
-			exit.WithError("Pause", err)
+				exit.WithError("Failed to get command runner", err)
-		}
+			}
 
-		if namespaces == nil {
+			cr, err := cruntime.New(cruntime.Config{Type: cc.KubernetesConfig.ContainerRuntime, Runner: r})
-			out.T(out.Pause, "Unpaused kubelet and {{.count}} containers", out.V{"count": len(ids)})
+			if err != nil {
-		} else {
+				exit.WithError("Failed runtime", err)
-			out.T(out.Pause, "Unpaused kubelet and {{.count}} containers in: {{.namespaces}}", out.V{"count": len(ids), "namespaces": strings.Join(namespaces, ", ")})
+			}
+
+			glog.Infof("namespaces: %v keys: %v", namespaces, viper.AllSettings())
+			if allNamespaces {
+				namespaces = nil //all
+			} else {
+				if len(namespaces) == 0 {
+					exit.WithCodeT(exit.BadUsage, "Use -A to specify all namespaces")
+				}
+			}
+
+			ids, err := cluster.Unpause(cr, r, namespaces)
+			if err != nil {
+				exit.WithError("Pause", err)
+			}
+
+			if namespaces == nil {
+				out.T(out.Pause, "Unpaused kubelet and {{.count}} containers", out.V{"count": len(ids)})
+			} else {
+				out.T(out.Pause, "Unpaused kubelet and {{.count}} containers in: {{.namespaces}}", out.V{"count": len(ids), "namespaces": strings.Join(namespaces, ", ")})
+			}
 		}
 
 	},

cmd/minikube/cmd/update-context.go

@@ -39,7 +39,7 @@ var updateContextCmd = &cobra.Command{
 			exit.WithError("Error getting client", err)
 		}
 		defer api.Close()
-		machineName := viper.GetString(config.MachineProfile)
+		machineName := viper.GetString(config.ProfileName)
 		ip, err := cluster.GetHostDriverIP(api, machineName)
 		if err != nil {
 			exit.WithError("Error host driver ip status", err)

deploy/iso/minikube-iso/configs/minikube_defconfig

@@ -68,3 +68,5 @@ BR2_PACKAGE_STRACE=y
 BR2_PACKAGE_SYSSTAT=y
 BR2_PACKAGE_HTOP=y
 BR2_PACKAGE_CONNTRACK_TOOLS=y
+BR2_PACKAGE_TAR=y
+BR2_PACKAGE_LZ4=y

deploy/iso/minikube-iso/package/Config.in

@@ -7,8 +7,8 @@ menu "System tools"
     source "$BR2_EXTERNAL_MINIKUBE_PATH/package/crictl-bin/Config.in"
     source "$BR2_EXTERNAL_MINIKUBE_PATH/package/automount/Config.in"
     source "$BR2_EXTERNAL_MINIKUBE_PATH/package/docker-bin/Config.in"
-    source "$BR2_EXTERNAL_MINIKUBE_PATH/package/cni-bin/Config.in"
+    source "$BR2_EXTERNAL_MINIKUBE_PATH/package/cni/Config.in"
-    source "$BR2_EXTERNAL_MINIKUBE_PATH/package/cni-plugins-bin/Config.in"
+    source "$BR2_EXTERNAL_MINIKUBE_PATH/package/cni-plugins/Config.in"
     source "$BR2_EXTERNAL_MINIKUBE_PATH/package/hyperv-daemons/Config.in"
     source "$BR2_EXTERNAL_MINIKUBE_PATH/package/gluster/Config.in"
     source "$BR2_EXTERNAL_MINIKUBE_PATH/package/vbox-guest/Config.in"

deploy/iso/minikube-iso/package/cni-bin/Config.in

@@ -1,4 +0,0 @@
-config BR2_PACKAGE_CNI_BIN
-	bool "cni-bin"
-	default y
-	depends on BR2_x86_64

deploy/iso/minikube-iso/package/cni-bin/cni-bin.mk

@@ -1,29 +0,0 @@
-################################################################################
-#
-# cni-bin
-#
-################################################################################
-
-CNI_BIN_VERSION = v0.6.0
-CNI_BIN_SITE = https://github.com/containernetworking/cni/releases/download/$(CNI_BIN_VERSION)
-CNI_BIN_SOURCE = cni-amd64-$(CNI_BIN_VERSION).tgz
-
-define CNI_BIN_INSTALL_TARGET_CMDS
-	$(INSTALL) -D -m 0755 \
-		$(@D)/noop \
-		$(TARGET_DIR)/opt/cni/bin/noop
-
-	ln -sf \
-		../../opt/cni/bin/noop \
-		$(TARGET_DIR)/usr/bin/noop
-
-	$(INSTALL) -D -m 0755 \
-		$(@D)/noop \
-		$(TARGET_DIR)/opt/cni/bin/cnitool
-
-	ln -sf \
-		../../opt/cni/bin/cnitool \
-		$(TARGET_DIR)/usr/bin/cnitool
-endef
-
-$(eval $(generic-package))

deploy/iso/minikube-iso/package/cni-plugins-bin/Config.in

@@ -1,4 +0,0 @@
-config BR2_PACKAGE_CNI_PLUGINS_BIN
-	bool "cni-plugins-bin"
-	default y
-	depends on BR2_x86_64

deploy/iso/minikube-iso/package/cni-plugins-bin/cni-plugins-bin.hash

@@ -1 +0,0 @@
-sha256 f04339a21b8edf76d415e7f17b620e63b8f37a76b2f706671587ab6464411f2d  cni-plugins-amd64-v0.6.0.tgz

deploy/iso/minikube-iso/package/cni-plugins/Config.in

@@ -0,0 +1,5 @@
+config BR2_PACKAGE_CNI_PLUGINS
+	bool "cni-plugins"
+	default y
+	depends on BR2_x86_64
+	depends on BR2_PACKAGE_HOST_GO_ARCH_SUPPORTS

deploy/iso/minikube-iso/package/cni-plugins/cni-plugins.hash

@@ -0,0 +1,4 @@
+sha256 f04339a21b8edf76d415e7f17b620e63b8f37a76b2f706671587ab6464411f2d  cni-plugins-amd64-v0.6.0.tgz
+sha256 8589670f7f9b211a351dfcd211d4fe0b961d77283a7415443dc188f3dbf05668  v0.6.0.tar.gz
+sha256 92c7599918be0a720ac020f137cdeac746dfa03da6b26e08a37132c5728c091f  v0.7.5.tar.gz
+sha256 9d1526ed965ac6562fd95a931ab2346b3c5efd58c2f569038ba3c530f7e66472  v0.8.5.tar.gz

deploy/iso/minikube-iso/package/cni-plugins/cni-plugins.mk

@@ -1,16 +1,39 @@
 ################################################################################
 #
-# cni-plugins-bin
+# cni-plugins
 #
 ################################################################################
 
-CNI_PLUGINS_BIN_VERSION = v0.6.0
+CNI_PLUGINS_VERSION = v0.8.5
-CNI_PLUGINS_BIN_SITE = https://github.com/containernetworking/plugins/releases/download/$(CNI_PLUGINS_BIN_VERSION)
+CNI_PLUGINS_SITE = https://github.com/containernetworking/plugins/archive
-CNI_PLUGINS_BIN_SOURCE = cni-plugins-amd64-$(CNI_PLUGINS_BIN_VERSION).tgz
+CNI_PLUGINS_SOURCE = $(CNI_PLUGINS_VERSION).tar.gz
+CNI_PLUGINS_LICENSE = Apache-2.0
+CNI_PLUGINS_LICENSE_FILES = LICENSE
 
-define CNI_PLUGINS_BIN_INSTALL_TARGET_CMDS
+CNI_PLUGINS_DEPENDENCIES = host-go
+
+CNI_PLUGINS_MAKE_ENV = \
+	CGO_ENABLED=0 \
+	GO111MODULE=off
+
+CNI_PLUGINS_BUILDFLAGS = -a -ldflags '-extldflags -static -X github.com/containernetworking/plugins/pkg/utils/buildversion.BuildVersion=$(CNI_PLUGINS_VERSION)'
+
+
+define CNI_PLUGINS_BUILD_CMDS
+	(cd $(@D); $(CNI_PLUGINS_MAKE_ENV) ./build_linux.sh $(CNI_PLUGINS_BUILDFLAGS))
+endef
+
+define CNI_PLUGINS_INSTALL_TARGET_CMDS
 	$(INSTALL) -D -m 0755 \
-		$(@D)/bridge \
+		$(@D)/bin/bandwidth \
+		$(TARGET_DIR)/opt/cni/bin/bandwidth
+
+	ln -sf \
+		../../opt/cni/bin/bandwidth \
+		$(TARGET_DIR)/usr/bin/bandwidth
+
+	$(INSTALL) -D -m 0755 \
+		$(@D)/bin/bridge \
 		$(TARGET_DIR)/opt/cni/bin/bridge
 
 	ln -sf \
@@ -18,7 +41,7 @@ define CNI_PLUGINS_BIN_INSTALL_TARGET_CMDS
 		$(TARGET_DIR)/usr/bin/bridge
 	
 	$(INSTALL) -D -m 0755 \
-		$(@D)/vlan \
+		$(@D)/bin/vlan \
 		$(TARGET_DIR)/opt/cni/bin/vlan
 
 	ln -sf \
@@ -26,7 +49,7 @@ define CNI_PLUGINS_BIN_INSTALL_TARGET_CMDS
 		$(TARGET_DIR)/usr/bin/vlan
 
 	$(INSTALL) -D -m 0755 \
-		$(@D)/tuning \
+		$(@D)/bin/tuning \
 		$(TARGET_DIR)/opt/cni/bin/tuning
 
 	ln -sf \
@@ -34,15 +57,7 @@ define CNI_PLUGINS_BIN_INSTALL_TARGET_CMDS
 		$(TARGET_DIR)/usr/bin/tuning
 
 	$(INSTALL) -D -m 0755 \
-		$(@D)/sample \
+		$(@D)/bin/ptp \
-		$(TARGET_DIR)/opt/cni/bin/sample
-
-	ln -sf \
-		../../opt/cni/bin/sample \
-		$(TARGET_DIR)/usr/bin/sample
-
-	$(INSTALL) -D -m 0755 \
-		$(@D)/ptp \
 		$(TARGET_DIR)/opt/cni/bin/ptp
 
 	ln -sf \
@@ -50,7 +65,7 @@ define CNI_PLUGINS_BIN_INSTALL_TARGET_CMDS
 		$(TARGET_DIR)/usr/bin/ptp
 
 	$(INSTALL) -D -m 0755 \
-		$(@D)/portmap \
+		$(@D)/bin/portmap \
 		$(TARGET_DIR)/opt/cni/bin/portmap
 
 	ln -sf \
@@ -58,7 +73,7 @@ define CNI_PLUGINS_BIN_INSTALL_TARGET_CMDS
 		$(TARGET_DIR)/usr/bin/portmap
 
 	$(INSTALL) -D -m 0755 \
-		$(@D)/macvlan \
+		$(@D)/bin/macvlan \
 		$(TARGET_DIR)/opt/cni/bin/macvlan
 
 	ln -sf \
@@ -66,7 +81,7 @@ define CNI_PLUGINS_BIN_INSTALL_TARGET_CMDS
 		$(TARGET_DIR)/usr/bin/macvlan
 
 	$(INSTALL) -D -m 0755 \
-		$(@D)/loopback \
+		$(@D)/bin/loopback \
 		$(TARGET_DIR)/opt/cni/bin/loopback
 
 	ln -sf \
@@ -74,7 +89,7 @@ define CNI_PLUGINS_BIN_INSTALL_TARGET_CMDS
 		$(TARGET_DIR)/usr/bin/loopback
 
 	$(INSTALL) -D -m 0755 \
-		$(@D)/ipvlan \
+		$(@D)/bin/ipvlan \
 		$(TARGET_DIR)/opt/cni/bin/ipvlan
 
 	ln -sf \
@@ -82,7 +97,7 @@ define CNI_PLUGINS_BIN_INSTALL_TARGET_CMDS
 		$(TARGET_DIR)/usr/bin/ipvlan
 
 	$(INSTALL) -D -m 0755 \
-		$(@D)/host-local \
+		$(@D)/bin/host-local \
 		$(TARGET_DIR)/opt/cni/bin/host-local
 
 	ln -sf \
@@ -90,7 +105,7 @@ define CNI_PLUGINS_BIN_INSTALL_TARGET_CMDS
 		$(TARGET_DIR)/usr/bin/host-local
 
 	$(INSTALL) -D -m 0755 \
-		$(@D)/flannel \
+		$(@D)/bin/flannel \
 		$(TARGET_DIR)/opt/cni/bin/flannel
 
 	ln -sf \
@@ -99,12 +114,20 @@ define CNI_PLUGINS_BIN_INSTALL_TARGET_CMDS
 
 
 	$(INSTALL) -D -m 0755 \
-		$(@D)/dhcp \
+		$(@D)/bin/dhcp \
 		$(TARGET_DIR)/opt/cni/bin/dhcp
 
 	ln -sf \
 		../../opt/cni/bin/dhcp \
 		$(TARGET_DIR)/usr/bin/dhcp
+
+	$(INSTALL) -D -m 0755 \
+		$(@D)/bin/firewall \
+		$(TARGET_DIR)/opt/cni/bin/firewall
+
+	ln -sf \
+		../../opt/cni/bin/firewall \
+		$(TARGET_DIR)/usr/bin/firewall
 endef
 
 $(eval $(generic-package))

deploy/iso/minikube-iso/package/cni/Config.in

@@ -0,0 +1,5 @@
+config BR2_PACKAGE_CNI
+	bool "cni"
+	default y
+	depends on BR2_x86_64
+	depends on BR2_PACKAGE_HOST_GO_ARCH_SUPPORTS

deploy/iso/minikube-iso/package/cni/cni.hash

@@ -1,3 +1,6 @@
 sha256 b1ae09833a238c51161918a8849031efdb46cf0068ea5b752e362d9836e2af7d  cni-v0.3.0.tgz
 sha256 84c9a0a41b59211d560bef14bf3f53bb370156f9ac7762270b3848fed96e1be8  cni-v0.4.0.tgz
 sha256 a7f84a742c8f3a95843b3cc636444742554a4853835649ec371a07c841daebab  cni-amd64-v0.6.0.tgz
+sha256 802f4a002b4eb774624a9dc1c859d3c9926eb2d862e66a673fc99cfc8bcd7494  v0.6.0.tar.gz
+sha256 78d57477d6b0ab9dc4d75ce9f275302d2f379206b5326503e57d9c08b76484c1  v0.7.0.tar.gz
+sha256 4517eabfd65aea2012dc48d057bf889a0a41ed9837387d95cd1e36c0dbddcfd4  v0.7.1.tar.gz

deploy/iso/minikube-iso/package/cni/cni.mk

@@ -0,0 +1,44 @@
+################################################################################
+#
+# cni
+#
+################################################################################
+
+CNI_VERSION = v0.7.1
+CNI_SITE = https://github.com/containernetworking/cni/archive
+CNI_SOURCE = $(CNI_VERSION).tar.gz
+CNI_LICENSE = Apache-2.0
+CNI_LICENSE_FILES = LICENSE
+
+CNI_DEPENDENCIES = host-go
+
+CNI_GOPATH = $(@D)/_output
+CNI_MAKE_ENV = \
+	CGO_ENABLED=0 \
+	GO111MODULE=off \
+	GOPATH="$(CNI_GOPATH)" \
+	GOBIN="$(CNI_GOPATH)/bin" \
+	PATH=$(CNI_GOPATH)/bin:$(BR_PATH)
+
+CNI_BUILDFLAGS = -a --ldflags '-extldflags \"-static\"'
+
+define CNI_CONFIGURE_CMDS
+        mkdir -p $(CNI_GOPATH)/src/github.com/containernetworking
+        ln -sf $(@D) $(CNI_GOPATH)/src/github.com/containernetworking/cni
+endef
+
+define CNI_BUILD_CMDS
+	(cd $(@D); $(CNI_MAKE_ENV) go build -o bin/cnitool $(CNI_BUILDFLAGS) ./cnitool)
+endef
+
+define CNI_INSTALL_TARGET_CMDS
+	$(INSTALL) -D -m 0755 \
+		$(@D)/bin/cnitool \
+		$(TARGET_DIR)/opt/cni/bin/cnitool
+
+	ln -sf \
+		../../opt/cni/bin/cnitool \
+		$(TARGET_DIR)/usr/bin/cnitool
+endef
+
+$(eval $(generic-package))

deploy/iso/minikube-iso/package/containerd-bin/config.toml

@@ -31,7 +31,7 @@ oom_score = 0
     enable_selinux = false
     sandbox_image = "k8s.gcr.io/pause:3.1"
     stats_collect_period = 10
-    systemd_cgroup = false
+    systemd_cgroup = true
     enable_tls_streaming = false
     max_container_log_line_size = 16384
     [plugins.cri.containerd]

deploy/iso/minikube-iso/package/crio-bin/crio.conf

@@ -120,7 +120,7 @@ seccomp_profile = ""
 apparmor_profile = "crio-default-1.16.1"
 
 # Cgroup management implementation used for the runtime.
-cgroup_manager = "cgroupfs"
+cgroup_manager = "systemd"
 
 # List of default capabilities for containers. If it is empty or commented out,
 # only the capabilities defined in the containers json file by the user/kube

deploy/iso/minikube-iso/package/docker-bin/daemon.json

@@ -0,0 +1,10 @@
+{
+  "exec-opts": [
+    "native.cgroupdriver=systemd"
+  ],
+  "log-driver": "json-file",
+  "log-opts": {
+    "max-size": "100m"
+  },
+  "storage-driver": "overlay2"
+}

deploy/iso/minikube-iso/package/docker-bin/docker-bin.mk

@@ -34,6 +34,12 @@ define DOCKER_BIN_INSTALL_TARGET_CMDS
 	$(INSTALL) -D -m 0755 \
 		$(@D)/docker-proxy \
 		$(TARGET_DIR)/bin/docker-proxy
+
+	# https://kubernetes.io/docs/setup/production-environment/container-runtimes/#docker
+
+	$(INSTALL) -Dm644 \
+		$(DOCKER_BIN_PKGDIR)/daemon.json \
+		$(TARGET_DIR)/etc/docker/daemon.json
 endef
 
 define DOCKER_BIN_INSTALL_INIT_SYSTEMD

deploy/iso/minikube-iso/package/falco-probe/falco-probe.hash

@@ -1,4 +1,5 @@
 # falco
 sha256 87c60273c35d544256e471b403497be33f24df662673338236ec92ba3fc1f8b7 0.19.0.tar.gz
+sha256 b873e3590e56ead740ed905108221f98da6100da3c5b7acf2355ea1cf628d931 0.20.0.tar.gz
 # sysdig
 sha256 6e477ac5fe9d3110b870bd4495f01541373a008c375a1934a2d1c46798b6bad6 146a431edf95829ac11bfd9c85ba3ef08789bffe.tar.gz

deploy/iso/minikube-iso/package/falco-probe/falco-probe.mk

@@ -4,7 +4,7 @@
 #
 ########################################################################
 
-FALCO_PROBE_VERSION = 0.19.0
+FALCO_PROBE_VERSION = 0.20.0
 FALCO_PROBE_SITE = https://github.com/falcosecurity/falco/archive
 FALCO_PROBE_SOURCE = $(FALCO_PROBE_VERSION).tar.gz
 FALCO_PROBE_DEPENDENCIES += ncurses libyaml
@@ -22,7 +22,6 @@ endef
 FALCO_PROBE_POST_EXTRACT_HOOKS += FALCO_PROBE_SYSDIG_SRC
 
 FALCO_PROBE_CONF_OPTS = -DFALCO_VERSION=$(FALCO_PROBE_VERSION)
-FALCO_PROBE_CONF_OPTS += -DSYSDIG_VERSION=$(FALCO_PROBE_SYSDIG_VERSION)
 FALCO_PROBE_CONF_OPTS += -DUSE_BUNDLED_DEPS=ON
 
 FALCO_PROBE_MAKE_OPTS = driver KERNELDIR=$(LINUX_DIR)

go.mod

@@ -3,6 +3,7 @@ module k8s.io/minikube
 go 1.13
 
 require (
+	cloud.google.com/go v0.45.1
 	github.com/Parallels/docker-machine-parallels v1.3.0
 	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
 	github.com/blang/semver v3.5.0+incompatible
@@ -11,6 +12,7 @@ require (
 	github.com/cheggaaa/pb/v3 v3.0.1
 	github.com/cloudfoundry-attic/jibber_jabber v0.0.0-20151120183258-bcc4c8345a21
 	github.com/cloudfoundry/jibber_jabber v0.0.0-20151120183258-bcc4c8345a21 // indirect
+	github.com/docker/cli v0.0.0-20200303162255-7d407207c304 // indirect
 	github.com/docker/docker v1.13.1
 	github.com/docker/go-units v0.4.0
 	github.com/docker/machine v0.7.1-0.20190718054102-a555e4f7a8f5 // version is 0.7.1 to pin to a555e4f7a8f5
@@ -23,15 +25,12 @@ require (
 	github.com/google/go-cmp v0.3.0
 	github.com/google/go-containerregistry v0.0.0-20200131185320-aec8da010de2
 	github.com/googleapis/gnostic v0.3.0 // indirect
-	github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce // indirect
 	github.com/hashicorp/go-getter v1.4.0
-	github.com/hashicorp/go-multierror v0.0.0-20160811015721-8c5f0ad93604 // indirect
 	github.com/hashicorp/go-retryablehttp v0.5.4
 	github.com/hooklift/assert v0.0.0-20170704181755-9d1defd6d214 // indirect
 	github.com/hooklift/iso9660 v0.0.0-20170318115843-1cf07e5970d8
 	github.com/imdario/mergo v0.3.8 // indirect
 	github.com/intel-go/cpuid v0.0.0-20181003105527-1a4a6f06a1c6 // indirect
-	github.com/jimmidyson/go-download v0.0.0-20161028105827-7f9a90c8c95b
 	github.com/johanneswuerbach/nfsexports v0.0.0-20181204082207-1aa528dcb345
 	github.com/juju/clock v0.0.0-20190205081909-9c5c9712527c
 	github.com/juju/errors v0.0.0-20190806202954-0232dcc7464d // indirect
@@ -65,17 +64,20 @@ require (
 	github.com/xeipuuv/gojsonschema v0.0.0-20160623135812-c539bca196be
 	github.com/zchee/go-vmnet v0.0.0-20161021174912-97ebf9174097
 	golang.org/x/build v0.0.0-20190927031335-2835ba2e683f
-	golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413
+	golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073
+	golang.org/x/net v0.0.0-20200301022130-244492dfa37a // indirect
 	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
 	golang.org/x/sys v0.0.0-20191010194322-b09406accb47
 	golang.org/x/text v0.3.2
+	google.golang.org/api v0.9.0
 	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
+	gotest.tools/v3 v3.0.2 // indirect
 	k8s.io/api v0.17.3
 	k8s.io/apimachinery v0.17.3
 	k8s.io/client-go v0.17.3
 	k8s.io/kubectl v0.0.0
 	k8s.io/kubernetes v1.17.3
-	k8s.io/utils v0.0.0-20200122174043-1e243dd1a584 // indirect
+	k8s.io/utils v0.0.0-20200229041039-0a110f9eb7ab // indirect
 	sigs.k8s.io/sig-storage-lib-external-provisioner v4.0.0+incompatible
 )
 

go.sum

@@ -144,6 +144,8 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm
 github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
 github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017 h1:2HQmlpI3yI9deH18Q6xiSOIjXD4sLI55Y/gfpa8/558=
 github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
+github.com/docker/cli v0.0.0-20200303162255-7d407207c304 h1:A7SYzidcyuQ/yS4wezWGYeUioUFJQk8HYWY9aMYTF4I=
+github.com/docker/cli v0.0.0-20200303162255-7d407207c304/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
 github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7 h1:Cvj7S8I4Xpx78KAl6TwTmMHuHlZ/0SM60NUneGJQ7IE=
@@ -346,12 +348,8 @@ github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:Fecb
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
-github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce h1:prjrVgOk2Yg6w+PflHoszQNLTUh4kaByUcEWM/9uin4=
-github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-cleanhttp v0.5.0 h1:wvCrVc9TjDls6+YGAF2hAifE1E5U1+b4tH6KdvN3Gig=
 github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
-github.com/hashicorp/go-multierror v0.0.0-20160811015721-8c5f0ad93604 h1:VIq8E7fMiC4h3agg0ya56L0jHn7QisZZcWZXVKJb9jQ=
-github.com/hashicorp/go-multierror v0.0.0-20160811015721-8c5f0ad93604/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I=
 github.com/hashicorp/go-retryablehttp v0.5.4 h1:1BZvpawXoJCWX6pNtow9+rpEj+3itIlutiqnntI6jOE=
 github.com/hashicorp/go-retryablehttp v0.5.4/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
 github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo=
@@ -386,8 +384,6 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt
 github.com/intel-go/cpuid v0.0.0-20181003105527-1a4a6f06a1c6 h1:XboatR7lasl05yel5hNXF7kQBw2oFUGdMztcgisfhNU=
 github.com/intel-go/cpuid v0.0.0-20181003105527-1a4a6f06a1c6/go.mod h1:RmeVYf9XrPRbRc3XIx0gLYA8qOFvNoPOfaEZduRlEp4=
 github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
-github.com/jimmidyson/go-download v0.0.0-20161028105827-7f9a90c8c95b h1:3TknJxYSK1eDe21QorC3C2Yz8jylk6vlJG9YABnFzkU=
-github.com/jimmidyson/go-download v0.0.0-20161028105827-7f9a90c8c95b/go.mod h1:I3WsAhNNoG7a/d8HMlYUywJJlfOs/+/83NEUjuDp4lc=
 github.com/jimstudt/http-authentication v0.0.0-20140401203705-3eca13d6893a/go.mod h1:wK6yTYYcgjHE1Z1QtXACPDjcFJyBskHEdagmnq3vsP8=
 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 h1:12VvqtR6Aowv3l/EQUlocDHW2Cp4G9WJVH7uyH8QFJE=
 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
@@ -735,6 +731,8 @@ golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 h1:ULYEB3JvPRE/IfO+9uO7vKV/xzVTO7XPAwm8xbf4w2g=
 golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 h1:xMPOj6Pz6UipU1wXLkrtqpHbR0AVFnyPEQq/wRWz9lM=
+golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -780,6 +778,8 @@ golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI=
 golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a h1:tImsplftrFpALCYumobsd0K86vlAs/eXGFms2txfJfA=
@@ -864,6 +864,7 @@ golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgw
 golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190706070813-72ffa07ba3db/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
 golang.org/x/tools v0.0.0-20190909030654-5b82db07426d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@@ -944,6 +945,8 @@ gotest.tools v2.1.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81
 gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 gotest.tools/gotestsum v0.3.5/go.mod h1:Mnf3e5FUzXbkCfynWBGOwLssY7gTQgCHObK9tMpAriY=
+gotest.tools/v3 v3.0.2 h1:kG1BFyqVHuQoVQiR1bWGnfz/fmHvvuiSPIV7rvl360E=
+gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
 grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
@@ -996,8 +999,8 @@ k8s.io/sample-apiserver v0.17.3/go.mod h1:cn/rvFIttGNqy1v88B5ZlDAbyyqDOoF7JHSwPi
 k8s.io/system-validators v1.0.4/go.mod h1:HgSgTg4NAGNoYYjKsUyk52gdNi2PVDswQ9Iyn66R7NI=
 k8s.io/utils v0.0.0-20191114184206-e782cd3c129f h1:GiPwtSzdP43eI1hpPCbROQCCIgCuiMMNF8YUVLF3vJo=
 k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
-k8s.io/utils v0.0.0-20200122174043-1e243dd1a584 h1:3tT5mBZNurtd5BoYrPBII3Sa8n7T2w405qdTQvr3vmY=
+k8s.io/utils v0.0.0-20200229041039-0a110f9eb7ab h1:I3f2hcBrepGRXI1z4sukzAb8w1R4eqbsHrAsx06LGYM=
-k8s.io/utils v0.0.0-20200122174043-1e243dd1a584/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
+k8s.io/utils v0.0.0-20200229041039-0a110f9eb7ab/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
 modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw=
 modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk=
 modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k=

hack/conformance_tests.sh

@@ -21,7 +21,7 @@
 #   conformance_tests.sh <path to minikube> <flags>
 #
 # Example:
-#   conformance_tests.sh ./out/minikube --vm-driver=hyperkit
+#   conformance_tests.sh ./out/minikube --driver=hyperkit
 set -ex -o pipefail
 
 readonly PROFILE_NAME="k8sconformance"

hack/images/kicbase.Dockerfile

@@ -52,13 +52,3 @@ RUN apt-get clean -y && rm -rf \
   /usr/share/man/* \
   /usr/share/local/* \
   RUN echo "kic! Build: ${COMMIT_SHA} Time :$(date)" > "/kic.txt"
-
-
-FROM busybox
-ARG KUBERNETES_VERSION
-COPY out/preloaded-images-k8s-$KUBERNETES_VERSION.tar /preloaded-images.tar
-RUN tar xvf /preloaded-images.tar -C /
-
-FROM base
-COPY --from=1 /var/lib/docker /var/lib/docker
-COPY --from=1 /var/lib/minikube/binaries /var/lib/minikube/binaries

hack/jenkins/common.sh

@@ -19,7 +19,7 @@
 
 # The script expects the following env variables:
 # OS_ARCH: The operating system and the architecture separated by a hyphen '-' (e.g. darwin-amd64, linux-amd64, windows-amd64)
-# VM_DRIVER: the vm-driver to use for the test
+# VM_DRIVER: the driver to use for the test
 # EXTRA_START_ARGS: additional flags to pass into minikube start
 # EXTRA_ARGS: additional flags to pass into minikube
 # JOB_NAME: the name of the logfile and check name to update on github
@@ -31,9 +31,9 @@ export KUBECONFIG="${TEST_HOME}/kubeconfig"
 export PATH=$PATH:"/usr/local/bin/:/usr/local/go/bin/:$GOPATH/bin"
 
 # installing golang so we could do go get for gopogh
-sudo ./installers/check_install_golang.sh "1.13.4" "/usr/local" || true
+sudo ./installers/check_install_golang.sh "1.13.6" "/usr/local" || true
 
-docker rm -f $(docker ps -aq) >/dev/null 2>&1 || true
+docker rm -f -v $(docker ps -aq) >/dev/null 2>&1 || true
 docker volume prune -f || true
 docker system df || true
 
@@ -285,7 +285,7 @@ if test -f "${TEST_OUT}"; then
 fi
 touch "${TEST_OUT}"
 ${SUDO_PREFIX}${E2E_BIN} \
-  -minikube-start-args="--vm-driver=${VM_DRIVER} ${EXTRA_START_ARGS}" \
+  -minikube-start-args="--driver=${VM_DRIVER} ${EXTRA_START_ARGS}" \
   -expected-default-driver="${EXPECTED_DEFAULT_DRIVER}" \
   -test.timeout=70m -test.v \
   ${EXTRA_TEST_ARGS} \

hack/jenkins/cron/cleanup_and_reboot_Darwin.sh

@@ -41,6 +41,13 @@ logger "cleanup_and_reboot is happening!"
 # kill jenkins to avoid an incoming request
 killall java
 
+# clean docker left overs
+docker rm -f -v $(docker ps -aq) >/dev/null 2>&1 || true
+docker volume prune -f || true
+docker volume ls || true
+docker system df || true
+
+
 # macOS specific cleanup
 sudo rm /var/db/dhcpd_leases || echo "could not clear dhcpd leases"
 sudo softwareupdate -i -a -R

hack/jenkins/cron/cleanup_and_reboot_Linux.sh

@@ -36,6 +36,12 @@ logger "cleanup_and_reboot is happening!"
 # kill jenkins to avoid an incoming request
 killall java
 
+# clean docker left overs
+docker rm -f -v $(docker ps -aq) >/dev/null 2>&1 || true
+docker volume prune -f || true
+docker volume ls || true
+docker system df || true
+
 # Linux-specific cleanup
 
 # disable localkube, kubelet

hack/jenkins/minikube_set_pending.sh

@@ -41,6 +41,7 @@ jobs=(
      'KVM_Linux'
      'none_Linux'
      'Docker_Linux'
+     'Docker_macOS'
      'Podman_Linux'
 )
 

hack/jenkins/osx_integration_tests_docker.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# This script runs the integration tests on an OSX machine for the Hyperkit Driver
+
+# The script expects the following env variables:
+# MINIKUBE_LOCATION: GIT_COMMIT from upstream build.
+# COMMIT: Actual commit ID from upstream build
+# EXTRA_BUILD_ARGS (optional): Extra args to be passed into the minikube integrations tests
+# access_token: The Github API access token. Injected by the Jenkins credential provider. 
+
+
+set -e
+
+OS_ARCH="darwin-amd64"
+VM_DRIVER="docker"
+JOB_NAME="Docker_macOS"
+EXTRA_START_ARGS=""
+EXPECTED_DEFAULT_DRIVER="hyperkit"
+
+
+# restart docker on mac for a fresh test
+osascript -e 'quit app "Docker"'; open -a Docker ; while [ -z "$(docker info 2> /dev/null )" ]; do printf "."; sleep 1; done; echo "" || true
+
+mkdir -p cron && gsutil -qm rsync "gs://minikube-builds/${MINIKUBE_LOCATION}/cron" cron || echo "FAILED TO GET CRON FILES"
+install cron/cleanup_and_reboot_Darwin.sh $HOME/cleanup_and_reboot.sh || echo "FAILED TO INSTALL CLEANUP"
+echo "*/30 * * * * $HOME/cleanup_and_reboot.sh" | crontab
+crontab -l
+
+source common.sh

hack/jenkins/windows_integration_test_hyperv.ps1

@@ -19,7 +19,7 @@ gsutil.cmd -m cp -r gs://minikube-builds/$env:MINIKUBE_LOCATION/testdata .
 
 ./out/minikube-windows-amd64.exe delete
 
-out/e2e-windows-amd64.exe --expected-default-driver=hyperv -minikube-start-args="--vm-driver=hyperv --hyperv-virtual-switch=primary-virtual-switch" -binary=out/minikube-windows-amd64.exe -test.v -test.timeout=65m
+out/e2e-windows-amd64.exe --expected-default-driver=hyperv -minikube-start-args="--driver=hyperv --hyperv-virtual-switch=primary-virtual-switch" -binary=out/minikube-windows-amd64.exe -test.v -test.timeout=65m
 $env:result=$lastexitcode
 # If the last exit code was 0->success, x>0->error
 If($env:result -eq 0){$env:status="success"}

hack/jenkins/windows_integration_test_virtualbox.ps1

@@ -19,7 +19,7 @@ gsutil.cmd -m cp -r gs://minikube-builds/$env:MINIKUBE_LOCATION/testdata .
 
 ./out/minikube-windows-amd64.exe delete
 
-out/e2e-windows-amd64.exe -minikube-start-args="--vm-driver=virtualbox" -expected-default-driver=hyperv -binary=out/minikube-windows-amd64.exe -test.v -test.timeout=30m
+out/e2e-windows-amd64.exe -minikube-start-args="--driver=virtualbox" -expected-default-driver=hyperv -binary=out/minikube-windows-amd64.exe -test.v -test.timeout=30m
 $env:result=$lastexitcode
 # If the last exit code was 0->success, x>0->error
 If($env:result -eq 0){$env:status="success"}

hack/preload-images/preload_images.go

@@ -25,6 +25,14 @@ import (
 	"strings"
 
 	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/drivers/kic"
+	"k8s.io/minikube/pkg/drivers/kic/oci"
+	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil"
+	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
+	"k8s.io/minikube/pkg/minikube/command"
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/driver"
+	"k8s.io/minikube/pkg/minikube/localpath"
 )
 
 const (
@@ -33,17 +41,28 @@ const (
 )
 
 var (
-	kubernetesVersion = ""
+	kubernetesVersion       = ""
-	tarballFilename   = ""
+	tarballFilename         = ""
+	dockerStorageDriver     = ""
+	preloadedTarballVersion = ""
+	containerRuntime        = ""
 )
 
 func init() {
 	flag.StringVar(&kubernetesVersion, "kubernetes-version", "", "desired kubernetes version, for example `v1.17.2`")
+	flag.StringVar(&dockerStorageDriver, "docker-storage-driver", "overlay2", "docker storage driver backend")
+	flag.StringVar(&preloadedTarballVersion, "preloaded-tarball-version", "", "preloaded tarball version")
+	flag.StringVar(&containerRuntime, "container-runtime", "docker", "container runtime")
+
 	flag.Parse()
-	tarballFilename = fmt.Sprintf("preloaded-images-k8s-%s.tar", kubernetesVersion)
+	tarballFilename = fmt.Sprintf("preloaded-images-k8s-%s-%s-%s-%s.tar.lz4", preloadedTarballVersion, kubernetesVersion, containerRuntime, dockerStorageDriver)
 }
 
 func main() {
+	if err := verifyDockerStorage(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
 	if err := executePreloadImages(); err != nil {
 		fmt.Println(err)
 		os.Exit(1)
@@ -56,42 +75,83 @@ func executePreloadImages() error {
 			fmt.Println(err)
 		}
 	}()
-	if err := startMinikube(); err != nil {
+
+	driver := kic.NewDriver(kic.Config{
+		KubernetesVersion: kubernetesVersion,
+		ContainerRuntime:  driver.Docker,
+		OCIBinary:         oci.Docker,
+		MachineName:       profile,
+		ImageDigest:       kic.BaseImage,
+		StorePath:         localpath.MiniPath(),
+		CPU:               2,
+		Memory:            4000,
+		APIServerPort:     8080,
+	})
+
+	baseDir := filepath.Dir(driver.GetSSHKeyPath())
+	defer os.Remove(baseDir)
+
+	if err := os.MkdirAll(baseDir, 0755); err != nil {
 		return err
 	}
+	if err := driver.Create(); err != nil {
+		return errors.Wrap(err, "creating kic driver")
+	}
+
+	// Now, get images to pull
+	imgs, err := images.Kubeadm("", kubernetesVersion)
+	if err != nil {
+		return errors.Wrap(err, "kubeadm images")
+	}
+
+	for _, img := range append(imgs, kic.OverlayImage) {
+		cmd := exec.Command("docker", "exec", profile, "docker", "pull", img)
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		if err := cmd.Run(); err != nil {
+			return errors.Wrapf(err, "downloading %s", img)
+		}
+	}
+
+	// Transfer in k8s binaries
+	kcfg := config.KubernetesConfig{
+		KubernetesVersion: kubernetesVersion,
+	}
+	runner := command.NewKICRunner(profile, driver.OCIBinary)
+	if err := bsutil.TransferBinaries(kcfg, runner); err != nil {
+		return errors.Wrap(err, "transferring k8s binaries")
+	}
+	// Create image tarball
 	if err := createImageTarball(); err != nil {
 		return err
 	}
 	return copyTarballToHost()
 }
 
-func startMinikube() error {
-	cmd := exec.Command(minikubePath, "start", "-p", profile, "--memory", "4000", "--kubernetes-version", kubernetesVersion, "--wait=false")
-	cmd.Stdout = os.Stdout
-	return cmd.Run()
-}
-
 func createImageTarball() error {
-	cmd := exec.Command(minikubePath, "ssh", "-p", profile, "--", "sudo", "tar", "cvf", tarballFilename, "/var/lib/docker", "/var/lib/minikube/binaries")
+	dirs := []string{
+		fmt.Sprintf("./lib/docker/%s", dockerStorageDriver),
+		"./lib/docker/image",
+		"./lib/minikube/binaries",
+	}
+	args := []string{"exec", profile, "sudo", "tar", "-I", "lz4", "-C", "/var", "-cvf", tarballFilename}
+	args = append(args, dirs...)
+	cmd := exec.Command("docker", args...)
 	cmd.Stdout = os.Stdout
-	return cmd.Run()
+	if err := cmd.Run(); err != nil {
+		return errors.Wrap(err, "creating image tarball")
+	}
+	return nil
 }
 
 func copyTarballToHost() error {
-	sshKey, err := runCmd([]string{minikubePath, "ssh-key", "-p", profile})
-	if err != nil {
-		return errors.Wrap(err, "getting ssh-key")
-	}
-
-	ip, err := runCmd([]string{minikubePath, "ip", "-p", profile})
-	if err != nil {
-		return errors.Wrap(err, "getting ip")
-	}
-
 	dest := filepath.Join("out/", tarballFilename)
-	args := []string{"scp", "-o", "StrictHostKeyChecking=no", "-i", sshKey, fmt.Sprintf("docker@%s:/home/docker/%s", ip, tarballFilename), dest}
+	cmd := exec.Command("docker", "cp", fmt.Sprintf("%s:/%s", profile, tarballFilename), dest)
-	_, err = runCmd(args)
+	cmd.Stdout = os.Stdout
-	return err
+	if err := cmd.Run(); err != nil {
+		return errors.Wrap(err, "copying tarball to host")
+	}
+	return nil
 }
 
 func deleteMinikube() error {
@@ -100,8 +160,15 @@ func deleteMinikube() error {
 	return cmd.Run()
 }
 
-func runCmd(command []string) (string, error) {
+func verifyDockerStorage() error {
-	cmd := exec.Command(command[0], command[1:]...)
+	cmd := exec.Command("docker", "info", "-f", "{{.Info.Driver}}")
 	output, err := cmd.Output()
-	return strings.Trim(string(output), "\n "), err
+	if err != nil {
+		return err
+	}
+	driver := strings.Trim(string(output), " \n")
+	if driver != dockerStorageDriver {
+		return fmt.Errorf("docker storage driver %s does not match requested %s", driver, dockerStorageDriver)
+	}
+	return nil
 }

pkg/addons/addons.go

@@ -30,6 +30,7 @@ import (
 	"k8s.io/minikube/pkg/minikube/assets"
 	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/driver"
 	"k8s.io/minikube/pkg/minikube/exit"
 	"k8s.io/minikube/pkg/minikube/machine"
 	"k8s.io/minikube/pkg/minikube/out"
@@ -118,6 +119,9 @@ func enableOrDisableAddon(name, val, profile string) error {
 
 	if alreadySet {
 		glog.Warningf("addon %s should already be in state %v", name, val)
+		if !enable {
+			return nil
+		}
 	}
 
 	if name == "istio" && enable {
@@ -243,7 +247,16 @@ func enableOrDisableStorageClasses(name, val, profile string) error {
 	}
 	defer api.Close()
 
-	if !machine.IsHostRunning(api, profile) {
+	cc, err := config.Load(profile)
+	if err != nil {
+		return errors.Wrap(err, "getting cluster")
+	}
+
+	cp, err := config.PrimaryControlPlane(*cc)
+	if err != nil {
+		return errors.Wrap(err, "getting control plane")
+	}
+	if !machine.IsHostRunning(api, driver.MachineName(*cc, cp)) {
 		glog.Warningf("%q is not running, writing %s=%v to disk and skipping enablement", profile, name, val)
 		return enableOrDisableAddon(name, val, profile)
 	}

pkg/drivers/kic/kic.go

@@ -17,11 +17,13 @@ limitations under the License.
 package kic
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"os/exec"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/docker/machine/libmachine/drivers"
 	"github.com/docker/machine/libmachine/log"
@@ -34,6 +36,7 @@ import (
 	"k8s.io/minikube/pkg/minikube/assets"
 	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/minikube/constants"
+	"k8s.io/minikube/pkg/minikube/download"
 )
 
 // Driver represents a kic driver https://minikube.sigs.k8s.io/docs/reference/drivers/docker
@@ -66,6 +69,7 @@ func (d *Driver) Create() error {
 		Name:          d.NodeConfig.MachineName,
 		Image:         d.NodeConfig.ImageDigest,
 		ClusterLabel:  oci.ProfileLabelKey + "=" + d.MachineName,
+		NodeLabel:     oci.NodeLabelKey + "=" + d.NodeConfig.MachineName,
 		CPUs:          strconv.Itoa(d.NodeConfig.CPU),
 		Memory:        strconv.Itoa(d.NodeConfig.Memory) + "mb",
 		Envs:          d.NodeConfig.Envs,
@@ -88,14 +92,23 @@ func (d *Driver) Create() error {
 			ContainerPort: constants.DockerDaemonPort,
 		},
 	)
-	err := oci.CreateContainerNode(params)
+	if err := oci.CreateContainerNode(params); err != nil {
-	if err != nil {
 		return errors.Wrap(err, "create kic node")
 	}
 
 	if err := d.prepareSSH(); err != nil {
 		return errors.Wrap(err, "prepare kic ssh")
 	}
+
+	t := time.Now()
+	glog.Infof("Starting extracting preloaded images to volume")
+	// Extract preloaded images to container
+	if err := oci.ExtractTarballToVolume(download.TarballPath(d.NodeConfig.KubernetesVersion), params.Name, BaseImage); err != nil {
+		glog.Infof("Unable to extract preloaded tarball to volume: %v", err)
+	} else {
+		glog.Infof("Took %f seconds to extract preloaded images to volume", time.Since(t).Seconds())
+	}
+
 	return nil
 }
 
@@ -124,10 +137,10 @@ func (d *Driver) prepareSSH() error {
 
 // DriverName returns the name of the driver
 func (d *Driver) DriverName() string {
-	if d.NodeConfig.OCIBinary == "podman" {
+	if d.NodeConfig.OCIBinary == oci.Podman {
-		return "podman"
+		return oci.Podman
 	}
-	return "docker"
+	return oci.Docker
 }
 
 // GetIP returns an IP or hostname that this host is available at
@@ -181,11 +194,18 @@ func (d *Driver) GetURL() (string, error) {
 // GetState returns the state that the host is in (running, stopped, etc)
 func (d *Driver) GetState() (state.State, error) {
 	if err := oci.PointToHostDockerDaemon(); err != nil {
-		return state.Error, errors.Wrap(err, "point host docker-daemon")
+		return state.Error, errors.Wrap(err, "point host docker daemon")
 	}
+	// allow no more than 2 seconds for this. when this takes long this means deadline passed
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel()
 
-	cmd := exec.Command(d.NodeConfig.OCIBinary, "inspect", "-f", "{{.State.Status}}", d.MachineName)
+	cmd := exec.CommandContext(ctx, d.NodeConfig.OCIBinary, "inspect", "-f", "{{.State.Status}}", d.MachineName)
 	out, err := cmd.CombinedOutput()
+	if ctx.Err() == context.DeadlineExceeded {
+		glog.Errorf("GetState for %s took longer than normal. Restarting your %s daemon might fix this issue.", d.MachineName, d.OCIBinary)
+		return state.Error, fmt.Errorf("inspect %s timeout", d.MachineName)
+	}
 	o := strings.TrimSpace(string(out))
 	if err != nil {
 		return state.Error, errors.Wrapf(err, "get container %s status", d.MachineName)
@@ -208,6 +228,9 @@ func (d *Driver) GetState() (state.State, error) {
 
 // Kill stops a host forcefully, including any containers that we are managing.
 func (d *Driver) Kill() error {
+	if err := oci.PointToHostDockerDaemon(); err != nil {
+		return errors.Wrap(err, "point host docker daemon")
+	}
 	cmd := exec.Command(d.NodeConfig.OCIBinary, "kill", d.MachineName)
 	if err := cmd.Run(); err != nil {
 		return errors.Wrapf(err, "killing kic node %s", d.MachineName)
@@ -217,6 +240,10 @@ func (d *Driver) Kill() error {
 
 // Remove will delete the Kic Node Container
 func (d *Driver) Remove() error {
+	if err := oci.PointToHostDockerDaemon(); err != nil {
+		return errors.Wrap(err, "point host docker daemon")
+	}
+
 	if _, err := oci.ContainerID(d.OCIBinary, d.MachineName); err != nil {
 		log.Warnf("could not find the container %s to remove it.", d.MachineName)
 	}
@@ -234,13 +261,14 @@ func (d *Driver) Remove() error {
 
 // Restart a host
 func (d *Driver) Restart() error {
+	if err := oci.PointToHostDockerDaemon(); err != nil {
+		return errors.Wrap(err, "point host docker daemon")
+	}
 	s, err := d.GetState()
 	if err != nil {
 		return errors.Wrap(err, "get kic state")
 	}
 	switch s {
-	case state.Paused:
-		return d.Unpause()
 	case state.Stopped:
 		return d.Start()
 	case state.Running, state.Error:
@@ -256,18 +284,12 @@ func (d *Driver) Restart() error {
 	return fmt.Errorf("restarted not implemented for kic state %s yet", s)
 }
 
-// Unpause a kic container
-func (d *Driver) Unpause() error {
-	cmd := exec.Command(d.NodeConfig.OCIBinary, "unpause", d.MachineName)
-	if err := cmd.Run(); err != nil {
-		return errors.Wrapf(err, "unpausing %s", d.MachineName)
-	}
-	return nil
-}
-
 // Start a _stopped_ kic container
 // not meant to be used for Create().
 func (d *Driver) Start() error {
+	if err := oci.PointToHostDockerDaemon(); err != nil {
+		return errors.Wrap(err, "point host docker daemon")
+	}
 	s, err := d.GetState()
 	if err != nil {
 		return errors.Wrap(err, "get kic state")
@@ -285,6 +307,9 @@ func (d *Driver) Start() error {
 
 // Stop a host gracefully, including any containers that we are managing.
 func (d *Driver) Stop() error {
+	if err := oci.PointToHostDockerDaemon(); err != nil {
+		return errors.Wrap(err, "point host docker daemon")
+	}
 	cmd := exec.Command(d.NodeConfig.OCIBinary, "stop", d.MachineName)
 	if err := cmd.Run(); err != nil {
 		return errors.Wrapf(err, "stopping %s", d.MachineName)

pkg/drivers/kic/oci/info.go

@@ -0,0 +1,245 @@
+/*
+Copyright 2019 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package oci
+
+import (
+	"encoding/json"
+	"os/exec"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+)
+
+// SysInfo Info represents common system Information between docker and podman that minikube cares
+type SysInfo struct {
+	CPUs        int   // CPUs is Number of CPUs
+	TotalMemory int64 // TotalMemory Total available ram
+}
+
+// DaemonInfo returns common docker/podman daemon system info that minikube cares about
+func DaemonInfo(ociBin string) (SysInfo, error) {
+	var info SysInfo
+	if ociBin == Podman {
+		p, err := podmanSystemInfo()
+		info.CPUs = p.Host.Cpus
+		info.TotalMemory = p.Host.MemTotal
+		return info, err
+	}
+	d, err := dockerSystemInfo()
+	info.CPUs = d.NCPU
+	info.TotalMemory = d.MemTotal
+	return info, err
+}
+
+// dockerSysInfo represents the output of docker system info --format '{{json .}}'
+type dockerSysInfo struct {
+	ID                string      `json:"ID"`
+	Containers        int         `json:"Containers"`
+	ContainersRunning int         `json:"ContainersRunning"`
+	ContainersPaused  int         `json:"ContainersPaused"`
+	ContainersStopped int         `json:"ContainersStopped"`
+	Images            int         `json:"Images"`
+	Driver            string      `json:"Driver"`
+	DriverStatus      [][]string  `json:"DriverStatus"`
+	SystemStatus      interface{} `json:"SystemStatus"`
+	Plugins           struct {
+		Volume        []string    `json:"Volume"`
+		Network       []string    `json:"Network"`
+		Authorization interface{} `json:"Authorization"`
+		Log           []string    `json:"Log"`
+	} `json:"Plugins"`
+	MemoryLimit        bool      `json:"MemoryLimit"`
+	SwapLimit          bool      `json:"SwapLimit"`
+	KernelMemory       bool      `json:"KernelMemory"`
+	KernelMemoryTCP    bool      `json:"KernelMemoryTCP"`
+	CPUCfsPeriod       bool      `json:"CpuCfsPeriod"`
+	CPUCfsQuota        bool      `json:"CpuCfsQuota"`
+	CPUShares          bool      `json:"CPUShares"`
+	CPUSet             bool      `json:"CPUSet"`
+	PidsLimit          bool      `json:"PidsLimit"`
+	IPv4Forwarding     bool      `json:"IPv4Forwarding"`
+	BridgeNfIptables   bool      `json:"BridgeNfIptables"`
+	BridgeNfIP6Tables  bool      `json:"BridgeNfIp6tables"`
+	Debug              bool      `json:"Debug"`
+	NFd                int       `json:"NFd"`
+	OomKillDisable     bool      `json:"OomKillDisable"`
+	NGoroutines        int       `json:"NGoroutines"`
+	SystemTime         time.Time `json:"SystemTime"`
+	LoggingDriver      string    `json:"LoggingDriver"`
+	CgroupDriver       string    `json:"CgroupDriver"`
+	NEventsListener    int       `json:"NEventsListener"`
+	KernelVersion      string    `json:"KernelVersion"`
+	OperatingSystem    string    `json:"OperatingSystem"`
+	OSType             string    `json:"OSType"`
+	Architecture       string    `json:"Architecture"`
+	IndexServerAddress string    `json:"IndexServerAddress"`
+	RegistryConfig     struct {
+		AllowNondistributableArtifactsCIDRs     []interface{} `json:"AllowNondistributableArtifactsCIDRs"`
+		AllowNondistributableArtifactsHostnames []interface{} `json:"AllowNondistributableArtifactsHostnames"`
+		InsecureRegistryCIDRs                   []string      `json:"InsecureRegistryCIDRs"`
+		IndexConfigs                            struct {
+			DockerIo struct {
+				Name     string        `json:"Name"`
+				Mirrors  []interface{} `json:"Mirrors"`
+				Secure   bool          `json:"Secure"`
+				Official bool          `json:"Official"`
+			} `json:"docker.io"`
+		} `json:"IndexConfigs"`
+		Mirrors []interface{} `json:"Mirrors"`
+	} `json:"RegistryConfig"`
+	NCPU              int           `json:"NCPU"`
+	MemTotal          int64         `json:"MemTotal"`
+	GenericResources  interface{}   `json:"GenericResources"`
+	DockerRootDir     string        `json:"DockerRootDir"`
+	HTTPProxy         string        `json:"HttpProxy"`
+	HTTPSProxy        string        `json:"HttpsProxy"`
+	NoProxy           string        `json:"NoProxy"`
+	Name              string        `json:"Name"`
+	Labels            []interface{} `json:"Labels"`
+	ExperimentalBuild bool          `json:"ExperimentalBuild"`
+	ServerVersion     string        `json:"ServerVersion"`
+	ClusterStore      string        `json:"ClusterStore"`
+	ClusterAdvertise  string        `json:"ClusterAdvertise"`
+	Runtimes          struct {
+		Runc struct {
+			Path string `json:"path"`
+		} `json:"runc"`
+	} `json:"Runtimes"`
+	DefaultRuntime string `json:"DefaultRuntime"`
+	Swarm          struct {
+		NodeID           string      `json:"NodeID"`
+		NodeAddr         string      `json:"NodeAddr"`
+		LocalNodeState   string      `json:"LocalNodeState"`
+		ControlAvailable bool        `json:"ControlAvailable"`
+		Error            string      `json:"Error"`
+		RemoteManagers   interface{} `json:"RemoteManagers"`
+	} `json:"Swarm"`
+	LiveRestoreEnabled bool   `json:"LiveRestoreEnabled"`
+	Isolation          string `json:"Isolation"`
+	InitBinary         string `json:"InitBinary"`
+	ContainerdCommit   struct {
+		ID       string `json:"ID"`
+		Expected string `json:"Expected"`
+	} `json:"ContainerdCommit"`
+	RuncCommit struct {
+		ID       string `json:"ID"`
+		Expected string `json:"Expected"`
+	} `json:"RuncCommit"`
+	InitCommit struct {
+		ID       string `json:"ID"`
+		Expected string `json:"Expected"`
+	} `json:"InitCommit"`
+	SecurityOptions []string    `json:"SecurityOptions"`
+	ProductLicense  string      `json:"ProductLicense"`
+	Warnings        interface{} `json:"Warnings"`
+	ClientInfo      struct {
+		Debug    bool          `json:"Debug"`
+		Plugins  []interface{} `json:"Plugins"`
+		Warnings interface{}   `json:"Warnings"`
+	} `json:"ClientInfo"`
+}
+
+// podmanSysInfo represents the output of podman system info --format '{{json .}}'
+type podmanSysInfo struct {
+	Host struct {
+		BuildahVersion string `json:"BuildahVersion"`
+		CgroupVersion  string `json:"CgroupVersion"`
+		Conmon         struct {
+			Package string `json:"package"`
+			Path    string `json:"path"`
+			Version string `json:"version"`
+		} `json:"Conmon"`
+		Distribution struct {
+			Distribution string `json:"distribution"`
+			Version      string `json:"version"`
+		} `json:"Distribution"`
+		MemFree    int   `json:"MemFree"`
+		MemTotal   int64 `json:"MemTotal"`
+		OCIRuntime struct {
+			Name    string `json:"name"`
+			Package string `json:"package"`
+			Path    string `json:"path"`
+			Version string `json:"version"`
+		} `json:"OCIRuntime"`
+		SwapFree    int    `json:"SwapFree"`
+		SwapTotal   int    `json:"SwapTotal"`
+		Arch        string `json:"arch"`
+		Cpus        int    `json:"cpus"`
+		Eventlogger string `json:"eventlogger"`
+		Hostname    string `json:"hostname"`
+		Kernel      string `json:"kernel"`
+		Os          string `json:"os"`
+		Rootless    bool   `json:"rootless"`
+		Uptime      string `json:"uptime"`
+	} `json:"host"`
+	Registries struct {
+		Search []string `json:"search"`
+	} `json:"registries"`
+	Store struct {
+		ConfigFile     string `json:"ConfigFile"`
+		ContainerStore struct {
+			Number int `json:"number"`
+		} `json:"ContainerStore"`
+		GraphDriverName string `json:"GraphDriverName"`
+		GraphOptions    struct {
+		} `json:"GraphOptions"`
+		GraphRoot   string `json:"GraphRoot"`
+		GraphStatus struct {
+			BackingFilesystem string `json:"Backing Filesystem"`
+			NativeOverlayDiff string `json:"Native Overlay Diff"`
+			SupportsDType     string `json:"Supports d_type"`
+			UsingMetacopy     string `json:"Using metacopy"`
+		} `json:"GraphStatus"`
+		ImageStore struct {
+			Number int `json:"number"`
+		} `json:"ImageStore"`
+		RunRoot    string `json:"RunRoot"`
+		VolumePath string `json:"VolumePath"`
+	} `json:"store"`
+}
+
+// dockerSystemInfo returns docker system info --format '{{json .}}'
+func dockerSystemInfo() (dockerSysInfo, error) {
+	var ds dockerSysInfo
+	if err := PointToHostDockerDaemon(); err != nil {
+		return ds, errors.Wrap(err, "point host docker-daemon")
+	}
+	cmd := exec.Command(Docker, "system", "info", "--format", "{{json .}}")
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return ds, errors.Wrap(err, "get docker system info")
+	}
+	if err := json.Unmarshal([]byte(strings.TrimSpace(string(out))), &ds); err != nil {
+		return ds, errors.Wrapf(err, "unmarshal docker system info")
+	}
+	return ds, nil
+}
+
+// podmanSysInfo returns podman system info --format '{{json .}}'
+func podmanSystemInfo() (podmanSysInfo, error) {
+	var ps podmanSysInfo
+	cmd := exec.Command(Podman, "system", "info", "--format", "'{{json .}}'")
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return ps, errors.Wrap(err, "get podman system info")
+	}
+	if err := json.Unmarshal([]byte(strings.TrimSpace(string(out))), &ps); err != nil {
+		return ps, errors.Wrapf(err, "unmarshal podman system info")
+	}
+	return ps, nil
+}

pkg/drivers/kic/oci/network.go

@@ -21,6 +21,7 @@ import (
 	"net"
 	"os/exec"
 	"runtime"
+	"strconv"
 	"strings"
 
 	"github.com/golang/glog"
@@ -43,7 +44,7 @@ func RoutableHostIPFromInside(ociBin string, containerName string) (net.IP, erro
 // digDNS will get the IP record for a dns
 func digDNS(ociBin, containerName, dns string) (net.IP, error) {
 	if err := PointToHostDockerDaemon(); err != nil {
-		return nil, errors.Wrap(err, "point host docker-daemon")
+		return nil, errors.Wrap(err, "point host docker daemon")
 	}
 	cmd := exec.Command(ociBin, "exec", "-t", containerName, "dig", "+short", dns)
 	out, err := cmd.CombinedOutput()
@@ -59,7 +60,7 @@ func digDNS(ociBin, containerName, dns string) (net.IP, error) {
 // gets the ip from user's host docker
 func dockerGatewayIP() (net.IP, error) {
 	if err := PointToHostDockerDaemon(); err != nil {
-		return nil, errors.Wrap(err, "point host docker-daemon")
+		return nil, errors.Wrap(err, "point host docker daemon")
 	}
 	cmd := exec.Command(Docker, "network", "ls", "--filter", "name=bridge", "--format", "{{.ID}}")
 	out, err := cmd.CombinedOutput()
@@ -77,3 +78,82 @@ func dockerGatewayIP() (net.IP, error) {
 	glog.Infof("got host ip for mount in container by inspect docker network: %s", ip.String())
 	return ip, nil
 }
+
+// HostPortBinding will return port mapping for a container using cli.
+// example : HostPortBinding("docker", "minikube", "22")
+// will return the docker assigned port:
+// 32769, nil
+// only supports TCP ports
+func HostPortBinding(ociBinary string, ociID string, contPort int) (int, error) {
+	if err := PointToHostDockerDaemon(); err != nil {
+		return 0, errors.Wrap(err, "point host docker daemon")
+	}
+	var out []byte
+	var err error
+	if ociBinary == Podman {
+		//podman inspect -f "{{range .NetworkSettings.Ports}}{{if eq .ContainerPort "80"}}{{.HostPort}}{{end}}{{end}}"
+		cmd := exec.Command(ociBinary, "inspect", "-f", fmt.Sprintf("{{range .NetworkSettings.Ports}}{{if eq .ContainerPort %s}}{{.HostPort}}{{end}}{{end}}", fmt.Sprint(contPort)), ociID)
+		out, err = cmd.CombinedOutput()
+		if err != nil {
+			return 0, errors.Wrapf(err, "get host-bind port %d for %q, output %s", contPort, ociID, out)
+		}
+	} else {
+		cmd := exec.Command(ociBinary, "inspect", "-f", fmt.Sprintf("'{{(index (index .NetworkSettings.Ports \"%d/tcp\") 0).HostPort}}'", contPort), ociID)
+		out, err = cmd.CombinedOutput()
+		if err != nil {
+			return 0, errors.Wrapf(err, "get host-bind port %d for %q, output %s", contPort, ociID, out)
+		}
+	}
+
+	o := strings.TrimSpace(string(out))
+	o = strings.Trim(o, "'")
+	p, err := strconv.Atoi(o)
+	if err != nil {
+		return p, errors.Wrapf(err, "convert host-port %q to number", p)
+	}
+	return p, nil
+}
+
+// ContainerIPs returns ipv4,ipv6, error of a container by their name
+func ContainerIPs(ociBinary string, name string) (string, string, error) {
+	if ociBinary == Podman {
+		return podmanConttainerIP(name)
+	}
+	return dockerContainerIP(name)
+}
+
+// podmanConttainerIP returns ipv4, ipv6 of container or error
+func podmanConttainerIP(name string) (string, string, error) {
+	cmd := exec.Command(Podman, "inspect",
+		"-f", "{{.NetworkSettings.IPAddress}}",
+		name)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return "", "", errors.Wrapf(err, "podman inspect ip %s", name)
+	}
+	output := strings.TrimSpace(string(out))
+	if err == nil && output == "" { // podman returns empty for 127.0.0.1
+		return DefaultBindIPV4, "", nil
+	}
+	return output, "", nil
+}
+
+// dockerContainerIP returns ipv4, ipv6 of container or error
+func dockerContainerIP(name string) (string, string, error) {
+	if err := PointToHostDockerDaemon(); err != nil {
+		return "", "", errors.Wrap(err, "point host docker daemon")
+	}
+	// retrieve the IP address of the node using docker inspect
+	lines, err := inspect(Docker, name, "{{range .NetworkSettings.Networks}}{{.IPAddress}},{{.GlobalIPv6Address}}{{end}}")
+	if err != nil {
+		return "", "", errors.Wrap(err, "inspecting NetworkSettings.Networks")
+	}
+	if len(lines) != 1 {
+		return "", "", errors.Errorf("IPs output should only be one line, got %d lines", len(lines))
+	}
+	ips := strings.Split(lines[0], ",")
+	if len(ips) != 2 {
+		return "", "", errors.Errorf("container addresses should have 2 values, got %d values: %+v", len(ips), ips)
+	}
+	return ips[0], ips[1], nil
+}

pkg/drivers/kic/oci/oci.go

@@ -17,9 +17,10 @@ limitations under the License.
 package oci
 
 import (
+	"context"
 	"os"
 	"path/filepath"
-	"strconv"
+	"time"
 
 	"bufio"
 	"bytes"
@@ -34,13 +35,13 @@ import (
 	"strings"
 )
 
-// DeleteAllContainersByLabel deletes all containers that have a specific label
+// DeleteContainersByLabel deletes all containers that have a specific label
 // if there no containers found with the given 	label, it will return nil
-func DeleteAllContainersByLabel(ociBin string, label string) []error {
+func DeleteContainersByLabel(ociBin string, label string) []error {
 	var deleteErrs []error
 	if ociBin == Docker {
 		if err := PointToHostDockerDaemon(); err != nil {
-			return []error{errors.Wrap(err, "point host docker-daemon")}
+			return []error{errors.Wrap(err, "point host docker daemon")}
 		}
 	}
 	cs, err := listContainersByLabel(ociBin, label)
@@ -51,10 +52,19 @@ func DeleteAllContainersByLabel(ociBin string, label string) []error {
 		return nil
 	}
 	for _, c := range cs {
+		_, err := ContainerStatus(ociBin, c)
+		// only try to delete if docker/podman inspect returns
+		// if it doesn't it means docker daemon is stuck and needs restart
+		if err != nil {
+			deleteErrs = append(deleteErrs, errors.Wrapf(err, "delete container %s: %s daemon is stuck. please try again!", c, ociBin))
+			glog.Errorf("%s daemon seems to be stuck. Please try restarting your %s.", ociBin, ociBin)
+			continue
+		}
 		cmd := exec.Command(ociBin, "rm", "-f", "-v", c)
 		if out, err := cmd.CombinedOutput(); err != nil {
 			deleteErrs = append(deleteErrs, errors.Wrapf(err, "delete container %s: output %s", c, out))
 		}
+
 	}
 	return deleteErrs
 }
@@ -62,7 +72,7 @@ func DeleteAllContainersByLabel(ociBin string, label string) []error {
 // CreateContainerNode creates a new container node
 func CreateContainerNode(p CreateParams) error {
 	if err := PointToHostDockerDaemon(); err != nil {
-		return errors.Wrap(err, "point host docker-daemon")
+		return errors.Wrap(err, "point host docker daemon")
 	}
 
 	runArgs := []string{
@@ -87,6 +97,8 @@ func CreateContainerNode(p CreateParams) error {
 		"--label", p.ClusterLabel,
 		// label the node with the role ID
 		"--label", fmt.Sprintf("%s=%s", nodeRoleLabelKey, p.Role),
+		// label th enode wuth the node ID
+		"--label", p.NodeLabel,
 	}
 
 	if p.OCIBinary == Podman { // enable execing in /var
@@ -99,7 +111,7 @@ func CreateContainerNode(p CreateParams) error {
 		runArgs = append(runArgs, "--volume", fmt.Sprintf("%s:/var:exec", hostVarVolPath))
 	}
 	if p.OCIBinary == Docker {
-		if err := createDockerVolume(p.Name); err != nil {
+		if err := createDockerVolume(p.Name, p.Name); err != nil {
 			return errors.Wrapf(err, "creating volume for %s container", p.Name)
 		}
 		glog.Infof("Successfully created a docker volume %s", p.Name)
@@ -141,7 +153,7 @@ func CreateContainerNode(p CreateParams) error {
 // CreateContainer creates a container with "docker/podman run"
 func createContainer(ociBinary string, image string, opts ...createOpt) ([]string, error) {
 	if err := PointToHostDockerDaemon(); err != nil {
-		return nil, errors.Wrap(err, "point host docker-daemon")
+		return nil, errors.Wrap(err, "point host docker daemon")
 	}
 
 	o := &createOpts{}
@@ -185,7 +197,7 @@ func createContainer(ociBinary string, image string, opts ...createOpt) ([]strin
 // Copy copies a local asset into the container
 func Copy(ociBinary string, ociID string, targetDir string, fName string) error {
 	if err := PointToHostDockerDaemon(); err != nil {
-		return errors.Wrap(err, "point host docker-daemon")
+		return errors.Wrap(err, "point host docker daemon")
 	}
 	if _, err := os.Stat(fName); os.IsNotExist(err) {
 		return errors.Wrapf(err, "error source %s does not exist", fName)
@@ -199,89 +211,10 @@ func Copy(ociBinary string, ociID string, targetDir string, fName string) error
 	return nil
 }
 
-// HostPortBinding will return port mapping for a container using cli.
-// example : HostPortBinding("docker", "minikube", "22")
-// will return the docker assigned port:
-// 32769, nil
-// only supports TCP ports
-func HostPortBinding(ociBinary string, ociID string, contPort int) (int, error) {
-	if err := PointToHostDockerDaemon(); err != nil {
-		return 0, errors.Wrap(err, "point host docker-daemon")
-	}
-	var out []byte
-	var err error
-	if ociBinary == Podman {
-		//podman inspect -f "{{range .NetworkSettings.Ports}}{{if eq .ContainerPort "80"}}{{.HostPort}}{{end}}{{end}}"
-		cmd := exec.Command(ociBinary, "inspect", "-f", fmt.Sprintf("{{range .NetworkSettings.Ports}}{{if eq .ContainerPort %s}}{{.HostPort}}{{end}}{{end}}", fmt.Sprint(contPort)), ociID)
-		out, err = cmd.CombinedOutput()
-		if err != nil {
-			return 0, errors.Wrapf(err, "get host-bind port %d for %q, output %s", contPort, ociID, out)
-		}
-	} else {
-		cmd := exec.Command(ociBinary, "inspect", "-f", fmt.Sprintf("'{{(index (index .NetworkSettings.Ports \"%d/tcp\") 0).HostPort}}'", contPort), ociID)
-		out, err = cmd.CombinedOutput()
-		if err != nil {
-			return 0, errors.Wrapf(err, "get host-bind port %d for %q, output %s", contPort, ociID, out)
-		}
-	}
-
-	o := strings.TrimSpace(string(out))
-	o = strings.Trim(o, "'")
-	p, err := strconv.Atoi(o)
-	if err != nil {
-		return p, errors.Wrapf(err, "convert host-port %q to number", p)
-	}
-	return p, nil
-}
-
-// ContainerIPs returns ipv4,ipv6, error of a container by their name
-func ContainerIPs(ociBinary string, name string) (string, string, error) {
-	if ociBinary == Podman {
-		return podmanConttainerIP(name)
-	}
-	return dockerContainerIP(name)
-}
-
-// podmanConttainerIP returns ipv4, ipv6 of container or error
-func podmanConttainerIP(name string) (string, string, error) {
-	cmd := exec.Command(Podman, "inspect",
-		"-f", "{{.NetworkSettings.IPAddress}}",
-		name)
-	out, err := cmd.CombinedOutput()
-	if err != nil {
-		return "", "", errors.Wrapf(err, "podman inspect ip %s", name)
-	}
-	output := strings.TrimSpace(string(out))
-	if err == nil && output == "" { // podman returns empty for 127.0.0.1
-		return DefaultBindIPV4, "", nil
-	}
-	return output, "", nil
-}
-
-// dockerContainerIP returns ipv4, ipv6 of container or error
-func dockerContainerIP(name string) (string, string, error) {
-	if err := PointToHostDockerDaemon(); err != nil {
-		return "", "", errors.Wrap(err, "point host docker-daemon")
-	}
-	// retrieve the IP address of the node using docker inspect
-	lines, err := inspect(Docker, name, "{{range .NetworkSettings.Networks}}{{.IPAddress}},{{.GlobalIPv6Address}}{{end}}")
-	if err != nil {
-		return "", "", errors.Wrap(err, "inspecting NetworkSettings.Networks")
-	}
-	if len(lines) != 1 {
-		return "", "", errors.Errorf("IPs output should only be one line, got %d lines", len(lines))
-	}
-	ips := strings.Split(lines[0], ",")
-	if len(ips) != 2 {
-		return "", "", errors.Errorf("container addresses should have 2 values, got %d values: %+v", len(ips), ips)
-	}
-	return ips[0], ips[1], nil
-}
-
 // ContainerID returns id of a container name
 func ContainerID(ociBinary string, nameOrID string) (string, error) {
 	if err := PointToHostDockerDaemon(); err != nil {
-		return "", errors.Wrap(err, "point host docker-daemon")
+		return "", errors.Wrap(err, "point host docker daemon")
 	}
 	cmd := exec.Command(ociBinary, "inspect", "-f", "{{.Id}}", nameOrID)
 	id, err := cmd.CombinedOutput()
@@ -299,7 +232,7 @@ func ListOwnedContainers(ociBinary string) ([]string, error) {
 // inspect return low-level information on containers
 func inspect(ociBinary string, containerNameOrID, format string) ([]string, error) {
 	if err := PointToHostDockerDaemon(); err != nil {
-		return nil, errors.Wrap(err, "point host docker-daemon")
+		return nil, errors.Wrap(err, "point host docker daemon")
 	}
 	cmd := exec.Command(ociBinary, "inspect",
 		"-f", format,
@@ -365,7 +298,7 @@ func generateMountBindings(mounts ...Mount) []string {
 // isUsernsRemapEnabled checks if userns-remap is enabled in docker
 func isUsernsRemapEnabled(ociBinary string) (bool, error) {
 	if err := PointToHostDockerDaemon(); err != nil {
-		return false, errors.Wrap(err, "point host docker-daemon")
+		return false, errors.Wrap(err, "point host docker daemon")
 	}
 	cmd := exec.Command(ociBinary, "info", "--format", "'{{json .SecurityOptions}}'")
 	var buff bytes.Buffer
@@ -427,9 +360,12 @@ func withPortMappings(portMappings []PortMapping) createOpt {
 // listContainersByLabel returns all the container names with a specified label
 func listContainersByLabel(ociBinary string, label string) ([]string, error) {
 	if err := PointToHostDockerDaemon(); err != nil {
-		return nil, errors.Wrap(err, "point host docker-daemon")
+		return nil, errors.Wrap(err, "point host docker daemon")
 	}
-	cmd := exec.Command(ociBinary, "ps", "-a", "--filter", fmt.Sprintf("label=%s", label), "--format", "{{.Names}}")
+	// allow no more than 5 seconds for docker ps
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	cmd := exec.CommandContext(ctx, ociBinary, "ps", "-a", "--filter", fmt.Sprintf("label=%s", label), "--format", "{{.Names}}")
 	stdout, err := cmd.Output()
 	s := bufio.NewScanner(bytes.NewReader(stdout))
 	var names []string
@@ -439,7 +375,6 @@ func listContainersByLabel(ociBinary string, label string) ([]string, error) {
 			names = append(names, n)
 		}
 	}
-
 	return names, err
 }
 
@@ -448,7 +383,7 @@ func listContainersByLabel(ociBinary string, label string) ([]string, error) {
 func PointToHostDockerDaemon() error {
 	p := os.Getenv(constants.MinikubeActiveDockerdEnv)
 	if p != "" {
-		glog.Infof("shell is pointing to docker inside minikube. will unset to use host")
+		glog.Infof("shell is pointing to dockerd inside minikube. will unset to use host")
 	}
 
 	for i := range constants.DockerDaemonEnvs {
@@ -461,3 +396,25 @@ func PointToHostDockerDaemon() error {
 	}
 	return nil
 }
+
+// ContainerStatus returns status of a container running,exited,...
+func ContainerStatus(ociBin string, name string) (string, error) {
+	if ociBin == Docker {
+		if err := PointToHostDockerDaemon(); err != nil {
+			return "", errors.Wrap(err, "point host docker daemon")
+		}
+	}
+	// allow no more than 2 seconds for this. when this takes long this means deadline passed
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel()
+	cmd := exec.CommandContext(ctx, ociBin, "inspect", name, "--format={{.State.Status}}")
+	out, err := cmd.CombinedOutput()
+	if ctx.Err() == context.DeadlineExceeded {
+		glog.Warningf("%s inspect %s took longer than normal. Restarting your %s daemon might fix this issue.", ociBin, name, ociBin)
+		return strings.TrimSpace(string(out)), fmt.Errorf("inspect %s timeout", name)
+	}
+	if err != nil {
+		return string(out), errors.Wrapf(err, "inspecting container: output %s", out)
+	}
+	return strings.TrimSpace(string(out)), nil
+}

pkg/drivers/kic/oci/types.go

@@ -25,6 +25,8 @@ const (
 	Podman = "podman"
 	// ProfileLabelKey is applied to any container or volume created by a specific minikube profile name.minikube.sigs.k8s.io=PROFILE_NAME
 	ProfileLabelKey = "name.minikube.sigs.k8s.io"
+	// NodeLabelKey is applied to each volume so it can be referred to by name
+	NodeLabelKey = "mode.minikube.sigs.k8s.io"
 	// NodeRoleKey is used to identify if it is control plane or worker
 	nodeRoleLabelKey = "role.minikube.sigs.k8s.io"
 	// CreatedByLabelKey is applied to any container/volume that is created by minikube created_by.minikube.sigs.k8s.io=true
@@ -35,7 +37,8 @@ const (
 type CreateParams struct {
 	Name          string            // used for container name and hostname
 	Image         string            // container image to use to create the node.
-	ClusterLabel  string            // label the containers we create using minikube so we can clean up
+	ClusterLabel  string            // label the clusters we create using minikube so we can clean up
+	NodeLabel     string            // label the nodes so we can clean up by node name
 	Role          string            // currently only role supported is control-plane
 	Mounts        []Mount           // volume mounts
 	APIServerPort int               // kubernetes api server port

pkg/drivers/kic/oci/volumes.go

@@ -19,9 +19,11 @@ package oci
 import (
 	"bufio"
 	"bytes"
+	"context"
 	"fmt"
 	"os/exec"
 	"strings"
+	"time"
 
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
@@ -34,7 +36,7 @@ func DeleteAllVolumesByLabel(ociBin string, label string) []error {
 	glog.Infof("trying to delete all %s volumes with label %s", ociBin, label)
 	if ociBin == Docker {
 		if err := PointToHostDockerDaemon(); err != nil {
-			return []error{errors.Wrap(err, "point host docker-daemon")}
+			return []error{errors.Wrap(err, "point host docker daemon")}
 		}
 	}
 
@@ -42,9 +44,15 @@ func DeleteAllVolumesByLabel(ociBin string, label string) []error {
 	if err != nil {
 		return []error{fmt.Errorf("listing volumes by label %q: %v", label, err)}
 	}
-
 	for _, v := range vs {
-		cmd := exec.Command(ociBin, "volume", "rm", "--force", v)
+		// allow no more than 3 seconds for this. when this takes long this means deadline passed
+		ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+		defer cancel()
+		cmd := exec.CommandContext(ctx, ociBin, "volume", "rm", "--force", v)
+		if ctx.Err() == context.DeadlineExceeded {
+			glog.Warningf("removing volume with label %s took longer than normal. Restarting your %s daemon might fix this issue.", label, ociBin)
+			deleteErrs = append(deleteErrs, fmt.Errorf("delete deadline exceeded for %s", label))
+		}
 		if out, err := cmd.CombinedOutput(); err != nil {
 			deleteErrs = append(deleteErrs, fmt.Errorf("deleting volume %s: output: %s", v, string(out)))
 		}
@@ -60,15 +68,22 @@ func PruneAllVolumesByLabel(ociBin string, label string) []error {
 	glog.Infof("trying to prune all %s volumes with label %s", ociBin, label)
 	if ociBin == Docker {
 		if err := PointToHostDockerDaemon(); err != nil {
-			return []error{errors.Wrap(err, "point host docker-daemon")}
+			return []error{errors.Wrap(err, "point host docker daemon")}
 		}
 	}
+	// allow no more than 3 seconds for this. when this takes long this means deadline passed
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
 
 	// try to prune afterwards just in case delete didn't go through
-	cmd := exec.Command(ociBin, "volume", "prune", "-f", "--filter", "label="+label)
+	cmd := exec.CommandContext(ctx, ociBin, "volume", "prune", "-f", "--filter", "label="+label)
 	if out, err := cmd.CombinedOutput(); err != nil {
 		deleteErrs = append(deleteErrs, errors.Wrapf(err, "prune volume by label %s: %s", label, string(out)))
 	}
+	if ctx.Err() == context.DeadlineExceeded {
+		glog.Warningf("pruning volume with label %s took longer than normal. Restarting your %s daemon might fix this issue.", label, ociBin)
+		deleteErrs = append(deleteErrs, fmt.Errorf("prune deadline exceeded for %s", label))
+	}
 	return deleteErrs
 }
 
@@ -88,14 +103,27 @@ func allVolumesByLabel(ociBin string, label string) ([]string, error) {
 	return vols, err
 }
 
-// createDockerVolume creates a docker volume to be attached to the container with correct labels and prefixes based on profile name
+// ExtractTarballToVolume runs a docker image imageName which extracts the tarball at tarballPath
-// Caution ! if volume already exists does NOT return an error and will not apply the minikube labels on it.
+// to the volume named volumeName
-// TODO: this should be fixed as a part of https://github.com/kubernetes/minikube/issues/6530
+func ExtractTarballToVolume(tarballPath, volumeName, imageName string) error {
-func createDockerVolume(name string) error {
 	if err := PointToHostDockerDaemon(); err != nil {
 		return errors.Wrap(err, "point host docker-daemon")
 	}
-	cmd := exec.Command(Docker, "volume", "create", name, "--label", fmt.Sprintf("%s=%s", ProfileLabelKey, name), "--label", fmt.Sprintf("%s=%s", CreatedByLabelKey, "true"))
+	cmd := exec.Command(Docker, "run", "--rm", "--entrypoint", "/usr/bin/tar", "-v", fmt.Sprintf("%s:/preloaded.tar:ro", tarballPath), "-v", fmt.Sprintf("%s:/extractDir", volumeName), imageName, "-I", "lz4", "-xvf", "/preloaded.tar", "-C", "/extractDir")
+	if out, err := cmd.CombinedOutput(); err != nil {
+		return errors.Wrapf(err, "output %s", string(out))
+	}
+	return nil
+}
+
+// createDockerVolume creates a docker volume to be attached to the container with correct labels and prefixes based on profile name
+// Caution ! if volume already exists does NOT return an error and will not apply the minikube labels on it.
+// TODO: this should be fixed as a part of https://github.com/kubernetes/minikube/issues/6530
+func createDockerVolume(profile string, nodeName string) error {
+	if err := PointToHostDockerDaemon(); err != nil {
+		return errors.Wrap(err, "point host docker daemon")
+	}
+	cmd := exec.Command(Docker, "volume", "create", nodeName, "--label", fmt.Sprintf("%s=%s", ProfileLabelKey, profile), "--label", fmt.Sprintf("%s=%s", CreatedByLabelKey, "true"))
 	if out, err := cmd.CombinedOutput(); err != nil {
 		return errors.Wrapf(err, "output %s", string(out))
 	}

pkg/drivers/kic/types.go

@@ -46,14 +46,16 @@ var (
 
 // Config is configuration for the kic driver used by registry
 type Config struct {
-	MachineName   string            // maps to the container name being created
+	MachineName       string            // maps to the container name being created
-	CPU           int               // Number of CPU cores assigned to the container
+	CPU               int               // Number of CPU cores assigned to the container
-	Memory        int               // max memory in MB
+	Memory            int               // max memory in MB
-	StorePath     string            // libmachine store path
+	StorePath         string            // libmachine store path
-	OCIBinary     string            // oci tool to use (docker, podman,...)
+	OCIBinary         string            // oci tool to use (docker, podman,...)
-	ImageDigest   string            // image name with sha to use for the node
+	ImageDigest       string            // image name with sha to use for the node
-	Mounts        []oci.Mount       // mounts
+	Mounts            []oci.Mount       // mounts
-	APIServerPort int               // kubernetes api server port inside the container
+	APIServerPort     int               // kubernetes api server port inside the container
-	PortMappings  []oci.PortMapping // container port mappings
+	PortMappings      []oci.PortMapping // container port mappings
-	Envs          map[string]string // key,value of environment variables passed to the node
+	Envs              map[string]string // key,value of environment variables passed to the node
+	KubernetesVersion string            // kubernetes version to install
+	ContainerRuntime  string            // container runtime kic is running
 }

pkg/minikube/bootstrapper/bsutil/binaries.go

@@ -18,23 +18,34 @@ limitations under the License.
 package bsutil
 
 import (
+	"fmt"
 	"os/exec"
 	"path"
 	"runtime"
+	"strings"
 
+	"github.com/golang/glog"
 	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
 	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
+	"k8s.io/minikube/pkg/minikube/download"
 	"k8s.io/minikube/pkg/minikube/machine"
 	"k8s.io/minikube/pkg/minikube/vmpath"
 )
 
 // TransferBinaries transfers all required Kubernetes binaries
 func TransferBinaries(cfg config.KubernetesConfig, c command.Runner) error {
+	ok, err := binariesExist(cfg, c)
+	if err == nil && ok {
+		glog.Info("Found k8s binaries, skipping transfer")
+		return nil
+	}
+	glog.Infof("Didn't find k8s binaries: %v\nInitiating transfer...", err)
+
 	dir := binRoot(cfg.KubernetesVersion)
-	_, err := c.RunCmd(exec.Command("sudo", "mkdir", "-p", dir))
+	_, err = c.RunCmd(exec.Command("sudo", "mkdir", "-p", dir))
 	if err != nil {
 		return err
 	}
@@ -43,7 +54,7 @@ func TransferBinaries(cfg config.KubernetesConfig, c command.Runner) error {
 	for _, name := range constants.KubernetesReleaseBinaries {
 		name := name
 		g.Go(func() error {
-			src, err := machine.CacheBinary(name, cfg.KubernetesVersion, "linux", runtime.GOARCH)
+			src, err := download.Binary(name, cfg.KubernetesVersion, "linux", runtime.GOARCH)
 			if err != nil {
 				return errors.Wrapf(err, "downloading %s", name)
 			}
@@ -58,6 +69,26 @@ func TransferBinaries(cfg config.KubernetesConfig, c command.Runner) error {
 	return g.Wait()
 }
 
+// binariesExist returns true if the binaries already exist
+func binariesExist(cfg config.KubernetesConfig, c command.Runner) (bool, error) {
+	dir := binRoot(cfg.KubernetesVersion)
+	rr, err := c.RunCmd(exec.Command("sudo", "ls", dir))
+	stdout := rr.Stdout.String()
+	if err != nil {
+		return false, err
+	}
+	foundBinaries := map[string]struct{}{}
+	for _, binary := range strings.Split(stdout, "\n") {
+		foundBinaries[binary] = struct{}{}
+	}
+	for _, name := range constants.KubernetesReleaseBinaries {
+		if _, ok := foundBinaries[name]; !ok {
+			return false, fmt.Errorf("didn't find preexisting %s", name)
+		}
+	}
+	return true, nil
+}
+
 // binRoot returns the persistent path binaries are stored in
 func binRoot(version string) string {
 	return path.Join(vmpath.GuestPersistentDir, "binaries", version)

pkg/minikube/bootstrapper/bsutil/extraconfig.go

@@ -70,6 +70,7 @@ var KubeadmExtraArgsWhitelist = map[int][]string{
 		"experimental-upload-certs",
 		"certificate-key",
 		"rootfs",
+		"skip-phases",
 	},
 	KubeadmConfigParam: {
 		"pod-network-cidr",

pkg/minikube/bootstrapper/certs.go

@@ -132,9 +132,8 @@ func SetupCerts(cmd command.Runner, k8s config.KubernetesConfig, n config.Node)
 		}
 	}
 
-	// configure CA certificates
+	if err := installCertSymlinks(cmd, caCerts); err != nil {
-	if err := configureCACerts(cmd, caCerts); err != nil {
+		return errors.Wrapf(err, "certificate symlinks")
-		return errors.Wrapf(err, "Configuring CA certs")
 	}
 	return nil
 }
@@ -320,9 +319,9 @@ func getSubjectHash(cr command.Runner, filePath string) (string, error) {
 	return stringHash, nil
 }
 
-// configureCACerts looks up and installs all uploaded PEM certificates in /usr/share/ca-certificates to system-wide certificate store (/etc/ssl/certs).
+// installCertSymlinks installs certs in /usr/share/ca-certificates into system-wide certificate store (/etc/ssl/certs).
 // OpenSSL binary required in minikube ISO
-func configureCACerts(cr command.Runner, caCerts map[string]string) error {
+func installCertSymlinks(cr command.Runner, caCerts map[string]string) error {
 	hasSSLBinary := true
 	_, err := cr.RunCmd(exec.Command("openssl", "version"))
 	if err != nil {
@@ -336,7 +335,8 @@ func configureCACerts(cr command.Runner, caCerts map[string]string) error {
 	for _, caCertFile := range caCerts {
 		dstFilename := path.Base(caCertFile)
 		certStorePath := path.Join(vmpath.GuestCertStoreDir, dstFilename)
-		cmd := fmt.Sprintf("test -f %s || ln -fs %s %s", caCertFile, certStorePath, caCertFile)
+		// If the cert really exists, add a named symlink
+		cmd := fmt.Sprintf("test -f %s && ln -fs %s %s", caCertFile, caCertFile, certStorePath)
 		if _, err := cr.RunCmd(exec.Command("sudo", "/bin/bash", "-c", cmd)); err != nil {
 			return errors.Wrapf(err, "create symlink for %s", caCertFile)
 		}

pkg/minikube/bootstrapper/certs_test.go

@@ -52,8 +52,8 @@ func TestSetupCerts(t *testing.T) {
 	}
 
 	expected := map[string]string{
-		`sudo /bin/bash -c "test -f /usr/share/ca-certificates/mycert.pem || ln -fs /etc/ssl/certs/mycert.pem /usr/share/ca-certificates/mycert.pem"`:             "-",
+		`sudo /bin/bash -c "test -f /usr/share/ca-certificates/mycert.pem && ln -fs /usr/share/ca-certificates/mycert.pem /etc/ssl/certs/mycert.pem"`:             "-",
-		`sudo /bin/bash -c "test -f /usr/share/ca-certificates/minikubeCA.pem || ln -fs /etc/ssl/certs/minikubeCA.pem /usr/share/ca-certificates/minikubeCA.pem"`: "-",
+		`sudo /bin/bash -c "test -f /usr/share/ca-certificates/minikubeCA.pem && ln -fs /usr/share/ca-certificates/minikubeCA.pem /etc/ssl/certs/minikubeCA.pem"`: "-",
 	}
 	f := command.NewFakeCommandRunner()
 	f.SetCommandToOutput(expected)

pkg/minikube/bootstrapper/kubeadm/kubeadm.go

@@ -36,7 +36,6 @@ import (
 	"github.com/docker/machine/libmachine/state"
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
-	"github.com/spf13/viper"
 	"k8s.io/client-go/kubernetes"
 	kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/minikube/pkg/drivers/kic"
@@ -65,7 +64,9 @@ type Bootstrapper struct {
 }
 
 // NewBootstrapper creates a new kubeadm.Bootstrapper
-func NewBootstrapper(api libmachine.API, name string) (*Bootstrapper, error) {
+// TODO(#6891): Remove node as an argument
+func NewBootstrapper(api libmachine.API, cc config.ClusterConfig, n config.Node) (*Bootstrapper, error) {
+	name := driver.MachineName(cc, n)
 	h, err := api.Load(name)
 	if err != nil {
 		return nil, errors.Wrap(err, "getting api client")
@@ -74,7 +75,7 @@ func NewBootstrapper(api libmachine.API, name string) (*Bootstrapper, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "command runner")
 	}
-	return &Bootstrapper{c: runner, contextName: viper.GetString(config.MachineProfile), k8sClient: nil}, nil
+	return &Bootstrapper{c: runner, contextName: cc.Name, k8sClient: nil}, nil
 }
 
 // GetKubeletStatus returns the kubelet status
@@ -268,7 +269,7 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time
 	port := cp.Port
 	if driver.IsKIC(cfg.Driver) {
 		ip = oci.DefaultBindIPV4
-		port, err = oci.HostPortBinding(cfg.Driver, driver.MachineName(cfg.Name, n.Name), port)
+		port, err = oci.HostPortBinding(cfg.Driver, driver.MachineName(cfg, n), port)
 		if err != nil {
 			return errors.Wrapf(err, "get host-bind port %d for container %s", port, cfg.Name)
 		}
@@ -338,9 +339,9 @@ func (k *Bootstrapper) restartCluster(cfg config.ClusterConfig) error {
 		port := n.Port
 		if driver.IsKIC(cfg.Driver) {
 			ip = oci.DefaultBindIPV4
-			port, err = oci.HostPortBinding(cfg.Driver, driver.MachineName(cfg.Name, n.Name), port)
+			port, err = oci.HostPortBinding(cfg.Driver, driver.MachineName(cfg, n), port)
 			if err != nil {
-				return errors.Wrapf(err, "get host-bind port %d for container %s", port, driver.MachineName(cfg.Name, n.Name))
+				return errors.Wrapf(err, "get host-bind port %d for container %s", port, driver.MachineName(cfg, n))
 			}
 		}
 		client, err := k.client(ip, port)

pkg/minikube/cluster/cache.go

@@ -24,8 +24,10 @@ import (
 	"github.com/spf13/viper"
 	"golang.org/x/sync/errgroup"
 	cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config"
+	"k8s.io/minikube/pkg/drivers/kic"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
+	"k8s.io/minikube/pkg/minikube/download"
 	"k8s.io/minikube/pkg/minikube/exit"
 	"k8s.io/minikube/pkg/minikube/image"
 	"k8s.io/minikube/pkg/minikube/localpath"
@@ -49,8 +51,7 @@ func BeginCacheRequiredImages(g *errgroup.Group, imageRepository string, k8sVers
 	})
 }
 
-// HandleDownloadOnly handles the download-only parameter
+func handleDownloadOnly(cacheGroup, kicGroup *errgroup.Group, k8sVersion string) {
-func HandleDownloadOnly(cacheGroup *errgroup.Group, k8sVersion string) {
 	// If --download-only, complete the remaining downloads and exit.
 	if !viper.GetBool("download-only") {
 		return
@@ -62,6 +63,7 @@ func HandleDownloadOnly(cacheGroup *errgroup.Group, k8sVersion string) {
 		exit.WithError("Failed to cache kubectl", err)
 	}
 	WaitCacheRequiredImages(cacheGroup)
+	waitDownloadKicArtifacts(kicGroup)
 	if err := saveImagesToTarFromConfig(); err != nil {
 		exit.WithError("Failed to cache images to tar", err)
 	}
@@ -77,7 +79,7 @@ func CacheKubectlBinary(k8sVerison string) (string, error) {
 		binary = "kubectl.exe"
 	}
 
-	return machine.CacheBinary(binary, k8sVerison, runtime.GOOS, runtime.GOARCH)
+	return download.Binary(binary, k8sVerison, runtime.GOOS, runtime.GOARCH)
 }
 
 // doCacheBinaries caches Kubernetes binaries in the foreground
@@ -85,7 +87,33 @@ func doCacheBinaries(k8sVersion string) error {
 	return machine.CacheBinariesForBootstrapper(k8sVersion, viper.GetString(cmdcfg.Bootstrapper))
 }
 
-// WaitCacheRequiredImages blocks until the required images are all cached.
+// beginDownloadKicArtifacts downloads the kic image + preload tarball, returns true if preload is available
+func beginDownloadKicArtifacts(g *errgroup.Group, k8sVersion, cRuntime string) bool {
+	glog.Info("Beginning downloading kic artifacts")
+	g.Go(func() error {
+		glog.Infof("Downloading %s to local daemon", kic.BaseImage)
+		return image.WriteImageToDaemon(kic.BaseImage)
+	})
+
+	if download.PreloadExists(k8sVersion, cRuntime) {
+		g.Go(func() error {
+			glog.Info("Caching tarball of preloaded images")
+			return download.Preload(k8sVersion, cRuntime)
+		})
+		return true
+	}
+	return false
+}
+
+func waitDownloadKicArtifacts(g *errgroup.Group) {
+	if err := g.Wait(); err != nil {
+		glog.Errorln("Error downloading kic artifacts: ", err)
+		return
+	}
+	glog.Info("Successfully downloaded all kic artifacts")
+}
+
+// waitCacheRequiredImages blocks until the required images are all cached.
 func WaitCacheRequiredImages(g *errgroup.Group) {
 	if !viper.GetBool(cacheImages) {
 		return

pkg/minikube/cluster/cluster.go

@@ -26,7 +26,7 @@ import (
 
 	"k8s.io/minikube/pkg/minikube/bootstrapper"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/kubeadm"
-	"k8s.io/minikube/pkg/minikube/driver"
+	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/exit"
 )
 
@@ -43,12 +43,13 @@ func init() {
 }
 
 // Bootstrapper returns a new bootstrapper for the cluster
-func Bootstrapper(api libmachine.API, bootstrapperName string, cluster string, nodeName string) (bootstrapper.Bootstrapper, error) {
+// TODO(#6891): Remove node as an argument
+func Bootstrapper(api libmachine.API, bootstrapperName string, cc config.ClusterConfig, n config.Node) (bootstrapper.Bootstrapper, error) {
 	var b bootstrapper.Bootstrapper
 	var err error
 	switch bootstrapperName {
 	case bootstrapper.Kubeadm:
-		b, err = kubeadm.NewBootstrapper(api, driver.MachineName(cluster, nodeName))
+		b, err = kubeadm.NewBootstrapper(api, cc, n)
 		if err != nil {
 			return nil, errors.Wrap(err, "getting a new kubeadm bootstrapper")
 		}

pkg/minikube/cluster/iso.go

@@ -1,30 +0,0 @@
-/*
-Copyright 2020 The Kubernetes Authors All rights reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package cluster
-
-import (
-	"k8s.io/minikube/pkg/minikube/config"
-	"k8s.io/minikube/pkg/minikube/driver"
-)
-
-// CacheISO downloads and caches ISO.
-func CacheISO(cfg config.ClusterConfig) error {
-	if driver.BareMetal(cfg.Driver) {
-		return nil
-	}
-	return cfg.Downloader.CacheMinikubeISOFromURL(cfg.MinikubeISO)
-}

pkg/minikube/command/ssh_runner.go

@@ -241,11 +241,13 @@ func (s *SSHRunner) sameFileExists(f assets.CopyableFile, dst string) (bool, err
 	if err != nil {
 		return false, err
 	}
+	glog.Infof("found %s: %d bytes, modified at %s", dst, dstSize, dstModTime)
 
 	// compare sizes and modtimes
 	if srcSize != dstSize {
 		return false, errors.New("source file and destination file are different sizes")
 	}
+
 	return srcModTime.Equal(dstModTime), nil
 }
 

pkg/minikube/config/config.go

@@ -41,8 +41,8 @@ const (
 	WantKubectlDownloadMsg = "WantKubectlDownloadMsg"
 	// WantNoneDriverWarning is the key for WantNoneDriverWarning
 	WantNoneDriverWarning = "WantNoneDriverWarning"
-	// MachineProfile is the key for MachineProfile
+	// ProfileName represents the key for the global profile parameter
-	MachineProfile = "profile"
+	ProfileName = "profile"
 	// ShowDriverDeprecationNotification is the key for ShowDriverDeprecationNotification
 	ShowDriverDeprecationNotification = "ShowDriverDeprecationNotification"
 	// ShowBootstrapperDeprecationNotification is the key for ShowBootstrapperDeprecationNotification

pkg/minikube/config/config_test.go

@@ -18,6 +18,7 @@ package config
 
 import (
 	"bytes"
+	"fmt"
 	"io/ioutil"
 	"os"
 	"reflect"
@@ -43,13 +44,13 @@ var configTestCases = []configTestCase{
     "ReminderWaitPeriodInHours": 99,
     "cpus": 4,
     "disk-size": "20g",
+    "driver": "test-driver",
     "log_dir": "/etc/hosts",
     "show-libmachine-logs": true,
-    "v": 5,
+    "v": 5
-    "vm-driver": "test-driver"
 }`,
 		config: map[string]interface{}{
-			"vm-driver":                 "test-driver",
+			"driver":                    "test-driver",
 			"cpus":                      4,
 			"disk-size":                 "20g",
 			"v":                         5,
@@ -130,7 +131,7 @@ func TestReadConfig(t *testing.T) {
 	}
 
 	expectedConfig := map[string]interface{}{
-		"vm-driver":            "test-driver",
+		"driver":               "test-driver",
 		"cpus":                 4,
 		"disk-size":            "20g",
 		"show-libmachine-logs": true,
@@ -149,7 +150,7 @@ func TestWriteConfig(t *testing.T) {
 	}
 
 	cfg := map[string]interface{}{
-		"vm-driver":            "test-driver",
+		"driver":               "test-driver",
 		"cpus":                 4,
 		"disk-size":            "20g",
 		"show-libmachine-logs": true,
@@ -179,6 +180,8 @@ func TestEncode(t *testing.T) {
 		if err != nil {
 			t.Errorf("Error encoding: %v", err)
 		}
+		fmt.Printf("%+v\n", b.String())
+		fmt.Printf("%+v\n", tt.data)
 		if b.String() != tt.data {
 			t.Errorf("Did not write config correctly, \n\n expected:\n %+v \n\n actual:\n %+v", tt.data, b.String())
 		}

pkg/minikube/config/profile.go

@@ -105,7 +105,7 @@ func SaveNode(cfg *ClusterConfig, node *Node) error {
 	if !update {
 		cfg.Nodes = append(cfg.Nodes, *node)
 	}
-	return SaveProfile(viper.GetString(MachineProfile), cfg)
+	return SaveProfile(viper.GetString(ProfileName), cfg)
 }
 
 // SaveProfile creates an profile out of the cfg and stores in $MINIKUBE_HOME/profiles/<profilename>/config.json

pkg/minikube/config/testdata/.minikube/config/valid_config.json

@@ -1,5 +1,5 @@
 {
-  "vm-driver": "kvm2",
+  "driver": "kvm2",
   "cpus": 4,
   "disk-size": "20g",
   "show-libmachine-logs": true,

pkg/minikube/config/types.go

@@ -20,7 +20,6 @@ import (
 	"net"
 
 	"github.com/blang/semver"
-	"k8s.io/minikube/pkg/util"
 )
 
 // Profile represents a minikube profile
@@ -49,13 +48,12 @@ type ClusterConfig struct {
 	HypervVirtualSwitch     string
 	HypervUseExternalSwitch bool
 	HypervExternalAdapter   string
-	KVMNetwork              string             // Only used by the KVM driver
+	KVMNetwork              string   // Only used by the KVM driver
-	KVMQemuURI              string             // Only used by kvm2
+	KVMQemuURI              string   // Only used by kvm2
-	KVMGPU                  bool               // Only used by kvm2
+	KVMGPU                  bool     // Only used by kvm2
-	KVMHidden               bool               // Only used by kvm2
+	KVMHidden               bool     // Only used by kvm2
-	Downloader              util.ISODownloader `json:"-"`
+	DockerOpt               []string // Each entry is formatted as KEY=VALUE.
-	DockerOpt               []string           // Each entry is formatted as KEY=VALUE.
+	DisableDriverMounts     bool     // Only used by virtualbox
-	DisableDriverMounts     bool               // Only used by virtualbox
 	NFSShare                []string
 	NFSSharesRoot           string
 	UUID                    string // Only used by hyperkit to restore the mac address

pkg/minikube/constants/constants.go

@@ -17,13 +17,11 @@ limitations under the License.
 package constants
 
 import (
-	"fmt"
 	"path/filepath"
 
 	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/client-go/util/homedir"
 	"k8s.io/minikube/pkg/minikube/localpath"
-	minikubeVersion "k8s.io/minikube/pkg/version"
 )
 
 const (
@@ -33,11 +31,8 @@ const (
 	NewestKubernetesVersion = "v1.17.3"
 	// OldestKubernetesVersion is the oldest Kubernetes version to test against
 	OldestKubernetesVersion = "v1.11.10"
-	// DefaultMachineName is the default name for the VM
+	// DefaultClusterName is the default nane for the k8s cluster
-	DefaultMachineName = "minikube"
+	DefaultClusterName = "minikube"
-	// DefaultNodeName is the default name for the kubeadm node within the VM
-	DefaultNodeName = "minikube"
-
 	// DockerDaemonPort is the port Docker daemon listening inside a minikube node (vm or container).
 	DockerDaemonPort = 2376
 	// APIServerPort is the default API server port
@@ -75,10 +70,6 @@ var (
 
 	// SHASuffix is the suffix of a SHA-256 checksum file
 	SHASuffix = ".sha256"
-	// DefaultISOURL is the default location of the minikube.iso file
-	DefaultISOURL = fmt.Sprintf("https://storage.googleapis.com/%s/minikube-%s.iso", minikubeVersion.GetISOPath(), minikubeVersion.GetISOVersion())
-	// DefaultISOSHAURL is the default location of the minikube.iso.sha256 file
-	DefaultISOSHAURL = DefaultISOURL + SHASuffix
 
 	// DockerDaemonEnvs is list of docker-daemon related environment variables.
 	DockerDaemonEnvs = [3]string{DockerHostEnv, DockerTLSVerifyEnv, DockerCertPathEnv}

pkg/minikube/download/binary.go

@@ -0,0 +1,83 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package download
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"runtime"
+
+	"github.com/blang/semver"
+	"github.com/golang/glog"
+	"github.com/hashicorp/go-getter"
+	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/minikube/localpath"
+)
+
+// binaryWithChecksumURL gets the location of a Kubernetes binary
+func binaryWithChecksumURL(binaryName, version, osName, archName string) (string, error) {
+	base := fmt.Sprintf("https://storage.googleapis.com/kubernetes-release/release/%s/bin/%s/%s/%s", version, osName, archName, binaryName)
+	v, err := semver.Make(version[1:])
+	if err != nil {
+		return "", err
+	}
+
+	if v.GTE(semver.MustParse("1.17.0")) {
+		return fmt.Sprintf("%s?checksum=file:%s.sha256", base, base), nil
+	}
+	return fmt.Sprintf("%s?checksum=file:%s.sha1", base, base), nil
+}
+
+// Binary will download a binary onto the host
+func Binary(binary, version, osName, archName string) (string, error) {
+	targetDir := localpath.MakeMiniPath("cache", osName, version)
+	targetFilepath := path.Join(targetDir, binary)
+
+	url, err := binaryWithChecksumURL(binary, version, osName, archName)
+	if err != nil {
+		return "", err
+	}
+
+	if _, err := os.Stat(targetFilepath); err == nil {
+		glog.Infof("Not caching binary, using %s", url)
+		return targetFilepath, nil
+	}
+
+	if err = os.MkdirAll(targetDir, 0777); err != nil {
+		return "", errors.Wrapf(err, "mkdir %s", targetDir)
+	}
+
+	client := &getter.Client{
+		Src:     url,
+		Dst:     targetFilepath,
+		Mode:    getter.ClientModeFile,
+		Options: []getter.ClientOption{getter.WithProgress(DefaultProgressBar)},
+	}
+
+	glog.Infof("Downloading: %+v", client)
+	if err := client.Get(); err != nil {
+		return "", errors.Wrapf(err, "download failed: %s", url)
+	}
+
+	if osName == runtime.GOOS && archName == runtime.GOARCH {
+		if err = os.Chmod(targetFilepath, 0755); err != nil {
+			return "", errors.Wrapf(err, "chmod +x %s", targetFilepath)
+		}
+	}
+	return targetFilepath, nil
+}

pkg/minikube/download/binary_test.go

@@ -0,0 +1,117 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package download
+
+import (
+	"io/ioutil"
+	"os"
+	"runtime"
+	"testing"
+)
+
+func TestCacheBinary(t *testing.T) {
+	oldMinikubeHome := os.Getenv("MINIKUBE_HOME")
+	defer os.Setenv("MINIKUBE_HOME", oldMinikubeHome)
+
+	minikubeHome, err := ioutil.TempDir("/tmp", "")
+	if err != nil {
+		t.Fatalf("error during creating tmp dir: %v", err)
+	}
+	defer os.RemoveAll(minikubeHome)
+	noWritePermDir, err := ioutil.TempDir("/tmp", "")
+	if err != nil {
+		t.Fatalf("error during creating tmp dir: %v", err)
+	}
+	defer os.RemoveAll(noWritePermDir)
+	err = os.Chmod(noWritePermDir, 0000)
+	if err != nil {
+		t.Fatalf("error (%v) during changing permissions of dir %v", err, noWritePermDir)
+	}
+
+	var tc = []struct {
+		desc, version, osName, archName   string
+		minikubeHome, binary, description string
+		err                               bool
+	}{
+		{
+			desc:         "ok kubeadm",
+			version:      "v1.16.0",
+			osName:       "linux",
+			archName:     runtime.GOARCH,
+			binary:       "kubeadm",
+			err:          false,
+			minikubeHome: minikubeHome,
+		},
+		{
+			desc:         "minikube home in dir without perms and arm runtime",
+			version:      "v1.16.0",
+			osName:       runtime.GOOS,
+			archName:     "arm",
+			binary:       "kubectl",
+			err:          true,
+			minikubeHome: noWritePermDir,
+		},
+		{
+			desc:         "minikube home in dir without perms",
+			version:      "v1.16.0",
+			osName:       runtime.GOOS,
+			archName:     runtime.GOARCH,
+			binary:       "kubectl",
+			err:          true,
+			minikubeHome: noWritePermDir,
+		},
+		{
+			desc:         "binary foo",
+			version:      "v1.16.0",
+			osName:       runtime.GOOS,
+			archName:     runtime.GOARCH,
+			binary:       "foo",
+			err:          true,
+			minikubeHome: minikubeHome,
+		},
+		{
+			desc:         "version 9000",
+			version:      "v9000",
+			osName:       runtime.GOOS,
+			archName:     runtime.GOARCH,
+			binary:       "foo",
+			err:          true,
+			minikubeHome: minikubeHome,
+		},
+		{
+			desc:         "bad os",
+			version:      "v1.16.0",
+			osName:       "no-such-os",
+			archName:     runtime.GOARCH,
+			binary:       "kubectl",
+			err:          true,
+			minikubeHome: minikubeHome,
+		},
+	}
+	for _, test := range tc {
+		t.Run(test.desc, func(t *testing.T) {
+			os.Setenv("MINIKUBE_HOME", test.minikubeHome)
+			_, err := Binary(test.binary, test.version, test.osName, test.archName)
+			if err != nil && !test.err {
+				t.Fatalf("Got unexpected error %v", err)
+			}
+			if err == nil && test.err {
+				t.Fatalf("Expected error but got %v", err)
+			}
+		})
+	}
+}

pkg/minikube/download/driver.go

@@ -0,0 +1,53 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package download
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/blang/semver"
+	"github.com/golang/glog"
+	"github.com/hashicorp/go-getter"
+	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/minikube/out"
+)
+
+func driverWithChecksumURL(name string, v semver.Version) string {
+	base := fmt.Sprintf("https://github.com/kubernetes/minikube/releases/download/v%s/%s", v, name)
+	return fmt.Sprintf("%s?checksum=file:%s.sha256", base, base)
+}
+
+// Driver downloads an arbitrary driver
+func Driver(name string, destination string, v semver.Version) error {
+	out.T(out.FileDownload, "Downloading driver {{.driver}}:", out.V{"driver": name})
+	os.Remove(destination)
+	url := driverWithChecksumURL(name, v)
+	client := &getter.Client{
+		Src:     url,
+		Dst:     destination,
+		Mode:    getter.ClientModeFile,
+		Options: []getter.ClientOption{getter.WithProgress(DefaultProgressBar)},
+	}
+
+	glog.Infof("Downloading: %+v", client)
+	if err := client.Get(); err != nil {
+		return errors.Wrapf(err, "download failed: %s", url)
+	}
+	// Give downloaded drivers a baseline decent file permission
+	return os.Chmod(destination, 0755)
+}

pkg/minikube/download/iso.go

@@ -0,0 +1,149 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package download
+
+import (
+	"fmt"
+	"net/url"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/golang/glog"
+	"github.com/hashicorp/go-getter"
+	"github.com/juju/mutex"
+	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/minikube/localpath"
+	"k8s.io/minikube/pkg/minikube/out"
+	"k8s.io/minikube/pkg/util/lock"
+	"k8s.io/minikube/pkg/version"
+)
+
+const fileScheme = "file"
+
+// DefaultISOURLs returns a list of ISO URL's to consult by default, in priority order
+func DefaultISOURLs() []string {
+	v := version.GetISOVersion()
+	return []string{
+		fmt.Sprintf("https://storage.googleapis.com/minikube/iso/minikube-%s.iso", v),
+		fmt.Sprintf("https://github.com/kubernetes/minikube/releases/download/%s/minikube-%s.iso", v, v),
+		fmt.Sprintf("https://kubernetes.oss-cn-hangzhou.aliyuncs.com/minikube/iso/minikube-%s.iso", v),
+	}
+}
+
+// LocalISOResource returns a local file:// URI equivalent for a local or remote ISO path
+func LocalISOResource(isoURL string) string {
+	u, err := url.Parse(isoURL)
+	if err != nil {
+		fake := "file://" + filepath.ToSlash(isoURL)
+		glog.Errorf("%s is not a URL! Returning %s", isoURL, fake)
+		return fake
+	}
+
+	if u.Scheme == fileScheme {
+		return isoURL
+	}
+
+	return fileURI(localISOPath(u))
+}
+
+// fileURI returns a file:// URI for a path
+func fileURI(path string) string {
+	return "file://" + filepath.ToSlash(path)
+}
+
+// localISOPath returns where an ISO should be stored locally
+func localISOPath(u *url.URL) string {
+	if u.Scheme == fileScheme {
+		return u.String()
+	}
+
+	return filepath.Join(localpath.MiniPath(), "cache", "iso", path.Base(u.Path))
+}
+
+// ISO downloads and returns the path to the downloaded ISO
+func ISO(urls []string) (string, error) {
+	out.T(out.ISODownload, "Downloading VM boot image ...")
+	errs := map[string]string{}
+
+	for _, url := range urls {
+		err := downloadISO(url)
+		if err != nil {
+			glog.Errorf("Unable to download %s: %v", url, err)
+			errs[url] = err.Error()
+			continue
+		}
+		return url, nil
+	}
+
+	var msg strings.Builder
+	msg.WriteString("unable to cache ISO: \n")
+	for u, err := range errs {
+		msg.WriteString(fmt.Sprintf("  %s: %s\n", u, err))
+	}
+
+	return "", fmt.Errorf(msg.String())
+}
+
+// downloadISO downloads an ISO URL
+func downloadISO(isoURL string) error {
+	u, err := url.Parse(isoURL)
+	if err != nil {
+		return errors.Wrapf(err, "url.parse %q", isoURL)
+	}
+
+	// It's already downloaded
+	if u.Scheme == fileScheme {
+		return nil
+	}
+
+	// Lock before we check for existence to avoid thundering herd issues
+	dst := localISOPath(u)
+	spec := lock.PathMutexSpec(dst)
+	spec.Timeout = 10 * time.Minute
+	glog.Infof("acquiring lock: %+v", spec)
+	releaser, err := mutex.Acquire(spec)
+	if err != nil {
+		return errors.Wrapf(err, "unable to acquire lock for %+v", spec)
+	}
+	defer releaser.Release()
+
+	if _, err := os.Stat(dst); err == nil {
+		return nil
+	}
+
+	urlWithChecksum := isoURL + "?checksum=file:" + isoURL + ".sha256"
+
+	// Predictable temp destination so that resume can function
+	tmpDst := dst + ".download"
+
+	opts := []getter.ClientOption{getter.WithProgress(DefaultProgressBar)}
+	client := &getter.Client{
+		Src:     urlWithChecksum,
+		Dst:     tmpDst,
+		Mode:    getter.ClientModeFile,
+		Options: opts,
+	}
+
+	glog.Infof("full url: %s", urlWithChecksum)
+	if err := client.Get(); err != nil {
+		return errors.Wrap(err, isoURL)
+	}
+	return os.Rename(tmpDst, dst)
+}

pkg/minikube/download/preload.go

@@ -0,0 +1,185 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package download
+
+import (
+	"context"
+	"crypto/md5"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path"
+
+	"cloud.google.com/go/storage"
+	"google.golang.org/api/option"
+
+	"github.com/golang/glog"
+	"github.com/hashicorp/go-getter"
+	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/minikube/localpath"
+	"k8s.io/minikube/pkg/minikube/out"
+)
+
+const (
+	// PreloadVersion is the current version of the preloaded tarball
+	PreloadVersion = "v1"
+	// PreloadBucket is the name of the GCS bucket where preloaded volume tarballs exist
+	PreloadBucket = "minikube-preloaded-volume-tarballs"
+)
+
+// returns name of the tarball
+func tarballName(k8sVersion string) string {
+	return fmt.Sprintf("preloaded-images-k8s-%s-%s-docker-overlay2.tar.lz4", PreloadVersion, k8sVersion)
+}
+
+// returns the name of the checksum file
+func checksumName(k8sVersion string) string {
+	return fmt.Sprintf("%s.checksum", tarballName(k8sVersion))
+}
+
+// returns target dir for all cached items related to preloading
+func targetDir() string {
+	return localpath.MakeMiniPath("cache", "preloaded-tarball")
+}
+
+// PreloadChecksumPath returns path to checksum file
+func PreloadChecksumPath(k8sVersion string) string {
+	return path.Join(targetDir(), checksumName(k8sVersion))
+}
+
+// TarballPath returns the path to the preloaded tarball
+func TarballPath(k8sVersion string) string {
+	return path.Join(targetDir(), tarballName(k8sVersion))
+}
+
+// remoteTarballURL returns the URL for the remote tarball in GCS
+func remoteTarballURL(k8sVersion string) string {
+	return fmt.Sprintf("https://storage.googleapis.com/%s/%s", PreloadBucket, tarballName(k8sVersion))
+}
+
+// PreloadExists returns true if there is a preloaded tarball that can be used
+func PreloadExists(k8sVersion, containerRuntime string) bool {
+	if containerRuntime != "docker" {
+		return false
+	}
+
+	// Omit remote check if tarball exists locally
+	targetPath := TarballPath(k8sVersion)
+	if _, err := os.Stat(targetPath); err == nil {
+		if err := verifyChecksum(k8sVersion); err == nil {
+			glog.Infof("Found %s in cache, no need to check remotely", targetPath)
+			return true
+		}
+	}
+
+	url := remoteTarballURL(k8sVersion)
+	resp, err := http.Head(url)
+	if err != nil {
+		glog.Warningf("%s fetch error: %v", url, err)
+		return false
+	}
+
+	// note: err won't be set if it's a 404
+	if resp.StatusCode != 200 {
+		glog.Warningf("%s status code: %d", url, resp.StatusCode)
+		return false
+	}
+
+	glog.Infof("Goody! %s exists!", url)
+	return true
+}
+
+// Preload caches the preloaded images tarball on the host machine
+func Preload(k8sVersion, containerRuntime string) error {
+	if containerRuntime != "docker" {
+		return nil
+	}
+	targetPath := TarballPath(k8sVersion)
+
+	if _, err := os.Stat(targetPath); err == nil {
+		if err := verifyChecksum(k8sVersion); err == nil {
+			glog.Infof("Found %s in cache, skipping downloading", targetPath)
+			return nil
+		}
+	}
+
+	// Make sure we support this k8s version
+	if !PreloadExists(k8sVersion, containerRuntime) {
+		glog.Infof("Preloaded tarball for k8s version %s does not exist", k8sVersion)
+		return nil
+	}
+
+	out.T(out.FileDownload, "Downloading preloaded images tarball for k8s {{.version}} ...", out.V{"version": k8sVersion})
+	url := remoteTarballURL(k8sVersion)
+	client := &getter.Client{
+		Src:     url,
+		Dst:     targetPath,
+		Mode:    getter.ClientModeFile,
+		Options: []getter.ClientOption{getter.WithProgress(DefaultProgressBar)},
+	}
+
+	glog.Infof("Downloading: %+v", client)
+	if err := client.Get(); err != nil {
+		return errors.Wrapf(err, "download failed: %s", url)
+	}
+	// Give downloaded drivers a baseline decent file permission
+	if err := os.Chmod(targetPath, 0755); err != nil {
+		return err
+	}
+	// Save checksum file locally
+	if err := saveChecksumFile(k8sVersion); err != nil {
+		return errors.Wrap(err, "saving checksum file")
+	}
+	return verifyChecksum(k8sVersion)
+}
+
+func saveChecksumFile(k8sVersion string) error {
+	ctx := context.Background()
+	client, err := storage.NewClient(ctx, option.WithoutAuthentication())
+	if err != nil {
+		return errors.Wrap(err, "getting storage client")
+	}
+	attrs, err := client.Bucket(PreloadBucket).Object(tarballName(k8sVersion)).Attrs(ctx)
+	if err != nil {
+		return errors.Wrap(err, "getting storage object")
+	}
+	checksum := attrs.MD5
+	return ioutil.WriteFile(PreloadChecksumPath(k8sVersion), checksum, 0644)
+}
+
+// verifyChecksum returns true if the checksum of the local binary matches
+// the checksum of the remote binary
+func verifyChecksum(k8sVersion string) error {
+	// get md5 checksum of tarball path
+	contents, err := ioutil.ReadFile(TarballPath(k8sVersion))
+	if err != nil {
+		return errors.Wrap(err, "reading tarball")
+	}
+	checksum := md5.Sum(contents)
+
+	remoteChecksum, err := ioutil.ReadFile(PreloadChecksumPath(k8sVersion))
+	if err != nil {
+		return errors.Wrap(err, "reading checksum file")
+	}
+
+	// create a slice of checksum, which is [16]byte
+	if string(remoteChecksum) != string(checksum[:]) {
+		return fmt.Errorf("checksum of %s does not match remote checksum (%s != %s)", TarballPath(k8sVersion), string(remoteChecksum), string(checksum[:]))
+	}
+	return nil
+}

pkg/minikube/download/progressbar.go

@@ -18,7 +18,7 @@ limitations under the License.
 // based on:
 // https://github.com/hashicorp/go-getter/blob/master/cmd/go-getter/progress_tracking.go
 
-package util
+package download
 
 import (
 	"io"

pkg/minikube/driver/driver.go

@@ -24,6 +24,7 @@ import (
 
 	"github.com/golang/glog"
 	"k8s.io/minikube/pkg/drivers/kic"
+	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/registry"
 )
 
@@ -85,7 +86,21 @@ func Supported(name string) bool {
 	return false
 }
 
-// IsKIC checks if the driver is a kubernetes in continer
+// MachineType returns appropriate machine name for the driver
+func MachineType(name string) string {
+	if IsKIC(name) {
+		return "container"
+	}
+
+	if IsVM(name) {
+		return "VM"
+	}
+
+	// none or mock
+	return "bare metal machine"
+}
+
+// IsKIC checks if the driver is a kubernetes in container
 func IsKIC(name string) bool {
 	return name == Docker || name == Podman
 }
@@ -108,11 +123,6 @@ func BareMetal(name string) bool {
 	return name == None || name == Mock
 }
 
-// MachineName return the name of the machine given proper config
-func MachineName(cluster string, node string) string {
-	return fmt.Sprintf("%s-%s", cluster, node)
-}
-
 // NeedsRoot returns true if driver needs to run with root privileges
 func NeedsRoot(name string) bool {
 	return name == None || name == Podman
@@ -217,3 +227,12 @@ func SetLibvirtURI(v string) {
 	os.Setenv("LIBVIRT_DEFAULT_URI", v)
 
 }
+
+// MachineName returns the name of the machine, as seen by the hypervisor given the cluster and node names
+func MachineName(cc config.ClusterConfig, n config.Node) string {
+	// For single node cluster, default to back to old naming
+	if len(cc.Nodes) == 1 || n.ControlPlane {
+		return cc.Name
+	}
+	return fmt.Sprintf("%s-%s", cc.Name, n.Name)
+}

pkg/minikube/driver/driver_test.go

@@ -58,6 +58,31 @@ func TestBareMetal(t *testing.T) {
 	}
 }
 
+func TestMachineType(t *testing.T) {
+	types := map[string]string{
+		Podman:       "container",
+		Docker:       "container",
+		Mock:         "bare metal machine",
+		None:         "bare metal machine",
+		KVM2:         "VM",
+		VirtualBox:   "VM",
+		HyperKit:     "VM",
+		VMware:       "VM",
+		VMwareFusion: "VM",
+		HyperV:       "VM",
+		Parallels:    "VM",
+	}
+
+	drivers := SupportedDrivers()
+	for _, driver := range drivers {
+		want := types[driver]
+		got := MachineType(driver)
+		if want != got {
+			t.Errorf("mismatched machine type for driver %s: want = %s got = %s", driver, want, got)
+		}
+	}
+}
+
 func TestFlagDefaults(t *testing.T) {
 	expected := FlagHints{CacheImages: true}
 	if diff := cmp.Diff(FlagDefaults(VirtualBox), expected); diff != "" {