Reorganize functional tests

cmd/minikube/cmd/start.go

@@ -28,6 +28,7 @@ import (
 	"os/user"
 	"regexp"
 	"runtime"
+	"sort"
 	"strconv"
 	"strings"
 
@@ -215,7 +216,7 @@ func runStart(cmd *cobra.Command, args []string) {
 			// Walk down the rest of the options
 			for _, alt := range alts {
 				// Skip non-default drivers
-				if !ds.Default {
+				if !alt.Default {
 					continue
 				}
 				out.WarningT("Startup with {{.old_driver}} driver failed, trying with alternate driver {{.new_driver}}: {{.error}}", out.V{"old_driver": ds.Name, "new_driver": alt.Name, "error": err})
@@ -589,7 +590,16 @@ func selectDriver(existing *config.ClusterConfig) (registry.DriverState, []regis
 	pick, alts, rejects := driver.Suggest(choices)
 	if pick.Name == "" {
 		out.Step(style.ThumbsDown, "Unable to pick a default driver. Here is what was considered, in preference order:")
+		sort.Slice(rejects, func(i, j int) bool {
+			if rejects[i].Priority == rejects[j].Priority {
+				return rejects[i].Preference > rejects[j].Preference
+			}
+			return rejects[i].Priority > rejects[j].Priority
+		})
 		for _, r := range rejects {
+			if !r.Default {
+				continue
+			}
 			out.Infof("{{ .name }}: {{ .rejection }}", out.V{"name": r.Name, "rejection": r.Rejection})
 			if r.Suggestion != "" {
 				out.Infof("{{ .name }}: Suggestion: {{ .suggestion}}", out.V{"name": r.Name, "suggestion": r.Suggestion})

cmd/minikube/cmd/unpause.go

@@ -39,8 +39,9 @@ import (
 
 // unpauseCmd represents the docker-pause command
 var unpauseCmd = &cobra.Command{
-	Use:   "unpause",
+	Use:     "unpause",
-	Short: "unpause Kubernetes",
+	Aliases: []string{"resume"},
+	Short:   "unpause Kubernetes",
 	Run: func(cmd *cobra.Command, args []string) {
 		cname := ClusterFlagValue()
 		register.SetEventLogPath(localpath.EventLog(cname))

deploy/addons/auto-pause/haproxy.cfg.tmpl

@@ -28,10 +28,10 @@ backend k8s-api-https
     #tcp-request inspect-delay 10s
     #tcp-request content lua.foo_action
     tcp-request inspect-delay 10s
-    tcp-request content lua.unpause 192.168.49.2 8080
+    tcp-request content lua.unpause {{.NetworkInfo.ControlPlaneNodeIP}} 8080
     tcp-request content reject if { var(req.blocked) -m bool }
     option tcplog
     option tcp-check
     default-server inter 10s downinter 5s rise 2 fall 2 slowstart 60s maxconn 250 maxqueue 256 weight 100
-    server k8s-api-1 192.168.49.2:8443 check
+    server k8s-api-1 {{.NetworkInfo.ControlPlaneNodeIP}}:{{.NetworkInfo.ControlPlaneNodePort}} check
 

hack/jenkins/installers/check_install_golang.sh

@@ -25,7 +25,22 @@ fi
 VERSION_TO_INSTALL=${1}
 INSTALL_PATH=${2}
 
-ARCH=${ARCH:=amd64}
+function current_arch() {
+  case $(arch) in
+  "x86_64")
+     echo "amd64"
+  ;;
+  "aarch64")
+    echo "arm64"
+  ;;
+  *)
+    echo "unexpected arch: $(arch). use amd64" 1>&2
+    echo "amd64"
+  ;;
+  esac
+}
+
+ARCH=${ARCH:=$(current_arch)}
 
 # installs or updates golang if right version doesn't exists
 function check_and_install_golang() {
@@ -62,7 +77,7 @@ function install_golang() {
   # using sudo because previously installed versions might have been installed by a different user.
   # as it was the case on jenkins VM.
   sudo curl -qL -O "https://storage.googleapis.com/golang/go${1}.${INSTALLOS}-${ARCH}.tar.gz" &&
-    sudo tar -xzf go${1}.${INSTALLOS}-amd64.tar.gz &&
+    sudo tar -xzf go${1}.${INSTALLOS}-${ARCH}.tar.gz &&
     sudo rm -rf "${2}/go" &&
     sudo mv go "${2}/" && sudo chown -R $(whoami): ${2}/go
   popd >/dev/null

netlify.toml

@@ -4,7 +4,7 @@ publish = "site/public/"
 command = "pwd && cd themes/docsy && git submodule update -f --init && cd ../.. && hugo"
 
 [build.environment]
-HUGO_VERSION = "0.68.3"
+HUGO_VERSION = "0.83.1"
 
 [context.production.environment]
 HUGO_ENV = "production"

pkg/addons/addons.go

@@ -185,6 +185,12 @@ https://github.com/kubernetes/minikube/issues/7332`, out.V{"driver_name": cc.Dri
 		exit.Error(reason.GuestCpConfig, "Error getting primary control plane", err)
 	}
 
+	// Persist images even if the machine is running so starting gets the correct images.
+	images, customRegistries, err := assets.SelectAndPersistImages(addon, cc)
+	if err != nil {
+		exit.Error(reason.HostSaveProfile, "Failed to persist images", err)
+	}
+
 	mName := config.MachineName(*cc, cp)
 	host, err := machine.LoadHost(api, mName)
 	if err != nil || !machine.IsRunning(api, mName) {
@@ -219,11 +225,12 @@ https://github.com/kubernetes/minikube/issues/7332`, out.V{"driver_name": cc.Dri
 	var networkInfo assets.NetworkInfo
 	if len(cc.Nodes) >= 1 {
 		networkInfo.ControlPlaneNodeIP = cc.Nodes[0].IP
+		networkInfo.ControlPlaneNodePort = cc.Nodes[0].Port
 	} else {
 		out.WarningT("At least needs control plane nodes to enable addon")
 	}
 
-	data := assets.GenerateTemplateData(addon, cc.KubernetesConfig, networkInfo)
+	data := assets.GenerateTemplateData(addon, cc.KubernetesConfig, networkInfo, images, customRegistries)
 	return enableOrDisableAddonInternal(cc, addon, runner, data, enable)
 }
 

pkg/minikube/assets/addons.go

@@ -42,7 +42,8 @@ type Addon struct {
 
 // NetworkInfo contains control plane node IP address used for add on template
 type NetworkInfo struct {
-	ControlPlaneNodeIP string
+	ControlPlaneNodeIP   string
+	ControlPlaneNodePort int
 }
 
 // NewAddon creates a new Addon
@@ -88,13 +89,13 @@ var Addons = map[string]*Addon{
 			"auto-pause-hook.yaml",
 			"0640"),
 		MustBinAsset(
-			"deploy/addons/auto-pause/haproxy.cfg",
+			"deploy/addons/auto-pause/haproxy.cfg.tmpl",
-			"/var/lib/minikube/",
+			vmpath.GuestPersistentDir,
 			"haproxy.cfg",
 			"0640"),
 		MustBinAsset(
 			"deploy/addons/auto-pause/unpause.lua",
-			"/var/lib/minikube/",
+			vmpath.GuestPersistentDir,
 			"unpause.lua",
 			"0640"),
 		MustBinAsset(
@@ -486,8 +487,8 @@ var Addons = map[string]*Addon{
 			"metallb-config.yaml",
 			"0640"),
 	}, false, "metallb", map[string]string{
-		"Speaker":    "metallb/speaker:v0.8.2@sha256:f1941498a28cdb332429e25d18233683da6949ecfc4f6dacf12b1416d7d38263",
+		"Speaker":    "metallb/speaker:v0.9.6@sha256:c66585a805bed1a3b829d8fb4a4aab9d87233497244ebff96f1b88f1e7f8f991",
-		"Controller": "metallb/controller:v0.8.2@sha256:5c050e59074e152711737d2bb9ede96dff67016c80cf25cdf5fc46109718a583",
+		"Controller": "metallb/controller:v0.9.6@sha256:fbfdb9d3f55976b0ee38f3309d83a4ca703efcf15d6ca7889cd8189142286502",
 	}, nil),
 	"ambassador": NewAddon([]*BinAsset{
 		MustBinAsset(
@@ -659,8 +660,110 @@ var Addons = map[string]*Addon{
 	}),
 }
 
+// parseMapString creates a map based on `str` which is encoded as <key1>=<value1>,<key2>=<value2>,...
+func parseMapString(str string) map[string]string {
+	mapResult := make(map[string]string)
+	if str == "" {
+		return mapResult
+	}
+	for _, pairText := range strings.Split(str, ",") {
+		vals := strings.Split(pairText, "=")
+		if len(vals) != 2 {
+			out.WarningT("Ignoring invalid pair entry {{.pair}}", out.V{"pair": pairText})
+			continue
+		}
+		mapResult[vals[0]] = vals[1]
+	}
+	return mapResult
+}
+
+// mergeMaps creates a map with the union of `sourceMap` and `overrideMap` where collisions take the value of `overrideMap`.
+func mergeMaps(sourceMap, overrideMap map[string]string) map[string]string {
+	result := make(map[string]string)
+	for name, value := range sourceMap {
+		result[name] = value
+	}
+	for name, value := range overrideMap {
+		result[name] = value
+	}
+	return result
+}
+
+// filterKeySpace creates a map of the values in `targetMap` where the keys are also in `keySpace`.
+func filterKeySpace(keySpace map[string]string, targetMap map[string]string) map[string]string {
+	result := make(map[string]string)
+	for name := range keySpace {
+		if value, ok := targetMap[name]; ok {
+			result[name] = value
+		}
+	}
+	return result
+}
+
+// overrideDefaults creates a copy of `defaultMap` where `overrideMap` replaces any of its values that `overrideMap` contains.
+func overrideDefaults(defaultMap, overrideMap map[string]string) map[string]string {
+	return mergeMaps(defaultMap, filterKeySpace(defaultMap, overrideMap))
+}
+
+// SelectAndPersistImages selects which images to use based on addon default images, previously persisted images, and newly requested images - which are then persisted for future enables.
+func SelectAndPersistImages(addon *Addon, cc *config.ClusterConfig) (images, customRegistries map[string]string, err error) {
+	addonDefaultImages := addon.Images
+	if addonDefaultImages == nil {
+		addonDefaultImages = make(map[string]string)
+	}
+
+	// Use previously configured custom images.
+	images = overrideDefaults(addonDefaultImages, cc.CustomAddonImages)
+	if viper.IsSet(config.AddonImages) {
+		// Parse the AddonImages flag if present.
+		newImages := parseMapString(viper.GetString(config.AddonImages))
+		for name, image := range newImages {
+			if image == "" {
+				out.WarningT("Ignoring empty custom image {{.name}}", out.V{"name": name})
+				delete(newImages, name)
+				continue
+			}
+			if _, ok := addonDefaultImages[name]; !ok {
+				out.WarningT("Ignoring unknown custom image {{.name}}", out.V{"name": name})
+			}
+		}
+		// Use newly configured custom images.
+		images = overrideDefaults(addonDefaultImages, newImages)
+		// Store custom addon images to be written.
+		cc.CustomAddonImages = mergeMaps(cc.CustomAddonImages, images)
+	}
+
+	// Use previously configured custom registries.
+	customRegistries = filterKeySpace(addonDefaultImages, cc.CustomAddonRegistries) // filter by images map because registry map may omit default registry.
+	if viper.IsSet(config.AddonRegistries) {
+		// Parse the AddonRegistries flag if present.
+		customRegistries = parseMapString(viper.GetString(config.AddonRegistries))
+		for name := range customRegistries {
+			if _, ok := addonDefaultImages[name]; !ok { // check images map because registry map may omitted default registry
+				out.WarningT("Ignoring unknown custom registry {{.name}}", out.V{"name": name})
+				delete(customRegistries, name)
+			}
+		}
+		// Since registry map may omit default registry, any previously set custom registries for these images must be cleared.
+		for name := range addonDefaultImages {
+			delete(cc.CustomAddonRegistries, name)
+		}
+		// Merge newly set registries into custom addon registries to be written.
+		cc.CustomAddonRegistries = mergeMaps(cc.CustomAddonRegistries, customRegistries)
+	}
+
+	err = nil
+	// If images or registries were specified, save the config afterward.
+	if viper.IsSet(config.AddonImages) || viper.IsSet(config.AddonRegistries) {
+		// Since these values are only set when a user enables an addon, it is safe to refer to the profile name.
+		err = config.Write(viper.GetString(config.ProfileName), cc)
+		// Whether err is nil or not we still return here.
+	}
+	return images, customRegistries, err
+}
+
 // GenerateTemplateData generates template data for template assets
-func GenerateTemplateData(addon *Addon, cfg config.KubernetesConfig, networkInfo NetworkInfo) interface{} {
+func GenerateTemplateData(addon *Addon, cfg config.KubernetesConfig, netInfo NetworkInfo, images, customRegistries map[string]string) interface{} {
 
 	a := runtime.GOARCH
 	// Some legacy docker images still need the -arch suffix
@@ -669,6 +772,7 @@ func GenerateTemplateData(addon *Addon, cfg config.KubernetesConfig, networkInfo
 	if runtime.GOARCH != "amd64" {
 		ea = "-" + runtime.GOARCH
 	}
+
 	opts := struct {
 		Arch                string
 		ExoticArch          string
@@ -687,57 +791,21 @@ func GenerateTemplateData(addon *Addon, cfg config.KubernetesConfig, networkInfo
 		LoadBalancerStartIP: cfg.LoadBalancerStartIP,
 		LoadBalancerEndIP:   cfg.LoadBalancerEndIP,
 		CustomIngressCert:   cfg.CustomIngressCert,
-		Images:              addon.Images,
+		Images:              images,
 		Registries:          addon.Registries,
-		CustomRegistries:    make(map[string]string),
+		CustomRegistries:    customRegistries,
 		NetworkInfo:         make(map[string]string),
 	}
 	if opts.ImageRepository != "" && !strings.HasSuffix(opts.ImageRepository, "/") {
 		opts.ImageRepository += "/"
 	}
-
-	// Network info for generating template
-	opts.NetworkInfo["ControlPlaneNodeIP"] = networkInfo.ControlPlaneNodeIP
-
-	if opts.Images == nil {
-		opts.Images = make(map[string]string) // Avoid nil access when rendering
-	}
-
-	images := viper.GetString(config.AddonImages)
-	if images != "" {
-		for _, image := range strings.Split(images, ",") {
-			vals := strings.Split(image, "=")
-			if len(vals) != 2 || vals[1] == "" {
-				out.WarningT("Ignoring invalid custom image {{.conf}}", out.V{"conf": image})
-				continue
-			}
-			if _, ok := opts.Images[vals[0]]; ok {
-				opts.Images[vals[0]] = vals[1]
-			} else {
-				out.WarningT("Ignoring unknown custom image {{.name}}", out.V{"name": vals[0]})
-			}
-		}
-	}
-
 	if opts.Registries == nil {
 		opts.Registries = make(map[string]string)
 	}
 
-	registries := viper.GetString(config.AddonRegistries)
+	// Network info for generating template
-	if registries != "" {
+	opts.NetworkInfo["ControlPlaneNodeIP"] = netInfo.ControlPlaneNodeIP
-		for _, registry := range strings.Split(registries, ",") {
+	opts.NetworkInfo["ControlPlaneNodePort"] = fmt.Sprint(netInfo.ControlPlaneNodePort)
-			vals := strings.Split(registry, "=")
-			if len(vals) != 2 {
-				out.WarningT("Ignoring invalid custom registry {{.conf}}", out.V{"conf": registry})
-				continue
-			}
-			if _, ok := opts.Images[vals[0]]; ok { // check images map because registry map may omitted default registry
-				opts.CustomRegistries[vals[0]] = vals[1]
-			} else {
-				out.WarningT("Ignoring unknown custom registry {{.name}}", out.V{"name": vals[0]})
-			}
-		}
-	}
 
 	// Append postfix "/" to registries
 	for k, v := range opts.Registries {

pkg/minikube/assets/addons_test.go

@@ -0,0 +1,144 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package assets
+
+import "testing"
+
+// mapsEqual returns true if and only if `a` contains all the same pairs as `b`.
+func mapsEqual(a, b map[string]string) bool {
+	for aKey, aValue := range a {
+		if bValue, ok := b[aKey]; !ok || aValue != bValue {
+			return false
+		}
+	}
+
+	for bKey := range b {
+		if _, ok := a[bKey]; !ok {
+			return false
+		}
+	}
+	return true
+}
+
+func TestParseMapString(t *testing.T) {
+	cases := map[string]map[string]string{
+		"Ardvark=1,B=2,Cantaloupe=3":         {"Ardvark": "1", "B": "2", "Cantaloupe": "3"},
+		"A=,B=2,C=":                          {"A": "", "B": "2", "C": ""},
+		"":                                   {},
+		"malformed,good=howdy,manyequals==,": {"good": "howdy"},
+	}
+	for actual, expected := range cases {
+		if parsedMap := parseMapString(actual); !mapsEqual(parsedMap, expected) {
+			t.Errorf("Parsed map from string \"%s\" differs from expected map: Actual: %v Expected: %v", actual, parsedMap, expected)
+		}
+	}
+}
+
+func TestMergeMaps(t *testing.T) {
+	type TestCase struct {
+		sourceMap   map[string]string
+		overrideMap map[string]string
+		expectedMap map[string]string
+	}
+	cases := []TestCase{
+		{
+			sourceMap:   map[string]string{"A": "1", "B": "2"},
+			overrideMap: map[string]string{"B": "7", "C": "3"},
+			expectedMap: map[string]string{"A": "1", "B": "7", "C": "3"},
+		},
+		{
+			sourceMap:   map[string]string{"B": "7", "C": "3"},
+			overrideMap: map[string]string{"A": "1", "B": "2"},
+			expectedMap: map[string]string{"A": "1", "B": "2", "C": "3"},
+		},
+		{
+			sourceMap:   map[string]string{"B": "7", "C": "3"},
+			overrideMap: map[string]string{},
+			expectedMap: map[string]string{"B": "7", "C": "3"},
+		},
+		{
+			sourceMap:   map[string]string{},
+			overrideMap: map[string]string{"B": "7", "C": "3"},
+			expectedMap: map[string]string{"B": "7", "C": "3"},
+		},
+	}
+	for _, test := range cases {
+		if actualMap := mergeMaps(test.sourceMap, test.overrideMap); !mapsEqual(actualMap, test.expectedMap) {
+			t.Errorf("Merging maps (source=%v, override=%v) differs from expected map: Actual: %v Expected: %v", test.sourceMap, test.overrideMap, actualMap, test.expectedMap)
+		}
+	}
+}
+
+func TestFilterKeySpace(t *testing.T) {
+	type TestCase struct {
+		keySpace    map[string]string
+		targetMap   map[string]string
+		expectedMap map[string]string
+	}
+	cases := []TestCase{
+		{
+			keySpace:    map[string]string{"A": "0", "B": ""},
+			targetMap:   map[string]string{"B": "1", "C": "2", "D": "3"},
+			expectedMap: map[string]string{"B": "1"},
+		},
+		{
+			keySpace:    map[string]string{},
+			targetMap:   map[string]string{"B": "1", "C": "2", "D": "3"},
+			expectedMap: map[string]string{},
+		},
+		{
+			keySpace:    map[string]string{"B": "1", "C": "2", "D": "3"},
+			targetMap:   map[string]string{},
+			expectedMap: map[string]string{},
+		},
+	}
+	for _, test := range cases {
+		if actualMap := filterKeySpace(test.keySpace, test.targetMap); !mapsEqual(actualMap, test.expectedMap) {
+			t.Errorf("Filtering keyspace of map (keyspace=%v, target=%v) differs from expected map: Actual: %v Expected: %v", test.keySpace, test.targetMap, actualMap, test.expectedMap)
+		}
+	}
+}
+
+func TestOverrideDefautls(t *testing.T) {
+	type TestCase struct {
+		defaultMap  map[string]string
+		overrideMap map[string]string
+		expectedMap map[string]string
+	}
+	cases := []TestCase{
+		{
+			defaultMap:  map[string]string{"A": "1", "B": "2", "C": "3"},
+			overrideMap: map[string]string{"B": "7", "C": "8"},
+			expectedMap: map[string]string{"A": "1", "B": "7", "C": "8"},
+		},
+		{
+			defaultMap:  map[string]string{"A": "1", "B": "2", "C": "3"},
+			overrideMap: map[string]string{"B": "7", "D": "8", "E": "9"},
+			expectedMap: map[string]string{"A": "1", "B": "7", "C": "3"},
+		},
+		{
+			defaultMap:  map[string]string{"A": "1", "B": "2", "C": "3"},
+			overrideMap: map[string]string{"B": "7", "D": "8", "E": "9"},
+			expectedMap: map[string]string{"A": "1", "B": "7", "C": "3"},
+		},
+	}
+	for _, test := range cases {
+		if actualMap := overrideDefaults(test.defaultMap, test.overrideMap); !mapsEqual(actualMap, test.expectedMap) {
+			t.Errorf("Override defaults (defaults=%v, overrides=%v) differs from expected map: Actual: %v Expected: %v", test.defaultMap, test.overrideMap, actualMap, test.expectedMap)
+		}
+	}
+}

pkg/minikube/bootstrapper/bsutil/kubeadm_test.go

@@ -123,23 +123,14 @@ func recentReleases(n int) ([]string, error) {
 }
 
 /**
-Need a separate test function to test the DNS server IP
-as v1.11 yaml file is very different compared to v1.12+.
 This test case has only 1 thing to test and that is the
 networking/dnsDomain value
 */
 func TestGenerateKubeadmYAMLDNS(t *testing.T) {
-	// test all testdata releases greater than v1.11
 	versions, err := recentReleases(0)
 	if err != nil {
 		t.Errorf("versions: %v", err)
 	}
-	for i, v := range versions {
-		if semver.Compare(v, "v1.11") <= 0 {
-			versions = versions[0:i]
-			break
-		}
-	}
 	fcr := command.NewFakeCommandRunner()
 	fcr.SetCommandToOutput(map[string]string{
 		"docker info --format {{.CgroupDriver}}": "systemd\n",

pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd-api-port.yaml

@@ -1,22 +0,0 @@
-apiVersion: kubeadm.k8s.io/v1alpha1
-kind: MasterConfiguration
-noTaintMaster: true
-api:
-  advertiseAddress: 1.1.1.1
-  bindPort: 12345
-  controlPlaneEndpoint: control-plane.minikube.internal
-kubernetesVersion: v1.11.0
-certificatesDir: /var/lib/minikube/certs
-networking:
-  serviceSubnet: 10.96.0.0/12
-etcd:
-  dataDir: /var/lib/minikube/etcd
-controllerManagerExtraArgs:
-  leader-elect: "false"
-schedulerExtraArgs:
-  leader-elect: "false"
-nodeName: "mk"
-apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
-criSocket: /run/containerd/containerd.sock
-apiServerExtraArgs:
-  enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"

pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd-pod-network-cidr.yaml

@@ -1,22 +0,0 @@
-apiVersion: kubeadm.k8s.io/v1alpha1
-kind: MasterConfiguration
-noTaintMaster: true
-api:
-  advertiseAddress: 1.1.1.1
-  bindPort: 8443
-  controlPlaneEndpoint: control-plane.minikube.internal
-kubernetesVersion: v1.11.0
-certificatesDir: /var/lib/minikube/certs
-networking:
-  serviceSubnet: 10.96.0.0/12
-etcd:
-  dataDir: /var/lib/minikube/etcd
-controllerManagerExtraArgs:
-  leader-elect: "false"
-schedulerExtraArgs:
-  leader-elect: "false"
-nodeName: "mk"
-apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
-criSocket: /run/containerd/containerd.sock
-apiServerExtraArgs:
-  enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"

pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd.yaml

@@ -1,22 +0,0 @@
-apiVersion: kubeadm.k8s.io/v1alpha1
-kind: MasterConfiguration
-noTaintMaster: true
-api:
-  advertiseAddress: 1.1.1.1
-  bindPort: 8443
-  controlPlaneEndpoint: control-plane.minikube.internal
-kubernetesVersion: v1.11.0
-certificatesDir: /var/lib/minikube/certs
-networking:
-  serviceSubnet: 10.96.0.0/12
-etcd:
-  dataDir: /var/lib/minikube/etcd
-controllerManagerExtraArgs:
-  leader-elect: "false"
-schedulerExtraArgs:
-  leader-elect: "false"
-nodeName: "mk"
-apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
-criSocket: /run/containerd/containerd.sock
-apiServerExtraArgs:
-  enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"

pkg/minikube/bootstrapper/bsutil/testdata/v1.11/crio-options-gates.yaml

@@ -1,30 +0,0 @@
-apiVersion: kubeadm.k8s.io/v1alpha1
-kind: MasterConfiguration
-noTaintMaster: true
-api:
-  advertiseAddress: 1.1.1.1
-  bindPort: 8443
-  controlPlaneEndpoint: control-plane.minikube.internal
-kubernetesVersion: v1.11.0
-certificatesDir: /var/lib/minikube/certs
-networking:
-  serviceSubnet: 10.96.0.0/12
-etcd:
-  dataDir: /var/lib/minikube/etcd
-controllerManagerExtraArgs:
-  leader-elect: "false"
-schedulerExtraArgs:
-  leader-elect: "false"
-nodeName: "mk"
-apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
-criSocket: /var/run/crio/crio.sock
-apiServerExtraArgs:
-  enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
-  fail-no-swap: "true"
-  feature-gates: "a=b"
-controllerManagerExtraArgs:
-  feature-gates: "a=b"
-  kube-api-burst: "32"
-schedulerExtraArgs:
-  feature-gates: "a=b"
-  scheduler-name: "mini-scheduler"

pkg/minikube/bootstrapper/bsutil/testdata/v1.11/crio.yaml

@@ -1,22 +0,0 @@
-apiVersion: kubeadm.k8s.io/v1alpha1
-kind: MasterConfiguration
-noTaintMaster: true
-api:
-  advertiseAddress: 1.1.1.1
-  bindPort: 8443
-  controlPlaneEndpoint: control-plane.minikube.internal
-kubernetesVersion: v1.11.0
-certificatesDir: /var/lib/minikube/certs
-networking:
-  serviceSubnet: 10.96.0.0/12
-etcd:
-  dataDir: /var/lib/minikube/etcd
-controllerManagerExtraArgs:
-  leader-elect: "false"
-schedulerExtraArgs:
-  leader-elect: "false"
-nodeName: "mk"
-apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
-criSocket: /var/run/crio/crio.sock
-apiServerExtraArgs:
-  enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"

pkg/minikube/bootstrapper/bsutil/testdata/v1.11/default.yaml

@@ -1,21 +0,0 @@
-apiVersion: kubeadm.k8s.io/v1alpha1
-kind: MasterConfiguration
-noTaintMaster: true
-api:
-  advertiseAddress: 1.1.1.1
-  bindPort: 8443
-  controlPlaneEndpoint: control-plane.minikube.internal
-kubernetesVersion: v1.11.0
-certificatesDir: /var/lib/minikube/certs
-networking:
-  serviceSubnet: 10.96.0.0/12
-etcd:
-  dataDir: /var/lib/minikube/etcd
-controllerManagerExtraArgs:
-  leader-elect: "false"
-schedulerExtraArgs:
-  leader-elect: "false"
-nodeName: "mk"
-apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
-apiServerExtraArgs:
-  enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"

pkg/minikube/bootstrapper/bsutil/testdata/v1.11/image-repository.yaml

@@ -1,22 +0,0 @@
-apiVersion: kubeadm.k8s.io/v1alpha1
-kind: MasterConfiguration
-noTaintMaster: true
-api:
-  advertiseAddress: 1.1.1.1
-  bindPort: 8443
-  controlPlaneEndpoint: control-plane.minikube.internal
-kubernetesVersion: v1.11.0
-certificatesDir: /var/lib/minikube/certs
-networking:
-  serviceSubnet: 10.96.0.0/12
-etcd:
-  dataDir: /var/lib/minikube/etcd
-controllerManagerExtraArgs:
-  leader-elect: "false"
-schedulerExtraArgs:
-  leader-elect: "false"
-nodeName: "mk"
-apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
-imageRepository: test/repo
-apiServerExtraArgs:
-  enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"

pkg/minikube/bootstrapper/bsutil/testdata/v1.11/options.yaml

@@ -1,26 +0,0 @@
-apiVersion: kubeadm.k8s.io/v1alpha1
-kind: MasterConfiguration
-noTaintMaster: true
-api:
-  advertiseAddress: 1.1.1.1
-  bindPort: 8443
-  controlPlaneEndpoint: control-plane.minikube.internal
-kubernetesVersion: v1.11.0
-certificatesDir: /var/lib/minikube/certs
-networking:
-  serviceSubnet: 10.96.0.0/12
-etcd:
-  dataDir: /var/lib/minikube/etcd
-controllerManagerExtraArgs:
-  leader-elect: "false"
-schedulerExtraArgs:
-  leader-elect: "false"
-nodeName: "mk"
-apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
-apiServerExtraArgs:
-  enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
-  fail-no-swap: "true"
-controllerManagerExtraArgs:
-  kube-api-burst: "32"
-schedulerExtraArgs:
-  scheduler-name: "mini-scheduler"

pkg/minikube/bootstrapper/images/images.go

@@ -28,9 +28,14 @@ import (
 
 // Pause returns the image name to pull for a given Kubernetes version
 func Pause(v semver.Version, mirror string) string {
+	// Note: changing this logic requires bumping the preload version
 	// Should match `PauseVersion` in:
+	// https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/constants/constants.go
+	pv := "3.4.1"
 	// https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/constants/constants_unix.go
-	pv := "3.2"
+	if semver.MustParseRange("<1.21.0-alpha.3")(v) {
+		pv = "3.2"
+	}
 	if semver.MustParseRange("<1.18.0-alpha.0")(v) {
 		pv = "3.1"
 	}
@@ -40,13 +45,14 @@ func Pause(v semver.Version, mirror string) string {
 // essentials returns images needed too bootstrap a Kubernetes
 func essentials(mirror string, v semver.Version) []string {
 	imgs := []string{
-		componentImage("kube-proxy", v, mirror),
+		// use the same order as: `kubeadm config images list`
-		componentImage("kube-scheduler", v, mirror),
-		componentImage("kube-controller-manager", v, mirror),
 		componentImage("kube-apiserver", v, mirror),
-		coreDNS(v, mirror),
+		componentImage("kube-controller-manager", v, mirror),
-		etcd(v, mirror),
+		componentImage("kube-scheduler", v, mirror),
+		componentImage("kube-proxy", v, mirror),
 		Pause(v, mirror),
+		etcd(v, mirror),
+		coreDNS(v, mirror),
 	}
 	return imgs
 }
@@ -58,13 +64,16 @@ func componentImage(name string, v semver.Version, mirror string) string {
 
 // coreDNS returns the images used for CoreDNS
 func coreDNS(v semver.Version, mirror string) string {
-	// Should match `CoreDNSVersion` in
+	// Note: changing this logic requires bumping the preload version
+	// Should match `CoreDNSImageName` and `CoreDNSVersion` in
 	// https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/constants/constants.go
-	cv := "1.7.0"
+	in := "coredns/coredns"
+	if semver.MustParseRange("<1.21.0-alpha.1")(v) {
+		in = "coredns"
+	}
+	cv := "v1.8.0"
 	switch v.Minor {
-	case 22:
+	case 20, 19:
-		cv = "1.8.0"
-	case 10, 20, 21:
 		cv = "1.7.0"
 	case 18:
 		cv = "1.6.7"
@@ -78,19 +87,20 @@ func coreDNS(v semver.Version, mirror string) string {
 		cv = "1.2.6"
 	case 12:
 		cv = "1.2.2"
-	case 11:
-		cv = "1.1.3"
 	}
-	return path.Join(kubernetesRepo(mirror), "coredns:"+cv)
+	return path.Join(kubernetesRepo(mirror), in+":"+cv)
 }
 
 // etcd returns the image used for etcd
 func etcd(v semver.Version, mirror string) string {
+	// Note: changing this logic requires bumping the preload version
 	// Should match `DefaultEtcdVersion` in:
 	// https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/constants/constants.go
-	ev := "3.4.13-0"
+	ev := "3.4.13-3"
 
 	switch v.Minor {
+	case 19, 20, 21:
+		ev = "3.4.13-0"
 	case 17, 18:
 		ev = "3.4.3-0"
 	case 16:
@@ -99,8 +109,6 @@ func etcd(v semver.Version, mirror string) string {
 		ev = "3.3.10"
 	case 12, 13:
 		ev = "3.2.24"
-	case 11:
-		ev = "3.2.18"
 	}
 
 	// An awkward special case for v1.19.0 - do not imitate unless necessary
@@ -113,6 +121,7 @@ func etcd(v semver.Version, mirror string) string {
 
 // auxiliary returns images that are helpful for running minikube
 func auxiliary(mirror string) []string {
+	// Note: changing this list requires bumping the preload version
 	return []string{
 		storageProvisioner(mirror),
 		dashboardFrontend(mirror),

pkg/minikube/bootstrapper/images/images_test.go

@@ -17,12 +17,71 @@ limitations under the License.
 package images
 
 import (
+	"strings"
 	"testing"
 
+	"github.com/blang/semver"
 	"github.com/google/go-cmp/cmp"
 	"k8s.io/minikube/pkg/version"
 )
 
+func TestEssentials(t *testing.T) {
+	var testCases = []struct {
+		version string
+		images  []string
+	}{
+		{"v1.18.0", strings.Split(strings.Trim(`
+k8s.gcr.io/kube-apiserver:v1.18.0
+k8s.gcr.io/kube-controller-manager:v1.18.0
+k8s.gcr.io/kube-scheduler:v1.18.0
+k8s.gcr.io/kube-proxy:v1.18.0
+k8s.gcr.io/pause:3.2
+k8s.gcr.io/etcd:3.4.3-0
+k8s.gcr.io/coredns:1.6.7
+`, "\n"), "\n")},
+		{"v1.19.0", strings.Split(strings.Trim(`
+k8s.gcr.io/kube-apiserver:v1.19.0
+k8s.gcr.io/kube-controller-manager:v1.19.0
+k8s.gcr.io/kube-scheduler:v1.19.0
+k8s.gcr.io/kube-proxy:v1.19.0
+k8s.gcr.io/pause:3.2
+k8s.gcr.io/etcd:3.4.9-1
+k8s.gcr.io/coredns:1.7.0
+`, "\n"), "\n")},
+		{"v1.20.0", strings.Split(strings.Trim(`
+k8s.gcr.io/kube-apiserver:v1.20.0
+k8s.gcr.io/kube-controller-manager:v1.20.0
+k8s.gcr.io/kube-scheduler:v1.20.0
+k8s.gcr.io/kube-proxy:v1.20.0
+k8s.gcr.io/pause:3.2
+k8s.gcr.io/etcd:3.4.13-0
+k8s.gcr.io/coredns:1.7.0
+`, "\n"), "\n")},
+		{"v1.21.0", strings.Split(strings.Trim(`
+k8s.gcr.io/kube-apiserver:v1.21.0
+k8s.gcr.io/kube-controller-manager:v1.21.0
+k8s.gcr.io/kube-scheduler:v1.21.0
+k8s.gcr.io/kube-proxy:v1.21.0
+k8s.gcr.io/pause:3.4.1
+k8s.gcr.io/etcd:3.4.13-0
+k8s.gcr.io/coredns/coredns:v1.8.0
+`, "\n"), "\n")},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.version, func(t *testing.T) {
+			v, err := semver.Make(strings.TrimPrefix(tc.version, "v"))
+			if err != nil {
+				t.Fatal(err)
+			}
+			want := tc.images
+			got := essentials("k8s.gcr.io", v)
+			if diff := cmp.Diff(want, got); diff != "" {
+				t.Errorf("images mismatch (-want +got):\n%s", diff)
+			}
+		})
+	}
+}
+
 func TestAuxiliary(t *testing.T) {
 	want := []string{
 		"gcr.io/k8s-minikube/storage-provisioner:" + version.GetStorageProvisionerVersion(),
@@ -46,3 +105,23 @@ func TestAuxiliaryMirror(t *testing.T) {
 		t.Errorf("images mismatch (-want +got):\n%s", diff)
 	}
 }
+
+func TestCNI(t *testing.T) {
+	// images used by k8s.io/minikube/pkg/minikube/cni
+	var testCases = []struct {
+		name     string
+		function func(string) string
+	}{
+		{"kindnet", KindNet},
+		{"calico-deployment", CalicoDeployment},
+		{"calico-daemonset", CalicoDaemonSet},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			img := tc.function("")
+			if img == "" {
+				t.Errorf("no image")
+			}
+		})
+	}
+}

pkg/minikube/bootstrapper/images/kubeadm.go

@@ -17,6 +17,7 @@ limitations under the License.
 package images
 
 import (
+	"fmt"
 	"strings"
 
 	"github.com/blang/semver"
@@ -29,6 +30,12 @@ func Kubeadm(mirror string, version string) ([]string, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "semver")
 	}
+	if v.Major > 1 {
+		return nil, fmt.Errorf("version too new: %v", v)
+	}
+	if semver.MustParseRange("<1.12.0-alpha.0")(v) {
+		return nil, fmt.Errorf("version too old: %v", v)
+	}
 	imgs := essentials(mirror, v)
 	imgs = append(imgs, auxiliary(mirror)...)
 	return imgs, nil

pkg/minikube/bootstrapper/images/kubeadm_test.go

@@ -28,9 +28,13 @@ func TestKubeadmImages(t *testing.T) {
 	tests := []struct {
 		version string
 		mirror  string
+		invalid bool
 		want    []string
 	}{
-		{"v1.17.0", "", []string{
+		{"invalid", "", true, nil},
+		{"v0.0.1", "", true, nil}, // too old
+		{"v2.0.0", "", true, nil}, // too new
+		{"v1.17.0", "", false, []string{
 			"k8s.gcr.io/kube-proxy:v1.17.0",
 			"k8s.gcr.io/kube-scheduler:v1.17.0",
 			"k8s.gcr.io/kube-controller-manager:v1.17.0",
@@ -42,7 +46,7 @@ func TestKubeadmImages(t *testing.T) {
 			"docker.io/kubernetesui/dashboard:v2.1.0",
 			"docker.io/kubernetesui/metrics-scraper:v1.0.4",
 		}},
-		{"v1.16.1", "mirror.k8s.io", []string{
+		{"v1.16.1", "mirror.k8s.io", false, []string{
 			"mirror.k8s.io/kube-proxy:v1.16.1",
 			"mirror.k8s.io/kube-scheduler:v1.16.1",
 			"mirror.k8s.io/kube-controller-manager:v1.16.1",
@@ -54,7 +58,7 @@ func TestKubeadmImages(t *testing.T) {
 			"mirror.k8s.io/dashboard:v2.1.0",
 			"mirror.k8s.io/metrics-scraper:v1.0.4",
 		}},
-		{"v1.15.0", "", []string{
+		{"v1.15.0", "", false, []string{
 			"k8s.gcr.io/kube-proxy:v1.15.0",
 			"k8s.gcr.io/kube-scheduler:v1.15.0",
 			"k8s.gcr.io/kube-controller-manager:v1.15.0",
@@ -66,7 +70,7 @@ func TestKubeadmImages(t *testing.T) {
 			"docker.io/kubernetesui/dashboard:v2.1.0",
 			"docker.io/kubernetesui/metrics-scraper:v1.0.4",
 		}},
-		{"v1.14.0", "", []string{
+		{"v1.14.0", "", false, []string{
 			"k8s.gcr.io/kube-proxy:v1.14.0",
 			"k8s.gcr.io/kube-scheduler:v1.14.0",
 			"k8s.gcr.io/kube-controller-manager:v1.14.0",
@@ -78,7 +82,7 @@ func TestKubeadmImages(t *testing.T) {
 			"docker.io/kubernetesui/dashboard:v2.1.0",
 			"docker.io/kubernetesui/metrics-scraper:v1.0.4",
 		}},
-		{"v1.13.0", "", []string{
+		{"v1.13.0", "", false, []string{
 			"k8s.gcr.io/kube-proxy:v1.13.0",
 			"k8s.gcr.io/kube-scheduler:v1.13.0",
 			"k8s.gcr.io/kube-controller-manager:v1.13.0",
@@ -90,7 +94,7 @@ func TestKubeadmImages(t *testing.T) {
 			"docker.io/kubernetesui/dashboard:v2.1.0",
 			"docker.io/kubernetesui/metrics-scraper:v1.0.4",
 		}},
-		{"v1.12.0", "", []string{
+		{"v1.12.0", "", false, []string{
 			"k8s.gcr.io/kube-proxy:v1.12.0",
 			"k8s.gcr.io/kube-scheduler:v1.12.0",
 			"k8s.gcr.io/kube-controller-manager:v1.12.0",
@@ -102,11 +106,16 @@ func TestKubeadmImages(t *testing.T) {
 			"docker.io/kubernetesui/dashboard:v2.1.0",
 			"docker.io/kubernetesui/metrics-scraper:v1.0.4",
 		}},
+		{"v1.11.0", "", true, nil},
+		{"v1.10.0", "", true, nil},
 	}
 	for _, tc := range tests {
 		got, err := Kubeadm(tc.mirror, tc.version)
-		if err != nil {
+		if err == nil && tc.invalid {
-			t.Fatalf("unexpected err: %v", err)
+			t.Fatalf("expected err (%s): %v", tc.version, got)
+		}
+		if err != nil && !tc.invalid {
+			t.Fatalf("unexpected err (%s): %v", tc.version, err)
 		}
 		sort.Strings(got)
 		sort.Strings(tc.want)

pkg/minikube/config/types.go

@@ -74,7 +74,9 @@ type ClusterConfig struct {
 	KubernetesConfig        KubernetesConfig
 	Nodes                   []Node
 	Addons                  map[string]bool
-	VerifyComponents        map[string]bool // map of components to verify and wait for after start.
+	CustomAddonImages       map[string]string // Maps image names to the image to use for addons. e.g. Dashboard -> k8s.gcr.io/echoserver:1.4 makes dashboard addon use echoserver for its Dashboard deployment.
+	CustomAddonRegistries   map[string]string // Maps image names to the registry to use for addons. See CustomAddonImages for example.
+	VerifyComponents        map[string]bool   // map of components to verify and wait for after start.
 	StartHostTimeout        time.Duration
 	ScheduledStop           *ScheduledStopConfig
 	ExposedPorts            []string // Only used by the docker and podman driver

pkg/minikube/download/download_test.go

@@ -85,7 +85,8 @@ func testPreloadDownloadPreventsMultipleDownload(t *testing.T) {
 		return nil, nil
 	}
 	checkPreloadExists = func(k8sVersion, containerRuntime string, forcePreload ...bool) bool { return true }
-	compareChecksum = func(k8sVersion, containerRuntime, path string) error { return nil }
+	getChecksum = func(k8sVersion, containerRuntime string) (string, error) { return "check", nil }
+	ensureChecksumValid = func(k8sVersion, containerRuntime, path string) error { return nil }
 
 	var group sync.WaitGroup
 	group.Add(2)

pkg/minikube/download/preload.go

@@ -42,7 +42,7 @@ const (
 	// PreloadVersion is the current version of the preloaded tarball
 	//
 	// NOTE: You may need to bump this version up when upgrading auxiliary docker images
-	PreloadVersion = "v10"
+	PreloadVersion = "v11"
 	// PreloadBucket is the name of the GCS bucket where preloaded volume tarballs exist
 	PreloadBucket = "minikube-preloaded-volume-tarballs"
 )
@@ -171,12 +171,8 @@ func Preload(k8sVersion, containerRuntime string) error {
 		return errors.Wrapf(err, "download failed: %s", url)
 	}
 
-	if err := saveChecksumFile(k8sVersion, containerRuntime); err != nil {
+	if err := ensureChecksumValid(k8sVersion, containerRuntime, targetPath); err != nil {
-		return errors.Wrap(err, "saving checksum file")
+		return err
-	}
-
-	if err := compareChecksum(k8sVersion, containerRuntime, targetPath); err != nil {
-		return errors.Wrap(err, "verify")
 	}
 
 	if realPath != "" {
@@ -203,7 +199,7 @@ func getStorageAttrs(name string) (*storage.ObjectAttrs, error) {
 	return attrs, nil
 }
 
-func getChecksum(k8sVersion, containerRuntime string) (string, error) {
+var getChecksum = func(k8sVersion, containerRuntime string) (string, error) {
 	klog.Infof("getting checksum for %s ...", TarballName(k8sVersion, containerRuntime))
 	attrs, err := getStorageAttrs(TarballName(k8sVersion, containerRuntime))
 	if err != nil {
@@ -246,4 +242,15 @@ func verifyChecksum(k8sVersion, containerRuntime, path string) error {
 	return nil
 }
 
-var compareChecksum = verifyChecksum
+// ensureChecksumValid saves and verifies local binary checksum matches remote binary checksum
+var ensureChecksumValid = func(k8sVersion, containerRuntime, targetPath string) error {
+	if err := saveChecksumFile(k8sVersion, containerRuntime); err != nil {
+		return errors.Wrap(err, "saving checksum file")
+	}
+
+	if err := verifyChecksum(k8sVersion, containerRuntime, targetPath); err != nil {
+		return errors.Wrap(err, "verify")
+	}
+
+	return nil
+}

pkg/minikube/node/start.go

@@ -21,6 +21,8 @@ import (
 	"net"
 	"os"
 	"os/exec"
+	"path"
+	"regexp"
 	"strconv"
 	"strings"
 	"sync"
@@ -60,6 +62,7 @@ import (
 	"k8s.io/minikube/pkg/minikube/proxy"
 	"k8s.io/minikube/pkg/minikube/reason"
 	"k8s.io/minikube/pkg/minikube/style"
+	"k8s.io/minikube/pkg/minikube/vmpath"
 	"k8s.io/minikube/pkg/util"
 	"k8s.io/minikube/pkg/util/retry"
 )
@@ -131,6 +134,10 @@ func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) {
 		if err := kapi.ScaleDeployment(starter.Cfg.Name, meta.NamespaceSystem, kconst.CoreDNSDeploymentName, 1); err != nil {
 			klog.Errorf("Unable to scale down deployment %q in namespace %q to 1 replica: %v", kconst.CoreDNSDeploymentName, meta.NamespaceSystem, err)
 		}
+		// inject {"host.minikube.internal": hostIP} record into CoreDNS
+		if err := addCoreDNSEntry(starter.Runner, "host.minikube.internal", hostIP.String(), *starter.Cfg); err != nil {
+			klog.Errorf("Unable to inject {%q: %s} record into CoreDNS: %v", "host.minikube.internal", hostIP.String(), err)
+		}
 	} else {
 		bs, err = cluster.Bootstrapper(starter.MachineAPI, viper.GetString(cmdcfg.Bootstrapper), *starter.Cfg, starter.Runner)
 		if err != nil {
@@ -669,3 +676,47 @@ func prepareNone() {
 		exit.Message(reason.HostHomeChown, "Failed to change permissions for {{.minikube_dir_path}}: {{.error}}", out.V{"minikube_dir_path": localpath.MiniPath(), "error": err})
 	}
 }
+
+// addCoreDNSEntry adds host name and IP record to the DNS by updating CoreDNS's ConfigMap.
+// ref: https://coredns.io/plugins/hosts/
+// note: there can be only one 'hosts' block in CoreDNS's ConfigMap (avoid "plugin/hosts: this plugin can only be used once per Server Block" error)
+func addCoreDNSEntry(runner command.Runner, name, ip string, cc config.ClusterConfig) error {
+	kubectl := kapi.KubectlBinaryPath(cc.KubernetesConfig.KubernetesVersion)
+	kubecfg := path.Join(vmpath.GuestPersistentDir, "kubeconfig")
+
+	// get current coredns configmap via kubectl
+	get := fmt.Sprintf("sudo %s --kubeconfig=%s -n kube-system get configmap coredns -o yaml", kubectl, kubecfg)
+	out, err := runner.RunCmd(exec.Command("/bin/bash", "-c", get))
+	if err != nil {
+		klog.Errorf("failed to get current CoreDNS ConfigMap: %v", err)
+		return err
+	}
+	cm := strings.TrimSpace(out.Stdout.String())
+
+	// check if this specific host entry already exists in coredns configmap, so not to duplicate/override it
+	host := regexp.MustCompile(fmt.Sprintf(`(?smU)^ *hosts {.*%s.*}`, name))
+	if host.MatchString(cm) {
+		klog.Infof("CoreDNS already contains %q host record, skipping...", name)
+		return nil
+	}
+
+	// inject hosts block with host record into coredns configmap
+	sed := fmt.Sprintf("sed '/^        forward . \\/etc\\/resolv.conf.*/i \\        hosts {\\n           %s %s\\n           fallthrough\\n        }'", ip, name)
+	// check if hosts block already exists in coredns configmap
+	hosts := regexp.MustCompile(`(?smU)^ *hosts {.*}`)
+	if hosts.MatchString(cm) {
+		// inject host record into existing coredns configmap hosts block instead
+		klog.Info("CoreDNS already contains hosts block, will inject host record there...")
+		sed = fmt.Sprintf("sed '/^        hosts {.*/a \\           %s %s'", ip, name)
+	}
+
+	// replace coredns configmap via kubectl
+	replace := fmt.Sprintf("sudo %s --kubeconfig=%s replace -f -", kubectl, kubecfg)
+	if _, err := runner.RunCmd(exec.Command("/bin/bash", "-c", fmt.Sprintf("%s | %s | %s", get, sed, replace))); err != nil {
+		klog.Errorf("failed to inject {%q: %s} host record into CoreDNS", name, ip)
+		return err
+	}
+	klog.Infof("{%q: %s} host record injected into CoreDNS", name, ip)
+
+	return nil
+}

pkg/minikube/out/out.go

@@ -32,7 +32,7 @@ import (
 
 	"github.com/Delta456/box-cli-maker/v2"
 	"github.com/briandowns/spinner"
-	isatty "github.com/mattn/go-isatty"
+	"github.com/mattn/go-isatty"
 
 	"k8s.io/klog/v2"
 	"k8s.io/minikube/pkg/minikube/localpath"
@@ -452,5 +452,8 @@ func applyTmpl(format string, a ...V) string {
 // Fmt applies formatting and translation
 func Fmt(format string, a ...V) string {
 	format = translate.T(format)
+	if len(a) == 0 {
+		return format
+	}
 	return applyTmpl(format, a...)
 }

pkg/minikube/registry/global.go

@@ -66,10 +66,16 @@ var (
 
 // DriverState is metadata relating to a driver and status
 type DriverState struct {
-	Name     string
+	// Name is the name of the driver used internally
-	Default  bool
+	Name string
+	// Default drivers are selected automatically
+	Default bool
+	// Preference is the original priority from driver
+	Preference Priority
+	// Priority is the effective priority with health
 	Priority Priority
-	State    State
+	// State is the state of driver and dependencies
+	State State
 	// Rejection is why we chose not to use this driver
 	Rejection string
 	// Suggestion is how the user could improve health
@@ -112,6 +118,7 @@ func Available(vm bool) []DriverState {
 		s := d.Status()
 		klog.Infof("%s default: %v priority: %d, state: %+v", d.Name, d.Default, d.Priority, s)
 
+		preference := d.Priority
 		priority := d.Priority
 		if !s.Healthy {
 			priority = Unhealthy
@@ -119,10 +126,10 @@ func Available(vm bool) []DriverState {
 
 		if vm {
 			if IsVM(d.Name) {
-				sts = append(sts, DriverState{Name: d.Name, Default: d.Default, Priority: priority, State: s})
+				sts = append(sts, DriverState{Name: d.Name, Default: d.Default, Preference: preference, Priority: priority, State: s})
 			}
 		} else {
-			sts = append(sts, DriverState{Name: d.Name, Default: d.Default, Priority: priority, State: s})
+			sts = append(sts, DriverState{Name: d.Name, Default: d.Default, Preference: preference, Priority: priority, State: s})
 		}
 	}
 

pkg/minikube/registry/global_test.go

@@ -93,16 +93,18 @@ func TestGlobalAvailable(t *testing.T) {
 
 	expected := []DriverState{
 		{
-			Name:     "healthy-bar",
+			Name:       "healthy-bar",
-			Default:  true,
+			Default:    true,
-			Priority: Default,
+			Preference: Default,
-			State:    State{Healthy: true},
+			Priority:   Default,
+			State:      State{Healthy: true},
 		},
 		{
-			Name:     "unhealthy-foo",
+			Name:       "unhealthy-foo",
-			Default:  true,
+			Default:    true,
-			Priority: Unhealthy,
+			Preference: Default,
-			State:    State{Healthy: false},
+			Priority:   Unhealthy,
+			State:      State{Healthy: false},
 		},
 	}
 

site/assets/scss/_variables_project.scss

@@ -23,7 +23,7 @@ $gray-800: #333 !default;
 $gray-900: #222 !default;
 $black: #000 !default;
 
-$primary: #f2771a !default;
+$primary: $mk-dark !default;
 $primary-light: $mk-light;
 $secondary: #403F4C;
 $success: #3772FF !default;
@@ -228,3 +228,16 @@ div.td-content {
 div.code-toolbar > .toolbar {
     top: -.3em !important;
 }
+
+.option-button {
+    border-radius: 0.2rem !important;
+    margin-right: 0.2rem;
+}
+
+.hide {
+    display: none !important;
+}
+
+.card-body-blue {
+    background: #f3f9fa;
+}

site/content/en/docs/benchmarks/imageBuild/benchmarkingProcess/_index.md

@@ -0,0 +1,89 @@
+---
+title: "About the Benchmarking Process"
+linkTitle: "About the Benchmarking Process"
+weight: 1
+--- 
+
+## What's the difference between the four images?
+In the benchmarking charts you'll see four images: Few Large Layers, Few Small Layers, Many Large Layers, and Many Small Layers
+
+All the images use the same base image: `gcr.io/buildpacks/builder:v1`
+
+#### Few vs Many
+Few will copy two files while many will copy 20 files.
+
+#### Small vs Large
+Small will copy a 20MB file while large will copy a 123MB file.
+
+Using this info you can see the following:
+- Few Large Layers: copies two 123MB files
+- Few Small Layers: copies two 20MB files
+- Many Large Layers: copies 20 123MB files
+- Many Small Layers: copies 20 20MB files
+
+Finally, as the last layer, a simplistic 11 line Go app is copied in.
+
+## Iterative vs Initial
+There are two graphs for each benchmark, iterative and inital.
+
+#### Inital
+Initial simulates loading the image for the first time.
+
+All existing images and cache is removed/cleared from minikube and Docker between runs to replicate what the user would experience when loading for the first time.
+
+#### Iterative
+Iterative simulates only the Go app (last layer of the image) changing.
+
+This is the exact use case of [Skaffold](https://github.com/GoogleContainerTools/skaffold), where if the user changes a file the Go binary is rebuilt and the image is re-loaded.
+
+Bewteen runs the cache and existing image is left alone, only the Go binary is changed.
+
+
+## How are the benchmarks conducted?
+```
+// Pseudo code of running docker-env benchmark
+
+startMininkube() // minikube start --container-runtime=docker
+
+for image in [fewLargeLayers, fewSmallLayers, ...] {
+	buildGoBinary()
+
+	// inital simulation
+	for i in runCount {
+		startTimer()
+
+		runDockerEnvImageLoad(image)
+
+		stopTimer()
+
+		verifyImageSuccessfullyLoaded()
+
+		storeTimeTaken()
+
+		removeImage()
+
+		clearDockerCache()
+	}
+
+	// iterative simulation
+	for i in runCount {
+		updateGoBinary()
+
+		startTimer()
+
+		runDockerEnvImageLoad(image)
+
+		stopTimer()
+
+		verifyImageSuccessfullyLoaded()
+
+		storeTimeTaken() // skip if first run
+	}
+
+	clearDockerCache()
+
+	calculateAndRecordAverageTime()
+}
+
+deleteMinikube() // minkube delete --all
+```

site/content/en/docs/benchmarks/imageBuild/containerdRuntime/_index.md

@@ -1,21 +0,0 @@
----
-title: "Containerd & Cri-o Runtime Benchmarks"
-linkTitle: "Containerd & Cri-o Runtime Benchmarks"
-weight: 1
----
-
-## Containerd
-![Iterative Summary](/images/benchmarks/containerdRuntime/iterativeSummary.png)
-![Initial Summary](/images/benchmarks/containerdRuntime/initialSummary.png)
-
-## Cri-o
-![Iterative Summary](/images/benchmarks/crioRuntime/iterativeSummary.png)
-![Initial Summary](/images/benchmarks/crioRuntime/initialSummary.png)
-
-## Containerd
-![Iterative](/images/benchmarks/containerdRuntime/iterative.png)
-![Initial](/images/benchmarks/containerdRuntime/initial.png)
-
-## Cri-o
-![Iterative](/images/benchmarks/crioRuntime/iterative.png)
-![Initial](/images/benchmarks/crioRuntime/initial.png)

site/content/en/docs/benchmarks/imageBuild/dockerRuntime/_index.md

@@ -1,17 +0,0 @@
----
-title: "Docker Runtime Benchmarks"
-linkTitle: "Docker Runtime Benchmarks"
-weight: 1
----
-
-## Linux
-![Iterative Summary](/images/benchmarks/dockerRuntime/iterativeSummary.png)
-![Initial Summary](/images/benchmarks/dockerRuntime/initialSummary.png)
-
-## Mac
-![Iterative Summary](/images/benchmarks/macArchitecture/iterative.png)
-![Initial Summary](/images/benchmarks/macArchitecture/initial.png)
-
-## Linux
-![Iterative](/images/benchmarks/dockerRuntime/iterative.png)
-![Initial](/images/benchmarks/dockerRuntime/initial.png)

site/content/en/docs/benchmarks/imageBuild/linux/_index.md

@@ -0,0 +1,24 @@
+---
+title: "Linux Benchmarks"
+linkTitle: "Linux Benchmarks"
+weight: 1
+---
+
+Benchmarking machine specs:
+- OS: Debian 10
+- Processor: 2.30 GHz 8-Core Intel Xeon
+- Memory: 32 GB
+- Storage: SSD
+
+## Docker Runtime
+![Docker Runtime  Iterative Loads](/images/benchmarks/dockerRuntime/iterative.png)
+![Linux Runtime Docker Initial Loads](/images/benchmarks/dockerRuntime/initial.png)
+
+
+## Containerd Runtime
+![Containerd Runtime Iterative Loads](/images/benchmarks/containerdRuntime/iterative.png)
+![Containerd Runtime Initial Loads](/images/benchmarks/containerdRuntime/initial.png)
+
+## Cri-o Runtime
+![Cri-o Runtime Iterative Loads](/images/benchmarks/crioRuntime/iterative.png)
+![Cri-o Runtime Initial Loads](/images/benchmarks/crioRuntime/initial.png)

site/content/en/docs/benchmarks/imageBuild/mac/_index.md

@@ -0,0 +1,14 @@
+---
+title: "Mac Benchmarks"
+linkTitle: "Mac Benchmarks"
+weight: 1
+---
+
+Benchmarking machine specs:
+- Model: Mac mini (Late 2014)
+- Processor: 2.6 GHz Dual-Core Intel Core i5
+- Memory: 8 GB 1600 MHz DDR3
+- Storage: SSD
+
+![Iterative Loads](/images/benchmarks/macArchitecture/iterative.png)
+![Initial Loads](/images/benchmarks/macArchitecture/initial.png)

site/content/en/docs/benchmarks/imageBuild/minikubeVsOthers/_index.md

@@ -1,8 +1,14 @@
 ---
-title: "minikube vs Others Benchmarks"
+title: "minikube vs kind vs k3d vs microk8s Benchmarks"
 linkTitle: "minikube vs Others Benchmarks"
 weight: 1
 ---
 
-![Iterative](/images/benchmarks/minikubeVsOthers/iterative.png)
+Benchmarking machine specs:
-![Initial](/images/benchmarks/minikubeVsOthers/initial.png)
+- OS: Debian 10
+- Processor: 2.30 GHz 8-Core Intel Xeon
+- Memory: 32 GB
+- Storage: SSD
+
+![Iterative Loads](/images/benchmarks/minikubeVsOthers/iterative.png)
+![Initial Loads](/images/benchmarks/minikubeVsOthers/initial.png)

site/content/en/docs/commands/unpause.md

@@ -17,6 +17,10 @@ unpause Kubernetes
 minikube unpause [flags]
 ```
 
+### Aliases
+
+[resume]
+
 ### Options
 
 ```

site/content/en/docs/contrib/tests.en.md

@@ -265,15 +265,8 @@ tests that the node name verification works as expected
 #### validateDeployAppToMultiNode
 deploys an app to a multinode cluster and makes sure all nodes can serve traffic
 
-## TestCNIFalse
+#### validatePodsPingHost
-checks that minikube returns and error
+uses app previously deplyed by validateDeployAppToMultiNode to verify its pods, located on different nodes, can resolve "host.minikube.internal".
-if container runtime is "containerd" or "crio"
-and --cni=false
-
-## TestCNIFalseForce
-checks that minikube returns not error
-if container runtime is "containerd" or "crio"
-and --cni=false, but --force=true
 
 ## TestNetworkPlugins
 tests all supported CNI options
@@ -339,6 +332,9 @@ runs the initial minikube start
 #### validateDeploying
 deploys an app the minikube cluster
 
+#### validateEnableAddonWhileActive
+makes sure addons can be enabled while cluster is active.
+
 #### validateStop
 tests minikube stop
 

site/content/en/docs/start/_index.md

@@ -20,61 +20,150 @@ All you need is Docker (or similarly compatible) container or a Virtual Machine
 
 <h2 class="step"><span class="fa-stack fa-1x"><i class="fa fa-circle fa-stack-2x"></i><strong class="fa-stack-1x text-primary">1</strong></span>Installation</h2>
 
-{{% tabs %}}
+{{% card %}}
-{{% linuxtab %}}
 
-For Linux users, we provide 3 easy download options (for each architecture):
+Click on the buttons that describe your target platform. For other architectures, see [the release page](https://github.com/kubernetes/minikube/releases/latest) for a complete list of minikube binaries.
 
-### amd64 / x86_64
+{{% quiz_row base="" name="Operating system" %}}
+{{% quiz_button option="Linux" %}} {{% quiz_button option="macOS" %}} {{% quiz_button option="Windows" %}}
+{{% /quiz_row %}}
 
-#### Binary download
+{{% quiz_row base="/Linux" name="Architecture" %}}
+{{% quiz_button option="x86-64" %}} {{% quiz_button option="ARM64" %}} {{% quiz_button option="ARMv7" %}} {{% quiz_button option="ppc64" %}} {{% quiz_button option="S390x" %}}
+{{% /quiz_row %}}
 
+{{% quiz_row base="/Linux/x86-64" name="Installer type" %}}
+{{% quiz_button option="Binary download" %}} {{% quiz_button option="Debian package" %}} {{% quiz_button option="RPM package" %}}
+{{% /quiz_row %}}
 
+{{% quiz_row base="/Linux/ARM64" name="Installer type" %}}
+{{% quiz_button option="Binary download" %}} {{% quiz_button option="Debian package" %}} {{% quiz_button option="RPM package" %}}
+{{% /quiz_row %}}
+
+{{% quiz_row base="/Linux/ppc64" name="Installer type" %}}
+{{% quiz_button option="Binary download" %}}
+{{% /quiz_row %}}
+
+{{% quiz_row base="/Linux/S390x" name="Installer type" %}}
+{{% quiz_button option="Binary download" %}}
+{{% /quiz_row %}}
+
+{{% quiz_row base="/Linux/ARMv7" name="Installer type" %}}
+{{% quiz_button option="Binary download" %}}
+{{% /quiz_row %}}
+
+{{% quiz_row base="/macOS" name="Architecture" %}}
+{{% quiz_button option="x86-64" %}} {{% quiz_button option="ARM64" %}}
+{{% /quiz_row %}}
+
+{{% quiz_row base="/macOS/x86-64" name="Installer type" %}}
+{{% quiz_button option="Binary download" %}} {{% quiz_button option="Homebrew" %}}
+{{% /quiz_row %}}
+
+{{% quiz_row base="/macOS/ARM64" name="Installer type" %}}
+{{% quiz_button option="Binary download" %}}
+{{% /quiz_row %}}
+
+{{% quiz_row base="/Windows" name="Architecture" %}}
+{{% quiz_button option="x86-64" %}}
+{{% /quiz_row %}}
+
+{{% quiz_row base="/Windows/x86-64" name="Installer type" %}}
+{{% quiz_button option=".exe download" %}} {{% quiz_button option="Windows Package Manager" %}} {{% quiz_button option="Chocolatey" %}}
+{{% /quiz_row %}}
+
+{{% quiz_instruction id="/Linux/x86-64/Binary download" %}}
 ```shell
- curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64
+curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64
- sudo install minikube-linux-amd64 /usr/local/bin/minikube
+sudo install minikube-linux-amd64 /usr/local/bin/minikube
 ```
+{{% /quiz_instruction %}}
 
-#### Debian package
+{{% quiz_instruction id="/Linux/x86-64/Debian package" %}}
-
 ```shell
 curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube_latest_amd64.deb
 sudo dpkg -i minikube_latest_amd64.deb
 ```
+{{% /quiz_instruction %}}
 
-#### RPM package
+{{% quiz_instruction id="/Linux/x86-64/RPM package" %}}
-
 ```shell
 curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-latest.x86_64.rpm
 sudo rpm -Uvh minikube-latest.x86_64.rpm
 ```
+{{% /quiz_instruction %}}
 
-### arm64 / aarch64
+{{% quiz_instruction id="/Linux/ARM64/Binary download" %}}
-
-#### Binary download
-
 ```shell
- curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-linux-arm64
+curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-linux-arm64
- sudo install minikube-linux-arm64 /usr/local/bin/minikube
+sudo install minikube-linux-arm64 /usr/local/bin/minikube
 ```
+{{% /quiz_instruction %}}
 
-#### Debian package
+{{% quiz_instruction id="/Linux/ARM64/Debian package" %}}
-
 ```shell
 curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube_latest_arm64.deb
 sudo dpkg -i minikube_latest_arm64.deb
 ```
+{{% /quiz_instruction %}}
 
-#### RPM package
+{{% quiz_instruction id="/Linux/ARM64/RPM package" %}}
-
 ```shell
 curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-latest.aarch64.rpm
 sudo rpm -Uvh minikube-latest.aarch64.rpm
 ```
+{{% /quiz_instruction %}}
 
-{{% /linuxtab %}}
+{{% quiz_instruction id="/Linux/ppc64/Binary download" %}}
-{{% mactab %}}
+```shell
+curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-linux-ppc64le
+sudo install minikube-linux-ppc64le /usr/local/bin/minikube
+```
+{{% /quiz_instruction %}}
 
+{{% quiz_instruction id="/Linux/ppc64/Debian package" %}}
+```shell
+curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube_latest_ppc64le.deb
+sudo dpkg -i minikube_latest_ppc64le.deb
+```
+{{% /quiz_instruction %}}
+
+{{% quiz_instruction id="/Linux/ppc64/RPM package" %}}
+```shell
+curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-latest.ppc64el.rpm
+sudo rpm -Uvh minikube-latest.ppc64el.rpm
+```
+{{% /quiz_instruction %}}
+
+{{% quiz_instruction id="/Linux/S390x/Binary download" %}}
+```shell
+curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-linux-s390x
+sudo install minikube-linux-s390x /usr/local/bin/minikube
+```
+{{% /quiz_instruction %}}
+
+{{% quiz_instruction id="/Linux/S390x/Debian package" %}}
+```shell
+curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube_latest_s390x.deb
+sudo dpkg -i minikube_latest_s390x.deb
+```
+{{% /quiz_instruction %}}
+
+{{% quiz_instruction id="/Linux/S390x/RPM package" %}}
+```shell
+curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-latest.s390x.rpm
+sudo rpm -Uvh minikube-latest.s390x.rpm
+```
+{{% /quiz_instruction %}}
+
+{{% quiz_instruction id="/Linux/ARMv7/Binary download" %}}
+```shell
+curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-linux-arm
+sudo install minikube-linux-arm /usr/local/bin/minikube
+```
+{{% /quiz_instruction %}}
+
+{{% quiz_instruction id="/macOS/x86-64/Homebrew" %}}
 If the [Brew Package Manager](https://brew.sh/) is installed:
 
 ```shell
@@ -87,49 +176,46 @@ If `which minikube` fails after installation via brew, you may have to remove th
 brew unlink minikube
 brew link minikube
 ```
+{{% /quiz_instruction %}}
 
-Otherwise, download minikube directly:
+{{% quiz_instruction id="/macOS/x86-64/Binary download" %}}
-
-### x86
-
 ```shell
 curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-darwin-amd64
 sudo install minikube-darwin-amd64 /usr/local/bin/minikube
 ```
+{{% /quiz_instruction %}}
 
-### ARM
+{{% quiz_instruction id="/macOS/ARM64/Binary download" %}}
-
 ```shell
 curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-darwin-arm64
 sudo install minikube-darwin-arm64 /usr/local/bin/minikube
 ```
+{{% /quiz_instruction %}}
 
-{{% /mactab %}}
+{{% quiz_instruction id="/Windows/x86-64/Windows Package Manager" %}}
-{{% windowstab %}}
-
-### Windows Package Manager
-
 If the [Windows Package Manager](https://docs.microsoft.com/en-us/windows/package-manager/) is installed, use the following command to install minikube:
 
 ```shell
 winget install minikube
 ```
+{{% /quiz_instruction %}}
 
-### Chocolatey
+{{% quiz_instruction id="/Windows/x86-64/Chocolatey" %}}
 If the [Chocolatey Package Manager](https://chocolatey.org/) is installed, use the following command:
 
 ```shell
 choco install minikube
 ```
+{{% /quiz_instruction %}}
 
-### Stand-alone Windows Installer
+{{% quiz_instruction id="/Windows/x86-64/.exe download" %}}
-Otherwise, download and run the [Windows installer](https://storage.googleapis.com/minikube/releases/latest/minikube-installer.exe)
+Download and run the stand-alone [minikube Windows installer](https://storage.googleapis.com/minikube/releases/latest/minikube-installer.exe).
 
+_If you used a CLI to perform the installation, you will need to close that CLI and open a new one before proceeding._
+{{% /quiz_instruction %}}
 
-_If you used a CLI to perform the installation, you will need to close that CLI and open a new one before proceeding._ 
+{{% /card %}}
 
-{{% /windowstab %}}
-{{% /tabs %}}
 
 <h2 class="step"><span class="fa-stack fa-1x"><i class="fa fa-circle fa-stack-2x"></i><strong class="fa-stack-1x text-primary">2</strong></span>Start your cluster</h2>
 

site/layouts/partials/hooks/body-end.html

@@ -1,3 +1,4 @@
 <!-- start: minikube override body-end partial -->
 <script language="javascript">initTabs();</script>
+<script language="javascript">initQuiz();</script>
 <!-- end: minikube override body-end partial -->

site/layouts/partials/hooks/head-end.html

@@ -2,4 +2,5 @@
 <link href="https://fonts.googleapis.com/css2?family=Lora&family=Open+Sans:wght@600;700&display=swap" rel="stylesheet">
 <link href="/css/tabs.css" rel="stylesheet">
 <script src="/js/tabs.js"></script>
-<!-- end: minikube override head-end partial -->
+<script src="/js/quiz.js"></script>
+<!-- end: minikube override head-end partial -->

site/layouts/shortcodes/card.html

@@ -0,0 +1 @@
+<div class="card"><div class="card-body card-body-blue">{{ .Inner }}</div></div>

site/layouts/shortcodes/quiz_button.html

@@ -0,0 +1 @@
+<button data-quiz-id="{{ .Parent.Get "base" }}/{{ .Get "option" }}" type="button" class="btn btn-outline-primary option-button">{{ .Get "option" }}</button>

site/layouts/shortcodes/quiz_instruction.html

@@ -0,0 +1,11 @@
+{{ $id := .Get "id" }}
+{{ $selected := split $id "/" }}
+
+{{ $os := index $selected 1 }}
+{{ $arch := index $selected 2 }}
+{{ $installer := index $selected 3 }}
+
+<div data-quiz-id="{{ with .Get "id"}}{{.}}{{end}}" class="quiz-instruction">
+<p>To install minikube on <b>{{ $arch }}</b> <b>{{ $os }}</b> using <b>{{ replace $installer "Binary" "binary" }}</b>:</p>
+{{ .Inner }}
+</div>

site/layouts/shortcodes/quiz_row.html

@@ -0,0 +1,8 @@
+{{ $level := .Get "base" | strings.Count "/" }}
+
+<div data-quiz-id="{{ .Get "base" }}" data-level="{{ $level }}" class="row option-row hide">
+    <div class="col-lg-2 my-auto">
+        <p>{{ with .Get "name"}}{{.}}{{end}}</p>
+    </div>
+    <div class="col-lg-10">{{ .Inner }}</div>
+</div>

site/static/js/quiz.js

@@ -0,0 +1,57 @@
+function selectQuizOption(selectedId) {
+    const currentLevel = selectedId.split('/').length - 1;
+    $('.option-row').each(function (i) {
+        const rowId = $(this).attr('data-quiz-id');
+        // don't hide option rows if it has a lower level
+        // e.g. when clicking "x86_64" under Linux, we don't want to hide the operating system row
+        if ($(this).attr('data-level') < currentLevel) {
+            return;
+        }
+        if (rowId === selectedId) {
+            $(this).removeClass('hide');
+            $(this).find('.option-button').removeClass('active');
+            return;
+        }
+        // hide all other option rows
+        $(this).addClass('hide');
+    });
+    // hide other answers
+    $('.quiz-instruction').addClass('hide');
+    // show the selected answer
+    $('.quiz-instruction[data-quiz-id=\'' + selectedId + '\']').removeClass('hide');
+
+    const buttons = $('.option-row[data-quiz-id=\'' + selectedId + '\']').find('.option-button');
+    // auto-select the first option for the user, to reduce the number of clicks
+    if (buttons.length > 0) {
+        const btn = buttons.first();
+        btn.addClass('active');
+        selectQuizOption(btn.attr('data-quiz-id'));
+    }
+}
+
+function initQuiz() {
+    try {
+        $('.option-button').click(function(e) {
+            $(this).parent().find('.option-button').removeClass('active');
+            $(this).addClass('active');
+            const dataContainerId = $(this).attr('data-quiz-id');
+
+            selectQuizOption(dataContainerId);
+        });
+        let userOS = getUserOS();
+        if (userOS === 'Mac') {
+            // use the name "macOS" to match the button
+            userOS = 'macOS';
+        }
+        $('.option-row[data-level=0]').removeClass('hide');
+        // auto-select the OS for user
+        const btn = $('.option-button[data-quiz-id=\'/' + userOS + '\']').first();
+        btn.addClass('active');
+        selectQuizOption(btn.attr('data-quiz-id'));
+    } catch(e) {
+        const elements = document.getElementsByClassName("quiz-instruction");
+        for (let element of elements) {
+            element.classList.remove("hide");
+        }
+    }
+}

site/static/js/tabs.js

@@ -17,7 +17,7 @@ function initTabs() {
       let tabSelector = getTabSelector(this);
       $(this).find('div'+tabSelector).addClass('active');
     })
- 
+
     $('.nav-tabs a').click(function(e){
       e.preventDefault();
       var tab = $(this).parent(),

test/integration/multinode_test.go

@@ -22,6 +22,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"net"
 	"os/exec"
 	"strings"
 	"testing"
@@ -48,6 +49,7 @@ func TestMultiNode(t *testing.T) {
 		}{
 			{"FreshStart2Nodes", validateMultiNodeStart},
 			{"DeployApp2Nodes", validateDeployAppToMultiNode},
+			{"PingHostFrom2Pods", validatePodsPingHost},
 			{"AddNode", validateAddNodeToMultiNode},
 			{"ProfileList", validateProfileListWithMultiNode},
 			{"CopyFile", validateCopyFileWithMultiNode},
@@ -154,9 +156,7 @@ func validateProfileListWithMultiNode(ctx context.Context, t *testing.T, profile
 				t.Errorf("expected the json of 'profile list' to not include profile or node in invalid profile but got *%q*. args: %q", rr.Stdout.String(), rr.Command())
 			}
 		}
-
 	}
-
 }
 
 // validateProfileListWithMultiNode make sure minikube profile list outputs correct with multinode clusters
@@ -464,6 +464,7 @@ func validateDeployAppToMultiNode(ctx context.Context, t *testing.T, profile str
 			t.Errorf("Pod %s could not resolve 'kubernetes.io': %v", name, err)
 		}
 	}
+
 	// verify both Pods could resolve "kubernetes.default"
 	// this one is also checked by k8s e2e node conformance tests:
 	// https://github.com/kubernetes/kubernetes/blob/f137c4777095b3972e2dd71a01365d47be459389/test/e2e_node/environment/conformance.go#L125-L179
@@ -473,6 +474,7 @@ func validateDeployAppToMultiNode(ctx context.Context, t *testing.T, profile str
 			t.Errorf("Pod %s could not resolve 'kubernetes.default': %v", name, err)
 		}
 	}
+
 	// verify both pods could resolve to a local service.
 	for _, name := range podNames {
 		_, err = Run(t, exec.CommandContext(ctx, Target(), "kubectl", "-p", profile, "--", "exec", name, "--", "nslookup", "kubernetes.default.svc.cluster.local"))
@@ -481,3 +483,33 @@ func validateDeployAppToMultiNode(ctx context.Context, t *testing.T, profile str
 		}
 	}
 }
+
+// validatePodsPingHost uses app previously deplyed by validateDeployAppToMultiNode to verify its pods, located on different nodes, can resolve "host.minikube.internal".
+func validatePodsPingHost(ctx context.Context, t *testing.T, profile string) {
+	// get Pod names
+	rr, err := Run(t, exec.CommandContext(ctx, Target(), "kubectl", "-p", profile, "--", "get", "pods", "-o", "jsonpath='{.items[*].metadata.name}'"))
+	if err != nil {
+		t.Errorf("failed get Pod names")
+	}
+	podNames := strings.Split(strings.Trim(rr.Stdout.String(), "'"), " ")
+
+	for _, name := range podNames {
+		// get host.minikube.internal ip as resolved by nslookup
+		if out, err := Run(t, exec.CommandContext(ctx, Target(), "kubectl", "-p", profile, "--", "exec", name, "--", "sh", "-c", "nslookup host.minikube.internal | awk 'NR==5' | cut -d' ' -f3")); err != nil {
+			t.Errorf("Pod %s could not resolve 'host.minikube.internal': %v", name, err)
+		} else {
+			hostIP := net.ParseIP(strings.TrimSpace(out.Stdout.String()))
+			// get node's eth0 network
+			if out, err := Run(t, exec.CommandContext(ctx, Target(), "ssh", "-p", profile, "ip -4 -br -o a s eth0 | tr -s ' ' | cut -d' ' -f3")); err != nil {
+				t.Errorf("Error getting eth0 IP of node %s: %v", profile, err)
+			} else {
+				if _, nodeNet, err := net.ParseCIDR(strings.TrimSpace(out.Stdout.String())); err != nil {
+					t.Errorf("Error parsing eth0 IP of node %s: %v", profile, err)
+					// check if host ip belongs to node's eth0 network
+				} else if !nodeNet.Contains(hostIP) {
+					t.Errorf("Pod %s resolved 'host.minikube.internal' to %s while node's eth0 network is %s", name, hostIP, nodeNet)
+				}
+			}
+		}
+	}
+}

test/integration/start_stop_delete_test.go

@@ -107,6 +107,7 @@ func TestStartStop(t *testing.T) {
 					}{
 						{"FirstStart", validateFirstStart},
 						{"DeployApp", validateDeploying},
+						{"EnableAddonWhileActive", validateEnableAddonWhileActive},
 						{"Stop", validateStop},
 						{"EnableAddonAfterStop", validateEnableAddonAfterStop},
 						{"SecondStart", validateSecondStart},
@@ -169,6 +170,31 @@ func validateDeploying(ctx context.Context, t *testing.T, profile string, tcName
 	}
 }
 
+// validateEnableAddonWhileActive makes sure addons can be enabled while cluster is active.
+func validateEnableAddonWhileActive(ctx context.Context, t *testing.T, profile string, tcName string, tcVersion string, startArgs []string) {
+	defer PostMortemLogs(t, profile)
+
+	// Enable an addon to assert it requests the correct image.
+	rr, err := Run(t, exec.CommandContext(ctx, Target(), "addons", "enable", "metrics-server", "-p", profile, "--images=MetricsServer=k8s.gcr.io/echoserver:1.4", "--registries=MetricsServer=fake.domain"))
+	if err != nil {
+		t.Errorf("failed to enable an addon post-stop. args %q: %v", rr.Command(), err)
+	}
+
+	if strings.Contains(tcName, "cni") {
+		t.Logf("WARNING: cni mode requires additional setup before pods can schedule :(")
+		return
+	}
+
+	rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "describe", "deploy/metrics-server", "-n", "kube-system"))
+	if err != nil {
+		t.Errorf("failed to get info on auto-pause deployments. args %q: %v", rr.Command(), err)
+	}
+	deploymentInfo := rr.Stdout.String()
+	if !strings.Contains(deploymentInfo, " fake.domain/k8s.gcr.io/echoserver:1.4") {
+		t.Errorf("addon did not load correct image. Expected to contain \" fake.domain/k8s.gcr.io/echoserver:1.4\". Addon deployment info: %s", deploymentInfo)
+	}
+}
+
 // validateStop tests minikube stop
 func validateStop(ctx context.Context, t *testing.T, profile string, tcName string, tcVersion string, startArgs []string) {
 	defer PostMortemLogs(t, profile)
@@ -190,7 +216,7 @@ func validateEnableAddonAfterStop(ctx context.Context, t *testing.T, profile str
 	}
 
 	// Enable an addon to assert it comes up afterwards
-	rr, err := Run(t, exec.CommandContext(ctx, Target(), "addons", "enable", "dashboard", "-p", profile))
+	rr, err := Run(t, exec.CommandContext(ctx, Target(), "addons", "enable", "dashboard", "-p", profile, "--images=MetricsScraper=k8s.gcr.io/echoserver:1.4"))
 	if err != nil {
 		t.Errorf("failed to enable an addon post-stop. args %q: %v", rr.Command(), err)
 	}
@@ -229,9 +255,20 @@ func validateAddonAfterStop(ctx context.Context, t *testing.T, profile string, t
 	defer PostMortemLogs(t, profile)
 	if strings.Contains(tcName, "cni") {
 		t.Logf("WARNING: cni mode requires additional setup before pods can schedule :(")
-	} else if _, err := PodWait(ctx, t, profile, "kubernetes-dashboard", "k8s-app=kubernetes-dashboard", Minutes(9)); err != nil {
+		return
+	}
+	if _, err := PodWait(ctx, t, profile, "kubernetes-dashboard", "k8s-app=kubernetes-dashboard", Minutes(9)); err != nil {
 		t.Errorf("failed waiting for 'addon dashboard' pod post-stop-start: %v", err)
 	}
+
+	rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "describe", "deploy/dashboard-metrics-scraper", "-n", "kubernetes-dashboard"))
+	if err != nil {
+		t.Errorf("failed to get info on kubernetes-dashboard deployments. args %q: %v", rr.Command(), err)
+	}
+	deploymentInfo := rr.Stdout.String()
+	if !strings.Contains(deploymentInfo, " k8s.gcr.io/echoserver:1.4") {
+		t.Errorf("addon did not load correct image. Expected to contain \" k8s.gcr.io/echoserver:1.4\". Addon deployment info: %s", deploymentInfo)
+	}
 }
 
 // validateKubernetesImages verifies that a restarted cluster contains all the necessary images

translations/de.json

@@ -241,6 +241,7 @@
 	"Failed to list cached images": "",
 	"Failed to list images": "",
 	"Failed to load image": "",
+	"Failed to persist images": "",
 	"Failed to pull image": "",
 	"Failed to reload cached images": "",
 	"Failed to remove image": "",
@@ -314,8 +315,8 @@
 	"If you are running minikube within a VM, consider using --driver=none:": "",
 	"If you are still interested to make {{.driver_name}} driver work. The following suggestions might help you get passed this issue:": "",
 	"If you don't want your credentials mounted into a specific pod, add a label with the `gcp-auth-skip-secret` key to your pod configuration.": "",
-	"Ignoring invalid custom image {{.conf}}": "",
+	"Ignoring empty custom image {{.name}}": "",
-	"Ignoring invalid custom registry {{.conf}}": "",
+	"Ignoring invalid pair entry {{.pair}}": "",
 	"Ignoring unknown custom image {{.name}}": "",
 	"Ignoring unknown custom registry {{.name}}": "",
 	"Images Commands:": "",

translations/es.json

@@ -246,6 +246,7 @@
 	"Failed to list cached images": "",
 	"Failed to list images": "",
 	"Failed to load image": "",
+	"Failed to persist images": "",
 	"Failed to pull image": "",
 	"Failed to reload cached images": "",
 	"Failed to remove image": "",
@@ -319,8 +320,8 @@
 	"If you are running minikube within a VM, consider using --driver=none:": "",
 	"If you are still interested to make {{.driver_name}} driver work. The following suggestions might help you get passed this issue:": "",
 	"If you don't want your credentials mounted into a specific pod, add a label with the `gcp-auth-skip-secret` key to your pod configuration.": "",
-	"Ignoring invalid custom image {{.conf}}": "",
+	"Ignoring empty custom image {{.name}}": "",
-	"Ignoring invalid custom registry {{.conf}}": "",
+	"Ignoring invalid pair entry {{.pair}}": "",
 	"Ignoring unknown custom image {{.name}}": "",
 	"Ignoring unknown custom registry {{.name}}": "",
 	"Images Commands:": "",

translations/fr.json

@@ -243,6 +243,7 @@
 	"Failed to list cached images": "",
 	"Failed to list images": "",
 	"Failed to load image": "",
+	"Failed to persist images": "",
 	"Failed to pull image": "",
 	"Failed to reload cached images": "",
 	"Failed to remove image": "",
@@ -316,8 +317,8 @@
 	"If you are running minikube within a VM, consider using --driver=none:": "",
 	"If you are still interested to make {{.driver_name}} driver work. The following suggestions might help you get passed this issue:": "",
 	"If you don't want your credentials mounted into a specific pod, add a label with the `gcp-auth-skip-secret` key to your pod configuration.": "",
-	"Ignoring invalid custom image {{.conf}}": "",
+	"Ignoring empty custom image {{.name}}": "",
-	"Ignoring invalid custom registry {{.conf}}": "",
+	"Ignoring invalid pair entry {{.pair}}": "",
 	"Ignoring unknown custom image {{.name}}": "",
 	"Ignoring unknown custom registry {{.name}}": "",
 	"Images Commands:": "",

translations/ja.json

@@ -234,6 +234,7 @@
 	"Failed to list cached images": "",
 	"Failed to list images": "",
 	"Failed to load image": "",
+	"Failed to persist images": "",
 	"Failed to pull image": "",
 	"Failed to reload cached images": "",
 	"Failed to remove image": "",
@@ -304,8 +305,8 @@
 	"If you are running minikube within a VM, consider using --driver=none:": "",
 	"If you are still interested to make {{.driver_name}} driver work. The following suggestions might help you get passed this issue:": "",
 	"If you don't want your credentials mounted into a specific pod, add a label with the `gcp-auth-skip-secret` key to your pod configuration.": "",
-	"Ignoring invalid custom image {{.conf}}": "",
+	"Ignoring empty custom image {{.name}}": "",
-	"Ignoring invalid custom registry {{.conf}}": "",
+	"Ignoring invalid pair entry {{.pair}}": "",
 	"Ignoring unknown custom image {{.name}}": "",
 	"Ignoring unknown custom registry {{.name}}": "",
 	"Images Commands:": "イメージ用コマンド:",

translations/ko.json

@@ -263,6 +263,7 @@
 	"Failed to list cached images": "캐시된 이미지를 조회하는 데 실패하였습니다",
 	"Failed to list images": "",
 	"Failed to load image": "",
+	"Failed to persist images": "",
 	"Failed to pull image": "",
 	"Failed to reload cached images": "캐시된 이미지를 다시 불러오는 데 실패하였습니다",
 	"Failed to remove image": "",
@@ -338,8 +339,8 @@
 	"If you are running minikube within a VM, consider using --driver=none:": "",
 	"If you are still interested to make {{.driver_name}} driver work. The following suggestions might help you get passed this issue:": "",
 	"If you don't want your credentials mounted into a specific pod, add a label with the `gcp-auth-skip-secret` key to your pod configuration.": "",
-	"Ignoring invalid custom image {{.conf}}": "",
+	"Ignoring empty custom image {{.name}}": "",
-	"Ignoring invalid custom registry {{.conf}}": "",
+	"Ignoring invalid pair entry {{.pair}}": "",
 	"Ignoring unknown custom image {{.name}}": "",
 	"Ignoring unknown custom registry {{.name}}": "",
 	"Images Commands:": "이미지 명령어",

translations/pl.json

@@ -251,6 +251,7 @@
 	"Failed to list cached images": "",
 	"Failed to list images": "",
 	"Failed to load image": "",
+	"Failed to persist images": "",
 	"Failed to pull image": "",
 	"Failed to reload cached images": "",
 	"Failed to remove image": "",
@@ -326,8 +327,8 @@
 	"If you are running minikube within a VM, consider using --driver=none:": "",
 	"If you are still interested to make {{.driver_name}} driver work. The following suggestions might help you get passed this issue:": "",
 	"If you don't want your credentials mounted into a specific pod, add a label with the `gcp-auth-skip-secret` key to your pod configuration.": "",
-	"Ignoring invalid custom image {{.conf}}": "",
+	"Ignoring empty custom image {{.name}}": "",
-	"Ignoring invalid custom registry {{.conf}}": "",
+	"Ignoring invalid pair entry {{.pair}}": "",
 	"Ignoring unknown custom image {{.name}}": "",
 	"Ignoring unknown custom registry {{.name}}": "",
 	"Images Commands:": "",

translations/strings.txt

@@ -226,6 +226,7 @@
 	"Failed to list cached images": "",
 	"Failed to list images": "",
 	"Failed to load image": "",
+	"Failed to persist images": "",
 	"Failed to pull image": "",
 	"Failed to reload cached images": "",
 	"Failed to remove image": "",
@@ -294,8 +295,8 @@
 	"If you are running minikube within a VM, consider using --driver=none:": "",
 	"If you are still interested to make {{.driver_name}} driver work. The following suggestions might help you get passed this issue:": "",
 	"If you don't want your credentials mounted into a specific pod, add a label with the `gcp-auth-skip-secret` key to your pod configuration.": "",
-	"Ignoring invalid custom image {{.conf}}": "",
+	"Ignoring empty custom image {{.name}}": "",
-	"Ignoring invalid custom registry {{.conf}}": "",
+	"Ignoring invalid pair entry {{.pair}}": "",
 	"Ignoring unknown custom image {{.name}}": "",
 	"Ignoring unknown custom registry {{.name}}": "",
 	"Images Commands:": "",

translations/zh-CN.json

@@ -312,6 +312,7 @@
 	"Failed to list cached images": "无法列出缓存镜像",
 	"Failed to list images": "",
 	"Failed to load image": "",
+	"Failed to persist images": "",
 	"Failed to pull image": "",
 	"Failed to reload cached images": "重新加载缓存镜像失败",
 	"Failed to remove image": "",
@@ -395,8 +396,8 @@
 	"If you are running minikube within a VM, consider using --driver=none:": "",
 	"If you are still interested to make {{.driver_name}} driver work. The following suggestions might help you get passed this issue:": "",
 	"If you don't want your credentials mounted into a specific pod, add a label with the `gcp-auth-skip-secret` key to your pod configuration.": "",
-	"Ignoring invalid custom image {{.conf}}": "",
+	"Ignoring empty custom image {{.name}}": "",
-	"Ignoring invalid custom registry {{.conf}}": "",
+	"Ignoring invalid pair entry {{.pair}}": "",
 	"Ignoring unknown custom image {{.name}}": "",
 	"Ignoring unknown custom registry {{.name}}": "",
 	"Images Commands:": "",