Merge branch 'master' of github.com:kubernetes/minikube into macstadium

.github/ISSUE_TEMPLATE/__en-US.md

@@ -9,16 +9,14 @@ about: Report an issue
 2. 
 3.
 
-<!--- TIP: Add the "--alsologtostderr" flag to the command-line for more logs --->
+**Full output of `minikube logs` command:**
-**Full output of failed command:** 
+<details>
-
+
-
+
-
+</details>
-**Full output of `minikube start` command used, if not already included:**
+
-
+<!--- TIP: Add the "--alsologtostderr" flag to the command-line for more logs --->
-
+**Full output of failed command:**
-
-**Optional: Full output of `minikube logs` command:**
 <details>
 
 

.github/workflows/master.yml

@@ -183,6 +183,105 @@ jobs:
           if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
           if [ "$numPass" -lt 36 ];then echo "*** Failed to pass at least 36 ! ***";exit 2;fi
           if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
+  functional_docker_containerd_ubuntu:
+    needs: [build_minikube]
+    env:
+      TIME_ELAPSED: time
+      JOB_NAME: "functional_docker_containerd_ubuntu"
+      GOPOGH_RESULT: ""
+      SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643
+    runs-on: ubuntu-18.04
+    steps:
+      - name: Install kubectl
+        shell: bash
+        run: |
+          curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/amd64/kubectl
+          sudo install kubectl /usr/local/bin/kubectl
+          kubectl version --client=true
+      - name: Docker Info
+        shell: bash
+        run: |
+          echo "--------------------------"
+          docker version || true
+          echo "--------------------------"
+          docker info || true
+          echo "--------------------------"
+          docker system df || true
+          echo "--------------------------"
+          docker system info --format='{{json .}}'|| true
+          echo "--------------------------"
+          docker ps || true
+          echo "--------------------------"
+      # go 1.14.6+ is needed because of this bug https://github.com/golang/go/issues/39308
+      - uses: actions/setup-go@v2
+        with:
+          go-version: ${{env.GO_VERSION}}
+          stable: true
+      - name: Install gopogh
+
+        shell: bash
+        run: |
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.6.0/gopogh-linux-amd64
+          sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
+      - name: Download Binaries
+        uses: actions/download-artifact@v1
+        with:
+          name: minikube_binaries
+      - name: Run Integration Test
+        continue-on-error: false
+        # bash {0} to allow test to continue to next step. in case of
+        shell: bash {0}
+        run: |
+          cd minikube_binaries
+          mkdir -p report
+          mkdir -p testhome
+          chmod a+x e2e-*
+          chmod a+x minikube-*
+          sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+          sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+          START_TIME=$(date -u +%s)
+          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args="--vm-driver=docker --container-runtime=containerd" -test.run TestFunctional -test.timeout=30m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
+          END_TIME=$(date -u +%s)
+          TIME_ELAPSED=$(($END_TIME-$START_TIME))
+          min=$((${TIME_ELAPSED}/60))
+          sec=$((${TIME_ELAPSED}%60))
+          TIME_ELAPSED="${min} min $sec seconds "
+          echo "TIME_ELAPSED=${TIME_ELAPSED}" >> $GITHUB_ENV
+      - name: Generate HTML Report
+        shell: bash
+        run: |
+          cd minikube_binaries
+          export PATH=${PATH}:`go env GOPATH`/bin
+          go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
+          STAT=$(gopogh -in ./report/testout.json -out_html ./report/testout.html -out_summary ./report/testout_summary.json -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
+          echo status: ${STAT}
+          FailNum=$(echo $STAT | jq '.NumberOfFail')
+          TestsNum=$(echo $STAT | jq '.NumberOfTests')
+          GOPOGH_RESULT="${JOB_NAME} : completed with ${FailNum} / ${TestsNum} failures in ${TIME_ELAPSED}"
+          echo "GOPOGH_RESULT=${GOPOGH_RESULT}" >> $GITHUB_ENV
+          echo 'STAT<<EOF' >> $GITHUB_ENV
+          echo "${STAT}" >> $GITHUB_ENV
+          echo 'EOF' >> $GITHUB_ENV
+      - uses: actions/upload-artifact@v1
+        with:
+          name: functional_docker_containerd_ubuntu
+          path: minikube_binaries/report
+      - name: The End Result functional_docker_containerd_ubuntu
+        shell: bash
+        run: |
+          echo ${GOPOGH_RESULT}
+          numFail=$(echo $STAT | jq '.NumberOfFail')
+          numPass=$(echo $STAT | jq '.NumberOfPass')
+          echo "*******************${numPass} Passes :) *******************"
+          echo $STAT | jq '.PassedTests' || true
+          echo "*******************************************************"
+          echo "---------------- ${numFail} Failures :( ----------------------------"
+          echo $STAT | jq '.FailedTests' || true
+          echo "-------------------------------------------------------"
+          if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
+          if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
+          if [ "$numPass" -lt 36 ];then echo "*** Failed to pass at least 36 ! ***";exit 2;fi
+          if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
   functional_virtualbox_macos:
     needs: [build_minikube]
     env:
@@ -781,6 +880,7 @@ jobs:
     needs:
       [
         functional_docker_ubuntu,
+        functional_docker_containerd_ubuntu,
         functional_docker_ubuntu_arm64,
         functional_virtualbox_macos,
         functional_docker_windows,
@@ -798,6 +898,7 @@ jobs:
           mkdir -p all_reports
           ls -lah
           cp -r ./functional_docker_ubuntu ./all_reports/
+          cp -r ./functional_docker_containerd_ubuntu ./all_reports/
           cp -r ./functional_docker_ubuntu_arm64 ./all_reports/
           cp -r ./functional_virtualbox_macos ./all_reports/
           cp -r ./functional_docker_windows ./all_reports/

.github/workflows/pr.yml

@@ -181,6 +181,105 @@ jobs:
           if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
           if [ "$numPass" -lt 36 ];then echo "*** Failed to pass at least 36 ! ***";exit 2;fi
           if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
+  functional_docker_containerd_ubuntu:
+    needs: [build_minikube]
+    env:
+      TIME_ELAPSED: time
+      JOB_NAME: "functional_docker_containerd_ubuntu"
+      GOPOGH_RESULT: ""
+      SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643
+    runs-on: ubuntu-18.04
+    steps:
+      - name: Install kubectl
+        shell: bash
+        run: |
+          curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/amd64/kubectl
+          sudo install kubectl /usr/local/bin/kubectl
+          kubectl version --client=true
+      - name: Docker Info
+        shell: bash
+        run: |
+          echo "--------------------------"
+          docker version || true
+          echo "--------------------------"
+          docker info || true
+          echo "--------------------------"
+          docker system df || true
+          echo "--------------------------"
+          docker system info --format='{{json .}}'|| true
+          echo "--------------------------"
+          docker ps || true
+          echo "--------------------------"
+      # go 1.14.6+ is needed because of this bug https://github.com/golang/go/issues/39308
+      - uses: actions/setup-go@v2
+        with:
+          go-version: ${{env.GO_VERSION}}
+          stable: true
+      - name: Install gopogh
+
+        shell: bash
+        run: |
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.6.0/gopogh-linux-amd64
+          sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
+      - name: Download Binaries
+        uses: actions/download-artifact@v1
+        with:
+          name: minikube_binaries
+      - name: Run Integration Test
+        continue-on-error: false
+        # bash {0} to allow test to continue to next step. in case of
+        shell: bash {0}
+        run: |
+          cd minikube_binaries
+          mkdir -p report
+          mkdir -p testhome
+          chmod a+x e2e-*
+          chmod a+x minikube-*
+          sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+          sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+          START_TIME=$(date -u +%s)
+          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args="--vm-driver=docker --container-runtime=containerd" -test.run TestFunctional -test.timeout=30m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
+          END_TIME=$(date -u +%s)
+          TIME_ELAPSED=$(($END_TIME-$START_TIME))
+          min=$((${TIME_ELAPSED}/60))
+          sec=$((${TIME_ELAPSED}%60))
+          TIME_ELAPSED="${min} min $sec seconds "
+          echo "TIME_ELAPSED=${TIME_ELAPSED}" >> $GITHUB_ENV
+      - name: Generate HTML Report
+        shell: bash
+        run: |
+          cd minikube_binaries
+          export PATH=${PATH}:`go env GOPATH`/bin
+          go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
+          STAT=$(gopogh -in ./report/testout.json -out_html ./report/testout.html -out_summary ./report/testout_summary.json -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
+          echo status: ${STAT}
+          FailNum=$(echo $STAT | jq '.NumberOfFail')
+          TestsNum=$(echo $STAT | jq '.NumberOfTests')
+          GOPOGH_RESULT="${JOB_NAME} : completed with ${FailNum} / ${TestsNum} failures in ${TIME_ELAPSED}"
+          echo "GOPOGH_RESULT=${GOPOGH_RESULT}" >> $GITHUB_ENV
+          echo 'STAT<<EOF' >> $GITHUB_ENV
+          echo "${STAT}" >> $GITHUB_ENV
+          echo 'EOF' >> $GITHUB_ENV
+      - uses: actions/upload-artifact@v1
+        with:
+          name: functional_docker_containerd_ubuntu
+          path: minikube_binaries/report
+      - name: The End Result functional_docker_containerd_ubuntu
+        shell: bash
+        run: |
+          echo ${GOPOGH_RESULT}
+          numFail=$(echo $STAT | jq '.NumberOfFail')
+          numPass=$(echo $STAT | jq '.NumberOfPass')
+          echo "*******************${numPass} Passes :) *******************"
+          echo $STAT | jq '.PassedTests' || true
+          echo "*******************************************************"
+          echo "---------------- ${numFail} Failures :( ----------------------------"
+          echo $STAT | jq '.FailedTests' || true
+          echo "-------------------------------------------------------"
+          if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
+          if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
+          if [ "$numPass" -lt 36 ];then echo "*** Failed to pass at least 36 ! ***";exit 2;fi
+          if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
   functional_virtualbox_macos:
     needs: [build_minikube]
     env:
@@ -779,6 +878,7 @@ jobs:
     needs:
       [
         functional_docker_ubuntu,
+        functional_docker_containerd_ubuntu,
         functional_docker_ubuntu_arm64,
         functional_virtualbox_macos,
         functional_docker_windows,
@@ -796,6 +896,7 @@ jobs:
           mkdir -p all_reports
           ls -lah
           cp -r ./functional_docker_ubuntu ./all_reports/
+          cp -r ./functional_docker_containerd_ubuntu ./all_reports/
           cp -r ./functional_docker_ubuntu_arm64 ./all_reports/
           cp -r ./functional_virtualbox_macos ./all_reports/
           cp -r ./functional_docker_windows ./all_reports/

CHANGELOG.md

@@ -1,5 +1,41 @@
 # Release Notes
 
+## Version 1.20.0-beta.0 - 2021-04-30
+
+Features:
+
+* New command: `build` to build images using minikube [#11164](https://github.com/kubernetes/minikube/pull/11164)
+* New command 'image pull': allow to load remote images directly without cache [#11127](https://github.com/kubernetes/minikube/pull/11127)
+* Add feature to opt-in to get notifications for beta releases [#11169](https://github.com/kubernetes/minikube/pull/11169)
+* UI: Add log file to GitHub issue output [#11158](https://github.com/kubernetes/minikube/pull/11158)
+
+Bug Fixes:
+
+* Ingress Addon: fix bug which the networking.k8s.io/v1 ingress is always rejected [#11189](https://github.com/kubernetes/minikube/pull/11189)
+* Improve how cni and cruntimes work together [#11185, #11209](https://github.com/kubernetes/minikube/pull/11209, https://github.com/kubernetes/minikube/pull/11185)
+* Docker driverr: support docker installed by Snap Package Manager [#11088](https://github.com/kubernetes/minikube/pull/11088)
+* Change 'minikube version --short' to only print the version without a prompt. [#11167](https://github.com/kubernetes/minikube/pull/11167)
+
+
+Thank you to our contributors for this release!
+
+- Anders F Björklund
+- Andriy Dzikh
+- Ed Vinyard
+- Hu Shuai
+- Ilya Zuyev
+- Kenta Iso
+- Medya Ghazizadeh
+- Michael Captain
+- Predrag Rogic
+- Sharif Elgamal
+- Steven Powell
+- Tobias Klauser
+- csiepka
+- hiroygo
+- 李龙峰
+
+
 ## Version 1.19.0 - 2021-04-09
 
 * allow Auto-Pause addon on VMs [#11019](https://github.com/kubernetes/minikube/pull/11019)

Makefile

@@ -14,8 +14,8 @@
 
 # Bump these on release - and please check ISO_VERSION for correctness.
 VERSION_MAJOR ?= 1
-VERSION_MINOR ?= 19
+VERSION_MINOR ?= 20
-VERSION_BUILD ?= 0
+VERSION_BUILD ?= 0-beta.0
 RAW_VERSION=$(VERSION_MAJOR).$(VERSION_MINOR).$(VERSION_BUILD)
 VERSION ?= v$(RAW_VERSION)
 
@@ -23,7 +23,7 @@ KUBERNETES_VERSION ?= $(shell egrep "DefaultKubernetesVersion =" pkg/minikube/co
 KIC_VERSION ?= $(shell egrep "Version =" pkg/drivers/kic/types.go | cut -d \" -f2)
 
 # Default to .0 for higher cache hit rates, as build increments typically don't require new ISO versions
-ISO_VERSION ?= v1.19.0-1619288095-11054
+ISO_VERSION ?= v1.20.0-beta.0
 # Dashes are valid in semver, but not Linux packaging. Use ~ to delimit alpha/beta
 DEB_VERSION ?= $(subst -,~,$(RAW_VERSION))
 DEB_REVISION ?= 0
@@ -855,7 +855,7 @@ site: site/themes/docsy/assets/vendor/bootstrap/package.js out/hugo/hugo ## Serv
 out/mkcmp:
 	GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o $@ cmd/performance/mkcmp/main.go
 
-.PHONY: deploy/kicbase/auto-pause # auto pause binary to be used for kic image work arround for not passing the whole repo as docker context
+.PHONY: deploy/kicbase/auto-pause # auto pause binary to be used for kic image work around for not passing the whole repo as docker context
 deploy/kicbase/auto-pause: $(SOURCE_GENERATED) $(SOURCE_FILES)
 	GOOS=linux GOARCH=$(GOARCH) go build -o $@ cmd/auto-pause/auto-pause.go
 

cmd/minikube/cmd/start.go

@@ -394,7 +394,7 @@ func startWithDriver(cmd *cobra.Command, starter node.Starter, existing *config.
 }
 
 func warnAboutMultiNodeCNI() {
-	out.WarningT("Cluster was created without any CNI, adding node to it might cause broken network.")
+	out.WarningT("Cluster was created without any CNI, adding a node to it might cause broken networking.")
 }
 
 func updateDriver(driverName string) {
@@ -1370,6 +1370,14 @@ func validateKubernetesVersion(old *config.ClusterConfig) {
 		exitIfNotForced(reason.KubernetesTooOld, "Kubernetes {{.version}} is not supported by this release of minikube", out.V{"version": nvs})
 	}
 
+	// If the version of Kubernetes has a known issue, print a warning out to the screen
+	if issue := reason.ProblematicK8sVersion(nvs); issue != nil {
+		out.WarningT(issue.Description, out.V{"version": nvs.String()})
+		if issue.URL != "" {
+			out.WarningT("For more information, see: {{.url}}", out.V{"url": issue.URL})
+		}
+	}
+
 	if old == nil || old.KubernetesConfig.KubernetesVersion == "" {
 		return
 	}

cmd/minikube/cmd/start_flags.go

@@ -249,6 +249,12 @@ func generateClusterConfig(cmd *cobra.Command, existing *config.ClusterConfig, k
 	var cc config.ClusterConfig
 	if existing != nil {
 		cc = updateExistingConfigFromFlags(cmd, existing)
+
+		// identify appropriate cni then configure cruntime accordingly
+		_, err := cni.New(&cc)
+		if err != nil {
+			return cc, config.Node{}, errors.Wrap(err, "cni")
+		}
 	} else {
 		klog.Info("no existing cluster config was found, will generate one from the flags ")
 		cc = generateNewConfigFromFlags(cmd, k8sVersion, drvName)

deploy/addons/olm/olm.yaml.tmpl

@@ -78,9 +78,9 @@ spec:
           command:
           - /bin/olm
           args:
-          - -namespace
+          - --namespace
           - $(OPERATOR_NAMESPACE)
-          - -writeStatusName
+          - --writeStatusName
           - ""
           image: {{.CustomRegistries.OLM  | default .ImageRepository | default .Registries.OLM }}{{.Images.OLM}}
           imagePullPolicy: IfNotPresent
@@ -113,7 +113,7 @@ spec:
           
       
       nodeSelector:
-        beta.kubernetes.io/os: linux
+        kubernetes.io/os: linux
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -143,6 +143,8 @@ spec:
           - '-namespace'
           - olm
           - -configmapServerImage=quay.io/operator-framework/configmap-operator-registry:latest
+          - -util-image
+          -  {{.CustomRegistries.OLM  | default .ImageRepository | default .Registries.OLM }}{{.Images.OLM}}
           image: {{.CustomRegistries.OLM  | default .ImageRepository | default .Registries.OLM }}{{.Images.OLM}}
           imagePullPolicy: IfNotPresent
           ports:
@@ -168,7 +170,7 @@ spec:
           
       
       nodeSelector:
-        beta.kubernetes.io/os: linux
+        kubernetes.io/os: linux
 ---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
@@ -222,7 +224,7 @@ metadata:
   name: packageserver
   namespace: olm
   labels:
-    olm.version: 0.14.1
+    olm.version: 0.17.0
 spec:
   displayName: Package Server
   description: Represents an Operator package that is available from a given CatalogSource which will resolve to a ClusterServiceVersion.
@@ -297,7 +299,7 @@ spec:
             spec:
               serviceAccountName: olm-operator-serviceaccount
               nodeSelector:
-                beta.kubernetes.io/os: linux
+                kubernetes.io/os: linux
               containers:
               - name: packageserver
                 command:
@@ -308,7 +310,7 @@ spec:
                 - --global-namespace
                 - olm
                 image: {{.CustomRegistries.OLM  | default .ImageRepository | default .Registries.OLM }}{{.Images.OLM}}
-                imagePullPolicy: IfNotPresent
+                imagePullPolicy: Always
                 ports:
                 - containerPort: 5443
                 livenessProbe:
@@ -326,8 +328,16 @@ spec:
                   requests:
                     cpu: 10m
                     memory: 50Mi
+                securityContext:
+                  runAsUser: 1000
+                volumeMounts:
+                - name: tmpfs
+                  mountPath: /tmp
+              volumes:
+              - name: tmpfs
+                emptyDir: {}
   maturity: alpha
-  version: 0.14.1
+  version: 0.17.0
   apiservicedefinitions:
     owned:
     - group: packages.operators.coreos.com

deploy/minikube/releases-beta.json

@@ -1,2 +1,10 @@
 [
+{
+      "name": "v1.20.0-beta.0",
+      "checksums": {
+          "darwin": "",
+          "linux": "",
+          "windows": ""
+      }
+  }
 ]

go.mod

@@ -3,9 +3,9 @@ module k8s.io/minikube
 go 1.16
 
 require (
-	cloud.google.com/go/storage v1.13.0
+	cloud.google.com/go/storage v1.15.0
 	contrib.go.opencensus.io/exporter/stackdriver v0.12.1
-	github.com/Azure/azure-sdk-for-go v43.0.0+incompatible
+	github.com/Azure/azure-sdk-for-go v43.3.0+incompatible
 	github.com/Delta456/box-cli-maker/v2 v2.2.1
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v0.16.0
 	github.com/Microsoft/hcsshim v0.8.15 // indirect
@@ -34,7 +34,7 @@ require (
 	github.com/google/slowjam v0.0.0-20200530021616-df27e642fe7b
 	github.com/google/uuid v1.2.0
 	github.com/hashicorp/go-getter v1.5.2
-	github.com/hashicorp/go-retryablehttp v0.6.8
+	github.com/hashicorp/go-retryablehttp v0.7.0
 	github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect
 	github.com/hooklift/assert v0.0.0-20170704181755-9d1defd6d214 // indirect
 	github.com/hooklift/iso9660 v0.0.0-20170318115843-1cf07e5970d8
@@ -68,7 +68,7 @@ require (
 	github.com/pmezard/go-difflib v1.0.0
 	github.com/russross/blackfriday v1.5.3-0.20200218234912-41c5fccfd6f6 // indirect
 	github.com/samalba/dockerclient v0.0.0-20160414174713-91d7393ff859 // indirect
-	github.com/shirou/gopsutil/v3 v3.21.3
+	github.com/shirou/gopsutil/v3 v3.21.4
 	github.com/spf13/cobra v1.1.3
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.7.1

go.sum

@@ -19,7 +19,6 @@ cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOY
 cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
 cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI=
 cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk=
-cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY=
 cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg=
 cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8=
 cloud.google.com/go v0.81.0 h1:at8Tk2zUz63cLPR0JPWm5vp77pEZmzxEQBEfRKn1VV8=
@@ -42,16 +41,17 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo
 cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
 cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
-cloud.google.com/go/storage v1.13.0 h1:amPvhCOI+Hltp6rPu+62YdwhIrjf+34PKVAL4HwgYwk=
+cloud.google.com/go/storage v1.15.0 h1:Ljj+ZXVEhCr/1+4ZhvtteN1ND7UUsNTlduGclLh8GO0=
-cloud.google.com/go/storage v1.13.0/go.mod h1:pqFyBUK3zZqMIIU5+8NaZq6/Ma3ClgUg9Hv5jfuJnvo=
+cloud.google.com/go/storage v1.15.0/go.mod h1:mjjQMoxxyGH7Jr8K5qrx6N2O0AHsczI61sMNn03GIZI=
 contrib.go.opencensus.io/exporter/stackdriver v0.12.1 h1:Dll2uFfOVI3fa8UzsHyP6z0M6fEc9ZTAMo+Y3z282Xg=
 contrib.go.opencensus.io/exporter/stackdriver v0.12.1/go.mod h1:iwB6wGarfphGGe/e5CWqyUk/cLzKnWsOKPVW3no6OTw=
 contrib.go.opencensus.io/resource v0.1.1/go.mod h1:F361eGI91LCmW1I/Saf+rX0+OFcigGlFvXwEGEnkRLA=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8=
 github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
-github.com/Azure/azure-sdk-for-go v43.0.0+incompatible h1:/wSNCu0e6EsHFR4Qa3vBEBbicaprEHMyyga9g8RTULI=
 github.com/Azure/azure-sdk-for-go v43.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
+github.com/Azure/azure-sdk-for-go v43.3.0+incompatible h1:o0G4JAsOzeVJEwU0Ud9bh+lUHPUc0GkFENJ02dk51Uo=
+github.com/Azure/azure-sdk-for-go v43.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
 github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
@@ -508,7 +508,6 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf
 github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
@@ -561,8 +560,8 @@ github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjh
 github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
 github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
-github.com/hashicorp/go-retryablehttp v0.6.8 h1:92lWxgpa+fF3FozM4B3UZtHZMJX8T5XT+TFdCxsPyWs=
+github.com/hashicorp/go-retryablehttp v0.7.0 h1:eu1EI/mbirUgP5C8hVsTNaGZreBDlYiwC1FZWkvQPQ4=
-github.com/hashicorp/go-retryablehttp v0.6.8/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
+github.com/hashicorp/go-retryablehttp v0.7.0/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
 github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
 github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo=
 github.com/hashicorp/go-safetemp v1.0.0/go.mod h1:oaerMy3BhqiTbVye6QuFhFtIceqFoDHxNAB65b+Rj1I=
@@ -918,8 +917,8 @@ github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
-github.com/shirou/gopsutil/v3 v3.21.3 h1:wgcdAHZS2H6qy4JFewVTtqfiYxFzCeEJod/mLztdPG8=
+github.com/shirou/gopsutil/v3 v3.21.4 h1:XB/+p+kVnyYLuPHCfa99lxz2aJyvVhnyd+FxZqH/k7M=
-github.com/shirou/gopsutil/v3 v3.21.3/go.mod h1:ghfMypLDrFSWN2c9cDYFLHyynQ+QUht0cv/18ZqVczw=
+github.com/shirou/gopsutil/v3 v3.21.4/go.mod h1:ghfMypLDrFSWN2c9cDYFLHyynQ+QUht0cv/18ZqVczw=
 github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
@@ -1197,7 +1196,6 @@ golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4Iltr
 golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210113205817-d3ed898aa8a3/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
@@ -1387,7 +1385,6 @@ golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4f
 golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY=
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -1426,7 +1423,6 @@ google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSr
 google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
 google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
 google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE=
-google.golang.org/api v0.38.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8=
 google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8=
 google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU=
 google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94=
@@ -1482,15 +1478,14 @@ google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6D
 google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210203152818-3206188e46ba/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=
-google.golang.org/genproto v0.0.0-20210413151531-c14fb6ef47c3 h1:K+7Ig5hjiLVA/i1UFUUbCGimWz5/Ey0lAQjT3QiLaPY=
 google.golang.org/genproto v0.0.0-20210413151531-c14fb6ef47c3/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A=
+google.golang.org/genproto v0.0.0-20210420162539-3c870d7478d2 h1:g2sJMUGCpeHZqTx8p3wsAWRS64nFq20i4dvJWcKGqvY=
+google.golang.org/genproto v0.0.0-20210420162539-3c870d7478d2/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A=
 google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=

hack/jenkins/release_update_releases_json.sh

@@ -51,7 +51,7 @@ if ! [[ "${VERSION_BUILD}" =~ ^[0-9]+$ ]]; then
   curl -X POST -u minikube-bot:${BOT_PASSWORD} -k   -d "{\"title\": \"update releases-beta.json to include ${TAGNAME}\",\"head\": \"minikube-bot:jenkins-releases.json-${TAGNAME}\",\"base\": \"master\"}" https://api.github.com/repos/kubernetes/minikube/pulls
 
   # Upload file to GCS so that minikube can see the new version
-  gsutil cp deploy/minikube/releases.json gs://minikube/releases-beta.json
+  gsutil cp deploy/minikube/releases-beta.json gs://minikube/releases-beta.json
 else
   #Prepends the new version to the release.json file
   sed -i "0,/{/s/{/{\n      \"name\": \"${TAGNAME}\",\n      \"checksums\": {\n          \"darwin\": \"${DARWIN_SHA256}\",\n          \"linux\": \"${LINUX_SHA256}\",\n          \"windows\": \"${WINDOWS_SHA256}\"\n      }\n  },\n  {"/ deploy/minikube/releases.json

pkg/drivers/kic/types.go

@@ -24,13 +24,13 @@ import (
 
 const (
 	// Version is the current version of kic
-	Version = "v0.0.20-1619543236-11212"
+	Version = "v0.0.21"
 	// SHA of the kic base image
-	baseImageSHA = "b7c6676a7c18654deb3258da55a95dec53983871e371186f217055256cda5b24"
+	baseImageSHA = "30ac715d2caf25e953e6ca9b6525d17bc8148236a6153208780f2c485bcdffed"
 	// The name of the GCR kicbase repository
-	gcrRepo = "gcr.io/k8s-minikube/kicbase-builds"
+	gcrRepo = "gcr.io/k8s-minikube/kicbase"
 	// The name of the Dockerhub kicbase repository
-	dockerhubRepo = "kicbase/build"
+	dockerhubRepo = "kicbase/stable"
 )
 
 var (

pkg/minikube/assets/addons.go

@@ -305,7 +305,7 @@ var Addons = map[string]*Addon{
 			"olm.yaml",
 			"0640"),
 	}, false, "olm", map[string]string{
-		"OLM":                        "operator-framework/olm:0.14.1@sha256:0d15ffb5d10a176ef6e831d7865f98d51255ea5b0d16403618c94a004d049373",
+		"OLM":                        "operator-framework/olm:v0.17.0@sha256:de396b540b82219812061d0d753440d5655250c621c753ed1dc67d6154741607",
 		"UpstreamCommunityOperators": "operator-framework/upstream-community-operators:07bbc13@sha256:cc7b3fdaa1ccdea5866fcd171669dc0ed88d3477779d8ed32e3712c827e38cc0",
 	}, map[string]string{
 		"OLM":                        "quay.io",

pkg/minikube/bootstrapper/bsutil/kverify/pod_ready.go

@@ -92,6 +92,7 @@ func waitPodCondition(cs *kubernetes.Clientset, name, namespace string, conditio
 			klog.Info(reason)
 			return true, nil
 		}
+		// return immediately: status == core.ConditionUnknown
 		if status == core.ConditionUnknown {
 			klog.Info(reason)
 			return false, fmt.Errorf(reason)
@@ -101,6 +102,7 @@ func waitPodCondition(cs *kubernetes.Clientset, name, namespace string, conditio
 			klog.Info(reason)
 			lap = time.Now()
 		}
+		// return immediately: status == core.ConditionFalse
 		return false, nil
 	}
 	if err := wait.PollImmediate(kconst.APICallRetryInterval, kconst.DefaultControlPlaneTimeout, checkCondition); err != nil {
@@ -114,13 +116,13 @@ func waitPodCondition(cs *kubernetes.Clientset, name, namespace string, conditio
 func podConditionStatus(cs *kubernetes.Clientset, name, namespace string, condition core.PodConditionType) (status core.ConditionStatus, reason string) {
 	pod, err := cs.CoreV1().Pods(namespace).Get(context.Background(), name, meta.GetOptions{})
 	if err != nil {
-		return core.ConditionUnknown, fmt.Sprintf("error getting pod %q in %q namespace: %v", name, namespace, err)
+		return core.ConditionUnknown, fmt.Sprintf("error getting pod %q in %q namespace (skipping!): %v", name, namespace, err)
 	}
 
 	// check if undelying node is Ready - in case we got stale data about the pod
 	if pod.Spec.NodeName != "" {
 		if status, reason := nodeConditionStatus(cs, pod.Spec.NodeName, core.NodeReady); status != core.ConditionTrue {
-			return core.ConditionUnknown, fmt.Sprintf("node %q hosting pod %q in %q namespace is currently not %q: %v", pod.Spec.NodeName, name, namespace, core.NodeReady, reason)
+			return core.ConditionUnknown, fmt.Sprintf("node %q hosting pod %q in %q namespace is currently not %q (skipping!): %v", pod.Spec.NodeName, name, namespace, core.NodeReady, reason)
 		}
 	}
 

pkg/minikube/bootstrapper/kubeadm/kubeadm.go

@@ -676,34 +676,6 @@ func (k *Bootstrapper) restartControlPlane(cfg config.ClusterConfig) error {
 		}
 	}
 
-	if cfg.VerifyComponents[kverify.ExtraKey] {
-		// after kubelet is restarted (with 'kubeadm init phase kubelet-start' above),
-		// it appears as to be immediately Ready as well as all kube-system pods (last observed state),
-		// then (after ~10sec) it realises it has some changes to apply, implying also pods restarts,
-		// and by that time we would exit completely, so we wait until kubelet begins restarting pods
-		klog.Info("waiting for restarted kubelet to initialise ...")
-		start := time.Now()
-		wait := func() error {
-			pods, err := client.CoreV1().Pods(meta.NamespaceSystem).List(context.Background(), meta.ListOptions{LabelSelector: "tier=control-plane"})
-			if err != nil {
-				return err
-			}
-			for _, pod := range pods.Items {
-				if ready, _ := kverify.IsPodReady(&pod); !ready {
-					return nil
-				}
-			}
-			return fmt.Errorf("kubelet not initialised")
-		}
-		_ = retry.Expo(wait, 250*time.Millisecond, 1*time.Minute)
-		klog.Infof("kubelet initialised")
-		klog.Infof("duration metric: took %s waiting for restarted kubelet to initialise ...", time.Since(start))
-
-		if err := kverify.WaitExtra(client, kverify.CorePodsLabels, kconst.DefaultControlPlaneTimeout); err != nil {
-			return errors.Wrap(err, "extra")
-		}
-	}
-
 	cr, err := cruntime.New(cruntime.Config{Type: cfg.KubernetesConfig.ContainerRuntime, Runner: k.c})
 	if err != nil {
 		return errors.Wrap(err, "runtime")
@@ -741,6 +713,35 @@ func (k *Bootstrapper) restartControlPlane(cfg config.ClusterConfig) error {
 		return errors.Wrap(err, "addons")
 	}
 
+	// must be called after applyCNI and `kubeadm phase addon all` (ie, coredns redeploy)
+	if cfg.VerifyComponents[kverify.ExtraKey] {
+		// after kubelet is restarted (with 'kubeadm init phase kubelet-start' above),
+		// it appears as to be immediately Ready as well as all kube-system pods (last observed state),
+		// then (after ~10sec) it realises it has some changes to apply, implying also pods restarts,
+		// and by that time we would exit completely, so we wait until kubelet begins restarting pods
+		klog.Info("waiting for restarted kubelet to initialise ...")
+		start := time.Now()
+		wait := func() error {
+			pods, err := client.CoreV1().Pods(meta.NamespaceSystem).List(context.Background(), meta.ListOptions{LabelSelector: "tier=control-plane"})
+			if err != nil {
+				return err
+			}
+			for _, pod := range pods.Items {
+				if ready, _ := kverify.IsPodReady(&pod); !ready {
+					return nil
+				}
+			}
+			return fmt.Errorf("kubelet not initialised")
+		}
+		_ = retry.Expo(wait, 250*time.Millisecond, 1*time.Minute)
+		klog.Infof("kubelet initialised")
+		klog.Infof("duration metric: took %s waiting for restarted kubelet to initialise ...", time.Since(start))
+
+		if err := kverify.WaitExtra(client, kverify.CorePodsLabels, kconst.DefaultControlPlaneTimeout); err != nil {
+			return errors.Wrap(err, "extra")
+		}
+	}
+
 	if err := bsutil.AdjustResourceLimits(k.c); err != nil {
 		klog.Warningf("unable to adjust resource limits: %v", err)
 	}
@@ -776,9 +777,17 @@ func (k *Bootstrapper) GenerateToken(cc config.ClusterConfig) (string, error) {
 	joinCmd := r.Stdout.String()
 	joinCmd = strings.Replace(joinCmd, "kubeadm", bsutil.InvokeKubeadm(cc.KubernetesConfig.KubernetesVersion), 1)
 	joinCmd = fmt.Sprintf("%s --ignore-preflight-errors=all", strings.TrimSpace(joinCmd))
-	if cc.KubernetesConfig.CRISocket != "" {
+
-		joinCmd = fmt.Sprintf("%s --cri-socket %s", joinCmd, cc.KubernetesConfig.CRISocket)
+	// avoid "Found multiple CRI sockets, please use --cri-socket to select one: /var/run/dockershim.sock, /var/run/crio/crio.sock" error
+	cr, err := cruntime.New(cruntime.Config{Type: cc.KubernetesConfig.ContainerRuntime, Runner: k.c, Socket: cc.KubernetesConfig.CRISocket})
+	if err != nil {
+		klog.Errorf("cruntime: %v", err)
 	}
+	sp := cr.SocketPath()
+	if sp == "" {
+		sp = kconst.DefaultDockerCRISocket
+	}
+	joinCmd = fmt.Sprintf("%s --cri-socket %s", joinCmd, sp)
 
 	return joinCmd, nil
 }

pkg/minikube/cni/cni.go

@@ -143,15 +143,6 @@ func chooseDefault(cc config.ClusterConfig) Manager {
 		return Bridge{}
 	}
 
-	if cc.KubernetesConfig.ContainerRuntime != "docker" {
-		if driver.IsKIC(cc.Driver) {
-			klog.Infof("%q driver + %s runtime found, recommending kindnet", cc.Driver, cc.KubernetesConfig.ContainerRuntime)
-			return KindNet{cc: cc}
-		}
-		klog.Infof("%q driver + %s runtime found, recommending bridge", cc.Driver, cc.KubernetesConfig.ContainerRuntime)
-		return Bridge{cc: cc}
-	}
-
 	if driver.BareMetal(cc.Driver) {
 		klog.Infof("Driver %s used, CNI unnecessary in this configuration, recommending no CNI", cc.Driver)
 		return Disabled{cc: cc}
@@ -164,6 +155,15 @@ func chooseDefault(cc config.ClusterConfig) Manager {
 		return KindNet{cc: cc}
 	}
 
+	if cc.KubernetesConfig.ContainerRuntime != "docker" {
+		if driver.IsKIC(cc.Driver) {
+			klog.Infof("%q driver + %s runtime found, recommending kindnet", cc.Driver, cc.KubernetesConfig.ContainerRuntime)
+			return KindNet{cc: cc}
+		}
+		klog.Infof("%q driver + %s runtime found, recommending bridge", cc.Driver, cc.KubernetesConfig.ContainerRuntime)
+		return Bridge{cc: cc}
+	}
+
 	klog.Infof("CNI unnecessary in this configuration, recommending no CNI")
 	return Disabled{cc: cc}
 }

pkg/minikube/cruntime/cruntime_test.go

@@ -486,6 +486,7 @@ func (f *FakeRunner) systemctl(args []string, root bool) (string, error) { // no
 	}
 
 	for _, svc := range svcs {
+		svc = strings.Replace(svc, ".service", "", 1)
 		state, ok := f.services[svc]
 		if !ok {
 			return out, fmt.Errorf("unknown fake service: %s", svc)
@@ -526,6 +527,11 @@ func (f *FakeRunner) systemctl(args []string, root bool) (string, error) { // no
 				return out, nil
 			}
 			return out, fmt.Errorf("%s cat unimplemented", svc)
+		case "enable":
+		case "disable":
+		case "mask":
+		case "unmask":
+			f.t.Logf("fake systemctl: %s %s: %v", svc, action, state)
 		default:
 			return out, fmt.Errorf("unimplemented fake action: %q", action)
 		}
@@ -587,7 +593,8 @@ func TestDisable(t *testing.T) {
 		runtime string
 		want    []string
 	}{
-		{"docker", []string{"sudo", "systemctl", "stop", "-f", "docker.socket", "sudo", "systemctl", "stop", "-f", "docker"}},
+		{"docker", []string{"sudo", "systemctl", "stop", "-f", "docker.socket", "sudo", "systemctl", "stop", "-f", "docker.service",
+			"sudo", "systemctl", "disable", "docker.socket", "sudo", "systemctl", "mask", "docker.service"}},
 		{"crio", []string{"sudo", "systemctl", "stop", "-f", "crio"}},
 		{"containerd", []string{"sudo", "systemctl", "stop", "-f", "containerd"}},
 	}

pkg/minikube/cruntime/docker.go

@@ -120,6 +120,14 @@ func (r *Docker) Enable(disOthers, forceSystemd bool) error {
 		return err
 	}
 
+	if err := r.Init.Unmask("docker.service"); err != nil {
+		return err
+	}
+
+	if err := r.Init.Enable("docker.socket"); err != nil {
+		klog.ErrorS(err, "Failed to enable", "service", "docker.socket")
+	}
+
 	if forceSystemd {
 		if err := r.forceSystemd(); err != nil {
 			return err
@@ -146,7 +154,14 @@ func (r *Docker) Disable() error {
 	if err := r.Init.ForceStop("docker.socket"); err != nil {
 		klog.ErrorS(err, "Failed to stop", "service", "docker.socket")
 	}
-	return r.Init.ForceStop("docker")
+	if err := r.Init.ForceStop("docker.service"); err != nil {
+		klog.ErrorS(err, "Failed to stop", "service", "docker.service")
+		return err
+	}
+	if err := r.Init.Disable("docker.socket"); err != nil {
+		klog.ErrorS(err, "Failed to disable", "service", "docker.socket")
+	}
+	return r.Init.Mask("docker.service")
 }
 
 // ImageExists checks if an image exists

pkg/minikube/download/iso.go

@@ -40,7 +40,7 @@ const fileScheme = "file"
 // DefaultISOURLs returns a list of ISO URL's to consult by default, in priority order
 func DefaultISOURLs() []string {
 	v := version.GetISOVersion()
-	isoBucket := "minikube-builds/iso/11054"
+	isoBucket := "minikube/iso"
 	return []string{
 		fmt.Sprintf("https://storage.googleapis.com/%s/minikube-%s.iso", isoBucket, v),
 		fmt.Sprintf("https://github.com/kubernetes/minikube/releases/download/%s/minikube-%s.iso", v, v),

pkg/minikube/out/out.go

@@ -113,15 +113,28 @@ func Styled(st style.Enum, format string, a ...V) {
 	}
 }
 
-// Boxed writes a stylized and templated message in a box to stdout
+func boxedCommon(printFunc func(format string, a ...interface{}), format string, a ...V) {
-func Boxed(format string, a ...V) {
 	str := Sprintf(style.None, format, a...)
 	str = strings.TrimSpace(str)
 	box := box.New(box.Config{Py: 1, Px: 4, Type: "Round"})
 	if useColor {
 		box.Config.Color = "Red"
 	}
-	box.Println("", str)
+	str = box.String("", str)
+	lines := strings.Split(str, "\n")
+	for _, line := range lines {
+		printFunc(line + "\n")
+	}
+}
+
+// Boxed writes a stylized and templated message in a box to stdout
+func Boxed(format string, a ...V) {
+	boxedCommon(String, format, a...)
+}
+
+// BoxedErr writes a stylized and templated message in a box to stderr
+func BoxedErr(format string, a ...V) {
+	boxedCommon(Err, format, a...)
 }
 
 // Sprintf is used for returning the string (doesn't write anything)
@@ -402,7 +415,7 @@ func displayGitHubIssueMessage() {
 	msg += Sprintf(style.Empty, "Please attach the following file to the GitHub issue:")
 	msg += Sprintf(style.Empty, "- {{.logPath}}", V{"logPath": logPath})
 
-	Boxed(msg)
+	BoxedErr(msg)
 }
 
 // applyTmpl applies formatting

pkg/minikube/out/out_style.go

@@ -54,7 +54,7 @@ func applyStyle(st style.Enum, useColor bool, format string) (string, bool) {
 func stylized(st style.Enum, useColor bool, format string, a ...V) (string, bool) {
 	var spinner bool
 	if a == nil {
-		a = []V{{}}
+		a = []V{}
 	}
 	format, spinner = applyStyle(st, useColor, format)
 	return Fmt(format, a...), spinner

pkg/minikube/reason/k8s.go

@@ -0,0 +1,58 @@
+/*
+Copyright 2021 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package reason
+
+import "github.com/blang/semver"
+
+// K8sIssue represents a known issue with a particular version of Kubernetes
+type K8sIssue struct {
+	// VersionAffected is the list of Kubernetes versions that has a particular issue
+	VersionsAffected []string
+	// Description is what will be printed to the user describing the issue
+	Description string
+	// URL is a link to an issue or documentation about the issue, optional.
+	URL string
+}
+
+var k8sIssues = []K8sIssue{
+	{
+		VersionsAffected: []string{
+			"1.18.16",
+			"1.18.17",
+			"1.19.8",
+			"1.19.9",
+			"1.20.3",
+			"1.20.4",
+			"1.20.5",
+			"1.21.0",
+		},
+		Description: "Kubernetes {{.version}} has a known performance issue on cluster startup. It might take 2 to 3 minutes for a cluster to start.",
+		URL:         "https://github.com/kubernetes/kubeadm/issues/2395",
+	},
+}
+
+// ProblematicK8sVersion checks for the supplied Kubernetes version and checks if there's a known issue with it.
+func ProblematicK8sVersion(v semver.Version) *K8sIssue {
+	for _, issue := range k8sIssues {
+		for _, va := range issue.VersionsAffected {
+			if va == v.String() {
+				return &issue
+			}
+		}
+	}
+	return nil
+}

pkg/minikube/reason/reason.go

@@ -14,22 +14,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-/*
-Copyright 2020 The Kubernetes Authors All rights reserved.
-
-Licensed under the Apache License, Version 2.0 (the Kind{ID: "License", ExitCode: });
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an Kind{ID: "AS IS", ExitCode: } BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
 package reason
 
 import (

pkg/minikube/sysinit/openrc.go

@@ -122,6 +122,11 @@ func (s *OpenRC) DisableNow(svc string) error {
 	return fmt.Errorf("disable now is not implemented for OpenRC! PRs to fix are welcomed")
 }
 
+// Mask does nothing
+func (s *OpenRC) Mask(svc string) error {
+	return nil
+}
+
 // Enable does nothing
 func (s *OpenRC) Enable(svc string) error {
 	return nil
@@ -132,6 +137,11 @@ func (s *OpenRC) EnableNow(svc string) error {
 	return fmt.Errorf("enable now is not implemented for OpenRC! PRs to fix are welcomed")
 }
 
+// Unmask does nothing
+func (s *OpenRC) Unmask(svc string) error {
+	return nil
+}
+
 // Restart restarts a service
 func (s *OpenRC) Restart(svc string) error {
 	rr, err := s.r.RunCmd(exec.Command("sudo", "service", svc, "restart"))

pkg/minikube/sysinit/sysinit.go

@@ -44,12 +44,18 @@ type Manager interface {
 	// Disable disables a service and stops it right after.
 	DisableNow(string) error
 
+	// Mask prevents a service from being started
+	Mask(string) error
+
 	// Enable enables a service
 	Enable(string) error
 
 	// EnableNow enables a service and starts it right after.
 	EnableNow(string) error
 
+	// Unmask allows a service to be started
+	Unmask(string) error
+
 	// Start starts a service idempotently
 	Start(string) error
 

pkg/minikube/sysinit/systemd.go

@@ -58,6 +58,12 @@ func (s *Systemd) DisableNow(svc string) error {
 	return err
 }
 
+// Mask prevents a service from being started
+func (s *Systemd) Mask(svc string) error {
+	_, err := s.r.RunCmd(exec.Command("sudo", "systemctl", "mask", svc))
+	return err
+}
+
 // Enable enables a service
 func (s *Systemd) Enable(svc string) error {
 	if svc == "kubelet" {
@@ -76,6 +82,12 @@ func (s *Systemd) EnableNow(svc string) error {
 	return err
 }
 
+// Unmask allows a service to be started
+func (s *Systemd) Unmask(svc string) error {
+	_, err := s.r.RunCmd(exec.Command("sudo", "systemctl", "unmask", svc))
+	return err
+}
+
 // Start starts a service
 func (s *Systemd) Start(svc string) error {
 	if err := s.daemonReload(); err != nil {

site/content/en/docs/commands/start.md

@@ -26,7 +26,7 @@ minikube start [flags]
       --apiserver-names strings           A set of apiserver names which are used in the generated certificate for kubernetes.  This can be used if you want to make the apiserver available from outside the machine
       --apiserver-port int                The apiserver listening port (default 8443)
       --auto-update-drivers               If set, automatically updates drivers to the latest version. Defaults to true. (default true)
-      --base-image string                 The base image to use for docker/podman drivers. Intended for local development. (default "gcr.io/k8s-minikube/kicbase-builds:v0.0.20-1619543236-11212@sha256:b7c6676a7c18654deb3258da55a95dec53983871e371186f217055256cda5b24")
+      --base-image string                 The base image to use for docker/podman drivers. Intended for local development. (default "gcr.io/k8s-minikube/kicbase:v0.0.21@sha256:30ac715d2caf25e953e6ca9b6525d17bc8148236a6153208780f2c485bcdffed")
       --cache-images                      If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --driver=none. (default true)
       --cni string                        CNI plug-in to use. Valid options: auto, bridge, calico, cilium, flannel, kindnet, or path to a CNI manifest (default: auto)
       --container-runtime string          The container runtime to be used (docker, cri-o, containerd). (default "docker")
@@ -64,7 +64,7 @@ minikube start [flags]
       --insecure-registry strings         Insecure Docker registries to pass to the Docker daemon.  The default service CIDR range will automatically be added.
       --install-addons                    If set, install addons. Defaults to true. (default true)
       --interactive                       Allow user prompts for more information (default true)
-      --iso-url strings                   Locations to fetch the minikube ISO from. (default [https://storage.googleapis.com/minikube-builds/iso/11054/minikube-v1.19.0-1619288095-11054.iso,https://github.com/kubernetes/minikube/releases/download/v1.19.0-1619288095-11054/minikube-v1.19.0-1619288095-11054.iso,https://kubernetes.oss-cn-hangzhou.aliyuncs.com/minikube/iso/minikube-v1.19.0-1619288095-11054.iso])
+      --iso-url strings                   Locations to fetch the minikube ISO from. (default [https://storage.googleapis.com/minikube/iso/minikube-v1.20.0-beta.0.iso,https://github.com/kubernetes/minikube/releases/download/v1.20.0-beta.0/minikube-v1.20.0-beta.0.iso,https://kubernetes.oss-cn-hangzhou.aliyuncs.com/minikube/iso/minikube-v1.20.0-beta.0.iso])
       --keep-context                      This will keep the existing kubectl context and will create a minikube context.
       --kubernetes-version string         The Kubernetes version that the minikube VM will use (ex: v1.2.3, 'stable' for v1.20.2, 'latest' for v1.20.5-rc.0). Defaults to 'stable'.
       --kvm-gpu                           Enable experimental NVIDIA GPU support in minikube

site/content/en/docs/contrib/tests.en.md

@@ -76,14 +76,17 @@ TestFunctional are functionality tests which can safely share a profile in paral
 validateNodeLabels checks if minikube cluster is created with correct kubernetes's node label
 
 #### validateLoadImage
-validateLoadImage makes sure that `minikube load image` works as expected
+validateLoadImage makes sure that `minikube image load` works as expected
 
 #### validateRemoveImage
-validateRemoveImage makes sures that `minikube rm image` works as expected
+validateRemoveImage makes sures that `minikube image rm` works as expected
 
 #### validateBuildImage
 validateBuildImage makes sures that `minikube image build` works as expected
 
+#### validateListImages
+validateListImages makes sures that `minikube image ls` works as expected
+
 #### validateDockerEnv
 check functionality of minikube after evaling docker-env
 
@@ -348,4 +351,4 @@ TestKubernetesUpgrade upgrades Kubernetes from oldest to newest
 ## TestMissingContainerUpgrade
 TestMissingContainerUpgrade tests a Docker upgrade where the underlying container is missing
 
-TEST COUNT: 111
+TEST COUNT: 112

test/integration/addons_test.go

@@ -462,7 +462,6 @@ func validateHelmTillerAddon(ctx context.Context, t *testing.T, profile string)
 
 // validateOlmAddon tests the OLM addon
 func validateOlmAddon(ctx context.Context, t *testing.T, profile string) {
-	t.Skipf("Skipping olm test till this timeout issue is solved https://github.com/operator-framework/operator-lifecycle-manager/issues/1534#issuecomment-632342257")
 	defer PostMortemLogs(t, profile)
 
 	client, err := kapi.Client(profile)

test/integration/functional_test.go

@@ -135,6 +135,7 @@ func TestFunctional(t *testing.T) {
 			{"LoadImage", validateLoadImage},
 			{"RemoveImage", validateRemoveImage},
 			{"BuildImage", validateBuildImage},
+			{"ListImages", validateListImages},
 		}
 		for _, tc := range tests {
 			tc := tc
@@ -157,11 +158,16 @@ func cleanupUnwantedImages(ctx context.Context, t *testing.T, profile string) {
 		t.Skipf("docker is not installed, cannot delete docker images")
 	} else {
 		t.Run("delete busybox image", func(t *testing.T) {
-			newImage := fmt.Sprintf("docker.io/library/busybox:%s", profile)
+			newImage := fmt.Sprintf("docker.io/library/busybox:load-%s", profile)
 			rr, err := Run(t, exec.CommandContext(ctx, "docker", "rmi", "-f", newImage))
 			if err != nil {
 				t.Logf("failed to remove image busybox from docker images. args %q: %v", rr.Command(), err)
 			}
+			newImage = fmt.Sprintf("docker.io/library/busybox:remove-%s", profile)
+			rr, err = Run(t, exec.CommandContext(ctx, "docker", "rmi", "-f", newImage))
+			if err != nil {
+				t.Logf("failed to remove image busybox from docker images. args %q: %v", rr.Command(), err)
+			}
 		})
 		t.Run("delete my-image image", func(t *testing.T) {
 			newImage := fmt.Sprintf("localhost/my-image:%s", profile)
@@ -198,7 +204,7 @@ func validateNodeLabels(ctx context.Context, t *testing.T, profile string) {
 	}
 }
 
-// validateLoadImage makes sure that `minikube load image` works as expected
+// validateLoadImage makes sure that `minikube image load` works as expected
 func validateLoadImage(ctx context.Context, t *testing.T, profile string) {
 	if NoneDriver() {
 		t.Skip("load image not available on none driver")
@@ -208,15 +214,15 @@ func validateLoadImage(ctx context.Context, t *testing.T, profile string) {
 	}
 	defer PostMortemLogs(t, profile)
 	// pull busybox
-	busybox := "busybox:latest"
+	busyboxImage := "busybox:latest"
-	rr, err := Run(t, exec.CommandContext(ctx, "docker", "pull", busybox))
+	rr, err := Run(t, exec.CommandContext(ctx, "docker", "pull", busyboxImage))
 	if err != nil {
 		t.Fatalf("failed to setup test (pull image): %v\n%s", err, rr.Output())
 	}
 
 	// tag busybox
-	newImage := fmt.Sprintf("docker.io/library/busybox:%s", profile)
+	newImage := fmt.Sprintf("docker.io/library/busybox:load-%s", profile)
-	rr, err = Run(t, exec.CommandContext(ctx, "docker", "tag", busybox, newImage))
+	rr, err = Run(t, exec.CommandContext(ctx, "docker", "tag", busyboxImage, newImage))
 	if err != nil {
 		t.Fatalf("failed to setup test (tag image) : %v\n%s", err, rr.Output())
 	}
@@ -232,13 +238,13 @@ func validateLoadImage(ctx context.Context, t *testing.T, profile string) {
 	if err != nil {
 		t.Fatalf("listing images: %v\n%s", err, rr.Output())
 	}
-	if !strings.Contains(rr.Output(), fmt.Sprintf("busybox:%s", profile)) {
+	if !strings.Contains(rr.Output(), fmt.Sprintf("busybox:load-%s", profile)) {
 		t.Fatalf("expected %s to be loaded into minikube but the image is not there", newImage)
 	}
 
 }
 
-// validateRemoveImage makes sures that `minikube rm image` works as expected
+// validateRemoveImage makes sures that `minikube image rm` works as expected
 func validateRemoveImage(ctx context.Context, t *testing.T, profile string) {
 	if NoneDriver() {
 		t.Skip("load image not available on none driver")
@@ -255,24 +261,32 @@ func validateRemoveImage(ctx context.Context, t *testing.T, profile string) {
 		t.Fatalf("failed to setup test (pull image): %v\n%s", err, rr.Output())
 	}
 
+	// tag busybox
+	newImage := fmt.Sprintf("docker.io/library/busybox:remove-%s", profile)
+	rr, err = Run(t, exec.CommandContext(ctx, "docker", "tag", busyboxImage, newImage))
+	if err != nil {
+		t.Fatalf("failed to setup test (tag image) : %v\n%s", err, rr.Output())
+	}
+
 	// try to load the image into minikube
-	rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "image", "load", busyboxImage))
+	rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "image", "load", newImage))
 	if err != nil {
 		t.Fatalf("loading image into minikube: %v\n%s", err, rr.Output())
 	}
 
 	// try to remove the image from minikube
-	rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "image", "rm", busyboxImage))
+	rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "image", "rm", newImage))
 	if err != nil {
 		t.Fatalf("removing image from minikube: %v\n%s", err, rr.Output())
 	}
+
 	// make sure the image was removed
 	rr, err = listImages(ctx, t, profile)
 	if err != nil {
 		t.Fatalf("listing images: %v\n%s", err, rr.Output())
 	}
-	if strings.Contains(rr.Output(), busyboxImage) {
+	if strings.Contains(rr.Output(), fmt.Sprintf("busybox:remove-%s", profile)) {
-		t.Fatalf("expected %s to be removed from minikube but the image is there", busyboxImage)
+		t.Fatalf("expected %s to be removed from minikube but the image is there", newImage)
 	}
 
 }
@@ -355,6 +369,36 @@ func startBuildkit(ctx context.Context, t *testing.T, profile string) {
 	}
 }
 
+// validateListImages makes sures that `minikube image ls` works as expected
+func validateListImages(ctx context.Context, t *testing.T, profile string) {
+	if NoneDriver() {
+		t.Skip("list images not available on none driver")
+	}
+	if GithubActionRunner() && runtime.GOOS == "darwin" {
+		t.Skip("skipping on github actions and darwin, as this test requires a running docker daemon")
+	}
+	defer PostMortemLogs(t, profile)
+
+	// try to list the images with minikube
+	rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "image", "ls"))
+	if err != nil {
+		t.Fatalf("listing image with minikube: %v\n%s", err, rr.Output())
+	}
+	if rr.Stdout.Len() > 0 {
+		t.Logf("(dbg) Stdout: %s:\n%s", rr.Command(), rr.Stdout)
+	}
+	if rr.Stderr.Len() > 0 {
+		t.Logf("(dbg) Stderr: %s:\n%s", rr.Command(), rr.Stderr)
+	}
+
+	list := rr.Output()
+	for _, theImage := range []string{"k8s.gcr.io/pause", "docker.io/kubernetesui/dashboard"} {
+		if !strings.Contains(list, theImage) {
+			t.Fatalf("expected %s to be listed with minikube but the image is not there", theImage)
+		}
+	}
+}
+
 // check functionality of minikube after evaling docker-env
 // TODO: Add validatePodmanEnv for crio runtime: #10231
 func validateDockerEnv(ctx context.Context, t *testing.T, profile string) {