Merge pull request #11426 from afbjorklund/images-versions

Update the bootstrapper images for k8s 1.21

pkg/minikube/bootstrapper/bsutil/kubeadm_test.go

@@ -123,23 +123,14 @@ func recentReleases(n int) ([]string, error) {
 }
 
 /**
-Need a separate test function to test the DNS server IP
-as v1.11 yaml file is very different compared to v1.12+.
 This test case has only 1 thing to test and that is the
 networking/dnsDomain value
 */
 func TestGenerateKubeadmYAMLDNS(t *testing.T) {
-	// test all testdata releases greater than v1.11
 	versions, err := recentReleases(0)
 	if err != nil {
 		t.Errorf("versions: %v", err)
 	}
-	for i, v := range versions {
-		if semver.Compare(v, "v1.11") <= 0 {
-			versions = versions[0:i]
-			break
-		}
-	}
 	fcr := command.NewFakeCommandRunner()
 	fcr.SetCommandToOutput(map[string]string{
 		"docker info --format {{.CgroupDriver}}": "systemd\n",

pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd-api-port.yaml

@@ -1,22 +0,0 @@
-apiVersion: kubeadm.k8s.io/v1alpha1
-kind: MasterConfiguration
-noTaintMaster: true
-api:
-  advertiseAddress: 1.1.1.1
-  bindPort: 12345
-  controlPlaneEndpoint: control-plane.minikube.internal
-kubernetesVersion: v1.11.0
-certificatesDir: /var/lib/minikube/certs
-networking:
-  serviceSubnet: 10.96.0.0/12
-etcd:
-  dataDir: /var/lib/minikube/etcd
-controllerManagerExtraArgs:
-  leader-elect: "false"
-schedulerExtraArgs:
-  leader-elect: "false"
-nodeName: "mk"
-apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
-criSocket: /run/containerd/containerd.sock
-apiServerExtraArgs:
-  enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"

pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd-pod-network-cidr.yaml

@@ -1,22 +0,0 @@
-apiVersion: kubeadm.k8s.io/v1alpha1
-kind: MasterConfiguration
-noTaintMaster: true
-api:
-  advertiseAddress: 1.1.1.1
-  bindPort: 8443
-  controlPlaneEndpoint: control-plane.minikube.internal
-kubernetesVersion: v1.11.0
-certificatesDir: /var/lib/minikube/certs
-networking:
-  serviceSubnet: 10.96.0.0/12
-etcd:
-  dataDir: /var/lib/minikube/etcd
-controllerManagerExtraArgs:
-  leader-elect: "false"
-schedulerExtraArgs:
-  leader-elect: "false"
-nodeName: "mk"
-apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
-criSocket: /run/containerd/containerd.sock
-apiServerExtraArgs:
-  enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"

pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd.yaml

@@ -1,22 +0,0 @@
-apiVersion: kubeadm.k8s.io/v1alpha1
-kind: MasterConfiguration
-noTaintMaster: true
-api:
-  advertiseAddress: 1.1.1.1
-  bindPort: 8443
-  controlPlaneEndpoint: control-plane.minikube.internal
-kubernetesVersion: v1.11.0
-certificatesDir: /var/lib/minikube/certs
-networking:
-  serviceSubnet: 10.96.0.0/12
-etcd:
-  dataDir: /var/lib/minikube/etcd
-controllerManagerExtraArgs:
-  leader-elect: "false"
-schedulerExtraArgs:
-  leader-elect: "false"
-nodeName: "mk"
-apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
-criSocket: /run/containerd/containerd.sock
-apiServerExtraArgs:
-  enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"

pkg/minikube/bootstrapper/bsutil/testdata/v1.11/crio-options-gates.yaml

@@ -1,30 +0,0 @@
-apiVersion: kubeadm.k8s.io/v1alpha1
-kind: MasterConfiguration
-noTaintMaster: true
-api:
-  advertiseAddress: 1.1.1.1
-  bindPort: 8443
-  controlPlaneEndpoint: control-plane.minikube.internal
-kubernetesVersion: v1.11.0
-certificatesDir: /var/lib/minikube/certs
-networking:
-  serviceSubnet: 10.96.0.0/12
-etcd:
-  dataDir: /var/lib/minikube/etcd
-controllerManagerExtraArgs:
-  leader-elect: "false"
-schedulerExtraArgs:
-  leader-elect: "false"
-nodeName: "mk"
-apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
-criSocket: /var/run/crio/crio.sock
-apiServerExtraArgs:
-  enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
-  fail-no-swap: "true"
-  feature-gates: "a=b"
-controllerManagerExtraArgs:
-  feature-gates: "a=b"
-  kube-api-burst: "32"
-schedulerExtraArgs:
-  feature-gates: "a=b"
-  scheduler-name: "mini-scheduler"

pkg/minikube/bootstrapper/bsutil/testdata/v1.11/crio.yaml

@@ -1,22 +0,0 @@
-apiVersion: kubeadm.k8s.io/v1alpha1
-kind: MasterConfiguration
-noTaintMaster: true
-api:
-  advertiseAddress: 1.1.1.1
-  bindPort: 8443
-  controlPlaneEndpoint: control-plane.minikube.internal
-kubernetesVersion: v1.11.0
-certificatesDir: /var/lib/minikube/certs
-networking:
-  serviceSubnet: 10.96.0.0/12
-etcd:
-  dataDir: /var/lib/minikube/etcd
-controllerManagerExtraArgs:
-  leader-elect: "false"
-schedulerExtraArgs:
-  leader-elect: "false"
-nodeName: "mk"
-apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
-criSocket: /var/run/crio/crio.sock
-apiServerExtraArgs:
-  enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"

pkg/minikube/bootstrapper/bsutil/testdata/v1.11/default.yaml

@@ -1,21 +0,0 @@
-apiVersion: kubeadm.k8s.io/v1alpha1
-kind: MasterConfiguration
-noTaintMaster: true
-api:
-  advertiseAddress: 1.1.1.1
-  bindPort: 8443
-  controlPlaneEndpoint: control-plane.minikube.internal
-kubernetesVersion: v1.11.0
-certificatesDir: /var/lib/minikube/certs
-networking:
-  serviceSubnet: 10.96.0.0/12
-etcd:
-  dataDir: /var/lib/minikube/etcd
-controllerManagerExtraArgs:
-  leader-elect: "false"
-schedulerExtraArgs:
-  leader-elect: "false"
-nodeName: "mk"
-apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
-apiServerExtraArgs:
-  enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"

pkg/minikube/bootstrapper/bsutil/testdata/v1.11/image-repository.yaml

@@ -1,22 +0,0 @@
-apiVersion: kubeadm.k8s.io/v1alpha1
-kind: MasterConfiguration
-noTaintMaster: true
-api:
-  advertiseAddress: 1.1.1.1
-  bindPort: 8443
-  controlPlaneEndpoint: control-plane.minikube.internal
-kubernetesVersion: v1.11.0
-certificatesDir: /var/lib/minikube/certs
-networking:
-  serviceSubnet: 10.96.0.0/12
-etcd:
-  dataDir: /var/lib/minikube/etcd
-controllerManagerExtraArgs:
-  leader-elect: "false"
-schedulerExtraArgs:
-  leader-elect: "false"
-nodeName: "mk"
-apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
-imageRepository: test/repo
-apiServerExtraArgs:
-  enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"

pkg/minikube/bootstrapper/bsutil/testdata/v1.11/options.yaml

@@ -1,26 +0,0 @@
-apiVersion: kubeadm.k8s.io/v1alpha1
-kind: MasterConfiguration
-noTaintMaster: true
-api:
-  advertiseAddress: 1.1.1.1
-  bindPort: 8443
-  controlPlaneEndpoint: control-plane.minikube.internal
-kubernetesVersion: v1.11.0
-certificatesDir: /var/lib/minikube/certs
-networking:
-  serviceSubnet: 10.96.0.0/12
-etcd:
-  dataDir: /var/lib/minikube/etcd
-controllerManagerExtraArgs:
-  leader-elect: "false"
-schedulerExtraArgs:
-  leader-elect: "false"
-nodeName: "mk"
-apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
-apiServerExtraArgs:
-  enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
-  fail-no-swap: "true"
-controllerManagerExtraArgs:
-  kube-api-burst: "32"
-schedulerExtraArgs:
-  scheduler-name: "mini-scheduler"

pkg/minikube/bootstrapper/images/images.go

@@ -28,9 +28,14 @@ import (
 
 // Pause returns the image name to pull for a given Kubernetes version
 func Pause(v semver.Version, mirror string) string {
+	// Note: changing this logic requires bumping the preload version
 	// Should match `PauseVersion` in:
+	// https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/constants/constants.go
+	pv := "3.4.1"
 	// https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/constants/constants_unix.go
-	pv := "3.2"
+	if semver.MustParseRange("<1.21.0-alpha.3")(v) {
+		pv = "3.2"
+	}
 	if semver.MustParseRange("<1.18.0-alpha.0")(v) {
 		pv = "3.1"
 	}
@@ -40,13 +45,14 @@ func Pause(v semver.Version, mirror string) string {
 // essentials returns images needed too bootstrap a Kubernetes
 func essentials(mirror string, v semver.Version) []string {
 	imgs := []string{
-		componentImage("kube-proxy", v, mirror),
+		// use the same order as: `kubeadm config images list`
-		componentImage("kube-scheduler", v, mirror),
-		componentImage("kube-controller-manager", v, mirror),
 		componentImage("kube-apiserver", v, mirror),
-		coreDNS(v, mirror),
+		componentImage("kube-controller-manager", v, mirror),
-		etcd(v, mirror),
+		componentImage("kube-scheduler", v, mirror),
+		componentImage("kube-proxy", v, mirror),
 		Pause(v, mirror),
+		etcd(v, mirror),
+		coreDNS(v, mirror),
 	}
 	return imgs
 }
@@ -58,13 +64,16 @@ func componentImage(name string, v semver.Version, mirror string) string {
 
 // coreDNS returns the images used for CoreDNS
 func coreDNS(v semver.Version, mirror string) string {
-	// Should match `CoreDNSVersion` in
+	// Note: changing this logic requires bumping the preload version
+	// Should match `CoreDNSImageName` and `CoreDNSVersion` in
 	// https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/constants/constants.go
-	cv := "1.7.0"
+	in := "coredns/coredns"
+	if semver.MustParseRange("<1.21.0-alpha.1")(v) {
+		in = "coredns"
+	}
+	cv := "v1.8.0"
 	switch v.Minor {
-	case 22:
+	case 20, 19:
-		cv = "1.8.0"
-	case 10, 20, 21:
 		cv = "1.7.0"
 	case 18:
 		cv = "1.6.7"
@@ -78,19 +87,20 @@ func coreDNS(v semver.Version, mirror string) string {
 		cv = "1.2.6"
 	case 12:
 		cv = "1.2.2"
-	case 11:
-		cv = "1.1.3"
 	}
-	return path.Join(kubernetesRepo(mirror), "coredns:"+cv)
+	return path.Join(kubernetesRepo(mirror), in+":"+cv)
 }
 
 // etcd returns the image used for etcd
 func etcd(v semver.Version, mirror string) string {
+	// Note: changing this logic requires bumping the preload version
 	// Should match `DefaultEtcdVersion` in:
 	// https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/constants/constants.go
-	ev := "3.4.13-0"
+	ev := "3.4.13-3"
 
 	switch v.Minor {
+	case 19, 20, 21:
+		ev = "3.4.13-0"
 	case 17, 18:
 		ev = "3.4.3-0"
 	case 16:
@@ -99,8 +109,6 @@ func etcd(v semver.Version, mirror string) string {
 		ev = "3.3.10"
 	case 12, 13:
 		ev = "3.2.24"
-	case 11:
-		ev = "3.2.18"
 	}
 
 	// An awkward special case for v1.19.0 - do not imitate unless necessary
@@ -113,6 +121,7 @@ func etcd(v semver.Version, mirror string) string {
 
 // auxiliary returns images that are helpful for running minikube
 func auxiliary(mirror string) []string {
+	// Note: changing this list requires bumping the preload version
 	return []string{
 		storageProvisioner(mirror),
 		dashboardFrontend(mirror),

pkg/minikube/bootstrapper/images/images_test.go

@@ -17,12 +17,71 @@ limitations under the License.
 package images
 
 import (
+	"strings"
 	"testing"
 
+	"github.com/blang/semver"
 	"github.com/google/go-cmp/cmp"
 	"k8s.io/minikube/pkg/version"
 )
 
+func TestEssentials(t *testing.T) {
+	var testCases = []struct {
+		version string
+		images  []string
+	}{
+		{"v1.18.0", strings.Split(strings.Trim(`
+k8s.gcr.io/kube-apiserver:v1.18.0
+k8s.gcr.io/kube-controller-manager:v1.18.0
+k8s.gcr.io/kube-scheduler:v1.18.0
+k8s.gcr.io/kube-proxy:v1.18.0
+k8s.gcr.io/pause:3.2
+k8s.gcr.io/etcd:3.4.3-0
+k8s.gcr.io/coredns:1.6.7
+`, "\n"), "\n")},
+		{"v1.19.0", strings.Split(strings.Trim(`
+k8s.gcr.io/kube-apiserver:v1.19.0
+k8s.gcr.io/kube-controller-manager:v1.19.0
+k8s.gcr.io/kube-scheduler:v1.19.0
+k8s.gcr.io/kube-proxy:v1.19.0
+k8s.gcr.io/pause:3.2
+k8s.gcr.io/etcd:3.4.9-1
+k8s.gcr.io/coredns:1.7.0
+`, "\n"), "\n")},
+		{"v1.20.0", strings.Split(strings.Trim(`
+k8s.gcr.io/kube-apiserver:v1.20.0
+k8s.gcr.io/kube-controller-manager:v1.20.0
+k8s.gcr.io/kube-scheduler:v1.20.0
+k8s.gcr.io/kube-proxy:v1.20.0
+k8s.gcr.io/pause:3.2
+k8s.gcr.io/etcd:3.4.13-0
+k8s.gcr.io/coredns:1.7.0
+`, "\n"), "\n")},
+		{"v1.21.0", strings.Split(strings.Trim(`
+k8s.gcr.io/kube-apiserver:v1.21.0
+k8s.gcr.io/kube-controller-manager:v1.21.0
+k8s.gcr.io/kube-scheduler:v1.21.0
+k8s.gcr.io/kube-proxy:v1.21.0
+k8s.gcr.io/pause:3.4.1
+k8s.gcr.io/etcd:3.4.13-0
+k8s.gcr.io/coredns/coredns:v1.8.0
+`, "\n"), "\n")},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.version, func(t *testing.T) {
+			v, err := semver.Make(strings.TrimPrefix(tc.version, "v"))
+			if err != nil {
+				t.Fatal(err)
+			}
+			want := tc.images
+			got := essentials("k8s.gcr.io", v)
+			if diff := cmp.Diff(want, got); diff != "" {
+				t.Errorf("images mismatch (-want +got):\n%s", diff)
+			}
+		})
+	}
+}
+
 func TestAuxiliary(t *testing.T) {
 	want := []string{
 		"gcr.io/k8s-minikube/storage-provisioner:" + version.GetStorageProvisionerVersion(),
@@ -46,3 +105,23 @@ func TestAuxiliaryMirror(t *testing.T) {
 		t.Errorf("images mismatch (-want +got):\n%s", diff)
 	}
 }
+
+func TestCNI(t *testing.T) {
+	// images used by k8s.io/minikube/pkg/minikube/cni
+	var testCases = []struct {
+		name     string
+		function func(string) string
+	}{
+		{"kindnet", KindNet},
+		{"calico-deployment", CalicoDeployment},
+		{"calico-daemonset", CalicoDaemonSet},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			img := tc.function("")
+			if img == "" {
+				t.Errorf("no image")
+			}
+		})
+	}
+}

pkg/minikube/bootstrapper/images/kubeadm.go

@@ -17,6 +17,7 @@ limitations under the License.
 package images
 
 import (
+	"fmt"
 	"strings"
 
 	"github.com/blang/semver"
@@ -29,6 +30,12 @@ func Kubeadm(mirror string, version string) ([]string, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "semver")
 	}
+	if v.Major > 1 {
+		return nil, fmt.Errorf("version too new: %v", v)
+	}
+	if semver.MustParseRange("<1.12.0-alpha.0")(v) {
+		return nil, fmt.Errorf("version too old: %v", v)
+	}
 	imgs := essentials(mirror, v)
 	imgs = append(imgs, auxiliary(mirror)...)
 	return imgs, nil

pkg/minikube/bootstrapper/images/kubeadm_test.go

@@ -28,9 +28,13 @@ func TestKubeadmImages(t *testing.T) {
 	tests := []struct {
 		version string
 		mirror  string
+		invalid bool
 		want    []string
 	}{
-		{"v1.17.0", "", []string{
+		{"invalid", "", true, nil},
+		{"v0.0.1", "", true, nil}, // too old
+		{"v2.0.0", "", true, nil}, // too new
+		{"v1.17.0", "", false, []string{
 			"k8s.gcr.io/kube-proxy:v1.17.0",
 			"k8s.gcr.io/kube-scheduler:v1.17.0",
 			"k8s.gcr.io/kube-controller-manager:v1.17.0",
@@ -42,7 +46,7 @@ func TestKubeadmImages(t *testing.T) {
 			"docker.io/kubernetesui/dashboard:v2.1.0",
 			"docker.io/kubernetesui/metrics-scraper:v1.0.4",
 		}},
-		{"v1.16.1", "mirror.k8s.io", []string{
+		{"v1.16.1", "mirror.k8s.io", false, []string{
 			"mirror.k8s.io/kube-proxy:v1.16.1",
 			"mirror.k8s.io/kube-scheduler:v1.16.1",
 			"mirror.k8s.io/kube-controller-manager:v1.16.1",
@@ -54,7 +58,7 @@ func TestKubeadmImages(t *testing.T) {
 			"mirror.k8s.io/dashboard:v2.1.0",
 			"mirror.k8s.io/metrics-scraper:v1.0.4",
 		}},
-		{"v1.15.0", "", []string{
+		{"v1.15.0", "", false, []string{
 			"k8s.gcr.io/kube-proxy:v1.15.0",
 			"k8s.gcr.io/kube-scheduler:v1.15.0",
 			"k8s.gcr.io/kube-controller-manager:v1.15.0",
@@ -66,7 +70,7 @@ func TestKubeadmImages(t *testing.T) {
 			"docker.io/kubernetesui/dashboard:v2.1.0",
 			"docker.io/kubernetesui/metrics-scraper:v1.0.4",
 		}},
-		{"v1.14.0", "", []string{
+		{"v1.14.0", "", false, []string{
 			"k8s.gcr.io/kube-proxy:v1.14.0",
 			"k8s.gcr.io/kube-scheduler:v1.14.0",
 			"k8s.gcr.io/kube-controller-manager:v1.14.0",
@@ -78,7 +82,7 @@ func TestKubeadmImages(t *testing.T) {
 			"docker.io/kubernetesui/dashboard:v2.1.0",
 			"docker.io/kubernetesui/metrics-scraper:v1.0.4",
 		}},
-		{"v1.13.0", "", []string{
+		{"v1.13.0", "", false, []string{
 			"k8s.gcr.io/kube-proxy:v1.13.0",
 			"k8s.gcr.io/kube-scheduler:v1.13.0",
 			"k8s.gcr.io/kube-controller-manager:v1.13.0",
@@ -90,7 +94,7 @@ func TestKubeadmImages(t *testing.T) {
 			"docker.io/kubernetesui/dashboard:v2.1.0",
 			"docker.io/kubernetesui/metrics-scraper:v1.0.4",
 		}},
-		{"v1.12.0", "", []string{
+		{"v1.12.0", "", false, []string{
 			"k8s.gcr.io/kube-proxy:v1.12.0",
 			"k8s.gcr.io/kube-scheduler:v1.12.0",
 			"k8s.gcr.io/kube-controller-manager:v1.12.0",
@@ -102,11 +106,16 @@ func TestKubeadmImages(t *testing.T) {
 			"docker.io/kubernetesui/dashboard:v2.1.0",
 			"docker.io/kubernetesui/metrics-scraper:v1.0.4",
 		}},
+		{"v1.11.0", "", true, nil},
+		{"v1.10.0", "", true, nil},
 	}
 	for _, tc := range tests {
 		got, err := Kubeadm(tc.mirror, tc.version)
-		if err != nil {
+		if err == nil && tc.invalid {
-			t.Fatalf("unexpected err: %v", err)
+			t.Fatalf("expected err (%s): %v", tc.version, got)
+		}
+		if err != nil && !tc.invalid {
+			t.Fatalf("unexpected err (%s): %v", tc.version, err)
 		}
 		sort.Strings(got)
 		sort.Strings(tc.want)

pkg/minikube/download/preload.go

@@ -42,7 +42,7 @@ const (
 	// PreloadVersion is the current version of the preloaded tarball
 	//
 	// NOTE: You may need to bump this version up when upgrading auxiliary docker images
-	PreloadVersion = "v10"
+	PreloadVersion = "v11"
 	// PreloadBucket is the name of the GCS bucket where preloaded volume tarballs exist
 	PreloadBucket = "minikube-preloaded-volume-tarballs"
 )