diff --git a/Makefile b/Makefile index aa5f603992..9576827c01 100644 --- a/Makefile +++ b/Makefile @@ -998,6 +998,12 @@ else go run update_kubernetes_version.go) endif +.PHONY: update-kubeadm-constants +update-kubeadm-constants: + (cd hack/update/kubeadm_constants && \ + go run update_kubeadm_constants.go) + gofmt -w pkg/minikube/constants/constants_kubeadm_images.go + .PHONY: stress stress: ## run the stress tests go test -test.v -test.timeout=2h ./test/stress -loops=10 | tee "./out/testout_$(COMMIT_SHORT).txt" diff --git a/hack/update/kubeadm_constants/update_kubeadm_constants.go b/hack/update/kubeadm_constants/update_kubeadm_constants.go new file mode 100644 index 0000000000..7399904d19 --- /dev/null +++ b/hack/update/kubeadm_constants/update_kubeadm_constants.go @@ -0,0 +1,208 @@ +/* +Copyright 2021 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "bytes" + "context" + "errors" + "flag" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "strings" + "text/template" + "time" + + "golang.org/x/mod/semver" + "k8s.io/klog/v2" + "k8s.io/minikube/hack/update" + "k8s.io/minikube/pkg/minikube/constants" +) + +const ( + // default context timeout + cxTimeout = 300 * time.Second + kubeadmReleaseURL = "https://storage.googleapis.com/kubernetes-release/release/%s/bin/linux/amd64/kubeadm" + kubeadmBinaryName = "kubeadm-linux-amd64-%s" + minikubeConstantsFilePath = "pkg/minikube/constants/constants_kubeadm_images.go" + kubeadmImagesTemplate = ` + {{- range $version, $element := .}} + "{{$version}}": { + {{- range $image, $tag := $element}} + "{{$image}}": "{{$tag}}", + {{- end}} + },{{- end}}` +) + +// Data contains kubeadm Images map +type Data struct { + ImageMap string `json:"ImageMap"` +} + +func main() { + + inputVersion := flag.Lookup("kubernetes-version").Value.String() + + imageVersions := make([]string, 0) + + // set a context with defined timeout + ctx, cancel := context.WithTimeout(context.Background(), cxTimeout) + defer cancel() + + if inputVersion == "latest" { + stableImageVersion, latestImageVersion, err := getK8sVersions(ctx, "kubernetes", "kubernetes") + if err != nil { + klog.Fatal(err) + } + imageVersions = append(imageVersions, stableImageVersion, latestImageVersion) + } else if semver.IsValid(inputVersion) { + imageVersions = append(imageVersions, inputVersion) + } else { + klog.Fatal(errors.New("invalid version")) + } + + for _, imageVersion := range imageVersions { + imageMapString, err := getKubeadmImagesMapString(imageVersion) + if err != nil { + klog.Fatalln(err) + } + + var data Data + schema := map[string]update.Item{ + minikubeConstantsFilePath: { + Replace: map[string]string{}, + }, + } + + majorMinorVersion := semver.MajorMinor(imageVersion) + + if _, ok := constants.KubeadmImages[majorMinorVersion]; !ok { + data = Data{ImageMap: imageMapString} + schema[minikubeConstantsFilePath].Replace[`KubeadmImages = .*`] = + `KubeadmImages = map[string]map[string]string{ {{.ImageMap}}` + } else { + data = Data{ImageMap: strings.TrimLeft(imageMapString, "\n")} + versionIdentifier := fmt.Sprintf(`"%s": {[^}]+},`, majorMinorVersion) + schema[minikubeConstantsFilePath].Replace[versionIdentifier] = "{{.ImageMap}}" + } + + update.Apply(ctx, schema, data, "", "", -1) + } +} + +func getKubeadmImagesMapString(version string) (string, error) { + url := fmt.Sprintf(kubeadmReleaseURL, version) + fileName := fmt.Sprintf(kubeadmBinaryName, version) + if err := downloadFile(url, fileName); err != nil { + klog.Errorf("failed to download kubeadm binary %s", err.Error()) + return "", err + } + + kubeadmCommand := fmt.Sprintf("./%s", fileName) + args := []string{"config", "images", "list"} + imageListString, err := executeCommand(kubeadmCommand, args...) + if err != nil { + klog.Errorf("failed to execute kubeadm command %s", kubeadmCommand) + return "", err + } + + if err := os.Remove(fileName); err != nil { + klog.Errorf("failed to remove binary %s", fileName) + } + + return formatKubeadmImageList(version, imageListString) +} + +func formatKubeadmImageList(version, data string) (string, error) { + templateData := make(map[string]map[string]string) + majorMinorVersion := semver.MajorMinor(version) + templateData[majorMinorVersion] = make(map[string]string) + lines := strings.Split(data, "\n") + for _, line := range lines { + imageTag := strings.Split(line, ":") + if len(imageTag) == 2 { + // removing the repo from image name + imageName := strings.Split(imageTag[0], "/") + imageTag[0] = strings.Join(imageName[1:], "/") + templateData[majorMinorVersion][imageTag[0]] = imageTag[1] + } + } + + imageTemplate := template.New("kubeadmImage") + t, err := imageTemplate.Parse(kubeadmImagesTemplate) + if err != nil { + klog.Errorf("failed to create kubeadm image map template %s", err.Error()) + return "", err + } + + var bytesBuffer bytes.Buffer + if err := t.Execute(&bytesBuffer, &templateData); err != nil { + return "", err + } + + return bytesBuffer.String(), nil +} + +func downloadFile(url, fileName string) error { + file, err := os.Create(fileName) + if err != nil { + return err + } + defer file.Close() + + response, err := http.Get(url) + if err != nil { + return err + } + defer response.Body.Close() + + if response.StatusCode != http.StatusOK { + return fmt.Errorf("non success status code, while downloading file: %s from: %s", fileName, url) + } + + if _, err := io.Copy(file, response.Body); err != nil { + return err + } + + return os.Chmod(fileName, os.ModePerm) +} + +func executeCommand(command string, args ...string) (string, error) { + output, err := exec.Command(command, args...).Output() + if err != nil { + return "", err + } + return string(output), nil +} + +// getK8sVersion returns Kubernetes versions. +func getK8sVersions(ctx context.Context, owner, repo string) (stable, latest string, err error) { + // get Kubernetes versions from GitHub Releases + stable, latest, err = update.GHReleases(ctx, owner, repo) + if err != nil { + return "", "", err + } + + if !semver.IsValid(stable) || !semver.IsValid(latest) { + return "", "", fmt.Errorf("invalid release obtained stable : %s, latest : %s", stable, latest) + } + + return stable, latest, nil +} diff --git a/hack/update/update.go b/hack/update/update.go index f4ba35bbaa..10c088ed95 100644 --- a/hack/update/update.go +++ b/hack/update/update.go @@ -62,6 +62,9 @@ func init() { if err := flag.Set("alsologtostderr", "true"); err != nil { klog.Warningf("Unable to set flag value for alsologtostderr: %v", err) } + + // used in update_kubeadm_constants.go + flag.String("kubernetes-version", "latest", "kubernetes-version") flag.Parse() defer klog.Flush() diff --git a/pkg/minikube/bootstrapper/images/images.go b/pkg/minikube/bootstrapper/images/images.go index ae2041aeb8..50dedec0c5 100644 --- a/pkg/minikube/bootstrapper/images/images.go +++ b/pkg/minikube/bootstrapper/images/images.go @@ -21,6 +21,8 @@ import ( "fmt" "path" + "k8s.io/minikube/pkg/minikube/constants" + "github.com/blang/semver/v4" "k8s.io/minikube/pkg/version" @@ -31,21 +33,15 @@ func Pause(v semver.Version, mirror string) string { // Note: changing this logic requires bumping the preload version // Should match `PauseVersion` in: // https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/constants/constants.go - pv := "3.6" // https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/constants/constants_unix.go - if semver.MustParseRange("<1.23.0-alpha.2")(v) { - pv = "3.5" + pv := "3.6" + majorMinorVersion := fmt.Sprintf("v%d.%d", v.Major, v.Minor) + imageName := "pause" + if pVersion, ok := constants.KubeadmImages[majorMinorVersion][imageName]; ok { + pv = pVersion } - if semver.MustParseRange("<1.22.0-alpha.3")(v) { - pv = "3.4.1" - } - if semver.MustParseRange("<1.21.0-alpha.3")(v) { - pv = "3.2" - } - if semver.MustParseRange("<1.18.0-alpha.0")(v) { - pv = "3.1" - } - return path.Join(kubernetesRepo(mirror), "pause:"+pv) + + return fmt.Sprintf("%s:%s", path.Join(kubernetesRepo(mirror), imageName), pv) } // essentials returns images needed too bootstrap a Kubernetes @@ -73,30 +69,19 @@ func coreDNS(v semver.Version, mirror string) string { // Note: changing this logic requires bumping the preload version // Should match `CoreDNSImageName` and `CoreDNSVersion` in // https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/constants/constants.go + + cv := "1.8.4" in := "coredns/coredns" if semver.MustParseRange("<1.21.0-alpha.1")(v) { in = "coredns" } - cv := "v1.8.4" - switch v.Minor { - case 21: - cv = "v1.8.0" - case 20, 19: - cv = "1.7.0" - case 18: - cv = "1.6.7" - case 17: - cv = "1.6.5" - case 16: - cv = "1.6.2" - case 15, 14: - cv = "1.3.1" - case 13: - cv = "1.2.6" - case 12: - cv = "1.2.2" + + majorMinorVersion := fmt.Sprintf("v%d.%d", v.Major, v.Minor) + if cVersion, ok := constants.KubeadmImages[majorMinorVersion][in]; ok { + cv = cVersion } - return path.Join(kubernetesRepo(mirror), in+":"+cv) + + return fmt.Sprintf("%s:%s", path.Join(kubernetesRepo(mirror), in), cv) } // etcd returns the image used for etcd @@ -105,26 +90,13 @@ func etcd(v semver.Version, mirror string) string { // Should match `DefaultEtcdVersion` in: // https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/constants/constants.go ev := "3.5.0-0" - - switch v.Minor { - case 19, 20, 21: - ev = "3.4.13-0" - case 17, 18: - ev = "3.4.3-0" - case 16: - ev = "3.3.15-0" - case 14, 15: - ev = "3.3.10" - case 12, 13: - ev = "3.2.24" + majorMinorVersion := fmt.Sprintf("v%d.%d", v.Major, v.Minor) + imageName := "etcd" + if eVersion, ok := constants.KubeadmImages[majorMinorVersion][imageName]; ok { + ev = eVersion } - // An awkward special case for v1.19.0 - do not imitate unless necessary - if v.Equals(semver.MustParse("1.19.0")) { - ev = "3.4.9-1" - } - - return path.Join(kubernetesRepo(mirror), "etcd:"+ev) + return fmt.Sprintf("%s:%s", path.Join(kubernetesRepo(mirror), imageName), ev) } // auxiliary returns images that are helpful for running minikube diff --git a/pkg/minikube/bootstrapper/images/repo_test.go b/pkg/minikube/bootstrapper/images/repo_test.go new file mode 100644 index 0000000000..e4e2b11aa8 --- /dev/null +++ b/pkg/minikube/bootstrapper/images/repo_test.go @@ -0,0 +1,70 @@ +/* +Copyright 2021 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package images + +import ( + "testing" + + "github.com/google/go-cmp/cmp" +) + +func Test_kubernetesRepo(t *testing.T) { + tests := []struct { + mirror string + want string + }{ + { + "", + DefaultKubernetesRepo, + }, + { + "mirror.k8s.io", + "mirror.k8s.io", + }, + } + for _, tc := range tests { + got := kubernetesRepo(tc.mirror) + if !cmp.Equal(got, tc.want) { + t.Errorf("mirror miss match, want: %s, got: %s", tc.want, got) + } + } + +} + +func Test_minikubeRepo(t *testing.T) { + tests := []struct { + mirror string + want string + }{ + { + "", + "gcr.io/k8s-minikube", + }, + { + "mirror.k8s.io", + "mirror.k8s.io/k8s-minikube", + }, + } + + for _, tc := range tests { + got := minikubeRepo(tc.mirror) + if !cmp.Equal(got, tc.want) { + t.Errorf("mirror miss match, want: %s, got: %s", tc.want, got) + } + } + +} diff --git a/pkg/minikube/constants/constants_kubeadm_images.go b/pkg/minikube/constants/constants_kubeadm_images.go new file mode 100644 index 0000000000..ceabf19bad --- /dev/null +++ b/pkg/minikube/constants/constants_kubeadm_images.go @@ -0,0 +1,130 @@ +/* +Copyright 2021 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package constants + +var ( + KubeadmImages = map[string]map[string]string{ + "v1.22": { + "coredns/coredns": "v1.8.4", + "etcd": "3.5.0-0", + "kube-apiserver": "v1.22.3", + "kube-controller-manager": "v1.22.3", + "kube-proxy": "v1.22.3", + "kube-scheduler": "v1.22.3", + "pause": "3.5", + }, + "v1.21": { + "coredns/coredns": "v1.8.0", + "etcd": "3.4.13-0", + "kube-apiserver": "v1.21.6", + "kube-controller-manager": "v1.21.6", + "kube-proxy": "v1.21.6", + "kube-scheduler": "v1.21.6", + "pause": "3.4.1", + }, + "v1.20": { + "coredns": "1.7.0", + "etcd": "3.4.13-0", + "kube-apiserver": "v1.20.12", + "kube-controller-manager": "v1.20.12", + "kube-proxy": "v1.20.12", + "kube-scheduler": "v1.20.12", + "pause": "3.2", + }, + "v1.19": { + "coredns": "1.7.0", + "etcd": "3.4.9-1", + "kube-apiserver": "v1.19.16", + "kube-controller-manager": "v1.19.16", + "kube-proxy": "v1.19.16", + "kube-scheduler": "v1.19.16", + "pause": "3.2", + }, + "v1.18": { + "coredns": "1.6.7", + "etcd": "3.4.3-0", + "kube-apiserver": "v1.18.20", + "kube-controller-manager": "v1.18.20", + "kube-proxy": "v1.18.20", + "kube-scheduler": "v1.18.20", + "pause": "3.2", + }, + "v1.17": { + "coredns": "1.6.5", + "etcd": "3.4.3-0", + "kube-apiserver": "v1.17.17", + "kube-controller-manager": "v1.17.17", + "kube-proxy": "v1.17.17", + "kube-scheduler": "v1.17.17", + "pause": "3.1", + }, + "v1.16": { + "coredns": "1.6.2", + "etcd": "3.3.15-0", + "kube-apiserver": "v1.16.15", + "kube-controller-manager": "v1.16.15", + "kube-proxy": "v1.16.15", + "kube-scheduler": "v1.16.15", + "pause": "3.1", + }, + "v1.15": { + "coredns": "1.3.1", + "etcd": "3.3.10", + "kube-apiserver": "v1.15.12", + "kube-controller-manager": "v1.15.12", + "kube-proxy": "v1.15.12", + "kube-scheduler": "v1.15.12", + "pause": "3.1", + }, + "v1.14": { + "coredns": "1.3.1", + "etcd": "3.3.10", + "kube-apiserver": "v1.14.10", + "kube-controller-manager": "v1.14.10", + "kube-proxy": "v1.14.10", + "kube-scheduler": "v1.14.10", + "pause": "3.1", + }, + "v1.13": { + "coredns": "1.2.6", + "etcd": "3.2.24", + "kube-apiserver": "v1.13.12", + "kube-controller-manager": "v1.13.12", + "kube-proxy": "v1.13.12", + "kube-scheduler": "v1.13.12", + "pause": "3.1", + }, + "v1.12": { + "coredns": "1.2.2", + "etcd": "3.2.24", + "kube-apiserver": "v1.22.3", + "kube-controller-manager": "v1.22.3", + "kube-proxy": "v1.22.3", + "kube-scheduler": "v1.22.3", + "pause": "3.1", + }, + "v1.11": { + "coredns": "1.1.3", + "etcd-amd64": "3.2.18", + "kube-apiserver-amd64": "v1.11.10", + "kube-controller-manager-amd64": "v1.11.10", + "kube-proxy-amd64": "v1.11.10", + "kube-scheduler-amd64": "v1.11.10", + "pause-amd64": "3.1", + }, + } +)