From 800be7edc71f0d9881d8518190590621180d02e0 Mon Sep 17 00:00:00 2001 From: Matt Rickard Date: Wed, 11 Oct 2017 10:49:31 -0700 Subject: [PATCH 1/4] Add convenience container-runtime flag for kubeadm To enable the cri-o runtime you may now just use `minikube start --container-runtime=cri-o --bootstrapper=kubeadm` or `minikube start --container-runtiume=crio --bootstrapper=kubeadm` or `minikube start --extra-config=kubelet.container-runtime=remote --extra-config=kubelet.container-runtime-endpoint=/var/run/crio.sock --extra-config=image-service-endpoint=/var/run/crio.sock` --- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 41 +++++++++++++++++-- .../bootstrapper/kubeadm/templates.go | 28 +++++++++---- pkg/minikube/bootstrapper/kubeadm/versions.go | 19 +++++++++ 3 files changed, 77 insertions(+), 11 deletions(-) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index fe400df844..3a9faf579a 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -27,6 +27,7 @@ import ( "github.com/docker/machine/libmachine" "github.com/docker/machine/libmachine/state" + "github.com/golang/glog" download "github.com/jimmidyson/go-download" "github.com/pkg/errors" "golang.org/x/sync/errgroup" @@ -178,6 +179,35 @@ func (k *KubeadmBootstrapper) SetupCerts(k8s bootstrapper.KubernetesConfig) erro return bootstrapper.SetupCerts(k.c, k8s) } +// SetContainerRuntime possibly sets the container runtime, if it hasn't already +// been specified by the extra-config option. It has a set of defaults known to +// work for a particular runtime. +func SetContainerRuntime(cfg map[string]string, runtime string) map[string]string { + if _, ok := cfg["container-runtime"]; ok { + glog.Infoln("Container runtime already set through extra options, ignoring --container-runtime flag.") + return cfg + } + + if runtime == "" { + glog.Infoln("Container runtime flag provided with no value, using defaults.") + return cfg + } + + switch runtime { + case "crio", "cri-o": + cfg["container-runtime"] = "remote" + cfg["container-runtime-endpoint"] = "/var/run/crio.sock" + cfg["image-service-endpoint"] = "/var/run/crio.sock" + cfg["runtime-request-timeout"] = "15m" + default: + cfg["container-runtime"] = runtime + } + + return cfg +} + +// NewKubeletConfig generates a new systemd unit containing a configured kubelet +// based on the options present in the KubernetesConfig. func NewKubeletConfig(k8s bootstrapper.KubernetesConfig) (string, error) { version, err := ParseKubernetesVersion(k8s.KubernetesVersion) if err != nil { @@ -189,14 +219,17 @@ func NewKubeletConfig(k8s bootstrapper.KubernetesConfig) (string, error) { return "", errors.Wrap(err, "generating extra configuration for kubelet") } + extraOpts = SetContainerRuntime(extraOpts, k8s.ContainerRuntime) extraFlags := convertToFlags(extraOpts) b := bytes.Buffer{} opts := struct { - ExtraOptions string - FeatureGates string + ExtraOptions string + FeatureGates string + ContainerRuntime string }{ - ExtraOptions: extraFlags, - FeatureGates: k8s.FeatureGates, + ExtraOptions: extraFlags, + FeatureGates: k8s.FeatureGates, + ContainerRuntime: k8s.ContainerRuntime, } if err := kubeletSystemdTemplate.Execute(&b, opts); err != nil { return "", err diff --git a/pkg/minikube/bootstrapper/kubeadm/templates.go b/pkg/minikube/bootstrapper/kubeadm/templates.go index f57789c1f0..3e7a16f27e 100644 --- a/pkg/minikube/bootstrapper/kubeadm/templates.go +++ b/pkg/minikube/bootstrapper/kubeadm/templates.go @@ -40,17 +40,31 @@ nodeName: {{.NodeName}} {{$val}}{{end}} {{end}}`)) -var kubeletSystemdTemplate = template.Must(template.New("kubeletSystemdTemplate").Parse(` +var kubeletSystemdTemplate = template.Must(template.New("kubeletSystemdTemplate").Funcs(template.FuncMap{ + "installWants": installWants, +}).Parse(` [Service] -Environment="KUBELET_KUBECONFIG_ARGS=--kubeconfig=/etc/kubernetes/kubelet.conf --require-kubeconfig=true" -Environment="KUBELET_SYSTEM_PODS_ARGS=--pod-manifest-path=/etc/kubernetes/manifests --allow-privileged=true" -Environment="KUBELET_DNS_ARGS=--cluster-dns=10.0.0.10 --cluster-domain=cluster.local" -Environment="KUBELET_CADVISOR_ARGS=--cadvisor-port=0" -Environment="KUBELET_CGROUP_ARGS=--cgroup-driver=cgroupfs" ExecStart= -ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_SYSTEM_PODS_ARGS $KUBELET_DNS_ARGS $KUBELET_CADVISOR_ARGS $KUBELET_CGROUP_ARGS {{.ExtraOptions}} {{if .FeatureGates}}--feature-gates={{.FeatureGates}}{{end}} +ExecStart=/usr/bin/kubelet {{.ExtraOptions}} {{if .FeatureGates}}--feature-gates={{.FeatureGates}}{{end}} + +[Install] +{{installWants .ContainerRuntime}} `)) +func installWants(containerRuntime string) string { + var wants string + switch containerRuntime { + case "": + wants = "docker.socket" + case "cri-o", "cri": + wants = "crio.service" + } + if wants != "" { + return fmt.Sprintf("Wants=%s", wants) + } + return "" +} + const kubeletService = ` [Unit] Description=kubelet: The Kubernetes Node Agent diff --git a/pkg/minikube/bootstrapper/kubeadm/versions.go b/pkg/minikube/bootstrapper/kubeadm/versions.go index 6e8ae8136b..f15b19f7b8 100644 --- a/pkg/minikube/bootstrapper/kubeadm/versions.go +++ b/pkg/minikube/bootstrapper/kubeadm/versions.go @@ -149,6 +149,17 @@ type VersionedExtraOption struct { GreaterThanOrEqual semver.Version } +// NewUnversionedOption returns a VersionedExtraOption that applies to all versions. +func NewUnversionedOption(component, k, v string) VersionedExtraOption { + return VersionedExtraOption{ + Option: util.ExtraOption{ + Component: component, + Key: k, + Value: v, + }, + } +} + var versionSpecificOpts = []VersionedExtraOption{ { Option: util.ExtraOption{ @@ -158,6 +169,14 @@ var versionSpecificOpts = []VersionedExtraOption{ }, GreaterThanOrEqual: semver.MustParse("1.8.0-alpha.0"), }, + NewUnversionedOption(Kubelet, "kubeconfig", "/etc/kubernetes/kubelet.conf"), + NewUnversionedOption(Kubelet, "require-kubeconfig", "true"), + NewUnversionedOption(Kubelet, "pod-manifest-path", "/etc/kubernetes/manifests"), + NewUnversionedOption(Kubelet, "allow-privileged", "true"), + NewUnversionedOption(Kubelet, "cluster-dns", "10.0.0.10"), + NewUnversionedOption(Kubelet, "cluster-domain", "cluster.local"), + NewUnversionedOption(Kubelet, "cadvisor-port", "0"), + NewUnversionedOption(Kubelet, "cgroup-driver", "cgroupfs"), } func VersionIsBetween(version, gte, lte semver.Version) bool { From 320b5cdb778e3f8b4089460e1dad571ac19423c7 Mon Sep 17 00:00:00 2001 From: Matt Rickard Date: Wed, 11 Oct 2017 14:06:00 -0700 Subject: [PATCH 2/4] Add windows virtualbox kubeadm crio test --- hack/jenkins/minikube_set_pending.sh | 2 +- ...tegration_test_virtualbox_kubeadm_crio.ps1 | 35 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 hack/jenkins/windows_integration_test_virtualbox_kubeadm_crio.ps1 diff --git a/hack/jenkins/minikube_set_pending.sh b/hack/jenkins/minikube_set_pending.sh index d1974bb3cd..f7d45be001 100755 --- a/hack/jenkins/minikube_set_pending.sh +++ b/hack/jenkins/minikube_set_pending.sh @@ -27,7 +27,7 @@ set -e set +x -for job in "Linux-KVM-Kubeadm" "OSX-Virtualbox-Kubeadm" "OSX-Virtualbox" "OSX-XHyve" "OSX-Hyperkit" "Linux-Virtualbox" "Linux-KVM" "Linux-KVM-Alt" "Linux-None" "Windows-Virtualbox" "Linux-Container"; do +for job in "Linux-KVM-Kubeadm" "OSX-Virtualbox-Kubeadm" "OSX-Virtualbox" "OSX-XHyve" "OSX-Hyperkit" "Linux-Virtualbox" "Linux-KVM" "Linux-KVM-Alt" "Linux-None" "Windows-Virtualbox" "Windows-Kubeadm-CRI-O" "Linux-Container"; do target_url="https://storage.googleapis.com/minikube-builds/logs/${ghprbPullId}/${job}.txt" curl "https://api.github.com/repos/kubernetes/minikube/statuses/${ghprbActualCommit}?access_token=$access_token" \ -H "Content-Type: application/json" \ diff --git a/hack/jenkins/windows_integration_test_virtualbox_kubeadm_crio.ps1 b/hack/jenkins/windows_integration_test_virtualbox_kubeadm_crio.ps1 new file mode 100644 index 0000000000..8d1077d320 --- /dev/null +++ b/hack/jenkins/windows_integration_test_virtualbox_kubeadm_crio.ps1 @@ -0,0 +1,35 @@ +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +mkdir -p out +gsutil.cmd cp gs://minikube-builds/$env:MINIKUBE_LOCATION/minikube-windows-amd64.exe out/ +gsutil.cmd cp gs://minikube-builds/$env:MINIKUBE_LOCATION/e2e-windows-amd64.exe out/ +gsutil.cmd cp -r gs://minikube-builds/$env:MINIKUBE_LOCATION/testdata . + + +./out/minikube-windows-amd64.exe delete +Remove-Item -Recurse -Force C:\Users\jenkins\.minikube + +out/e2e-windows-amd64.exe --% -minikube-start-args="--vm-driver=virtualbox --container-runtime=cri-o" -minikube-args="--v=10 --logtostderr --bootstrapper=kubeadm" -test.v -test.timeout=30m -binary=out/minikube-windows-amd64.exe + +$env:result=$lastexitcode +# If the last exit code was 0->success, x>0->error +If($env:result -eq 0){$env:status="success"} +Else {$env:status="failure"} + +$env:target_url="https://storage.googleapis.com/minikube-builds/logs/$env:MINIKUBE_LOCATION/Windows-Kubeadm-CRI-O.txt" +$json = "{`"state`": `"$env:status`", `"description`": `"Jenkins`", `"target_url`": `"$env:target_url`", `"context`": `"Windows-Kubeadm-CRI-O`"}" +Invoke-WebRequest -Uri "https://api.github.com/repos/kubernetes/minikube/statuses/$env:COMMIT`?access_token=$env:access_token" -Body $json -ContentType "application/json" -Method Post -usebasicparsing + +Exit $env:result From 1a38ef74b23834bf51b17f9e2d5834d9025eb4ce Mon Sep 17 00:00:00 2001 From: Matt Rickard Date: Wed, 11 Oct 2017 14:26:42 -0700 Subject: [PATCH 3/4] Use path instead of filepath for cross platform --- pkg/minikube/assets/vm_assets.go | 4 ++-- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/minikube/assets/vm_assets.go b/pkg/minikube/assets/vm_assets.go index 30cbe58745..27d668a7ea 100644 --- a/pkg/minikube/assets/vm_assets.go +++ b/pkg/minikube/assets/vm_assets.go @@ -20,7 +20,7 @@ import ( "bytes" "io" "os" - "path/filepath" + "path" "github.com/pkg/errors" ) @@ -65,7 +65,7 @@ type FileAsset struct { } func NewMemoryAssetTarget(d []byte, targetPath, permissions string) *MemoryAsset { - return NewMemoryAsset(d, filepath.Dir(targetPath), filepath.Base(targetPath), permissions) + return NewMemoryAsset(d, path.Dir(targetPath), path.Base(targetPath), permissions) } func NewFileAsset(assetName, targetDir, targetName, permissions string) (*FileAsset, error) { diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 3a9faf579a..7646827c83 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -21,7 +21,7 @@ import ( "crypto" "fmt" "os" - "path/filepath" + "path" "strings" "time" @@ -361,7 +361,7 @@ func generateConfig(k8s bootstrapper.KubernetesConfig) (string, error) { func maybeDownloadAndCache(binary, version string) (string, error) { targetDir := constants.MakeMiniPath("cache", version) - targetFilepath := filepath.Join(targetDir, binary) + targetFilepath := path.Join(targetDir, binary) _, err := os.Stat(targetFilepath) // If it exists, do no verification and continue From 37456d22e097120bf7116b3fc53b427673013a95 Mon Sep 17 00:00:00 2001 From: Matt Rickard Date: Tue, 17 Oct 2017 10:08:07 -0700 Subject: [PATCH 4/4] Review feedback --- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 17 ------ .../bootstrapper/kubeadm/kubeadm_test.go | 52 ------------------- .../bootstrapper/kubeadm/templates.go | 20 +------ 3 files changed, 2 insertions(+), 87 deletions(-) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 7646827c83..a20cce60ba 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -302,23 +302,6 @@ sudo systemctl start kubelet return nil } -func parseFeatureGates(featureGates string) (map[string]string, error) { - if featureGates == "" { - return nil, nil - } - fgMap := map[string]string{} - fg := strings.Split(featureGates, ",") - for _, f := range fg { - kv := strings.SplitN(f, "=", 2) - if len(kv) != 2 { - return nil, fmt.Errorf("Invalid feature gate format: %s", f) - } - fgMap[kv[0]] = kv[1] - } - - return fgMap, nil -} - func generateConfig(k8s bootstrapper.KubernetesConfig) (string, error) { version, err := ParseKubernetesVersion(k8s.KubernetesVersion) if err != nil { diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go index 574ebe7266..5d598ba02a 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go @@ -17,7 +17,6 @@ limitations under the License. package kubeadm import ( - "reflect" "testing" "k8s.io/minikube/pkg/minikube/bootstrapper" @@ -233,54 +232,3 @@ schedulerExtraArgs: }) } } - -func TestParseFeatureGates(t *testing.T) { - tests := []struct { - description string - fg string - expected map[string]string - shouldErr bool - }{ - { - description: "no feature gates", - }, - { - description: "one feature gate", - fg: "AppArmor=true", - expected: map[string]string{ - "AppArmor": "true", - }, - }, - { - description: "two feature gates", - fg: "AppArmor=true,HugePages=true", - expected: map[string]string{ - "AppArmor": "true", - "HugePages": "true", - }, - }, - { - description: "missing value pair", - fg: "AppArmor=true,HugePages", - shouldErr: true, - }, - } - - for _, test := range tests { - t.Run(test.description, func(t *testing.T) { - actual, err := parseFeatureGates(test.fg) - t.Logf("%+v", actual) - if err == nil && test.shouldErr { - t.Errorf("Expected error but got none: fg: %v", actual) - return - } - if err != nil && !test.shouldErr { - t.Errorf("Unexpected error: %s", err) - return - } - if !reflect.DeepEqual(actual, test.expected) { - t.Errorf("Actual not equal expected: Actual: %v Expected: %v", actual, test.expected) - } - }) - } -} diff --git a/pkg/minikube/bootstrapper/kubeadm/templates.go b/pkg/minikube/bootstrapper/kubeadm/templates.go index 3e7a16f27e..6dbd445b0e 100644 --- a/pkg/minikube/bootstrapper/kubeadm/templates.go +++ b/pkg/minikube/bootstrapper/kubeadm/templates.go @@ -40,31 +40,15 @@ nodeName: {{.NodeName}} {{$val}}{{end}} {{end}}`)) -var kubeletSystemdTemplate = template.Must(template.New("kubeletSystemdTemplate").Funcs(template.FuncMap{ - "installWants": installWants, -}).Parse(` +var kubeletSystemdTemplate = template.Must(template.New("kubeletSystemdTemplate").Parse(` [Service] ExecStart= ExecStart=/usr/bin/kubelet {{.ExtraOptions}} {{if .FeatureGates}}--feature-gates={{.FeatureGates}}{{end}} [Install] -{{installWants .ContainerRuntime}} +{{if or (eq .ContainerRuntime "cri-o") (eq .ContainerRuntime "cri")}}Wants=crio.service{{else}}Wants=docker.socket{{end}} `)) -func installWants(containerRuntime string) string { - var wants string - switch containerRuntime { - case "": - wants = "docker.socket" - case "cri-o", "cri": - wants = "crio.service" - } - if wants != "" { - return fmt.Sprintf("Wants=%s", wants) - } - return "" -} - const kubeletService = ` [Unit] Description=kubelet: The Kubernetes Node Agent