diff --git a/hack/jenkins/minikube_set_pending.sh b/hack/jenkins/minikube_set_pending.sh index d1974bb3cd..f7d45be001 100755 --- a/hack/jenkins/minikube_set_pending.sh +++ b/hack/jenkins/minikube_set_pending.sh @@ -27,7 +27,7 @@ set -e set +x -for job in "Linux-KVM-Kubeadm" "OSX-Virtualbox-Kubeadm" "OSX-Virtualbox" "OSX-XHyve" "OSX-Hyperkit" "Linux-Virtualbox" "Linux-KVM" "Linux-KVM-Alt" "Linux-None" "Windows-Virtualbox" "Linux-Container"; do +for job in "Linux-KVM-Kubeadm" "OSX-Virtualbox-Kubeadm" "OSX-Virtualbox" "OSX-XHyve" "OSX-Hyperkit" "Linux-Virtualbox" "Linux-KVM" "Linux-KVM-Alt" "Linux-None" "Windows-Virtualbox" "Windows-Kubeadm-CRI-O" "Linux-Container"; do target_url="https://storage.googleapis.com/minikube-builds/logs/${ghprbPullId}/${job}.txt" curl "https://api.github.com/repos/kubernetes/minikube/statuses/${ghprbActualCommit}?access_token=$access_token" \ -H "Content-Type: application/json" \ diff --git a/hack/jenkins/windows_integration_test_virtualbox_kubeadm_crio.ps1 b/hack/jenkins/windows_integration_test_virtualbox_kubeadm_crio.ps1 new file mode 100644 index 0000000000..8d1077d320 --- /dev/null +++ b/hack/jenkins/windows_integration_test_virtualbox_kubeadm_crio.ps1 @@ -0,0 +1,35 @@ +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +mkdir -p out +gsutil.cmd cp gs://minikube-builds/$env:MINIKUBE_LOCATION/minikube-windows-amd64.exe out/ +gsutil.cmd cp gs://minikube-builds/$env:MINIKUBE_LOCATION/e2e-windows-amd64.exe out/ +gsutil.cmd cp -r gs://minikube-builds/$env:MINIKUBE_LOCATION/testdata . + + +./out/minikube-windows-amd64.exe delete +Remove-Item -Recurse -Force C:\Users\jenkins\.minikube + +out/e2e-windows-amd64.exe --% -minikube-start-args="--vm-driver=virtualbox --container-runtime=cri-o" -minikube-args="--v=10 --logtostderr --bootstrapper=kubeadm" -test.v -test.timeout=30m -binary=out/minikube-windows-amd64.exe + +$env:result=$lastexitcode +# If the last exit code was 0->success, x>0->error +If($env:result -eq 0){$env:status="success"} +Else {$env:status="failure"} + +$env:target_url="https://storage.googleapis.com/minikube-builds/logs/$env:MINIKUBE_LOCATION/Windows-Kubeadm-CRI-O.txt" +$json = "{`"state`": `"$env:status`", `"description`": `"Jenkins`", `"target_url`": `"$env:target_url`", `"context`": `"Windows-Kubeadm-CRI-O`"}" +Invoke-WebRequest -Uri "https://api.github.com/repos/kubernetes/minikube/statuses/$env:COMMIT`?access_token=$env:access_token" -Body $json -ContentType "application/json" -Method Post -usebasicparsing + +Exit $env:result diff --git a/pkg/minikube/assets/vm_assets.go b/pkg/minikube/assets/vm_assets.go index 30cbe58745..27d668a7ea 100644 --- a/pkg/minikube/assets/vm_assets.go +++ b/pkg/minikube/assets/vm_assets.go @@ -20,7 +20,7 @@ import ( "bytes" "io" "os" - "path/filepath" + "path" "github.com/pkg/errors" ) @@ -65,7 +65,7 @@ type FileAsset struct { } func NewMemoryAssetTarget(d []byte, targetPath, permissions string) *MemoryAsset { - return NewMemoryAsset(d, filepath.Dir(targetPath), filepath.Base(targetPath), permissions) + return NewMemoryAsset(d, path.Dir(targetPath), path.Base(targetPath), permissions) } func NewFileAsset(assetName, targetDir, targetName, permissions string) (*FileAsset, error) { diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index fe400df844..a20cce60ba 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -21,12 +21,13 @@ import ( "crypto" "fmt" "os" - "path/filepath" + "path" "strings" "time" "github.com/docker/machine/libmachine" "github.com/docker/machine/libmachine/state" + "github.com/golang/glog" download "github.com/jimmidyson/go-download" "github.com/pkg/errors" "golang.org/x/sync/errgroup" @@ -178,6 +179,35 @@ func (k *KubeadmBootstrapper) SetupCerts(k8s bootstrapper.KubernetesConfig) erro return bootstrapper.SetupCerts(k.c, k8s) } +// SetContainerRuntime possibly sets the container runtime, if it hasn't already +// been specified by the extra-config option. It has a set of defaults known to +// work for a particular runtime. +func SetContainerRuntime(cfg map[string]string, runtime string) map[string]string { + if _, ok := cfg["container-runtime"]; ok { + glog.Infoln("Container runtime already set through extra options, ignoring --container-runtime flag.") + return cfg + } + + if runtime == "" { + glog.Infoln("Container runtime flag provided with no value, using defaults.") + return cfg + } + + switch runtime { + case "crio", "cri-o": + cfg["container-runtime"] = "remote" + cfg["container-runtime-endpoint"] = "/var/run/crio.sock" + cfg["image-service-endpoint"] = "/var/run/crio.sock" + cfg["runtime-request-timeout"] = "15m" + default: + cfg["container-runtime"] = runtime + } + + return cfg +} + +// NewKubeletConfig generates a new systemd unit containing a configured kubelet +// based on the options present in the KubernetesConfig. func NewKubeletConfig(k8s bootstrapper.KubernetesConfig) (string, error) { version, err := ParseKubernetesVersion(k8s.KubernetesVersion) if err != nil { @@ -189,14 +219,17 @@ func NewKubeletConfig(k8s bootstrapper.KubernetesConfig) (string, error) { return "", errors.Wrap(err, "generating extra configuration for kubelet") } + extraOpts = SetContainerRuntime(extraOpts, k8s.ContainerRuntime) extraFlags := convertToFlags(extraOpts) b := bytes.Buffer{} opts := struct { - ExtraOptions string - FeatureGates string + ExtraOptions string + FeatureGates string + ContainerRuntime string }{ - ExtraOptions: extraFlags, - FeatureGates: k8s.FeatureGates, + ExtraOptions: extraFlags, + FeatureGates: k8s.FeatureGates, + ContainerRuntime: k8s.ContainerRuntime, } if err := kubeletSystemdTemplate.Execute(&b, opts); err != nil { return "", err @@ -269,23 +302,6 @@ sudo systemctl start kubelet return nil } -func parseFeatureGates(featureGates string) (map[string]string, error) { - if featureGates == "" { - return nil, nil - } - fgMap := map[string]string{} - fg := strings.Split(featureGates, ",") - for _, f := range fg { - kv := strings.SplitN(f, "=", 2) - if len(kv) != 2 { - return nil, fmt.Errorf("Invalid feature gate format: %s", f) - } - fgMap[kv[0]] = kv[1] - } - - return fgMap, nil -} - func generateConfig(k8s bootstrapper.KubernetesConfig) (string, error) { version, err := ParseKubernetesVersion(k8s.KubernetesVersion) if err != nil { @@ -328,7 +344,7 @@ func generateConfig(k8s bootstrapper.KubernetesConfig) (string, error) { func maybeDownloadAndCache(binary, version string) (string, error) { targetDir := constants.MakeMiniPath("cache", version) - targetFilepath := filepath.Join(targetDir, binary) + targetFilepath := path.Join(targetDir, binary) _, err := os.Stat(targetFilepath) // If it exists, do no verification and continue diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go index 574ebe7266..5d598ba02a 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go @@ -17,7 +17,6 @@ limitations under the License. package kubeadm import ( - "reflect" "testing" "k8s.io/minikube/pkg/minikube/bootstrapper" @@ -233,54 +232,3 @@ schedulerExtraArgs: }) } } - -func TestParseFeatureGates(t *testing.T) { - tests := []struct { - description string - fg string - expected map[string]string - shouldErr bool - }{ - { - description: "no feature gates", - }, - { - description: "one feature gate", - fg: "AppArmor=true", - expected: map[string]string{ - "AppArmor": "true", - }, - }, - { - description: "two feature gates", - fg: "AppArmor=true,HugePages=true", - expected: map[string]string{ - "AppArmor": "true", - "HugePages": "true", - }, - }, - { - description: "missing value pair", - fg: "AppArmor=true,HugePages", - shouldErr: true, - }, - } - - for _, test := range tests { - t.Run(test.description, func(t *testing.T) { - actual, err := parseFeatureGates(test.fg) - t.Logf("%+v", actual) - if err == nil && test.shouldErr { - t.Errorf("Expected error but got none: fg: %v", actual) - return - } - if err != nil && !test.shouldErr { - t.Errorf("Unexpected error: %s", err) - return - } - if !reflect.DeepEqual(actual, test.expected) { - t.Errorf("Actual not equal expected: Actual: %v Expected: %v", actual, test.expected) - } - }) - } -} diff --git a/pkg/minikube/bootstrapper/kubeadm/templates.go b/pkg/minikube/bootstrapper/kubeadm/templates.go index f57789c1f0..6dbd445b0e 100644 --- a/pkg/minikube/bootstrapper/kubeadm/templates.go +++ b/pkg/minikube/bootstrapper/kubeadm/templates.go @@ -42,13 +42,11 @@ nodeName: {{.NodeName}} var kubeletSystemdTemplate = template.Must(template.New("kubeletSystemdTemplate").Parse(` [Service] -Environment="KUBELET_KUBECONFIG_ARGS=--kubeconfig=/etc/kubernetes/kubelet.conf --require-kubeconfig=true" -Environment="KUBELET_SYSTEM_PODS_ARGS=--pod-manifest-path=/etc/kubernetes/manifests --allow-privileged=true" -Environment="KUBELET_DNS_ARGS=--cluster-dns=10.0.0.10 --cluster-domain=cluster.local" -Environment="KUBELET_CADVISOR_ARGS=--cadvisor-port=0" -Environment="KUBELET_CGROUP_ARGS=--cgroup-driver=cgroupfs" ExecStart= -ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_SYSTEM_PODS_ARGS $KUBELET_DNS_ARGS $KUBELET_CADVISOR_ARGS $KUBELET_CGROUP_ARGS {{.ExtraOptions}} {{if .FeatureGates}}--feature-gates={{.FeatureGates}}{{end}} +ExecStart=/usr/bin/kubelet {{.ExtraOptions}} {{if .FeatureGates}}--feature-gates={{.FeatureGates}}{{end}} + +[Install] +{{if or (eq .ContainerRuntime "cri-o") (eq .ContainerRuntime "cri")}}Wants=crio.service{{else}}Wants=docker.socket{{end}} `)) const kubeletService = ` diff --git a/pkg/minikube/bootstrapper/kubeadm/versions.go b/pkg/minikube/bootstrapper/kubeadm/versions.go index 6e8ae8136b..f15b19f7b8 100644 --- a/pkg/minikube/bootstrapper/kubeadm/versions.go +++ b/pkg/minikube/bootstrapper/kubeadm/versions.go @@ -149,6 +149,17 @@ type VersionedExtraOption struct { GreaterThanOrEqual semver.Version } +// NewUnversionedOption returns a VersionedExtraOption that applies to all versions. +func NewUnversionedOption(component, k, v string) VersionedExtraOption { + return VersionedExtraOption{ + Option: util.ExtraOption{ + Component: component, + Key: k, + Value: v, + }, + } +} + var versionSpecificOpts = []VersionedExtraOption{ { Option: util.ExtraOption{ @@ -158,6 +169,14 @@ var versionSpecificOpts = []VersionedExtraOption{ }, GreaterThanOrEqual: semver.MustParse("1.8.0-alpha.0"), }, + NewUnversionedOption(Kubelet, "kubeconfig", "/etc/kubernetes/kubelet.conf"), + NewUnversionedOption(Kubelet, "require-kubeconfig", "true"), + NewUnversionedOption(Kubelet, "pod-manifest-path", "/etc/kubernetes/manifests"), + NewUnversionedOption(Kubelet, "allow-privileged", "true"), + NewUnversionedOption(Kubelet, "cluster-dns", "10.0.0.10"), + NewUnversionedOption(Kubelet, "cluster-domain", "cluster.local"), + NewUnversionedOption(Kubelet, "cadvisor-port", "0"), + NewUnversionedOption(Kubelet, "cgroup-driver", "cgroupfs"), } func VersionIsBetween(version, gte, lte semver.Version) bool {