Merge pull request #2052 from r2d4/cri-o

Add convenience container-runtime flag for kubeadm

hack/jenkins/minikube_set_pending.sh

@@ -27,7 +27,7 @@
 set -e
 set +x
 
-for job in "Linux-KVM-Kubeadm" "OSX-Virtualbox-Kubeadm" "OSX-Virtualbox" "OSX-XHyve" "OSX-Hyperkit" "Linux-Virtualbox" "Linux-KVM" "Linux-KVM-Alt" "Linux-None" "Windows-Virtualbox" "Linux-Container"; do
+for job in "Linux-KVM-Kubeadm" "OSX-Virtualbox-Kubeadm" "OSX-Virtualbox" "OSX-XHyve" "OSX-Hyperkit" "Linux-Virtualbox" "Linux-KVM" "Linux-KVM-Alt" "Linux-None" "Windows-Virtualbox" "Windows-Kubeadm-CRI-O" "Linux-Container"; do
   target_url="https://storage.googleapis.com/minikube-builds/logs/${ghprbPullId}/${job}.txt"
   curl "https://api.github.com/repos/kubernetes/minikube/statuses/${ghprbActualCommit}?access_token=$access_token" \
     -H "Content-Type: application/json" \

hack/jenkins/windows_integration_test_virtualbox_kubeadm_crio.ps1

@@ -0,0 +1,35 @@
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+mkdir -p out
+gsutil.cmd cp gs://minikube-builds/$env:MINIKUBE_LOCATION/minikube-windows-amd64.exe out/
+gsutil.cmd cp gs://minikube-builds/$env:MINIKUBE_LOCATION/e2e-windows-amd64.exe out/
+gsutil.cmd cp -r gs://minikube-builds/$env:MINIKUBE_LOCATION/testdata .
+
+
+./out/minikube-windows-amd64.exe delete
+Remove-Item -Recurse -Force C:\Users\jenkins\.minikube
+
+out/e2e-windows-amd64.exe --% -minikube-start-args="--vm-driver=virtualbox --container-runtime=cri-o" -minikube-args="--v=10 --logtostderr --bootstrapper=kubeadm" -test.v -test.timeout=30m -binary=out/minikube-windows-amd64.exe
+
+$env:result=$lastexitcode
+# If the last exit code was 0->success, x>0->error
+If($env:result -eq 0){$env:status="success"}
+Else {$env:status="failure"}
+
+$env:target_url="https://storage.googleapis.com/minikube-builds/logs/$env:MINIKUBE_LOCATION/Windows-Kubeadm-CRI-O.txt"
+$json = "{`"state`": `"$env:status`", `"description`": `"Jenkins`", `"target_url`": `"$env:target_url`", `"context`": `"Windows-Kubeadm-CRI-O`"}"
+Invoke-WebRequest -Uri "https://api.github.com/repos/kubernetes/minikube/statuses/$env:COMMIT`?access_token=$env:access_token" -Body $json -ContentType "application/json" -Method Post -usebasicparsing
+
+Exit $env:result

pkg/minikube/assets/vm_assets.go

@@ -20,7 +20,7 @@ import (
 	"bytes"
 	"io"
 	"os"
-	"path/filepath"
+	"path"
 
 	"github.com/pkg/errors"
 )
@@ -65,7 +65,7 @@ type FileAsset struct {
 }
 
 func NewMemoryAssetTarget(d []byte, targetPath, permissions string) *MemoryAsset {
-	return NewMemoryAsset(d, filepath.Dir(targetPath), filepath.Base(targetPath), permissions)
+	return NewMemoryAsset(d, path.Dir(targetPath), path.Base(targetPath), permissions)
 }
 
 func NewFileAsset(assetName, targetDir, targetName, permissions string) (*FileAsset, error) {

pkg/minikube/bootstrapper/kubeadm/kubeadm.go

@@ -21,12 +21,13 @@ import (
 	"crypto"
 	"fmt"
 	"os"
-	"path/filepath"
+	"path"
 	"strings"
 	"time"
 
 	"github.com/docker/machine/libmachine"
 	"github.com/docker/machine/libmachine/state"
+	"github.com/golang/glog"
 	download "github.com/jimmidyson/go-download"
 	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
@@ -178,6 +179,35 @@ func (k *KubeadmBootstrapper) SetupCerts(k8s bootstrapper.KubernetesConfig) erro
 	return bootstrapper.SetupCerts(k.c, k8s)
 }
 
+// SetContainerRuntime possibly sets the container runtime, if it hasn't already
+// been specified by the extra-config option.  It has a set of defaults known to
+// work for a particular runtime.
+func SetContainerRuntime(cfg map[string]string, runtime string) map[string]string {
+	if _, ok := cfg["container-runtime"]; ok {
+		glog.Infoln("Container runtime already set through extra options, ignoring --container-runtime flag.")
+		return cfg
+	}
+
+	if runtime == "" {
+		glog.Infoln("Container runtime flag provided with no value, using defaults.")
+		return cfg
+	}
+
+	switch runtime {
+	case "crio", "cri-o":
+		cfg["container-runtime"] = "remote"
+		cfg["container-runtime-endpoint"] = "/var/run/crio.sock"
+		cfg["image-service-endpoint"] = "/var/run/crio.sock"
+		cfg["runtime-request-timeout"] = "15m"
+	default:
+		cfg["container-runtime"] = runtime
+	}
+
+	return cfg
+}
+
+// NewKubeletConfig generates a new systemd unit containing a configured kubelet
+// based on the options present in the KubernetesConfig.
 func NewKubeletConfig(k8s bootstrapper.KubernetesConfig) (string, error) {
 	version, err := ParseKubernetesVersion(k8s.KubernetesVersion)
 	if err != nil {
@@ -189,14 +219,17 @@ func NewKubeletConfig(k8s bootstrapper.KubernetesConfig) (string, error) {
 		return "", errors.Wrap(err, "generating extra configuration for kubelet")
 	}
 
+	extraOpts = SetContainerRuntime(extraOpts, k8s.ContainerRuntime)
 	extraFlags := convertToFlags(extraOpts)
 	b := bytes.Buffer{}
 	opts := struct {
-		ExtraOptions string
-		FeatureGates string
+		ExtraOptions     string
+		FeatureGates     string
+		ContainerRuntime string
 	}{
-		ExtraOptions: extraFlags,
-		FeatureGates: k8s.FeatureGates,
+		ExtraOptions:     extraFlags,
+		FeatureGates:     k8s.FeatureGates,
+		ContainerRuntime: k8s.ContainerRuntime,
 	}
 	if err := kubeletSystemdTemplate.Execute(&b, opts); err != nil {
 		return "", err
@@ -269,23 +302,6 @@ sudo systemctl start kubelet
 	return nil
 }
 
-func parseFeatureGates(featureGates string) (map[string]string, error) {
-	if featureGates == "" {
-		return nil, nil
-	}
-	fgMap := map[string]string{}
-	fg := strings.Split(featureGates, ",")
-	for _, f := range fg {
-		kv := strings.SplitN(f, "=", 2)
-		if len(kv) != 2 {
-			return nil, fmt.Errorf("Invalid feature gate format: %s", f)
-		}
-		fgMap[kv[0]] = kv[1]
-	}
-
-	return fgMap, nil
-}
-
 func generateConfig(k8s bootstrapper.KubernetesConfig) (string, error) {
 	version, err := ParseKubernetesVersion(k8s.KubernetesVersion)
 	if err != nil {
@@ -328,7 +344,7 @@ func generateConfig(k8s bootstrapper.KubernetesConfig) (string, error) {
 
 func maybeDownloadAndCache(binary, version string) (string, error) {
 	targetDir := constants.MakeMiniPath("cache", version)
-	targetFilepath := filepath.Join(targetDir, binary)
+	targetFilepath := path.Join(targetDir, binary)
 
 	_, err := os.Stat(targetFilepath)
 	// If it exists, do no verification and continue

pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go

@@ -17,7 +17,6 @@ limitations under the License.
 package kubeadm
 
 import (
-	"reflect"
 	"testing"
 
 	"k8s.io/minikube/pkg/minikube/bootstrapper"
@@ -233,54 +232,3 @@ schedulerExtraArgs:
 		})
 	}
 }
-
-func TestParseFeatureGates(t *testing.T) {
-	tests := []struct {
-		description string
-		fg          string
-		expected    map[string]string
-		shouldErr   bool
-	}{
-		{
-			description: "no feature gates",
-		},
-		{
-			description: "one feature gate",
-			fg:          "AppArmor=true",
-			expected: map[string]string{
-				"AppArmor": "true",
-			},
-		},
-		{
-			description: "two feature gates",
-			fg:          "AppArmor=true,HugePages=true",
-			expected: map[string]string{
-				"AppArmor":  "true",
-				"HugePages": "true",
-			},
-		},
-		{
-			description: "missing value pair",
-			fg:          "AppArmor=true,HugePages",
-			shouldErr:   true,
-		},
-	}
-
-	for _, test := range tests {
-		t.Run(test.description, func(t *testing.T) {
-			actual, err := parseFeatureGates(test.fg)
-			t.Logf("%+v", actual)
-			if err == nil && test.shouldErr {
-				t.Errorf("Expected error but got none: fg: %v", actual)
-				return
-			}
-			if err != nil && !test.shouldErr {
-				t.Errorf("Unexpected error: %s", err)
-				return
-			}
-			if !reflect.DeepEqual(actual, test.expected) {
-				t.Errorf("Actual not equal expected: Actual: %v Expected: %v", actual, test.expected)
-			}
-		})
-	}
-}

pkg/minikube/bootstrapper/kubeadm/templates.go

@@ -42,13 +42,11 @@ nodeName: {{.NodeName}}
 
 var kubeletSystemdTemplate = template.Must(template.New("kubeletSystemdTemplate").Parse(`
 [Service]
-Environment="KUBELET_KUBECONFIG_ARGS=--kubeconfig=/etc/kubernetes/kubelet.conf --require-kubeconfig=true"
-Environment="KUBELET_SYSTEM_PODS_ARGS=--pod-manifest-path=/etc/kubernetes/manifests --allow-privileged=true"
-Environment="KUBELET_DNS_ARGS=--cluster-dns=10.0.0.10 --cluster-domain=cluster.local"
-Environment="KUBELET_CADVISOR_ARGS=--cadvisor-port=0"
-Environment="KUBELET_CGROUP_ARGS=--cgroup-driver=cgroupfs"
 ExecStart=
-ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_SYSTEM_PODS_ARGS $KUBELET_DNS_ARGS $KUBELET_CADVISOR_ARGS $KUBELET_CGROUP_ARGS {{.ExtraOptions}} {{if .FeatureGates}}--feature-gates={{.FeatureGates}}{{end}}
+ExecStart=/usr/bin/kubelet {{.ExtraOptions}} {{if .FeatureGates}}--feature-gates={{.FeatureGates}}{{end}}
+
+[Install]
+{{if or (eq .ContainerRuntime "cri-o") (eq .ContainerRuntime "cri")}}Wants=crio.service{{else}}Wants=docker.socket{{end}}
 `))
 
 const kubeletService = `

pkg/minikube/bootstrapper/kubeadm/versions.go

@@ -149,6 +149,17 @@ type VersionedExtraOption struct {
 	GreaterThanOrEqual semver.Version
 }
 
+// NewUnversionedOption returns a VersionedExtraOption that applies to all versions.
+func NewUnversionedOption(component, k, v string) VersionedExtraOption {
+	return VersionedExtraOption{
+		Option: util.ExtraOption{
+			Component: component,
+			Key:       k,
+			Value:     v,
+		},
+	}
+}
+
 var versionSpecificOpts = []VersionedExtraOption{
 	{
 		Option: util.ExtraOption{
@@ -158,6 +169,14 @@ var versionSpecificOpts = []VersionedExtraOption{
 		},
 		GreaterThanOrEqual: semver.MustParse("1.8.0-alpha.0"),
 	},
+	NewUnversionedOption(Kubelet, "kubeconfig", "/etc/kubernetes/kubelet.conf"),
+	NewUnversionedOption(Kubelet, "require-kubeconfig", "true"),
+	NewUnversionedOption(Kubelet, "pod-manifest-path", "/etc/kubernetes/manifests"),
+	NewUnversionedOption(Kubelet, "allow-privileged", "true"),
+	NewUnversionedOption(Kubelet, "cluster-dns", "10.0.0.10"),
+	NewUnversionedOption(Kubelet, "cluster-domain", "cluster.local"),
+	NewUnversionedOption(Kubelet, "cadvisor-port", "0"),
+	NewUnversionedOption(Kubelet, "cgroup-driver", "cgroupfs"),
 }
 
 func VersionIsBetween(version, gte, lte semver.Version) bool {