kubernetes
/
minikube
mirror of https://github.com/kubernetes/minikube.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

remove kicbs

pull/6338/head
Medya Gh 2020-01-17 13:45:19 -08:00
parent 13d55c3dde
commit 0b2d8051bb
9 changed files with 214 additions and 536 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
cmd/minikube/cmd/root.go
View File

@ -32,7 +32,6 @@ import (
"k8s.io/kubectl/pkg/util/templates" "k8s.io/kubectl/pkg/util/templates"
configCmd "k8s.io/minikube/cmd/minikube/cmd/config" configCmd "k8s.io/minikube/cmd/minikube/cmd/config"
"k8s.io/minikube/pkg/minikube/bootstrapper" "k8s.io/minikube/pkg/minikube/bootstrapper"
"k8s.io/minikube/pkg/minikube/bootstrapper/kicbs"
"k8s.io/minikube/pkg/minikube/bootstrapper/kubeadm" "k8s.io/minikube/pkg/minikube/bootstrapper/kubeadm"
"k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/config"
"k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/constants"
@ -274,16 +273,9 @@ func getClusterBootstrapper(api libmachine.API, bootstrapperName string) (bootst
if err != nil { if err != nil {
return nil, errors.Wrap(err, "getting a new kubeadm bootstrapper") return nil, errors.Wrap(err, "getting a new kubeadm bootstrapper")
} }
case bootstrapper.KIC:
b, err = kicbs.NewBootstrapper(api)
if err != nil {
return nil, errors.Wrap(err, "getting a new kic bootstrapper")
}
default: default:
return nil, fmt.Errorf("unknown bootstrapper: %s", bootstrapperName) return nil, fmt.Errorf("unknown bootstrapper: %s", bootstrapperName)
} }
return b, nil return b, nil
} }

14
cmd/minikube/cmd/start.go
View File

@ -366,7 +366,7 @@ func runStart(cmd *cobra.Command, args []string) {
bs := setupKubeAdm(machineAPI, config) bs := setupKubeAdm(machineAPI, config)
// pull images or restart cluster // pull images or restart cluster
bootstrapCluster(bs, cr, mRunner, config.KubernetesConfig, preExists, isUpgrade) bootstrapCluster(bs, cr, mRunner, config, preExists, isUpgrade)
configureMounts() configureMounts()
// enable addons with start command // enable addons with start command
@ -383,7 +383,7 @@ func runStart(cmd *cobra.Command, args []string) {
// Skip pre-existing, because we already waited for health // Skip pre-existing, because we already waited for health
if viper.GetBool(waitUntilHealthy) && !preExists { if viper.GetBool(waitUntilHealthy) && !preExists {
if err := bs.WaitForCluster(config.KubernetesConfig, viper.GetDuration(waitTimeout)); err != nil { if err := bs.WaitForCluster(config, viper.GetDuration(waitTimeout)); err != nil {
exit.WithError("Wait failed", err) exit.WithError("Wait failed", err)
} }
} }
@ -1027,6 +1027,10 @@ func autoSetDriverOptions(cmd *cobra.Command, drvName string) (err error) {
viper.Set(cacheImages, hints.CacheImages) viper.Set(cacheImages, hints.CacheImages)
} }
if !cmd.Flags().Changed(enableDefaultCNI) {
viper.Set(enableDefaultCNI, hints.EnableDefaultCNI)
}
if !cmd.Flags().Changed(containerRuntime) && hints.ContainerRuntime != "" { if !cmd.Flags().Changed(containerRuntime) && hints.ContainerRuntime != "" {
viper.Set(containerRuntime, hints.ContainerRuntime) viper.Set(containerRuntime, hints.ContainerRuntime)
glog.Infof("auto set %s to %q.", containerRuntime, hints.ContainerRuntime) glog.Infof("auto set %s to %q.", containerRuntime, hints.ContainerRuntime)
@ -1287,16 +1291,16 @@ func configureRuntimes(runner cruntime.CommandRunner, drvName string, k8s cfg.Ku
} }
// bootstrapCluster starts Kubernetes using the chosen bootstrapper // bootstrapCluster starts Kubernetes using the chosen bootstrapper
func bootstrapCluster(bs bootstrapper.Bootstrapper, r cruntime.Manager, runner command.Runner, kc cfg.KubernetesConfig, preexisting bool, isUpgrade bool) { func bootstrapCluster(bs bootstrapper.Bootstrapper, r cruntime.Manager, runner command.Runner, c cfg.MachineConfig, preexisting bool, isUpgrade bool) {
if isUpgrade || !preexisting { if isUpgrade || !preexisting {
out.T(out.Pulling, "Pulling images ...") out.T(out.Pulling, "Pulling images ...")
if err := bs.PullImages(kc); err != nil { if err := bs.PullImages(c.KubernetesConfig); err != nil {
out.T(out.FailureType, "Unable to pull images, which may be OK: {{.error}}", out.V{"error": err}) out.T(out.FailureType, "Unable to pull images, which may be OK: {{.error}}", out.V{"error": err})
} }
} }
out.T(out.Launch, "Launching Kubernetes ... ") out.T(out.Launch, "Launching Kubernetes ... ")
if err := bs.StartCluster(kc); err != nil { if err := bs.StartCluster(c); err != nil {
exit.WithLogEntries("Error starting cluster", err, logs.FindProblems(r, bs, runner)) exit.WithLogEntries("Error starting cluster", err, logs.FindProblems(r, bs, runner))
} }
} }

5
pkg/minikube/bootstrapper/bootstrapper.go
View File

@ -37,10 +37,10 @@ type LogOptions struct {
type Bootstrapper interface { type Bootstrapper interface {
// PullImages pulls images necessary for a cluster. Success should not be required. // PullImages pulls images necessary for a cluster. Success should not be required.
PullImages(config.KubernetesConfig) error PullImages(config.KubernetesConfig) error
StartCluster(config.KubernetesConfig) error StartCluster(config.MachineConfig) error
UpdateCluster(config.MachineConfig) error UpdateCluster(config.MachineConfig) error
DeleteCluster(config.KubernetesConfig) error DeleteCluster(config.KubernetesConfig) error
WaitForCluster(config.KubernetesConfig, time.Duration) error WaitForCluster(config.MachineConfig, time.Duration) error
// LogCommands returns a map of log type to a command which will display that log. // LogCommands returns a map of log type to a command which will display that log.
LogCommands(LogOptions) map[string]string LogCommands(LogOptions) map[string]string
SetupCerts(cfg config.KubernetesConfig) error SetupCerts(cfg config.KubernetesConfig) error
@ -51,7 +51,6 @@ type Bootstrapper interface {
const ( const (
// Kubeadm is the kubeadm bootstrapper type // Kubeadm is the kubeadm bootstrapper type
Kubeadm = "kubeadm" Kubeadm = "kubeadm"
KIC = "kic"
) )
// GetCachedBinaryList returns the list of binaries // GetCachedBinaryList returns the list of binaries

4
pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
View File

@ -56,8 +56,8 @@ func APIServerProcess(runner command.Runner, start time.Time, timeout time.Durat
} }
// SystemPods verifies essential pods for running kurnetes is running // SystemPods verifies essential pods for running kurnetes is running
func SystemPods(client *kubernetes.Clientset, start time.Time, ip string, port int, timeout time.Duration) error { func SystemPods(client *kubernetes.Clientset, start time.Time, timeout time.Duration) error {
glog.Infof("waiting for kube-system pods to appear %s...", net.JoinHostPort(ip, fmt.Sprint(port))) glog.Info("waiting for kube-system pods to appear ...")
pStart := time.Now() pStart := time.Now()
podStart := time.Time{} podStart := time.Time{}
podList := func() (bool, error) { podList := func() (bool, error) {

134
pkg/minikube/bootstrapper/kicbs/cni.go
View File

@ -1,134 +0,0 @@
/*
Copyright 2019 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package kicbs bootstrapper for kic
package kicbs
const defaultCNIManifest = `
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: kindnet
rules:
- apiGroups:
- policy
resources:
- podsecuritypolicies
verbs:
- use
resourceNames:
- kindnet
- apiGroups:
- ""
resources:
- nodes
verbs:
- list
- watch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: kindnet
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: kindnet
subjects:
- kind: ServiceAccount
name: kindnet
namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: kindnet
namespace: kube-system
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: kindnet
namespace: kube-system
labels:
tier: node
app: kindnet
k8s-app: kindnet
spec:
selector:
matchLabels:
app: kindnet
template:
metadata:
labels:
tier: node
app: kindnet
k8s-app: kindnet
spec:
hostNetwork: true
tolerations:
- operator: Exists
effect: NoSchedule
serviceAccountName: kindnet
containers:
- name: kindnet-cni
image: kindest/kindnetd:0.5.3
env:
- name: HOST_IP
valueFrom:
fieldRef:
fieldPath: status.hostIP
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: POD_SUBNET
value: 10.244.0.0/16
volumeMounts:
- name: cni-cfg
mountPath: /etc/cni/net.d
- name: xtables-lock
mountPath: /run/xtables.lock
readOnly: false
- name: lib-modules
mountPath: /lib/modules
readOnly: true
resources:
requests:
cpu: "100m"
memory: "50Mi"
limits:
cpu: "100m"
memory: "50Mi"
securityContext:
privileged: false
capabilities:
add: ["NET_RAW", "NET_ADMIN"]
volumes:
- name: cni-cfg
hostPath:
path: /etc/cni/net.d
- name: xtables-lock
hostPath:
path: /run/xtables.lock
type: FileOrCreate
- name: lib-modules
hostPath:
path: /lib/modules
---
`

346
pkg/minikube/bootstrapper/kicbs/kicbs.go
View File

@ -1,346 +0,0 @@
/*
Copyright 2019 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package kicbs bootstrapper for kic
package kicbs
import (
"fmt"
"net"
"os/exec"
"strings"
"time"
"github.com/blang/semver"
"github.com/docker/machine/libmachine"
"github.com/golang/glog"
"github.com/pkg/errors"
"github.com/spf13/viper"
"k8s.io/client-go/kubernetes"
kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/minikube/pkg/drivers/kic"
"k8s.io/minikube/pkg/kapi"
"k8s.io/minikube/pkg/minikube/assets"
"k8s.io/minikube/pkg/minikube/bootstrapper"
"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil"
"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/kverify"
"k8s.io/minikube/pkg/minikube/bootstrapper/images"
"k8s.io/minikube/pkg/minikube/command"
"k8s.io/minikube/pkg/minikube/config"
"k8s.io/minikube/pkg/minikube/constants"
"k8s.io/minikube/pkg/minikube/cruntime"
"k8s.io/minikube/pkg/minikube/machine"
"k8s.io/minikube/pkg/minikube/out"
"k8s.io/minikube/pkg/minikube/vmpath"
)
// Bootstrapper is a bootstrapper using kicbs
type Bootstrapper struct {
c command.Runner
k8sClient *kubernetes.Clientset // kubernetes client used to verify pods inside cluster
contextName string
}
// NewBootstrapper creates a new kicbs.Bootstrapper
func NewBootstrapper(api libmachine.API) (*Bootstrapper, error) {
name := viper.GetString(config.MachineProfile)
h, err := api.Load(name)
if err != nil {
return nil, errors.Wrap(err, "getting api client")
}
runner, err := machine.CommandRunner(h)
if err != nil {
return nil, errors.Wrap(err, "command runner")
}
return &Bootstrapper{c: runner, contextName: name}, nil
}
// UpdateCluster updates the cluster
func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error {
images, err := images.Kubeadm(cfg.KubernetesConfig.ImageRepository, cfg.KubernetesConfig.KubernetesVersion)
if err != nil {
return errors.Wrap(err, "kic images")
}
if cfg.KubernetesConfig.ShouldLoadCachedImages {
if err := machine.LoadImages(&cfg, k.c, images, constants.ImageCacheDir); err != nil {
out.FailureT("Unable to load cached images: {{.error}}", out.V{"error": err})
}
}
r, err := cruntime.New(cruntime.Config{Type: cfg.ContainerRuntime, Socket: cfg.KubernetesConfig.CRISocket, Runner: k.c})
if err != nil {
return errors.Wrap(err, "runtime")
}
kubeadmCfg, err := bsutil.GenerateKubeadmYAML(cfg.KubernetesConfig, r)
if err != nil {
return errors.Wrap(err, "generating kubeadm cfg")
}
kubeletCfg, err := bsutil.NewKubeletConfig(cfg.KubernetesConfig, r)
if err != nil {
return errors.Wrap(err, "generating kubelet config")
}
kubeletService, err := bsutil.NewKubeletService(cfg.KubernetesConfig)
if err != nil {
return errors.Wrap(err, "generating kubelet service")
}
glog.Infof("kubelet %s config:\n%+v", kubeletCfg, cfg.KubernetesConfig)
stopCmd := exec.Command("/bin/bash", "-c", "pgrep kubelet && sudo systemctl stop kubelet")
// stop kubelet to avoid "Text File Busy" error
if rr, err := k.c.RunCmd(stopCmd); err != nil {
glog.Warningf("unable to stop kubelet: %s command: %q output: %q", err, rr.Command(), rr.Output())
}
if err := bsutil.TransferBinaries(cfg.KubernetesConfig, k.c); err != nil {
return errors.Wrap(err, "downloading binaries")
}
cniFile := []byte(defaultCNIManifest)
files := bsutil.ConfigFileAssets(cfg.KubernetesConfig, kubeadmCfg, kubeletCfg, kubeletService, cniFile)
if err := bsutil.AddAddons(&files, assets.GenerateTemplateData(cfg.KubernetesConfig)); err != nil {
return errors.Wrap(err, "adding addons")
}
for _, f := range files {
if err := k.c.Copy(f); err != nil {
return errors.Wrapf(err, "copy")
}
}
if _, err := k.c.RunCmd(exec.Command("/bin/bash", "-c", "sudo systemctl daemon-reload && sudo systemctl start kubelet")); err != nil {
return errors.Wrap(err, "starting kubelet")
}
return nil
}
// SetupCerts generates the certs the cluster
func (k *Bootstrapper) SetupCerts(cfg config.KubernetesConfig) error {
return bootstrapper.SetupCerts(k.c, cfg)
}
// PullImages downloads images that will be used by Kubernetes
func (k *Bootstrapper) PullImages(k8s config.KubernetesConfig) error {
version, err := bsutil.ParseKubernetesVersion(k8s.KubernetesVersion)
if err != nil {
return errors.Wrap(err, "parsing kubernetes version")
}
if version.LT(semver.MustParse("1.11.0")) {
return fmt.Errorf("pull command is not supported by kubeadm v%s", version)
}
rr, err := k.c.RunCmd(exec.Command("/bin/bash", "-c", fmt.Sprintf("%s config images pull --config %s", bsutil.InvokeKubeadm(k8s.KubernetesVersion), bsutil.KubeadmYamlPath)))
if err != nil {
return errors.Wrapf(err, "running cmd: %q", rr.Command())
}
return nil
}
// StartCluster starts the cluster
func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error {
k8s.NodeIP = kic.DefaultBindIPV4
err := bsutil.ExistingConfig(k.c)
if err == nil { // if there is an existing cluster don't reconfigure it
return k.restartCluster(k8s)
}
glog.Infof("existence check: %v", err)
start := time.Now()
glog.Infof("StartCluster: %+v", k8s)
defer func() {
glog.Infof("StartCluster complete in %s", time.Since(start))
}()
extraFlags := bsutil.CreateFlagsFromExtraArgs(k8s.ExtraOptions)
r, err := cruntime.New(cruntime.Config{Type: k8s.ContainerRuntime})
if err != nil {
return err
}
ignore := []string{
fmt.Sprintf("DirAvailable-%s", strings.Replace(vmpath.GuestManifestsDir, "/", "-", -1)),
fmt.Sprintf("DirAvailable-%s", strings.Replace(vmpath.GuestPersistentDir, "/", "-", -1)),
fmt.Sprintf("DirAvailable-%s", strings.Replace(bsutil.EtcdDataDir(), "/", "-", -1)),
"FileAvailable--etc-kubernetes-manifests-kube-scheduler.yaml",
"FileAvailable--etc-kubernetes-manifests-kube-apiserver.yaml",
"FileAvailable--etc-kubernetes-manifests-kube-controller-manager.yaml",
"FileAvailable--etc-kubernetes-manifests-etcd.yaml",
"FileContent--proc-sys-net-bridge-bridge-nf-call-iptables", // for kic only
"Port-10250", // For "none" users who already have a kubelet online
"Swap", // For "none" users who have swap configured
"SystemVerification", // For kic on linux example error: "modprobe: FATAL: Module configs not found in directory /lib/modules/5.2.17-1rodete3-amd64"
}
ignore = append(ignore, bsutil.SkipAdditionalPreflights[r.Name()]...)
c := exec.Command("/bin/bash", "-c", fmt.Sprintf("%s init --config %s %s --ignore-preflight-errors=%s", bsutil.InvokeKubeadm(k8s.KubernetesVersion), bsutil.KubeadmYamlPath, extraFlags, strings.Join(ignore, ",")))
glog.Infof("starting kubeadm init")
if rr, err := k.c.RunCmd(c); err != nil {
return errors.Wrapf(err, "init failed. cmd: %q output: %q", rr.Command(), rr.Output())
}
glog.Infof("applying kic overlay network")
if err := k.applyOverlayNetwork(); err != nil {
return errors.Wrap(err, "applying kic overlay network")
}
glog.Infof("Skipping Configuring cluster permissions for kic...")
if err := bsutil.AdjustResourceLimits(k.c); err != nil {
glog.Warningf("unable to adjust resource limits: %v", err)
}
return nil
}
// restartCluster restarts the Kubernetes cluster configured by kubeadm
func (k *Bootstrapper) restartCluster(k8s config.KubernetesConfig) error {
glog.Infof("restartCluster start")
start := time.Now()
defer func() {
glog.Infof("restartCluster took %s", time.Since(start))
}()
version, err := bsutil.ParseKubernetesVersion(k8s.KubernetesVersion)
if err != nil {
return errors.Wrap(err, "parsing kubernetes version")
}
phase := "alpha"
controlPlane := "controlplane"
if version.GTE(semver.MustParse("1.13.0")) {
phase = "init"
controlPlane = "control-plane"
}
baseCmd := fmt.Sprintf("%s %s", bsutil.InvokeKubeadm(k8s.KubernetesVersion), phase)
cmds := []string{
fmt.Sprintf("%s phase certs all --config %s", baseCmd, bsutil.KubeadmYamlPath),
fmt.Sprintf("%s phase kubeconfig all --config %s", baseCmd, bsutil.KubeadmYamlPath),
fmt.Sprintf("%s phase %s all --config %s", baseCmd, controlPlane, bsutil.KubeadmYamlPath),
fmt.Sprintf("%s phase etcd local --config %s", baseCmd, bsutil.KubeadmYamlPath),
}
// Run commands one at a time so that it is easier to root cause failures.
for _, c := range cmds {
rr, err := k.c.RunCmd(exec.Command("/bin/bash", "-c", c))
if err != nil {
return errors.Wrapf(err, "running cmd: %s", rr.Command())
}
}
// We must ensure that the apiserver is healthy before proceeding
if err := kverify.APIServerProcess(k.c, time.Now(), kconst.DefaultControlPlaneTimeout); err != nil {
return errors.Wrap(err, "apiserver healthz")
}
client, err := k.client(k8s)
if err != nil {
return errors.Wrap(err, "getting k8s client")
}
if err := kverify.SystemPods(client, time.Now(), k8s.NodeIP, k8s.NodePort, kconst.DefaultControlPlaneTimeout); err != nil {
return errors.Wrap(err, "system pods")
}
// Explicitly re-enable kubeadm addons (proxy, coredns) so that they will check for IP or configuration changes.
if rr, err := k.c.RunCmd(exec.Command("/bin/bash", "-c", fmt.Sprintf("%s phase addon all --config %s", baseCmd, bsutil.KubeadmYamlPath))); err != nil {
return errors.Wrapf(err, fmt.Sprintf("addon phase cmd:%q", rr.Command()))
}
if err := bsutil.AdjustResourceLimits(k.c); err != nil {
glog.Warningf("unable to adjust resource limits: %v", err)
}
return nil
}
// WaitForCluster blocks until the cluster appears to be healthy
func (k *Bootstrapper) WaitForCluster(k8s config.KubernetesConfig, timeout time.Duration) error {
start := time.Now()
out.T(out.Waiting, "Waiting for cluster to come online ...")
if err := kverify.APIServerProcess(k.c, start, timeout); err != nil {
return errors.Wrap(err, "wait for api proc")
}
if err := kverify.APIServerIsRunning(start, "127.0.0.1", k8s.NodePort, timeout); err != nil {
return err
}
c, err := k.client(k8s) // getting kubernetes client before polling.
if err != nil {
return errors.Wrap(err, "get k8s client")
}
if err := kverify.SystemPods(c, start, "127.0.0.1", k8s.NodePort, timeout); err != nil {
return errors.Wrap(err, "wait for system pods")
}
return nil
}
func (k *Bootstrapper) DeleteCluster(config.KubernetesConfig) error {
return fmt.Errorf("the DeleteCluster is not implemented in kicbs yet")
}
func (k *Bootstrapper) LogCommands(bootstrapper.LogOptions) map[string]string {
return map[string]string{}
}
func (k *Bootstrapper) GetKubeletStatus() (string, error) {
return "", fmt.Errorf("the GetKubeletStatus is not implemented in kicbs yet")
}
func (k *Bootstrapper) GetAPIServerStatus(net.IP, int) (string, error) {
return "", fmt.Errorf("the GetAPIServerStatus is not implemented in kicbs yet")
}
// client sets and returns a Kubernetes client to use to speak to a kubeadm launched apiserver
func (k *Bootstrapper) client(k8s config.KubernetesConfig) (*kubernetes.Clientset, error) {
if k.k8sClient != nil {
return k.k8sClient, nil
}
config, err := kapi.ClientConfig(k.contextName)
if err != nil {
return nil, errors.Wrap(err, "client config")
}
endpoint := fmt.Sprintf("https://%s", net.JoinHostPort("127.0.0.1", fmt.Sprint(k8s.NodePort)))
if config.Host != endpoint {
glog.Errorf("Overriding stale ClientConfig host %s with %s", config.Host, endpoint)
config.Host = endpoint
}
c, err := kubernetes.NewForConfig(config)
if err == nil {
k.k8sClient = c
}
return c, err
}
// applyOverlayNetwork applies the CNI plugin needed to make kic work
func (k *Bootstrapper) applyOverlayNetwork() error {
cmd := exec.Command(
"kubectl", "create", "--kubeconfig=/etc/kubernetes/admin.conf",
"-f", bsutil.DefaultCNIConfigPath,
)
if rr, err := k.c.RunCmd(cmd); err != nil {
return errors.Wrapf(err, "cmd: %s output: %s", rr.Command(), rr.Output())
}
return nil
}

118
pkg/minikube/bootstrapper/kubeadm/default_cni.go
View File

@ -42,3 +42,121 @@ const defaultCNIConfig = `
} }
} }
` `
// kicCNIConfig is the cni plugin needed for kic
// uses cni plugin created by kind https://github.com/kubernetes-sigs/kind/blob/03a4b519067dc308308cce735065c47a6fda1583/pkg/build/node/cni.go
const kicCNIConfig = `
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: kindnet
rules:
- apiGroups:
- policy
resources:
- podsecuritypolicies
verbs:
- use
resourceNames:
- kindnet
- apiGroups:
- ""
resources:
- nodes
verbs:
- list
- watch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: kindnet
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: kindnet
subjects:
- kind: ServiceAccount
name: kindnet
namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: kindnet
namespace: kube-system
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: kindnet
namespace: kube-system
labels:
tier: node
app: kindnet
k8s-app: kindnet
spec:
selector:
matchLabels:
app: kindnet
template:
metadata:
labels:
tier: node
app: kindnet
k8s-app: kindnet
spec:
hostNetwork: true
tolerations:
- operator: Exists
effect: NoSchedule
serviceAccountName: kindnet
containers:
- name: kindnet-cni
image: kindest/kindnetd:0.5.3
env:
- name: HOST_IP
valueFrom:
fieldRef:
fieldPath: status.hostIP
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: POD_SUBNET
value: 10.244.0.0/16
volumeMounts:
- name: cni-cfg
mountPath: /etc/cni/net.d
- name: xtables-lock
mountPath: /run/xtables.lock
readOnly: false
- name: lib-modules
mountPath: /lib/modules
readOnly: true
resources:
requests:
cpu: "100m"
memory: "50Mi"
limits:
cpu: "100m"
memory: "50Mi"
securityContext:
privileged: false
capabilities:
add: ["NET_RAW", "NET_ADMIN"]
volumes:
- name: cni-cfg
hostPath:
path: /etc/cni/net.d
- name: xtables-lock
hostPath:
path: /run/xtables.lock
type: FileOrCreate
- name: lib-modules
hostPath:
path: /lib/modules
---
`

117
pkg/minikube/bootstrapper/kubeadm/kubeadm.go
View File

@ -36,6 +36,7 @@ import (
"github.com/spf13/viper" "github.com/spf13/viper"
"k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes"
kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants" kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/minikube/pkg/drivers/kic"
"k8s.io/minikube/pkg/kapi" "k8s.io/minikube/pkg/kapi"
"k8s.io/minikube/pkg/minikube/assets" "k8s.io/minikube/pkg/minikube/assets"
"k8s.io/minikube/pkg/minikube/bootstrapper" "k8s.io/minikube/pkg/minikube/bootstrapper"
@ -46,6 +47,7 @@ import (
"k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/config"
"k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/constants"
"k8s.io/minikube/pkg/minikube/cruntime" "k8s.io/minikube/pkg/minikube/cruntime"
"k8s.io/minikube/pkg/minikube/driver"
"k8s.io/minikube/pkg/minikube/machine" "k8s.io/minikube/pkg/minikube/machine"
"k8s.io/minikube/pkg/minikube/out" "k8s.io/minikube/pkg/minikube/out"
"k8s.io/minikube/pkg/minikube/vmpath" "k8s.io/minikube/pkg/minikube/vmpath"
@ -139,26 +141,29 @@ func (k *Bootstrapper) createCompatSymlinks() error {
} }
// StartCluster starts the cluster // StartCluster starts the cluster
func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error { func (k *Bootstrapper) StartCluster(cfg config.MachineConfig) error {
if driver.IsKIC(cfg.VMDriver) {
cfg.KubernetesConfig.NodeIP = kic.DefaultBindIPV4
}
err := bsutil.ExistingConfig(k.c) err := bsutil.ExistingConfig(k.c)
if err == nil { // if there is an existing cluster don't reconfigure it if err == nil { // if there is an existing cluster don't reconfigure it
return k.restartCluster(k8s) return k.restartCluster(cfg)
} }
glog.Infof("existence check: %v", err) glog.Infof("existence check: %v", err)
start := time.Now() start := time.Now()
glog.Infof("StartCluster: %+v", k8s) glog.Infof("StartCluster: %+v", cfg)
defer func() { defer func() {
glog.Infof("StartCluster complete in %s", time.Since(start)) glog.Infof("StartCluster complete in %s", time.Since(start))
}() }()
version, err := bsutil.ParseKubernetesVersion(k8s.KubernetesVersion) // version, err := bsutil.ParseKubernetesVersion(cfg.KubernetesConfig.KubernetesVersion)
if err != nil { // if err != nil {
return errors.Wrap(err, "parsing kubernetes version") // return errors.Wrap(err, "parsing kubernetes version")
} // }
extraFlags := bsutil.CreateFlagsFromExtraArgs(k8s.ExtraOptions) extraFlags := bsutil.CreateFlagsFromExtraArgs(cfg.KubernetesConfig.ExtraOptions)
r, err := cruntime.New(cruntime.Config{Type: k8s.ContainerRuntime}) r, err := cruntime.New(cruntime.Config{Type: cfg.KubernetesConfig.ContainerRuntime})
if err != nil { if err != nil {
return err return err
} }
@ -173,31 +178,44 @@ func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error {
"FileAvailable--etc-kubernetes-manifests-etcd.yaml", "FileAvailable--etc-kubernetes-manifests-etcd.yaml",
"Port-10250", // For "none" users who already have a kubelet online "Port-10250", // For "none" users who already have a kubelet online
"Swap", // For "none" users who have swap configured "Swap", // For "none" users who have swap configured
"SystemVerification",
} }
ignore = append(ignore, bsutil.SkipAdditionalPreflights[r.Name()]...) ignore = append(ignore, bsutil.SkipAdditionalPreflights[r.Name()]...)
// if cfg.KubernetesConfig.EnableDefaultCNI {
// // if err := k.applyOverlayNetwork(); err != nil {
// // return errors.Wrap(err, "applying overlay network")
// // }
// }
// Allow older kubeadm versions to function with newer Docker releases. // Allow older kubeadm versions to function with newer Docker releases.
if version.LT(semver.MustParse("1.13.0")) { // For kic on linux example error: "modprobe: FATAL: Module configs not found in directory /lib/modules/5.2.17-1rodete3-amd64"
glog.Infof("Older Kubernetes release detected (%s), disabling SystemVerification check.", version) // fmt.Println("medya dbg: ", cfg.VMDriver)
ignore = append(ignore, "SystemVerification") // if version.LT(semver.MustParse("1.13.0")) || driver.IsKIC(cfg.VMDriver) {
} // glog.Infof("Older Kubernetes release detected (%s), disabling SystemVerification check.", version)
// ignore = append(ignore, "SystemVerification")
// }
c := exec.Command("/bin/bash", "-c", fmt.Sprintf("%s init --config %s %s --ignore-preflight-errors=%s", bsutil.InvokeKubeadm(k8s.KubernetesVersion), bsutil.KubeadmYamlPath, extraFlags, strings.Join(ignore, ","))) fmt.Printf("medya dbg:ignore is %+v ", ignore)
if rr, err := k.c.RunCmd(c); err != nil { c := exec.Command("/bin/bash", "-c", fmt.Sprintf("%s init --config %s %s --ignore-preflight-errors=%s", bsutil.InvokeKubeadm(cfg.KubernetesConfig.KubernetesVersion), bsutil.KubeadmYamlPath, extraFlags, strings.Join(ignore, ",")))
return errors.Wrapf(err, "init failed. cmd: %q", rr.Command()) rr, err := k.c.RunCmd(c)
if err != nil {
return errors.Wrapf(err, "init failed. output: %q", rr.Output())
} }
fmt.Println(rr.Output())
glog.Infof("Configuring cluster permissions ...") if !driver.IsKIC(cfg.VMDriver) { // TODO: skip for both after verifications https://github.com/kubernetes/minikube/issues/6239
elevate := func() error { glog.Infof("Configuring cluster permissions ...")
client, err := k.client(k8s) elevate := func() error {
if err != nil { client, err := k.client(cfg)
return err if err != nil {
return err
}
return bsutil.ElevateKubeSystemPrivileges(client)
} }
return bsutil.ElevateKubeSystemPrivileges(client)
}
if err := retry.Expo(elevate, time.Millisecond*500, 120*time.Second); err != nil { if err := retry.Expo(elevate, time.Millisecond*500, 120*time.Second); err != nil {
return errors.Wrap(err, "timed out waiting to elevate kube-system RBAC privileges") return errors.Wrap(err, "timed out waiting to elevate kube-system RBAC privileges")
}
} }
if err := bsutil.AdjustResourceLimits(k.c); err != nil { if err := bsutil.AdjustResourceLimits(k.c); err != nil {
@ -208,7 +226,7 @@ func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error {
} }
// client sets and returns a Kubernetes client to use to speak to a kubeadm launched apiserver // client sets and returns a Kubernetes client to use to speak to a kubeadm launched apiserver
func (k *Bootstrapper) client(k8s config.KubernetesConfig) (*kubernetes.Clientset, error) { func (k *Bootstrapper) client(cfg config.MachineConfig) (*kubernetes.Clientset, error) {
if k.k8sClient != nil { if k.k8sClient != nil {
return k.k8sClient, nil return k.k8sClient, nil
} }
@ -218,7 +236,8 @@ func (k *Bootstrapper) client(k8s config.KubernetesConfig) (*kubernetes.Clientse
return nil, errors.Wrap(err, "client config") return nil, errors.Wrap(err, "client config")
} }
endpoint := fmt.Sprintf("https://%s", net.JoinHostPort(k8s.NodeIP, strconv.Itoa(k8s.NodePort))) ip, port := k.clientEndpointAddr(cfg)
endpoint := fmt.Sprintf("https://%s", net.JoinHostPort(ip, strconv.Itoa(port)))
if config.Host != endpoint { if config.Host != endpoint {
glog.Errorf("Overriding stale ClientConfig host %s with %s", config.Host, endpoint) glog.Errorf("Overriding stale ClientConfig host %s with %s", config.Host, endpoint)
config.Host = endpoint config.Host = endpoint
@ -231,26 +250,27 @@ func (k *Bootstrapper) client(k8s config.KubernetesConfig) (*kubernetes.Clientse
} }
// WaitForCluster blocks until the cluster appears to be healthy // WaitForCluster blocks until the cluster appears to be healthy
func (k *Bootstrapper) WaitForCluster(k8s config.KubernetesConfig, timeout time.Duration) error { func (k *Bootstrapper) WaitForCluster(cfg config.MachineConfig, timeout time.Duration) error {
start := time.Now() start := time.Now()
ip, port := k.clientEndpointAddr(cfg)
out.T(out.Waiting, "Waiting for cluster to come online ...") out.T(out.Waiting, "Waiting for cluster to come online ...")
if err := kverify.APIServerProcess(k.c, start, timeout); err != nil { if err := kverify.APIServerProcess(k.c, start, timeout); err != nil {
return err return err
} }
if err := kverify.APIServerIsRunning(start, k8s.NodeIP, k8s.NodePort, timeout); err != nil { if err := kverify.APIServerIsRunning(start, ip, port, timeout); err != nil {
return err return err
} }
c, err := k.client(k8s) c, err := k.client(cfg)
if err != nil { if err != nil {
return errors.Wrap(err, "get k8s client") return errors.Wrap(err, "get k8s client")
} }
return kverify.SystemPods(c, start, k8s.NodeIP, k8s.NodePort, timeout) return kverify.SystemPods(c, start, timeout)
} }
// restartCluster restarts the Kubernetes cluster configured by kubeadm // restartCluster restarts the Kubernetes cluster configured by kubeadm
func (k *Bootstrapper) restartCluster(k8s config.KubernetesConfig) error { func (k *Bootstrapper) restartCluster(cfg config.MachineConfig) error {
glog.Infof("restartCluster start") glog.Infof("restartCluster start")
start := time.Now() start := time.Now()
@ -258,7 +278,7 @@ func (k *Bootstrapper) restartCluster(k8s config.KubernetesConfig) error {
glog.Infof("restartCluster took %s", time.Since(start)) glog.Infof("restartCluster took %s", time.Since(start))
}() }()
version, err := bsutil.ParseKubernetesVersion(k8s.KubernetesVersion) version, err := bsutil.ParseKubernetesVersion(cfg.KubernetesConfig.KubernetesVersion)
if err != nil { if err != nil {
return errors.Wrap(err, "parsing kubernetes version") return errors.Wrap(err, "parsing kubernetes version")
} }
@ -274,7 +294,7 @@ func (k *Bootstrapper) restartCluster(k8s config.KubernetesConfig) error {
glog.Errorf("failed to create compat symlinks: %v", err) glog.Errorf("failed to create compat symlinks: %v", err)
} }
baseCmd := fmt.Sprintf("%s %s", bsutil.InvokeKubeadm(k8s.KubernetesVersion), phase) baseCmd := fmt.Sprintf("%s %s", bsutil.InvokeKubeadm(cfg.KubernetesConfig.KubernetesVersion), phase)
cmds := []string{ cmds := []string{
fmt.Sprintf("%s phase certs all --config %s", baseCmd, bsutil.KubeadmYamlPath), fmt.Sprintf("%s phase certs all --config %s", baseCmd, bsutil.KubeadmYamlPath),
fmt.Sprintf("%s phase kubeconfig all --config %s", baseCmd, bsutil.KubeadmYamlPath), fmt.Sprintf("%s phase kubeconfig all --config %s", baseCmd, bsutil.KubeadmYamlPath),
@ -295,12 +315,12 @@ func (k *Bootstrapper) restartCluster(k8s config.KubernetesConfig) error {
return errors.Wrap(err, "apiserver healthz") return errors.Wrap(err, "apiserver healthz")
} }
client, err := k.client(k8s) client, err := k.client(cfg)
if err != nil { if err != nil {
return errors.Wrap(err, "getting k8s client") return errors.Wrap(err, "getting k8s client")
} }
if err := kverify.SystemPods(client, time.Now(), k8s.NodeIP, k8s.NodePort, kconst.DefaultControlPlaneTimeout); err != nil { if err := kverify.SystemPods(client, time.Now(), kconst.DefaultControlPlaneTimeout); err != nil {
return errors.Wrap(err, "system pods") return errors.Wrap(err, "system pods")
} }
@ -403,6 +423,10 @@ func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error {
var cniFile []byte = nil var cniFile []byte = nil
if cfg.KubernetesConfig.EnableDefaultCNI { if cfg.KubernetesConfig.EnableDefaultCNI {
cniFile = []byte(defaultCNIConfig) cniFile = []byte(defaultCNIConfig)
if driver.IsKIC(cfg.VMDriver) {
cniFile = []byte(kicCNIConfig)
}
} }
files := bsutil.ConfigFileAssets(cfg.KubernetesConfig, kubeadmCfg, kubeletCfg, kubeletService, cniFile) files := bsutil.ConfigFileAssets(cfg.KubernetesConfig, kubeadmCfg, kubeletCfg, kubeletService, cniFile)
@ -420,3 +444,24 @@ func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error {
} }
return nil return nil
} }
// // applyOverlayNetwork applies the CNI plugin needed to make kic work
// func (k *Bootstrapper) applyOverlayNetwork() error {
// cmd := exec.Command(
// "kubectl", "create", "--kubeconfig=/var/lib/minikube/kubeconfig",
// "-f", bsutil.DefaultCNIConfigPath,
// )
// if rr, err := k.c.RunCmd(cmd); err != nil {
// return errors.Wrapf(err, "cmd: %s output: %s", rr.Command(), rr.Output())
// }
// return nil
// }
// clientEndpointAddr returns ip and port accessible for the kubernetes clients to talk to the cluster
func (k *Bootstrapper) clientEndpointAddr(cfg config.MachineConfig) (string, int) {
if driver.IsKIC(cfg.VMDriver) {
// because docker container ip on non-linux is not accesible
return kic.DefaultBindIPV4, cfg.KubernetesConfig.NodePort
}
return cfg.KubernetesConfig.NodeIP, cfg.KubernetesConfig.NodePort
}

4
pkg/minikube/driver/driver.go
View File

@ -24,7 +24,6 @@ import (
"github.com/golang/glog" "github.com/golang/glog"
"k8s.io/minikube/pkg/drivers/kic" "k8s.io/minikube/pkg/drivers/kic"
"k8s.io/minikube/pkg/minikube/bootstrapper"
"k8s.io/minikube/pkg/minikube/registry" "k8s.io/minikube/pkg/minikube/registry"
) )
@ -100,6 +99,7 @@ type FlagHints struct {
CacheImages bool CacheImages bool
ContainerRuntime string ContainerRuntime string
Bootstrapper string Bootstrapper string
EnableDefaultCNI bool
} }
// FlagDefaults returns suggested defaults based on a driver // FlagDefaults returns suggested defaults based on a driver
@ -110,8 +110,8 @@ func FlagDefaults(name string) FlagHints {
// only for kic, till other run-times are available we auto-set containerd. // only for kic, till other run-times are available we auto-set containerd.
if name == Docker { if name == Docker {
fh.ContainerRuntime = "containerd" fh.ContainerRuntime = "containerd"
fh.Bootstrapper = bootstrapper.KIC
fh.ExtraOptions = append(fh.ExtraOptions, fmt.Sprintf("kubeadm.pod-network-cidr=%s", kic.DefaultPodCIDR)) fh.ExtraOptions = append(fh.ExtraOptions, fmt.Sprintf("kubeadm.pod-network-cidr=%s", kic.DefaultPodCIDR))
fh.EnableDefaultCNI = true
} }
return fh return fh
} }