diff --git a/cmd/minikube/cmd/root.go b/cmd/minikube/cmd/root.go index 2a1d012b29..344aee7eff 100644 --- a/cmd/minikube/cmd/root.go +++ b/cmd/minikube/cmd/root.go @@ -32,7 +32,6 @@ import ( "k8s.io/kubectl/pkg/util/templates" configCmd "k8s.io/minikube/cmd/minikube/cmd/config" "k8s.io/minikube/pkg/minikube/bootstrapper" - "k8s.io/minikube/pkg/minikube/bootstrapper/kicbs" "k8s.io/minikube/pkg/minikube/bootstrapper/kubeadm" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/constants" @@ -274,16 +273,9 @@ func getClusterBootstrapper(api libmachine.API, bootstrapperName string) (bootst if err != nil { return nil, errors.Wrap(err, "getting a new kubeadm bootstrapper") } - case bootstrapper.KIC: - b, err = kicbs.NewBootstrapper(api) - if err != nil { - return nil, errors.Wrap(err, "getting a new kic bootstrapper") - } - default: return nil, fmt.Errorf("unknown bootstrapper: %s", bootstrapperName) } - return b, nil } diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index ebb5158124..ba3df0215c 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -366,7 +366,7 @@ func runStart(cmd *cobra.Command, args []string) { bs := setupKubeAdm(machineAPI, config) // pull images or restart cluster - bootstrapCluster(bs, cr, mRunner, config.KubernetesConfig, preExists, isUpgrade) + bootstrapCluster(bs, cr, mRunner, config, preExists, isUpgrade) configureMounts() // enable addons with start command @@ -383,7 +383,7 @@ func runStart(cmd *cobra.Command, args []string) { // Skip pre-existing, because we already waited for health if viper.GetBool(waitUntilHealthy) && !preExists { - if err := bs.WaitForCluster(config.KubernetesConfig, viper.GetDuration(waitTimeout)); err != nil { + if err := bs.WaitForCluster(config, viper.GetDuration(waitTimeout)); err != nil { exit.WithError("Wait failed", err) } } @@ -1027,6 +1027,10 @@ func autoSetDriverOptions(cmd *cobra.Command, drvName string) (err error) { viper.Set(cacheImages, hints.CacheImages) } + if !cmd.Flags().Changed(enableDefaultCNI) { + viper.Set(enableDefaultCNI, hints.EnableDefaultCNI) + } + if !cmd.Flags().Changed(containerRuntime) && hints.ContainerRuntime != "" { viper.Set(containerRuntime, hints.ContainerRuntime) glog.Infof("auto set %s to %q.", containerRuntime, hints.ContainerRuntime) @@ -1287,16 +1291,16 @@ func configureRuntimes(runner cruntime.CommandRunner, drvName string, k8s cfg.Ku } // bootstrapCluster starts Kubernetes using the chosen bootstrapper -func bootstrapCluster(bs bootstrapper.Bootstrapper, r cruntime.Manager, runner command.Runner, kc cfg.KubernetesConfig, preexisting bool, isUpgrade bool) { +func bootstrapCluster(bs bootstrapper.Bootstrapper, r cruntime.Manager, runner command.Runner, c cfg.MachineConfig, preexisting bool, isUpgrade bool) { if isUpgrade || !preexisting { out.T(out.Pulling, "Pulling images ...") - if err := bs.PullImages(kc); err != nil { + if err := bs.PullImages(c.KubernetesConfig); err != nil { out.T(out.FailureType, "Unable to pull images, which may be OK: {{.error}}", out.V{"error": err}) } } out.T(out.Launch, "Launching Kubernetes ... ") - if err := bs.StartCluster(kc); err != nil { + if err := bs.StartCluster(c); err != nil { exit.WithLogEntries("Error starting cluster", err, logs.FindProblems(r, bs, runner)) } } diff --git a/pkg/minikube/bootstrapper/bootstrapper.go b/pkg/minikube/bootstrapper/bootstrapper.go index 9b45006176..f247c84684 100644 --- a/pkg/minikube/bootstrapper/bootstrapper.go +++ b/pkg/minikube/bootstrapper/bootstrapper.go @@ -37,10 +37,10 @@ type LogOptions struct { type Bootstrapper interface { // PullImages pulls images necessary for a cluster. Success should not be required. PullImages(config.KubernetesConfig) error - StartCluster(config.KubernetesConfig) error + StartCluster(config.MachineConfig) error UpdateCluster(config.MachineConfig) error DeleteCluster(config.KubernetesConfig) error - WaitForCluster(config.KubernetesConfig, time.Duration) error + WaitForCluster(config.MachineConfig, time.Duration) error // LogCommands returns a map of log type to a command which will display that log. LogCommands(LogOptions) map[string]string SetupCerts(cfg config.KubernetesConfig) error @@ -51,7 +51,6 @@ type Bootstrapper interface { const ( // Kubeadm is the kubeadm bootstrapper type Kubeadm = "kubeadm" - KIC = "kic" ) // GetCachedBinaryList returns the list of binaries diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go index 40cb5c4678..00374eb23b 100644 --- a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go +++ b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go @@ -56,8 +56,8 @@ func APIServerProcess(runner command.Runner, start time.Time, timeout time.Durat } // SystemPods verifies essential pods for running kurnetes is running -func SystemPods(client *kubernetes.Clientset, start time.Time, ip string, port int, timeout time.Duration) error { - glog.Infof("waiting for kube-system pods to appear %s...", net.JoinHostPort(ip, fmt.Sprint(port))) +func SystemPods(client *kubernetes.Clientset, start time.Time, timeout time.Duration) error { + glog.Info("waiting for kube-system pods to appear ...") pStart := time.Now() podStart := time.Time{} podList := func() (bool, error) { diff --git a/pkg/minikube/bootstrapper/kicbs/cni.go b/pkg/minikube/bootstrapper/kicbs/cni.go deleted file mode 100644 index caa4534b47..0000000000 --- a/pkg/minikube/bootstrapper/kicbs/cni.go +++ /dev/null @@ -1,134 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package kicbs bootstrapper for kic -package kicbs - -const defaultCNIManifest = ` ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: kindnet -rules: - - apiGroups: - - policy - resources: - - podsecuritypolicies - verbs: - - use - resourceNames: - - kindnet - - apiGroups: - - "" - resources: - - nodes - verbs: - - list - - watch ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: kindnet -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: kindnet -subjects: -- kind: ServiceAccount - name: kindnet - namespace: kube-system ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: kindnet - namespace: kube-system ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: kindnet - namespace: kube-system - labels: - tier: node - app: kindnet - k8s-app: kindnet -spec: - selector: - matchLabels: - app: kindnet - template: - metadata: - labels: - tier: node - app: kindnet - k8s-app: kindnet - spec: - hostNetwork: true - tolerations: - - operator: Exists - effect: NoSchedule - serviceAccountName: kindnet - containers: - - name: kindnet-cni - image: kindest/kindnetd:0.5.3 - env: - - name: HOST_IP - valueFrom: - fieldRef: - fieldPath: status.hostIP - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: POD_SUBNET - value: 10.244.0.0/16 - volumeMounts: - - name: cni-cfg - mountPath: /etc/cni/net.d - - name: xtables-lock - mountPath: /run/xtables.lock - readOnly: false - - name: lib-modules - mountPath: /lib/modules - readOnly: true - resources: - requests: - cpu: "100m" - memory: "50Mi" - limits: - cpu: "100m" - memory: "50Mi" - securityContext: - privileged: false - capabilities: - add: ["NET_RAW", "NET_ADMIN"] - volumes: - - name: cni-cfg - hostPath: - path: /etc/cni/net.d - - name: xtables-lock - hostPath: - path: /run/xtables.lock - type: FileOrCreate - - name: lib-modules - hostPath: - path: /lib/modules - ---- -` diff --git a/pkg/minikube/bootstrapper/kicbs/kicbs.go b/pkg/minikube/bootstrapper/kicbs/kicbs.go deleted file mode 100644 index ea7728a042..0000000000 --- a/pkg/minikube/bootstrapper/kicbs/kicbs.go +++ /dev/null @@ -1,346 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package kicbs bootstrapper for kic -package kicbs - -import ( - "fmt" - "net" - "os/exec" - "strings" - "time" - - "github.com/blang/semver" - "github.com/docker/machine/libmachine" - "github.com/golang/glog" - "github.com/pkg/errors" - "github.com/spf13/viper" - "k8s.io/client-go/kubernetes" - kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/minikube/pkg/drivers/kic" - "k8s.io/minikube/pkg/kapi" - "k8s.io/minikube/pkg/minikube/assets" - "k8s.io/minikube/pkg/minikube/bootstrapper" - "k8s.io/minikube/pkg/minikube/bootstrapper/bsutil" - "k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/kverify" - "k8s.io/minikube/pkg/minikube/bootstrapper/images" - "k8s.io/minikube/pkg/minikube/command" - "k8s.io/minikube/pkg/minikube/config" - "k8s.io/minikube/pkg/minikube/constants" - "k8s.io/minikube/pkg/minikube/cruntime" - "k8s.io/minikube/pkg/minikube/machine" - "k8s.io/minikube/pkg/minikube/out" - "k8s.io/minikube/pkg/minikube/vmpath" -) - -// Bootstrapper is a bootstrapper using kicbs -type Bootstrapper struct { - c command.Runner - k8sClient *kubernetes.Clientset // kubernetes client used to verify pods inside cluster - contextName string -} - -// NewBootstrapper creates a new kicbs.Bootstrapper -func NewBootstrapper(api libmachine.API) (*Bootstrapper, error) { - name := viper.GetString(config.MachineProfile) - h, err := api.Load(name) - if err != nil { - return nil, errors.Wrap(err, "getting api client") - } - runner, err := machine.CommandRunner(h) - if err != nil { - return nil, errors.Wrap(err, "command runner") - } - return &Bootstrapper{c: runner, contextName: name}, nil -} - -// UpdateCluster updates the cluster -func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error { - images, err := images.Kubeadm(cfg.KubernetesConfig.ImageRepository, cfg.KubernetesConfig.KubernetesVersion) - if err != nil { - return errors.Wrap(err, "kic images") - } - - if cfg.KubernetesConfig.ShouldLoadCachedImages { - if err := machine.LoadImages(&cfg, k.c, images, constants.ImageCacheDir); err != nil { - out.FailureT("Unable to load cached images: {{.error}}", out.V{"error": err}) - } - } - r, err := cruntime.New(cruntime.Config{Type: cfg.ContainerRuntime, Socket: cfg.KubernetesConfig.CRISocket, Runner: k.c}) - if err != nil { - return errors.Wrap(err, "runtime") - } - kubeadmCfg, err := bsutil.GenerateKubeadmYAML(cfg.KubernetesConfig, r) - if err != nil { - return errors.Wrap(err, "generating kubeadm cfg") - } - - kubeletCfg, err := bsutil.NewKubeletConfig(cfg.KubernetesConfig, r) - if err != nil { - return errors.Wrap(err, "generating kubelet config") - } - - kubeletService, err := bsutil.NewKubeletService(cfg.KubernetesConfig) - if err != nil { - return errors.Wrap(err, "generating kubelet service") - } - - glog.Infof("kubelet %s config:\n%+v", kubeletCfg, cfg.KubernetesConfig) - - stopCmd := exec.Command("/bin/bash", "-c", "pgrep kubelet && sudo systemctl stop kubelet") - // stop kubelet to avoid "Text File Busy" error - if rr, err := k.c.RunCmd(stopCmd); err != nil { - glog.Warningf("unable to stop kubelet: %s command: %q output: %q", err, rr.Command(), rr.Output()) - } - - if err := bsutil.TransferBinaries(cfg.KubernetesConfig, k.c); err != nil { - return errors.Wrap(err, "downloading binaries") - } - - cniFile := []byte(defaultCNIManifest) - - files := bsutil.ConfigFileAssets(cfg.KubernetesConfig, kubeadmCfg, kubeletCfg, kubeletService, cniFile) - - if err := bsutil.AddAddons(&files, assets.GenerateTemplateData(cfg.KubernetesConfig)); err != nil { - return errors.Wrap(err, "adding addons") - } - - for _, f := range files { - if err := k.c.Copy(f); err != nil { - return errors.Wrapf(err, "copy") - } - } - - if _, err := k.c.RunCmd(exec.Command("/bin/bash", "-c", "sudo systemctl daemon-reload && sudo systemctl start kubelet")); err != nil { - return errors.Wrap(err, "starting kubelet") - } - return nil -} - -// SetupCerts generates the certs the cluster -func (k *Bootstrapper) SetupCerts(cfg config.KubernetesConfig) error { - return bootstrapper.SetupCerts(k.c, cfg) -} - -// PullImages downloads images that will be used by Kubernetes -func (k *Bootstrapper) PullImages(k8s config.KubernetesConfig) error { - version, err := bsutil.ParseKubernetesVersion(k8s.KubernetesVersion) - if err != nil { - return errors.Wrap(err, "parsing kubernetes version") - } - if version.LT(semver.MustParse("1.11.0")) { - return fmt.Errorf("pull command is not supported by kubeadm v%s", version) - } - - rr, err := k.c.RunCmd(exec.Command("/bin/bash", "-c", fmt.Sprintf("%s config images pull --config %s", bsutil.InvokeKubeadm(k8s.KubernetesVersion), bsutil.KubeadmYamlPath))) - if err != nil { - return errors.Wrapf(err, "running cmd: %q", rr.Command()) - } - return nil -} - -// StartCluster starts the cluster -func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error { - k8s.NodeIP = kic.DefaultBindIPV4 - err := bsutil.ExistingConfig(k.c) - if err == nil { // if there is an existing cluster don't reconfigure it - return k.restartCluster(k8s) - } - glog.Infof("existence check: %v", err) - - start := time.Now() - glog.Infof("StartCluster: %+v", k8s) - defer func() { - glog.Infof("StartCluster complete in %s", time.Since(start)) - }() - - extraFlags := bsutil.CreateFlagsFromExtraArgs(k8s.ExtraOptions) - r, err := cruntime.New(cruntime.Config{Type: k8s.ContainerRuntime}) - if err != nil { - return err - } - - ignore := []string{ - fmt.Sprintf("DirAvailable-%s", strings.Replace(vmpath.GuestManifestsDir, "/", "-", -1)), - fmt.Sprintf("DirAvailable-%s", strings.Replace(vmpath.GuestPersistentDir, "/", "-", -1)), - fmt.Sprintf("DirAvailable-%s", strings.Replace(bsutil.EtcdDataDir(), "/", "-", -1)), - "FileAvailable--etc-kubernetes-manifests-kube-scheduler.yaml", - "FileAvailable--etc-kubernetes-manifests-kube-apiserver.yaml", - "FileAvailable--etc-kubernetes-manifests-kube-controller-manager.yaml", - "FileAvailable--etc-kubernetes-manifests-etcd.yaml", - "FileContent--proc-sys-net-bridge-bridge-nf-call-iptables", // for kic only - "Port-10250", // For "none" users who already have a kubelet online - "Swap", // For "none" users who have swap configured - "SystemVerification", // For kic on linux example error: "modprobe: FATAL: Module configs not found in directory /lib/modules/5.2.17-1rodete3-amd64" - } - ignore = append(ignore, bsutil.SkipAdditionalPreflights[r.Name()]...) - - c := exec.Command("/bin/bash", "-c", fmt.Sprintf("%s init --config %s %s --ignore-preflight-errors=%s", bsutil.InvokeKubeadm(k8s.KubernetesVersion), bsutil.KubeadmYamlPath, extraFlags, strings.Join(ignore, ","))) - glog.Infof("starting kubeadm init") - if rr, err := k.c.RunCmd(c); err != nil { - return errors.Wrapf(err, "init failed. cmd: %q output: %q", rr.Command(), rr.Output()) - } - - glog.Infof("applying kic overlay network") - if err := k.applyOverlayNetwork(); err != nil { - return errors.Wrap(err, "applying kic overlay network") - } - - glog.Infof("Skipping Configuring cluster permissions for kic...") - - if err := bsutil.AdjustResourceLimits(k.c); err != nil { - glog.Warningf("unable to adjust resource limits: %v", err) - } - - return nil -} - -// restartCluster restarts the Kubernetes cluster configured by kubeadm -func (k *Bootstrapper) restartCluster(k8s config.KubernetesConfig) error { - glog.Infof("restartCluster start") - - start := time.Now() - defer func() { - glog.Infof("restartCluster took %s", time.Since(start)) - }() - - version, err := bsutil.ParseKubernetesVersion(k8s.KubernetesVersion) - if err != nil { - return errors.Wrap(err, "parsing kubernetes version") - } - - phase := "alpha" - controlPlane := "controlplane" - if version.GTE(semver.MustParse("1.13.0")) { - phase = "init" - controlPlane = "control-plane" - } - - baseCmd := fmt.Sprintf("%s %s", bsutil.InvokeKubeadm(k8s.KubernetesVersion), phase) - cmds := []string{ - fmt.Sprintf("%s phase certs all --config %s", baseCmd, bsutil.KubeadmYamlPath), - fmt.Sprintf("%s phase kubeconfig all --config %s", baseCmd, bsutil.KubeadmYamlPath), - fmt.Sprintf("%s phase %s all --config %s", baseCmd, controlPlane, bsutil.KubeadmYamlPath), - fmt.Sprintf("%s phase etcd local --config %s", baseCmd, bsutil.KubeadmYamlPath), - } - - // Run commands one at a time so that it is easier to root cause failures. - for _, c := range cmds { - rr, err := k.c.RunCmd(exec.Command("/bin/bash", "-c", c)) - if err != nil { - return errors.Wrapf(err, "running cmd: %s", rr.Command()) - } - } - - // We must ensure that the apiserver is healthy before proceeding - if err := kverify.APIServerProcess(k.c, time.Now(), kconst.DefaultControlPlaneTimeout); err != nil { - return errors.Wrap(err, "apiserver healthz") - } - - client, err := k.client(k8s) - if err != nil { - return errors.Wrap(err, "getting k8s client") - } - - if err := kverify.SystemPods(client, time.Now(), k8s.NodeIP, k8s.NodePort, kconst.DefaultControlPlaneTimeout); err != nil { - return errors.Wrap(err, "system pods") - } - - // Explicitly re-enable kubeadm addons (proxy, coredns) so that they will check for IP or configuration changes. - if rr, err := k.c.RunCmd(exec.Command("/bin/bash", "-c", fmt.Sprintf("%s phase addon all --config %s", baseCmd, bsutil.KubeadmYamlPath))); err != nil { - return errors.Wrapf(err, fmt.Sprintf("addon phase cmd:%q", rr.Command())) - } - - if err := bsutil.AdjustResourceLimits(k.c); err != nil { - glog.Warningf("unable to adjust resource limits: %v", err) - } - return nil -} - -// WaitForCluster blocks until the cluster appears to be healthy -func (k *Bootstrapper) WaitForCluster(k8s config.KubernetesConfig, timeout time.Duration) error { - start := time.Now() - out.T(out.Waiting, "Waiting for cluster to come online ...") - if err := kverify.APIServerProcess(k.c, start, timeout); err != nil { - return errors.Wrap(err, "wait for api proc") - } - - if err := kverify.APIServerIsRunning(start, "127.0.0.1", k8s.NodePort, timeout); err != nil { - return err - } - - c, err := k.client(k8s) // getting kubernetes client before polling. - if err != nil { - return errors.Wrap(err, "get k8s client") - } - - if err := kverify.SystemPods(c, start, "127.0.0.1", k8s.NodePort, timeout); err != nil { - return errors.Wrap(err, "wait for system pods") - } - - return nil -} - -func (k *Bootstrapper) DeleteCluster(config.KubernetesConfig) error { - return fmt.Errorf("the DeleteCluster is not implemented in kicbs yet") -} - -func (k *Bootstrapper) LogCommands(bootstrapper.LogOptions) map[string]string { - return map[string]string{} -} - -func (k *Bootstrapper) GetKubeletStatus() (string, error) { - return "", fmt.Errorf("the GetKubeletStatus is not implemented in kicbs yet") -} -func (k *Bootstrapper) GetAPIServerStatus(net.IP, int) (string, error) { - return "", fmt.Errorf("the GetAPIServerStatus is not implemented in kicbs yet") -} - -// client sets and returns a Kubernetes client to use to speak to a kubeadm launched apiserver -func (k *Bootstrapper) client(k8s config.KubernetesConfig) (*kubernetes.Clientset, error) { - if k.k8sClient != nil { - return k.k8sClient, nil - } - - config, err := kapi.ClientConfig(k.contextName) - if err != nil { - return nil, errors.Wrap(err, "client config") - } - - endpoint := fmt.Sprintf("https://%s", net.JoinHostPort("127.0.0.1", fmt.Sprint(k8s.NodePort))) - if config.Host != endpoint { - glog.Errorf("Overriding stale ClientConfig host %s with %s", config.Host, endpoint) - config.Host = endpoint - } - c, err := kubernetes.NewForConfig(config) - if err == nil { - k.k8sClient = c - } - return c, err -} - -// applyOverlayNetwork applies the CNI plugin needed to make kic work -func (k *Bootstrapper) applyOverlayNetwork() error { - cmd := exec.Command( - "kubectl", "create", "--kubeconfig=/etc/kubernetes/admin.conf", - "-f", bsutil.DefaultCNIConfigPath, - ) - if rr, err := k.c.RunCmd(cmd); err != nil { - return errors.Wrapf(err, "cmd: %s output: %s", rr.Command(), rr.Output()) - } - return nil -} diff --git a/pkg/minikube/bootstrapper/kubeadm/default_cni.go b/pkg/minikube/bootstrapper/kubeadm/default_cni.go index da896eee8d..8eae93c0df 100644 --- a/pkg/minikube/bootstrapper/kubeadm/default_cni.go +++ b/pkg/minikube/bootstrapper/kubeadm/default_cni.go @@ -42,3 +42,121 @@ const defaultCNIConfig = ` } } ` + +// kicCNIConfig is the cni plugin needed for kic +// uses cni plugin created by kind https://github.com/kubernetes-sigs/kind/blob/03a4b519067dc308308cce735065c47a6fda1583/pkg/build/node/cni.go +const kicCNIConfig = ` +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kindnet +rules: + - apiGroups: + - policy + resources: + - podsecuritypolicies + verbs: + - use + resourceNames: + - kindnet + - apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kindnet +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kindnet +subjects: +- kind: ServiceAccount + name: kindnet + namespace: kube-system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kindnet + namespace: kube-system +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kindnet + namespace: kube-system + labels: + tier: node + app: kindnet + k8s-app: kindnet +spec: + selector: + matchLabels: + app: kindnet + template: + metadata: + labels: + tier: node + app: kindnet + k8s-app: kindnet + spec: + hostNetwork: true + tolerations: + - operator: Exists + effect: NoSchedule + serviceAccountName: kindnet + containers: + - name: kindnet-cni + image: kindest/kindnetd:0.5.3 + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: POD_SUBNET + value: 10.244.0.0/16 + volumeMounts: + - name: cni-cfg + mountPath: /etc/cni/net.d + - name: xtables-lock + mountPath: /run/xtables.lock + readOnly: false + - name: lib-modules + mountPath: /lib/modules + readOnly: true + resources: + requests: + cpu: "100m" + memory: "50Mi" + limits: + cpu: "100m" + memory: "50Mi" + securityContext: + privileged: false + capabilities: + add: ["NET_RAW", "NET_ADMIN"] + volumes: + - name: cni-cfg + hostPath: + path: /etc/cni/net.d + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + - name: lib-modules + hostPath: + path: /lib/modules + +--- +` diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 323e5e7609..d5e455174f 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -36,6 +36,7 @@ import ( "github.com/spf13/viper" "k8s.io/client-go/kubernetes" kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/minikube/pkg/drivers/kic" "k8s.io/minikube/pkg/kapi" "k8s.io/minikube/pkg/minikube/assets" "k8s.io/minikube/pkg/minikube/bootstrapper" @@ -46,6 +47,7 @@ import ( "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/cruntime" + "k8s.io/minikube/pkg/minikube/driver" "k8s.io/minikube/pkg/minikube/machine" "k8s.io/minikube/pkg/minikube/out" "k8s.io/minikube/pkg/minikube/vmpath" @@ -139,26 +141,29 @@ func (k *Bootstrapper) createCompatSymlinks() error { } // StartCluster starts the cluster -func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error { +func (k *Bootstrapper) StartCluster(cfg config.MachineConfig) error { + if driver.IsKIC(cfg.VMDriver) { + cfg.KubernetesConfig.NodeIP = kic.DefaultBindIPV4 + } err := bsutil.ExistingConfig(k.c) if err == nil { // if there is an existing cluster don't reconfigure it - return k.restartCluster(k8s) + return k.restartCluster(cfg) } glog.Infof("existence check: %v", err) start := time.Now() - glog.Infof("StartCluster: %+v", k8s) + glog.Infof("StartCluster: %+v", cfg) defer func() { glog.Infof("StartCluster complete in %s", time.Since(start)) }() - version, err := bsutil.ParseKubernetesVersion(k8s.KubernetesVersion) - if err != nil { - return errors.Wrap(err, "parsing kubernetes version") - } + // version, err := bsutil.ParseKubernetesVersion(cfg.KubernetesConfig.KubernetesVersion) + // if err != nil { + // return errors.Wrap(err, "parsing kubernetes version") + // } - extraFlags := bsutil.CreateFlagsFromExtraArgs(k8s.ExtraOptions) - r, err := cruntime.New(cruntime.Config{Type: k8s.ContainerRuntime}) + extraFlags := bsutil.CreateFlagsFromExtraArgs(cfg.KubernetesConfig.ExtraOptions) + r, err := cruntime.New(cruntime.Config{Type: cfg.KubernetesConfig.ContainerRuntime}) if err != nil { return err } @@ -173,31 +178,44 @@ func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error { "FileAvailable--etc-kubernetes-manifests-etcd.yaml", "Port-10250", // For "none" users who already have a kubelet online "Swap", // For "none" users who have swap configured + "SystemVerification", } ignore = append(ignore, bsutil.SkipAdditionalPreflights[r.Name()]...) + // if cfg.KubernetesConfig.EnableDefaultCNI { + // // if err := k.applyOverlayNetwork(); err != nil { + // // return errors.Wrap(err, "applying overlay network") + // // } + // } // Allow older kubeadm versions to function with newer Docker releases. - if version.LT(semver.MustParse("1.13.0")) { - glog.Infof("Older Kubernetes release detected (%s), disabling SystemVerification check.", version) - ignore = append(ignore, "SystemVerification") - } + // For kic on linux example error: "modprobe: FATAL: Module configs not found in directory /lib/modules/5.2.17-1rodete3-amd64" + // fmt.Println("medya dbg: ", cfg.VMDriver) + // if version.LT(semver.MustParse("1.13.0")) || driver.IsKIC(cfg.VMDriver) { + // glog.Infof("Older Kubernetes release detected (%s), disabling SystemVerification check.", version) + // ignore = append(ignore, "SystemVerification") + // } - c := exec.Command("/bin/bash", "-c", fmt.Sprintf("%s init --config %s %s --ignore-preflight-errors=%s", bsutil.InvokeKubeadm(k8s.KubernetesVersion), bsutil.KubeadmYamlPath, extraFlags, strings.Join(ignore, ","))) - if rr, err := k.c.RunCmd(c); err != nil { - return errors.Wrapf(err, "init failed. cmd: %q", rr.Command()) + fmt.Printf("medya dbg:ignore is %+v ", ignore) + c := exec.Command("/bin/bash", "-c", fmt.Sprintf("%s init --config %s %s --ignore-preflight-errors=%s", bsutil.InvokeKubeadm(cfg.KubernetesConfig.KubernetesVersion), bsutil.KubeadmYamlPath, extraFlags, strings.Join(ignore, ","))) + rr, err := k.c.RunCmd(c) + if err != nil { + return errors.Wrapf(err, "init failed. output: %q", rr.Output()) } + fmt.Println(rr.Output()) - glog.Infof("Configuring cluster permissions ...") - elevate := func() error { - client, err := k.client(k8s) - if err != nil { - return err + if !driver.IsKIC(cfg.VMDriver) { // TODO: skip for both after verifications https://github.com/kubernetes/minikube/issues/6239 + glog.Infof("Configuring cluster permissions ...") + elevate := func() error { + client, err := k.client(cfg) + if err != nil { + return err + } + return bsutil.ElevateKubeSystemPrivileges(client) } - return bsutil.ElevateKubeSystemPrivileges(client) - } - if err := retry.Expo(elevate, time.Millisecond*500, 120*time.Second); err != nil { - return errors.Wrap(err, "timed out waiting to elevate kube-system RBAC privileges") + if err := retry.Expo(elevate, time.Millisecond*500, 120*time.Second); err != nil { + return errors.Wrap(err, "timed out waiting to elevate kube-system RBAC privileges") + } } if err := bsutil.AdjustResourceLimits(k.c); err != nil { @@ -208,7 +226,7 @@ func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error { } // client sets and returns a Kubernetes client to use to speak to a kubeadm launched apiserver -func (k *Bootstrapper) client(k8s config.KubernetesConfig) (*kubernetes.Clientset, error) { +func (k *Bootstrapper) client(cfg config.MachineConfig) (*kubernetes.Clientset, error) { if k.k8sClient != nil { return k.k8sClient, nil } @@ -218,7 +236,8 @@ func (k *Bootstrapper) client(k8s config.KubernetesConfig) (*kubernetes.Clientse return nil, errors.Wrap(err, "client config") } - endpoint := fmt.Sprintf("https://%s", net.JoinHostPort(k8s.NodeIP, strconv.Itoa(k8s.NodePort))) + ip, port := k.clientEndpointAddr(cfg) + endpoint := fmt.Sprintf("https://%s", net.JoinHostPort(ip, strconv.Itoa(port))) if config.Host != endpoint { glog.Errorf("Overriding stale ClientConfig host %s with %s", config.Host, endpoint) config.Host = endpoint @@ -231,26 +250,27 @@ func (k *Bootstrapper) client(k8s config.KubernetesConfig) (*kubernetes.Clientse } // WaitForCluster blocks until the cluster appears to be healthy -func (k *Bootstrapper) WaitForCluster(k8s config.KubernetesConfig, timeout time.Duration) error { +func (k *Bootstrapper) WaitForCluster(cfg config.MachineConfig, timeout time.Duration) error { start := time.Now() + ip, port := k.clientEndpointAddr(cfg) out.T(out.Waiting, "Waiting for cluster to come online ...") if err := kverify.APIServerProcess(k.c, start, timeout); err != nil { return err } - if err := kverify.APIServerIsRunning(start, k8s.NodeIP, k8s.NodePort, timeout); err != nil { + if err := kverify.APIServerIsRunning(start, ip, port, timeout); err != nil { return err } - c, err := k.client(k8s) + c, err := k.client(cfg) if err != nil { return errors.Wrap(err, "get k8s client") } - return kverify.SystemPods(c, start, k8s.NodeIP, k8s.NodePort, timeout) + return kverify.SystemPods(c, start, timeout) } // restartCluster restarts the Kubernetes cluster configured by kubeadm -func (k *Bootstrapper) restartCluster(k8s config.KubernetesConfig) error { +func (k *Bootstrapper) restartCluster(cfg config.MachineConfig) error { glog.Infof("restartCluster start") start := time.Now() @@ -258,7 +278,7 @@ func (k *Bootstrapper) restartCluster(k8s config.KubernetesConfig) error { glog.Infof("restartCluster took %s", time.Since(start)) }() - version, err := bsutil.ParseKubernetesVersion(k8s.KubernetesVersion) + version, err := bsutil.ParseKubernetesVersion(cfg.KubernetesConfig.KubernetesVersion) if err != nil { return errors.Wrap(err, "parsing kubernetes version") } @@ -274,7 +294,7 @@ func (k *Bootstrapper) restartCluster(k8s config.KubernetesConfig) error { glog.Errorf("failed to create compat symlinks: %v", err) } - baseCmd := fmt.Sprintf("%s %s", bsutil.InvokeKubeadm(k8s.KubernetesVersion), phase) + baseCmd := fmt.Sprintf("%s %s", bsutil.InvokeKubeadm(cfg.KubernetesConfig.KubernetesVersion), phase) cmds := []string{ fmt.Sprintf("%s phase certs all --config %s", baseCmd, bsutil.KubeadmYamlPath), fmt.Sprintf("%s phase kubeconfig all --config %s", baseCmd, bsutil.KubeadmYamlPath), @@ -295,12 +315,12 @@ func (k *Bootstrapper) restartCluster(k8s config.KubernetesConfig) error { return errors.Wrap(err, "apiserver healthz") } - client, err := k.client(k8s) + client, err := k.client(cfg) if err != nil { return errors.Wrap(err, "getting k8s client") } - if err := kverify.SystemPods(client, time.Now(), k8s.NodeIP, k8s.NodePort, kconst.DefaultControlPlaneTimeout); err != nil { + if err := kverify.SystemPods(client, time.Now(), kconst.DefaultControlPlaneTimeout); err != nil { return errors.Wrap(err, "system pods") } @@ -403,6 +423,10 @@ func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error { var cniFile []byte = nil if cfg.KubernetesConfig.EnableDefaultCNI { cniFile = []byte(defaultCNIConfig) + if driver.IsKIC(cfg.VMDriver) { + cniFile = []byte(kicCNIConfig) + } + } files := bsutil.ConfigFileAssets(cfg.KubernetesConfig, kubeadmCfg, kubeletCfg, kubeletService, cniFile) @@ -420,3 +444,24 @@ func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error { } return nil } + +// // applyOverlayNetwork applies the CNI plugin needed to make kic work +// func (k *Bootstrapper) applyOverlayNetwork() error { +// cmd := exec.Command( +// "kubectl", "create", "--kubeconfig=/var/lib/minikube/kubeconfig", +// "-f", bsutil.DefaultCNIConfigPath, +// ) +// if rr, err := k.c.RunCmd(cmd); err != nil { +// return errors.Wrapf(err, "cmd: %s output: %s", rr.Command(), rr.Output()) +// } +// return nil +// } + +// clientEndpointAddr returns ip and port accessible for the kubernetes clients to talk to the cluster +func (k *Bootstrapper) clientEndpointAddr(cfg config.MachineConfig) (string, int) { + if driver.IsKIC(cfg.VMDriver) { + // because docker container ip on non-linux is not accesible + return kic.DefaultBindIPV4, cfg.KubernetesConfig.NodePort + } + return cfg.KubernetesConfig.NodeIP, cfg.KubernetesConfig.NodePort +} diff --git a/pkg/minikube/driver/driver.go b/pkg/minikube/driver/driver.go index 006e346d64..9d32c14da3 100644 --- a/pkg/minikube/driver/driver.go +++ b/pkg/minikube/driver/driver.go @@ -24,7 +24,6 @@ import ( "github.com/golang/glog" "k8s.io/minikube/pkg/drivers/kic" - "k8s.io/minikube/pkg/minikube/bootstrapper" "k8s.io/minikube/pkg/minikube/registry" ) @@ -100,6 +99,7 @@ type FlagHints struct { CacheImages bool ContainerRuntime string Bootstrapper string + EnableDefaultCNI bool } // FlagDefaults returns suggested defaults based on a driver @@ -110,8 +110,8 @@ func FlagDefaults(name string) FlagHints { // only for kic, till other run-times are available we auto-set containerd. if name == Docker { fh.ContainerRuntime = "containerd" - fh.Bootstrapper = bootstrapper.KIC fh.ExtraOptions = append(fh.ExtraOptions, fmt.Sprintf("kubeadm.pod-network-cidr=%s", kic.DefaultPodCIDR)) + fh.EnableDefaultCNI = true } return fh }