autoconfigure cni and cri with proper cgroup driver
Predrag Rogic
parent
735ac02aac
commit
e59d6217a8
|
|
@ -9,7 +9,7 @@ EnvironmentFile=-/etc/sysconfig/containerd
|
|||
EnvironmentFile=-/etc/sysconfig/containerd.minikube
|
||||
EnvironmentFile=/var/run/minikube/env
|
||||
Environment=GOTRACEBACK=crash
|
||||
ExecStartPre=-/sbin/modprobe overlay
|
||||
ExecStartPre=/sbin/modprobe overlay
|
||||
ExecStart=/usr/bin/containerd \
|
||||
$CONTAINERD_OPTIONS \
|
||||
$CONTAINERD_MINIKUBE_OPTIONS \
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ EnvironmentFile=-/etc/sysconfig/containerd
|
|||
EnvironmentFile=-/etc/sysconfig/containerd.minikube
|
||||
EnvironmentFile=/var/run/minikube/env
|
||||
Environment=GOTRACEBACK=crash
|
||||
ExecStartPre=-/sbin/modprobe overlay
|
||||
ExecStartPre=/sbin/modprobe overlay
|
||||
ExecStart=/usr/bin/containerd \
|
||||
$CONTAINERD_OPTIONS \
|
||||
$CONTAINERD_MINIKUBE_OPTIONS \
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ import (
|
|||
"k8s.io/minikube/pkg/minikube/command"
|
||||
"k8s.io/minikube/pkg/minikube/config"
|
||||
"k8s.io/minikube/pkg/minikube/cruntime"
|
||||
"k8s.io/minikube/pkg/minikube/detect"
|
||||
"k8s.io/minikube/pkg/minikube/localpath"
|
||||
"k8s.io/minikube/pkg/minikube/sysinit"
|
||||
"k8s.io/minikube/pkg/util"
|
||||
|
|
@ -93,7 +94,8 @@ func generateTarball(kubernetesVersion, containerRuntime, tarballFilename string
|
|||
if err != nil {
|
||||
return errors.Wrap(err, "failed create new runtime")
|
||||
}
|
||||
if err := cr.Enable(true, false, false); err != nil {
|
||||
|
||||
if err := cr.Enable(true, detect.CgroupDriver(), false); err != nil {
|
||||
return errors.Wrap(err, "enable container runtime")
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ package cni
|
|||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"os/exec"
|
||||
"path"
|
||||
"path/filepath"
|
||||
|
|
@ -34,7 +35,6 @@ import (
|
|||
"k8s.io/minikube/pkg/minikube/config"
|
||||
"k8s.io/minikube/pkg/minikube/constants"
|
||||
"k8s.io/minikube/pkg/minikube/driver"
|
||||
"k8s.io/minikube/pkg/minikube/sysinit"
|
||||
"k8s.io/minikube/pkg/minikube/vmpath"
|
||||
)
|
||||
|
||||
|
|
@ -176,10 +176,6 @@ func manifestAsset(b []byte) assets.CopyableFile {
|
|||
|
||||
// applyManifest applies a CNI manifest
|
||||
func applyManifest(cc config.ClusterConfig, r Runner, f assets.CopyableFile) error {
|
||||
if err := NameLoopback(r); err != nil {
|
||||
klog.Warningf("unable to name loopback interface in applyManifest: %v", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
|
|
@ -198,12 +194,12 @@ func applyManifest(cc config.ClusterConfig, r Runner, f assets.CopyableFile) err
|
|||
return nil
|
||||
}
|
||||
|
||||
// NameLoopback ensures loopback has a name in its config file in /etc/cni/net.d
|
||||
// cri-o is leaving it out atm (https://github.com/cri-o/cri-o/pull/6273)
|
||||
// ConfigureLoopback ensures loopback has expected version ("1.0.0") and valid name ("loopback") in its config file in /etc/cni/net.d
|
||||
// cri-o is leaving name out atm (https://github.com/cri-o/cri-o/pull/6273)
|
||||
// avoid errors like:
|
||||
// Failed to create pod sandbox: rpc error: code = Unknown desc = [failed to set up sandbox container "..." network for pod "...": networkPlugin cni failed to set up pod "..." network: missing network name:,
|
||||
// failed to clean up sandbox container "..." network for pod "...": networkPlugin cni failed to teardown pod "..." network: missing network name]
|
||||
func NameLoopback(r Runner) error {
|
||||
func ConfigureLoopback(r Runner) error {
|
||||
loopback := "/etc/cni/net.d/*loopback.conf*" // usually: 200-loopback.conf
|
||||
// turn { "cniVersion": "0.3.1", "type": "loopback" }
|
||||
// into { "cniVersion": "0.3.1", "name": "loopback", "type": "loopback" }
|
||||
|
|
@ -213,38 +209,90 @@ func NameLoopback(r Runner) error {
|
|||
}
|
||||
if _, err := r.RunCmd(exec.Command(
|
||||
"sudo", "find", filepath.Dir(loopback), "-maxdepth", "1", "-type", "f", "-name", filepath.Base(loopback), "-exec", "sh", "-c",
|
||||
`grep -q loopback {} && ( grep -q name {} || sudo sed -i '/"type": "loopback"/i \ \ \ \ "name": "loopback",' {} )`, ";")); err != nil {
|
||||
`grep -q loopback {} && ( grep -q name {} || sudo sed -i '/"type": "loopback"/i \ \ \ \ "name": "loopback",' {} ) && sudo sed -i 's|"cniVersion": ".*"|"cniVersion": "1.0.0"|g' {}`, ";")); err != nil {
|
||||
return fmt.Errorf("unable to patch loopback config %q: %v", loopback, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DisableBridgeCNIs disables all default bridge CNIs on a node (designated by runner) by changing extension to "mk_disabled" of *bridge* config file(s) found in /etc/cni/net.d.
|
||||
// It's usually called before deploying new CNI or on restarts, to avoid conflicts and flip-flopping of pods' ip addresses.
|
||||
// ConfigureDefaultBridgeCNIs configures all default bridge CNIs on a node (designated by runner).
|
||||
// If network plugin is set (could be, eg "cni" or "kubenet"), it will disable all default bridges by changing extension to "mk_disabled" of *bridge* config file(s) found in /etc/cni/net.d to avoid conflicts.
|
||||
// Otherwise, it will change ip address range to match DefaultPodCIDR in all *bridge* config file(s) found in /etc/cni/net.d.
|
||||
// It's usually called before deploying new CNI and on node restarts, to avoid conflicts and flip-flopping of pods' ip addresses.
|
||||
// It is caller's responsibility to restart container runtime for these changes to take effect.
|
||||
// ref: https://github.com/containernetworking/cni/blob/main/libcni/conf.go
|
||||
func DisableAllBridgeCNIs(r Runner, cc config.ClusterConfig) error {
|
||||
// ref: https://kubernetes.io/docs/tasks/administer-cluster/migrating-from-dockershim/troubleshooting-cni-plugin-related-errors/
|
||||
func ConfigureDefaultBridgeCNIs(r Runner, networkPlugin string) error {
|
||||
if networkPlugin != "" {
|
||||
return disableAllBridgeCNIs(r)
|
||||
}
|
||||
|
||||
return configureAllBridgeCNIs(r, DefaultPodCIDR)
|
||||
}
|
||||
|
||||
func disableAllBridgeCNIs(r Runner) error {
|
||||
path := "/etc/cni/net.d"
|
||||
|
||||
out, err := r.RunCmd(exec.Command(
|
||||
"sudo", "find", path, "-maxdepth", "1", "-type", "f", "-name", "*bridge*", "-not", "-name", "*.mk_disabled", "-printf", "%p|", "-exec", "sh", "-c",
|
||||
"sudo", "find", path, "-maxdepth", "1", "-type", "f", "-name", "*bridge*", "-not", "-name", "*.mk_disabled", "-printf", "%p, ", "-exec", "sh", "-c",
|
||||
`sudo mv {} {}.mk_disabled`, ";"))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to disable all bridge cni configs in %q: %v", path, err)
|
||||
}
|
||||
configs := strings.Trim(out.Stdout.String(), "|")
|
||||
configs := strings.Trim(out.Stdout.String(), ", ")
|
||||
if len(configs) == 0 {
|
||||
klog.Infof("no bridge cni config found in %q - nothing to disable", configs, path)
|
||||
klog.Infof("no bridge cni configs found in %q - nothing to disable", configs, path)
|
||||
return nil
|
||||
}
|
||||
svc := cc.KubernetesConfig.ContainerRuntime
|
||||
klog.Infof("disabled [%s] bridge cni config(s) in %q, now restarting selected %q container runtime", configs, path, svc)
|
||||
|
||||
if svc == "cri-o" {
|
||||
svc = "crio"
|
||||
}
|
||||
if err := sysinit.New(r).Restart(svc); err != nil {
|
||||
klog.Warningf("failed to restart %q container runtime service in %q: %v", svc, cc.Name, err)
|
||||
return err
|
||||
}
|
||||
klog.Infof("disabled [%s] bridge cni config(s)", configs)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func configureAllBridgeCNIs(r Runner, cidr string) error {
|
||||
path := "/etc/cni/net.d"
|
||||
configs := ""
|
||||
|
||||
// non-podman configs:
|
||||
out, err := r.RunCmd(exec.Command(
|
||||
"sudo", "find", path, "-maxdepth", "1", "-type", "f", "-name", "*bridge*", "-not", "-name", "*podman*", "-not", "-name", "*.mk_disabled", "-printf", "%p, ", "-exec", "sh", "-c",
|
||||
// remove ipv6 entries to avoid "failed to set bridge addr: could not add IP address to \"cni0\": permission denied"
|
||||
// ref: https://github.com/cri-o/cri-o/issues/3555
|
||||
// then also remove trailing comma after ipv4 elements, if any
|
||||
// ie, this will transform from, eg:
|
||||
// from: "ranges": [ [{ "subnet": "10.85.0.0/16" }], [{ "subnet": "1100:200::/24" }] ]
|
||||
// to: "ranges": [ [{ "subnet": "10.244.0.0/16" }] ]
|
||||
// getting something similar to https://github.com/cri-o/cri-o/blob/main/contrib/cni/11-crio-ipv4-bridge.conflist
|
||||
fmt.Sprintf(`sudo sed -i -r -e '/"dst": ".*:.*"/d' -e 's|^(.*)"dst": (.*)[,*]$|\1"dst": \2|g' -e '/"subnet": ".*:.*"/d' -e 's|^(.*)"subnet": ".*"(.*)[,*]$|\1"subnet": "%s"\2|g' {}`, cidr), ";"))
|
||||
if err != nil {
|
||||
klog.Errorf("failed to configure non-podman bridge cni configs in %q: %v", path, err)
|
||||
} else {
|
||||
configs = out.Stdout.String()
|
||||
}
|
||||
|
||||
// podman config(s):
|
||||
// ref: https://github.com/containers/podman/blob/main/cni/87-podman-bridge.conflist
|
||||
ip, ipnet, err := net.ParseCIDR(cidr)
|
||||
if err != nil || ip.To4() == nil {
|
||||
klog.Errorf("cidr %q is not valid ipv4 address: %v", cidr, err)
|
||||
} else {
|
||||
gateway := ip.Mask(ipnet.Mask)
|
||||
gateway[3]++
|
||||
out, err = r.RunCmd(exec.Command(
|
||||
"sudo", "find", path, "-maxdepth", "1", "-type", "f", "-name", "*bridge*", "-name", "*podman*", "-not", "-name", "*.mk_disabled", "-printf", "%p, ", "-exec", "sh", "-c",
|
||||
fmt.Sprintf(`sudo sed -i -r -e 's|^(.*)"subnet": ".*"(.*)$|\1"subnet": "%s"\2|g' -e 's|^(.*)"gateway": ".*"(.*)$|\1"gateway": "%s"\2|g' {}`, cidr, gateway), ";"))
|
||||
if err != nil {
|
||||
klog.Errorf("failed to configure podman bridge cni configs in %q: %v", path, err)
|
||||
} else {
|
||||
configs += out.Stdout.String()
|
||||
}
|
||||
}
|
||||
|
||||
if len(strings.Trim(configs, ", ")) == 0 {
|
||||
klog.Infof("no bridge cni configs found in %q - nothing to configure", configs, path)
|
||||
return nil
|
||||
}
|
||||
klog.Infof("configured [%s] bridge cni config(s)", configs)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -115,6 +115,7 @@ type KubernetesConfig struct {
|
|||
APIServerNames []string
|
||||
APIServerIPs []net.IP
|
||||
DNSDomain string
|
||||
CgroupDriver string
|
||||
ContainerRuntime string
|
||||
CRISocket string
|
||||
NetworkPlugin string
|
||||
|
|
@ -144,6 +145,7 @@ type Node struct {
|
|||
IP string
|
||||
Port int
|
||||
KubernetesVersion string
|
||||
CgroupDriver string
|
||||
ContainerRuntime string
|
||||
ControlPlane bool
|
||||
Worker bool
|
||||
|
|
|
|||
|
|
@ -54,6 +54,7 @@ const (
|
|||
SSHPort = 22
|
||||
// RegistryAddonPort os the default registry addon port
|
||||
RegistryAddonPort = 5000
|
||||
|
||||
// Containerd is the default name and spelling for the containerd container runtime
|
||||
Containerd = "containerd"
|
||||
// CRIO is the default name and spelling for the cri-o container runtime
|
||||
|
|
@ -63,6 +64,12 @@ const (
|
|||
// DefaultContainerRuntime is our default container runtime
|
||||
DefaultContainerRuntime = ""
|
||||
|
||||
// cgroup drivers
|
||||
DefaultCgroupDriver = "systemd"
|
||||
CgroupfsCgroupDriver = "cgroupfs"
|
||||
SystemdCgroupDriver = "systemd"
|
||||
UnknownCgroupDriver = ""
|
||||
|
||||
// APIServerName is the default API server name
|
||||
APIServerName = "minikubeCA"
|
||||
// ClusterDNSDomain is the default DNS domain
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ import (
|
|||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
|
|
@ -37,6 +38,7 @@ import (
|
|||
"k8s.io/minikube/pkg/minikube/cni"
|
||||
"k8s.io/minikube/pkg/minikube/command"
|
||||
"k8s.io/minikube/pkg/minikube/config"
|
||||
"k8s.io/minikube/pkg/minikube/constants"
|
||||
"k8s.io/minikube/pkg/minikube/download"
|
||||
"k8s.io/minikube/pkg/minikube/style"
|
||||
"k8s.io/minikube/pkg/minikube/sysinit"
|
||||
|
|
@ -127,18 +129,36 @@ func (r *Containerd) Available() error {
|
|||
}
|
||||
|
||||
// generateContainerdConfig sets up /etc/containerd/config.toml & /etc/containerd/containerd.conf.d/02-containerd.conf
|
||||
func generateContainerdConfig(cr CommandRunner, imageRepository string, kv semver.Version, forceSystemd bool, insecureRegistry []string, inUserNamespace bool) error {
|
||||
func generateContainerdConfig(cr CommandRunner, imageRepository string, kv semver.Version, cgroupDriver string, insecureRegistry []string, inUserNamespace bool) error {
|
||||
pauseImage := images.Pause(kv, imageRepository)
|
||||
if _, err := cr.RunCmd(exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo sed -e 's|^.*sandbox_image = .*$|sandbox_image = \"%s\"|' -i %s", pauseImage, containerdConfigFile))); err != nil {
|
||||
if _, err := cr.RunCmd(exec.Command("sh", "-c", fmt.Sprintf(`sudo sed -i -r 's|^( *)sandbox_image = .*$|\1sandbox_image = %q|' %s`, pauseImage, containerdConfigFile))); err != nil {
|
||||
return errors.Wrap(err, "update sandbox_image")
|
||||
}
|
||||
if _, err := cr.RunCmd(exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo sed -e 's|^.*restrict_oom_score_adj = .*$|restrict_oom_score_adj = %t|' -i %s", inUserNamespace, containerdConfigFile))); err != nil {
|
||||
if _, err := cr.RunCmd(exec.Command("sh", "-c", fmt.Sprintf(`sudo sed -i -r 's|^( *)restrict_oom_score_adj = .*$|\1restrict_oom_score_adj = %t|' %s`, inUserNamespace, containerdConfigFile))); err != nil {
|
||||
return errors.Wrap(err, "update restrict_oom_score_adj")
|
||||
}
|
||||
if _, err := cr.RunCmd(exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo sed -e 's|^.*SystemdCgroup = .*$|SystemdCgroup = %t|' -i %s", forceSystemd, containerdConfigFile))); err != nil {
|
||||
return errors.Wrap(err, "update SystemdCgroup")
|
||||
// configure cgroup driver
|
||||
if cgroupDriver != constants.UnknownCgroupDriver {
|
||||
klog.Infof("configuring containerd to use %q as cgroup driver...", cgroupDriver)
|
||||
useSystemd := cgroupDriver == constants.SystemdCgroupDriver
|
||||
if _, err := cr.RunCmd(exec.Command("sh", "-c", fmt.Sprintf(`sudo sed -i -r 's|^( *)SystemdCgroup = .*$|\1SystemdCgroup = %t|g' %s`, useSystemd, containerdConfigFile))); err != nil {
|
||||
return errors.Wrap(err, "configuring SystemdCgroup")
|
||||
}
|
||||
}
|
||||
if _, err := cr.RunCmd(exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo sed -e 's|^.*conf_dir = .*$|conf_dir = \"%s\"|' -i %s", cni.DefaultConfDir, containerdConfigFile))); err != nil {
|
||||
// handle deprecated features
|
||||
// ref: https://github.com/containerd/containerd/blob/main/RELEASES.md#deprecated-features
|
||||
if _, err := cr.RunCmd(exec.Command("sh", "-c", fmt.Sprintf(`sudo sed -i -r 's|"io.containerd.runtime.v1.linux"|"io.containerd.runc.v2"|g' %s`, containerdConfigFile))); err != nil {
|
||||
return errors.Wrap(err, "configuring io.containerd.runtime version")
|
||||
}
|
||||
if _, err := cr.RunCmd(exec.Command("sh", "-c", fmt.Sprintf(`sudo sed -i -r 's|"io.containerd.runc.v1"|"io.containerd.runc.v2"|g' %s`, containerdConfigFile))); err != nil {
|
||||
return errors.Wrap(err, "configuring io.containerd.runc version")
|
||||
}
|
||||
// ensure conf_dir is using '/etc/cni/net.d'
|
||||
// TODO (@prezha): this should be removed (ie, not needed) once we remove "hardcoded" '/etc/cni/net.mk' folder in minikube distro
|
||||
if _, err := cr.RunCmd(exec.Command("sh", "-c", `sudo rm -rf /etc/cni/net.mk`)); err != nil {
|
||||
return fmt.Errorf("unable to remove /etc/cni/net.mk directory: %v", err)
|
||||
}
|
||||
if _, err := cr.RunCmd(exec.Command("sh", "-c", fmt.Sprintf(`sudo sed -i -r 's|^( *)conf_dir = .*$|\1conf_dir = %q|g' %s`, cni.DefaultConfDir, containerdConfigFile))); err != nil {
|
||||
return errors.Wrap(err, "update conf_dir")
|
||||
}
|
||||
|
||||
|
|
@ -176,7 +196,7 @@ func generateContainerdConfig(cr CommandRunner, imageRepository string, kv semve
|
|||
|
||||
// Enable idempotently enables containerd on a host
|
||||
// It is also called by docker.Enable() - if bound to containerd, to enforce proper containerd configuration completed by service restart.
|
||||
func (r *Containerd) Enable(disOthers, forceSystemd, inUserNamespace bool) error {
|
||||
func (r *Containerd) Enable(disOthers bool, cgroupDriver string, inUserNamespace bool) error {
|
||||
if inUserNamespace {
|
||||
if err := CheckKernelCompatibility(r.Runner, 5, 11); err != nil {
|
||||
// For using overlayfs
|
||||
|
|
@ -195,13 +215,25 @@ func (r *Containerd) Enable(disOthers, forceSystemd, inUserNamespace bool) error
|
|||
if err := populateCRIConfig(r.Runner, r.SocketPath()); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := generateContainerdConfig(r.Runner, r.ImageRepository, r.KubernetesVersion, forceSystemd, r.InsecureRegistry, inUserNamespace); err != nil {
|
||||
|
||||
if err := generateContainerdConfig(r.Runner, r.ImageRepository, r.KubernetesVersion, cgroupDriver, r.InsecureRegistry, inUserNamespace); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := enableIPForwarding(r.Runner); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// TODO (@prezha): remove this hack after proper version update in minikube release
|
||||
// ref: https://github.com/containerd/containerd/blob/main/RELEASES.md#kubernetes-support
|
||||
targetVersion := "1.6.14"
|
||||
currentVersion, err := r.Version()
|
||||
if err == nil && semver.MustParse(targetVersion).GT(semver.MustParse(currentVersion)) {
|
||||
klog.Infof("replacing original containerd with v%s-%s-%s", targetVersion, runtime.GOOS, runtime.GOARCH)
|
||||
if err := updateContainerdBinary(r.Runner, targetVersion, runtime.GOOS, runtime.GOARCH); err != nil {
|
||||
klog.Warningf("unable to replace original containerd with v%s-%s-%s: %v", targetVersion, runtime.GOOS, runtime.GOARCH, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Otherwise, containerd will fail API requests with 'Unimplemented'
|
||||
return r.Init.Restart("containerd")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ import (
|
|||
"k8s.io/minikube/pkg/minikube/cni"
|
||||
"k8s.io/minikube/pkg/minikube/command"
|
||||
"k8s.io/minikube/pkg/minikube/config"
|
||||
"k8s.io/minikube/pkg/minikube/constants"
|
||||
"k8s.io/minikube/pkg/minikube/download"
|
||||
"k8s.io/minikube/pkg/minikube/style"
|
||||
"k8s.io/minikube/pkg/minikube/sysinit"
|
||||
|
|
@ -72,10 +73,15 @@ func generateCRIOConfig(cr CommandRunner, imageRepository string, kv semver.Vers
|
|||
return nil
|
||||
}
|
||||
|
||||
func (r *CRIO) forceSystemd() error {
|
||||
c := exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo sed -e 's|^.*cgroup_manager = .*$|cgroup_manager = \"systemd\"|' -i %s", crioConfigFile))
|
||||
func (r *CRIO) setCGroup(driver string) error {
|
||||
if driver == constants.UnknownCgroupDriver {
|
||||
return fmt.Errorf("unable to configure cri-o to use unknown cgroup driver")
|
||||
}
|
||||
|
||||
klog.Infof("configuring cri-o to use %q as cgroup driver...", driver)
|
||||
c := exec.Command("/bin/bash", "-c", fmt.Sprintf(`sudo sed -i -r 's|^( *)cgroup_manager = .*$|\1cgroup_manager = %q|' %s`, driver, crioConfigFile))
|
||||
if _, err := r.Runner.RunCmd(c); err != nil {
|
||||
return errors.Wrap(err, "force systemd")
|
||||
return errors.Wrap(err, "configuring cgroup_manager")
|
||||
}
|
||||
|
||||
return nil
|
||||
|
|
@ -185,7 +191,7 @@ Environment="_CRIO_ROOTLESS=1"
|
|||
}
|
||||
|
||||
// Enable idempotently enables CRIO on a host
|
||||
func (r *CRIO) Enable(disOthers, forceSystemd, inUserNamespace bool) error {
|
||||
func (r *CRIO) Enable(disOthers bool, cgroupDriver string, inUserNamespace bool) error {
|
||||
if disOthers {
|
||||
if err := disableOthers(r, r.Runner); err != nil {
|
||||
klog.Warningf("disableOthers: %v", err)
|
||||
|
|
@ -200,10 +206,8 @@ func (r *CRIO) Enable(disOthers, forceSystemd, inUserNamespace bool) error {
|
|||
if err := enableIPForwarding(r.Runner); err != nil {
|
||||
return err
|
||||
}
|
||||
if forceSystemd {
|
||||
if err := r.forceSystemd(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := r.setCGroup(cgroupDriver); err != nil {
|
||||
return err
|
||||
}
|
||||
if inUserNamespace {
|
||||
if err := CheckKernelCompatibility(r.Runner, 5, 11); err != nil {
|
||||
|
|
|
|||
|
|
@ -81,7 +81,7 @@ type Manager interface {
|
|||
// Version retrieves the current version of this runtime
|
||||
Version() (string, error)
|
||||
// Enable idempotently enables this runtime on a host
|
||||
Enable(bool, bool, bool) error
|
||||
Enable(bool, string, bool) error
|
||||
// Disable idempotently disables this runtime on a host
|
||||
Disable() error
|
||||
// Active returns whether or not a runtime is active on a host
|
||||
|
|
@ -351,3 +351,27 @@ func ConfigureNetworkPlugin(r Manager, cr CommandRunner, networkPlugin string) e
|
|||
}
|
||||
return dockerConfigureNetworkPlugin(*dm, cr, networkPlugin)
|
||||
}
|
||||
|
||||
// updateCRIDockerdBinary updates cri-dockerd to version
|
||||
func updateCRIDockerdBinary(cr CommandRunner, version, arch string) error {
|
||||
curl := fmt.Sprintf("curl -sSfL https://github.com/Mirantis/cri-dockerd/releases/download/v%s/cri-dockerd-%s.%s.tgz | tar -xz -C /tmp", version, version, arch)
|
||||
if _, err := cr.RunCmd(exec.Command("sudo", "sh", "-c", curl)); err != nil {
|
||||
return fmt.Errorf("unable to download cri-dockerd version %s: %v", version, err)
|
||||
}
|
||||
if _, err := cr.RunCmd(exec.Command("sudo", "mv", "/tmp/cri-dockerd/cri-dockerd", "/usr/bin/cri-dockerd")); err != nil {
|
||||
return fmt.Errorf("unable to install cri-dockerd version %s: %v", version, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// updateContainerdBinary updates containerd to version
|
||||
func updateContainerdBinary(cr CommandRunner, version, os, arch string) error {
|
||||
curl := fmt.Sprintf("curl -sSfL https://github.com/containerd/containerd/releases/download/v%s/containerd-%s-%s-%s.tar.gz | tar -xz -C /tmp", version, version, os, arch)
|
||||
if _, err := cr.RunCmd(exec.Command("sudo", "sh", "-c", curl)); err != nil {
|
||||
return fmt.Errorf("unable to download containerd version %s: %v", version, err)
|
||||
}
|
||||
if _, err := cr.RunCmd(exec.Command("sudo", "sh", "-c", "mv /tmp/bin/* /usr/bin/")); err != nil { // note: has to run in subshell because of wildcard!
|
||||
return fmt.Errorf("unable to install containerd version %s: %v", version, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ import (
|
|||
"k8s.io/klog/v2"
|
||||
"k8s.io/minikube/pkg/minikube/assets"
|
||||
"k8s.io/minikube/pkg/minikube/command"
|
||||
"k8s.io/minikube/pkg/minikube/constants"
|
||||
)
|
||||
|
||||
func TestName(t *testing.T) {
|
||||
|
|
@ -699,7 +700,7 @@ func TestEnable(t *testing.T) {
|
|||
if err != nil {
|
||||
t.Fatalf("New(%s): %v", tc.runtime, err)
|
||||
}
|
||||
err = cr.Enable(true, false, false)
|
||||
err = cr.Enable(true, constants.CgroupfsCgroupDriver, false)
|
||||
if err != nil {
|
||||
t.Errorf("%s disable unexpected error: %v", tc.runtime, err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -34,9 +34,9 @@ import (
|
|||
"k8s.io/klog/v2"
|
||||
"k8s.io/minikube/pkg/minikube/assets"
|
||||
"k8s.io/minikube/pkg/minikube/bootstrapper/images"
|
||||
"k8s.io/minikube/pkg/minikube/cni"
|
||||
"k8s.io/minikube/pkg/minikube/command"
|
||||
"k8s.io/minikube/pkg/minikube/config"
|
||||
"k8s.io/minikube/pkg/minikube/constants"
|
||||
"k8s.io/minikube/pkg/minikube/docker"
|
||||
"k8s.io/minikube/pkg/minikube/download"
|
||||
"k8s.io/minikube/pkg/minikube/image"
|
||||
|
|
@ -127,7 +127,7 @@ func (r *Docker) Active() bool {
|
|||
}
|
||||
|
||||
// Enable idempotently enables Docker on a host
|
||||
func (r *Docker) Enable(disOthers, forceSystemd, inUserNamespace bool) error {
|
||||
func (r *Docker) Enable(disOthers bool, cgroupDriver string, inUserNamespace bool) error {
|
||||
if inUserNamespace {
|
||||
return errors.New("inUserNamespace must not be true for docker")
|
||||
}
|
||||
|
|
@ -137,14 +137,6 @@ func (r *Docker) Enable(disOthers, forceSystemd, inUserNamespace bool) error {
|
|||
klog.Warningf("disableOthers: %v", err)
|
||||
}
|
||||
}
|
||||
// check if containerd is still alive (hopefully because being being bound to docker), and if so: ensure it's configured properly by calling its Enable() method
|
||||
cdr, errCdR := New(Config{Type: "containerd", Runner: r.Runner})
|
||||
if errCdR == nil && cdr.Active() {
|
||||
errCdR = cdr.Enable(false, forceSystemd, inUserNamespace)
|
||||
}
|
||||
if errCdR != nil {
|
||||
klog.Warningf("cannot ensure containerd (as bound to docker) is configured properly and reloaded - cluster might be unstable: %v", errCdR)
|
||||
}
|
||||
|
||||
if err := populateCRIConfig(r.Runner, r.SocketPath()); err != nil {
|
||||
return err
|
||||
|
|
@ -158,10 +150,8 @@ func (r *Docker) Enable(disOthers, forceSystemd, inUserNamespace bool) error {
|
|||
klog.ErrorS(err, "Failed to enable", "service", "docker.socket")
|
||||
}
|
||||
|
||||
if forceSystemd {
|
||||
if err := r.forceSystemd(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := r.setCGroup(cgroupDriver); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := r.Init.Restart("docker"); err != nil {
|
||||
|
|
@ -169,6 +159,16 @@ func (r *Docker) Enable(disOthers, forceSystemd, inUserNamespace bool) error {
|
|||
}
|
||||
|
||||
if r.CRIService != "" {
|
||||
// TODO (@prezha): remove this hack after proper version update in minikube release
|
||||
// deploy/iso/minikube-iso/arch/x86_64/package/cri-dockerd/cri-dockerd.*
|
||||
// deploy/iso/minikube-iso/arch/aarch64/package/cri-dockerd-aarch64/cri-dockerd.*
|
||||
// note: https://github.com/Mirantis/cri-dockerd/blob/master/Makefile changed => also needs updating .mk files?!
|
||||
targetVersion := "0.2.6"
|
||||
klog.Infof("replacing original cri-dockerd with v%s-%s", targetVersion, runtime.GOARCH)
|
||||
if err := updateCRIDockerdBinary(r.Runner, targetVersion, runtime.GOARCH); err != nil {
|
||||
klog.Warningf("unable to replace original cri-dockerd with v%s-%s: %v", targetVersion, runtime.GOARCH, err)
|
||||
}
|
||||
|
||||
if err := r.Init.Enable(r.CRIService); err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -515,18 +515,23 @@ func (r *Docker) SystemLogCmd(len int) string {
|
|||
return fmt.Sprintf("sudo journalctl -u docker -n %d", len)
|
||||
}
|
||||
|
||||
// ForceSystemd forces the docker daemon to use systemd as cgroup manager
|
||||
func (r *Docker) forceSystemd() error {
|
||||
klog.Infof("Forcing docker to use systemd as cgroup manager...")
|
||||
daemonConfig := `{
|
||||
"exec-opts": ["native.cgroupdriver=systemd"],
|
||||
// setCGroup configures the docker daemon to use driver as cgroup manager
|
||||
// ref: https://docs.docker.com/engine/reference/commandline/dockerd/#options-for-the-runtime
|
||||
func (r *Docker) setCGroup(driver string) error {
|
||||
if driver == constants.UnknownCgroupDriver {
|
||||
return fmt.Errorf("unable to configure docker to use unknown cgroup driver")
|
||||
}
|
||||
|
||||
klog.Infof("configuring docker to use %q as cgroup driver...", driver)
|
||||
daemonConfig := fmt.Sprintf(`{
|
||||
"exec-opts": ["native.cgroupdriver=%s"],
|
||||
"log-driver": "json-file",
|
||||
"log-opts": {
|
||||
"max-size": "100m"
|
||||
},
|
||||
"storage-driver": "overlay2"
|
||||
}
|
||||
`
|
||||
`, driver)
|
||||
ma := assets.NewMemoryAsset([]byte(daemonConfig), "/etc/docker", "daemon.json", "0644")
|
||||
return r.Runner.Copy(ma)
|
||||
}
|
||||
|
|
@ -680,25 +685,18 @@ const (
|
|||
)
|
||||
|
||||
func dockerConfigureNetworkPlugin(r Docker, cr CommandRunner, networkPlugin string) error {
|
||||
// $ cri-dockerd --version
|
||||
// cri-dockerd 0.2.6 (d8accf7)
|
||||
// $ cri-dockerd --help | grep -i cni
|
||||
// --cni-bin-dir string A comma-separated list of full paths of directories in which to search for CNI plugin binaries. (default "/opt/cni/bin")
|
||||
// --cni-cache-dir string The full path of the directory in which CNI should store cache files. (default "/var/lib/cni/cache")
|
||||
// --cni-conf-dir string The full path of the directory in which to search for CNI config files (default "/etc/cni/net.d")
|
||||
// --network-plugin string The name of the network plugin to be invoked for various events in kubelet/pod lifecycle. (default "cni")
|
||||
args := " --hairpin-mode=hairpin-veth"
|
||||
// if network plugin is not selected - use default "cni"
|
||||
if networkPlugin == "" {
|
||||
// no-op plugin
|
||||
return nil
|
||||
networkPlugin = "cni"
|
||||
}
|
||||
|
||||
if err := cni.NameLoopback(r.Runner); err != nil {
|
||||
klog.Warningf("unable to name loopback interface in dockerConfigureNetworkPlugin: %v", err)
|
||||
}
|
||||
|
||||
args := ""
|
||||
// The CNI configuration is handled by CRI in 1.24+
|
||||
// ref: https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/#installation
|
||||
if networkPlugin == "cni" && r.KubernetesVersion.GTE(semver.MustParse("1.24.0-alpha.2")) {
|
||||
args += " --cni-bin-dir=" + CNIBinDir
|
||||
args += " --cni-cache-dir=" + CNICacheDir
|
||||
args += " --cni-conf-dir=" + cni.DefaultConfDir
|
||||
}
|
||||
args += " --hairpin-mode=hairpin-veth"
|
||||
|
||||
opts := struct {
|
||||
NetworkPlugin string
|
||||
ExtraArguments string
|
||||
|
|
@ -707,15 +705,6 @@ func dockerConfigureNetworkPlugin(r Docker, cr CommandRunner, networkPlugin stri
|
|||
ExtraArguments: args,
|
||||
}
|
||||
|
||||
// TODO (@prezha): remove once cri-dockerd is updated in future minikube release:
|
||||
// deploy/iso/minikube-iso/arch/x86_64/package/cri-dockerd/cri-dockerd.*
|
||||
// deploy/iso/minikube-iso/arch/aarch64/package/cri-dockerd-aarch64/cri-dockerd.*
|
||||
// note: https://github.com/Mirantis/cri-dockerd/blob/master/Makefile changed => also needs updating .mk files?!
|
||||
klog.Infof("replacing original cri-dockerd with v0.2.6-%s", runtime.GOARCH)
|
||||
if err := downloadCRIDockerdBinary(cr, "0.2.6", runtime.GOARCH); err != nil {
|
||||
klog.Warningf("unable to replace original cri-dockerd with v0.2.6-%s: %v", runtime.GOARCH, err)
|
||||
}
|
||||
|
||||
const CRIDockerServiceConfFile = "/etc/systemd/system/cri-docker.service.d/10-cni.conf"
|
||||
var CRIDockerServiceConfTemplate = template.Must(template.New("criDockerServiceConfTemplate").Parse(`[Service]
|
||||
ExecStart=
|
||||
|
|
@ -736,21 +725,3 @@ ExecStart=/usr/bin/cri-dockerd --container-runtime-endpoint fd:// --network-plug
|
|||
}
|
||||
return r.Init.Restart("cri-docker")
|
||||
}
|
||||
|
||||
// download cri-dockerd version
|
||||
func downloadCRIDockerdBinary(cr CommandRunner, version, arch string) error {
|
||||
curl := fmt.Sprintf("curl -sSfL https://github.com/Mirantis/cri-dockerd/releases/download/v%s/cri-dockerd-%s.%s.tgz | tar -xz -C /tmp", version, version, arch)
|
||||
if _, err := cr.RunCmd(exec.Command("sudo", "sh", "-c", curl)); err != nil {
|
||||
return fmt.Errorf("unable to download new cri-dockerd: %v", err)
|
||||
}
|
||||
if _, err := cr.RunCmd(exec.Command("sudo", "mv", "/usr/bin/cri-dockerd", "/usr/bin/cri-dockerd-org")); err != nil {
|
||||
return fmt.Errorf("unable to backup org cri-dockerd: %v", err)
|
||||
}
|
||||
if _, err := cr.RunCmd(exec.Command("sudo", "mv", "/tmp/cri-dockerd/cri-dockerd", "/usr/bin/cri-dockerd")); err != nil {
|
||||
if _, err := cr.RunCmd(exec.Command("sudo", "mv", "/usr/bin/cri-dockerd", "/usr/bin/cri-dockerd-org")); err != nil {
|
||||
return fmt.Errorf("unable to install new cri-dockerd and restore org cri-dockerd - it's broken!: %v", err)
|
||||
}
|
||||
return fmt.Errorf("unable to install new cri-dockerd: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,7 +28,9 @@ import (
|
|||
"github.com/klauspost/cpuid"
|
||||
"github.com/spf13/viper"
|
||||
"golang.org/x/sys/cpu"
|
||||
"golang.org/x/sys/unix"
|
||||
"k8s.io/klog/v2"
|
||||
"k8s.io/minikube/pkg/minikube/constants"
|
||||
"k8s.io/minikube/pkg/minikube/localpath"
|
||||
)
|
||||
|
||||
|
|
@ -147,3 +149,50 @@ func SocketVMNetInstalled() bool {
|
|||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// CgroupDriver returns detected cgroup driver as configured on host os.
|
||||
// If unable to detect, it will return constants.DefaultCgroupDriver instead.
|
||||
func CgroupDriver() string {
|
||||
switch cgroupVersion() {
|
||||
case "v1":
|
||||
klog.Infof("detected %q cgroup driver on host os", constants.CgroupfsCgroupDriver)
|
||||
return constants.CgroupfsCgroupDriver
|
||||
case "v2":
|
||||
klog.Infof("detected %q cgroup driver on host os", constants.SystemdCgroupDriver)
|
||||
return constants.SystemdCgroupDriver
|
||||
default:
|
||||
klog.Warningf("unable to detect host's os cgroup driver - will continue and try with %q as per default, but things might break", constants.DefaultCgroupDriver)
|
||||
return constants.DefaultCgroupDriver // try with default rather than just give up
|
||||
}
|
||||
}
|
||||
|
||||
// cgroupVersion returns cgroup version as set on the linux OS host machine (where minikube runs).
|
||||
// Possible options are: "v1", "v2" or "" (unknown).
|
||||
// ref: https://kubernetes.io/docs/concepts/architecture/cgroups/#check-cgroup-version
|
||||
// ref: https://man7.org/linux/man-pages/man7/cgroups.7.html
|
||||
func cgroupVersion() string {
|
||||
if runtime.GOOS != "linux" {
|
||||
return ""
|
||||
}
|
||||
|
||||
// check '/sys/fs/cgroup' or '/sys/fs/cgroup/unified' type
|
||||
var stat unix.Statfs_t
|
||||
if err := unix.Statfs("/sys/fs/cgroup", &stat); err != nil {
|
||||
return ""
|
||||
}
|
||||
// fallback, but could be misleading
|
||||
if stat.Type != unix.TMPFS_MAGIC && stat.Type != unix.CGROUP_SUPER_MAGIC && stat.Type != unix.CGROUP2_SUPER_MAGIC {
|
||||
if err := unix.Statfs("/sys/fs/cgroup/unified", &stat); err != nil {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
switch stat.Type {
|
||||
case unix.TMPFS_MAGIC, unix.CGROUP_SUPER_MAGIC: // tmpfs, cgroupfs
|
||||
return "v1"
|
||||
case unix.CGROUP2_SUPER_MAGIC: // cgroup2fs
|
||||
return "v2"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -50,6 +50,7 @@ import (
|
|||
"k8s.io/minikube/pkg/minikube/config"
|
||||
"k8s.io/minikube/pkg/minikube/constants"
|
||||
"k8s.io/minikube/pkg/minikube/cruntime"
|
||||
"k8s.io/minikube/pkg/minikube/detect"
|
||||
"k8s.io/minikube/pkg/minikube/driver"
|
||||
"k8s.io/minikube/pkg/minikube/exit"
|
||||
"k8s.io/minikube/pkg/minikube/kubeconfig"
|
||||
|
|
@ -439,8 +440,14 @@ func configureRuntimes(runner cruntime.CommandRunner, cc config.ClusterConfig, k
|
|||
}
|
||||
}
|
||||
|
||||
// if cni is used, ensure all default CNI(s) are disabled on every node start
|
||||
if err := cni.DisableAllBridgeCNIs(runner, cc); err != nil {
|
||||
// ensure loopback is properly configured
|
||||
if err := cni.ConfigureLoopback(runner); err != nil {
|
||||
klog.Warningf("unable to name loopback interface in dockerConfigureNetworkPlugin: %v", err)
|
||||
}
|
||||
|
||||
// ensure all default CNI(s) are properly configured on each and every node (re)start
|
||||
// make sure container runtime is restarted afterwards for these changes to take effect
|
||||
if err := cni.ConfigureDefaultBridgeCNIs(runner, cc.KubernetesConfig.NetworkPlugin); err != nil {
|
||||
klog.Errorf("unable to disable preinstalled bridge CNI(s): %v", err)
|
||||
}
|
||||
|
||||
|
|
@ -451,7 +458,24 @@ func configureRuntimes(runner cruntime.CommandRunner, cc config.ClusterConfig, k
|
|||
}
|
||||
|
||||
inUserNamespace := strings.Contains(cc.KubernetesConfig.FeatureGates, "KubeletInUserNamespace=true")
|
||||
err = cr.Enable(disableOthers, forceSystemd(kv), inUserNamespace)
|
||||
// for docker container runtime: ensure containerd is properly configured by calling Enable(), as docker could be bound to containerd
|
||||
// it will also "soft" start containerd, but it will not disable others; docker will disable containerd if not used in the next step
|
||||
if co.Type == "docker" {
|
||||
containerd, err := cruntime.New(cruntime.Config{
|
||||
Type: "containerd",
|
||||
Socket: "", // use default
|
||||
Runner: co.Runner,
|
||||
ImageRepository: co.ImageRepository,
|
||||
KubernetesVersion: co.KubernetesVersion,
|
||||
InsecureRegistry: co.InsecureRegistry})
|
||||
if err == nil {
|
||||
err = containerd.Enable(false, cgroupDriver(cc), inUserNamespace) // do not disableOthers, as it's not primary cr
|
||||
}
|
||||
if err != nil {
|
||||
klog.Warningf("cannot ensure containerd is configured properly and reloaded for docker - cluster might be unstable: %v", err)
|
||||
}
|
||||
}
|
||||
err = cr.Enable(disableOthers, cgroupDriver(cc), inUserNamespace)
|
||||
if err != nil {
|
||||
exit.Error(reason.RuntimeEnable, "Failed to enable container runtime", err)
|
||||
}
|
||||
|
|
@ -471,20 +495,40 @@ func configureRuntimes(runner cruntime.CommandRunner, cc config.ClusterConfig, k
|
|||
return cr
|
||||
}
|
||||
|
||||
func forceSystemd(kv semver.Version) bool {
|
||||
// starting from k8s v1.22: "kubeadm clusters should be using the systemd driver"
|
||||
// ref: https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.22.md#no-really-you-must-read-this-before-you-upgrade
|
||||
// ref: https://kubernetes.io/docs/setup/production-environment/container-runtimes/#cgroup-drivers
|
||||
// ref: https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/configure-cgroup-driver/
|
||||
// still, respect user preference, if present
|
||||
if viper.IsSet("force-systemd") {
|
||||
return viper.GetBool("force-systemd")
|
||||
// cgroupDriver returns cgroup driver that should be used to further configure container runtime, node(s) and cluster.
|
||||
// It is based on:
|
||||
// - (forced) user preference (set via flags or env), if present, or
|
||||
// - existing k8s cluster config, if present, or
|
||||
// - host os config detection, if possible, or
|
||||
// - constants.DefaultCgroupDriver, otherwise.
|
||||
// Possible mappings are: "v1" (legacy) cgroups => "cgroupfs", "v2" (unified) cgroups => "systemd" and "" (unknown) cgroups => constants.DefaultCgroupDriver.
|
||||
// Note: starting from k8s v1.22, "kubeadm clusters should be using the systemd driver":
|
||||
// ref: https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.22.md#no-really-you-must-read-this-before-you-upgrade
|
||||
// ref: https://kubernetes.io/docs/setup/production-environment/container-runtimes/#cgroup-drivers
|
||||
// ref: https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/configure-cgroup-driver/
|
||||
func cgroupDriver(cc config.ClusterConfig) string {
|
||||
klog.Info("detecting cgroup driver to use...")
|
||||
|
||||
// check flags for user preference
|
||||
if viper.GetBool("force-systemd") {
|
||||
klog.Infof("using %q cgroup driver as enforced via flags", constants.SystemdCgroupDriver)
|
||||
return constants.SystemdCgroupDriver
|
||||
}
|
||||
|
||||
// check env for user preference
|
||||
env := os.Getenv(constants.MinikubeForceSystemdEnv)
|
||||
if force, err := strconv.ParseBool(env); env != "" && err == nil {
|
||||
return force
|
||||
if force, err := strconv.ParseBool(env); env != "" && err == nil && force {
|
||||
klog.Infof("using %q cgroup driver as enforced via env", constants.SystemdCgroupDriver)
|
||||
return constants.SystemdCgroupDriver
|
||||
}
|
||||
return kv.GTE(semver.Version{Major: 1, Minor: 22})
|
||||
|
||||
// check current cluster config
|
||||
if cc.KubernetesConfig.CgroupDriver != "" {
|
||||
klog.Infof("using %q cgroup driver from existing cluster config", cc.KubernetesConfig.CgroupDriver)
|
||||
return cc.KubernetesConfig.CgroupDriver
|
||||
}
|
||||
|
||||
return detect.CgroupDriver()
|
||||
}
|
||||
|
||||
func pathExists(runner cruntime.CommandRunner, path string) (bool, error) {
|
||||
|
|
|
|||
|
|
@ -57,14 +57,13 @@ func TestNetworkPlugins(t *testing.T) {
|
|||
namespace string
|
||||
hairpin bool
|
||||
}{
|
||||
// for containerd and crio runtimes kindnet CNI is used by default and hairpin is enabled
|
||||
{"auto", []string{}, "", "", "", ContainerRuntime() != "docker"},
|
||||
{"auto", []string{}, "", "", "", true},
|
||||
{"kubenet", []string{"--network-plugin=kubenet"}, "kubenet", "", "", true},
|
||||
{"bridge", []string{"--cni=bridge"}, "cni", "", "", true},
|
||||
{"enable-default-cni", []string{"--enable-default-cni=true"}, "cni", "", "", true},
|
||||
{"flannel", []string{"--cni=flannel"}, "cni", "app=flannel", "kube-flannel", true},
|
||||
{"kindnet", []string{"--cni=kindnet"}, "cni", "app=kindnet", "kube-system", true},
|
||||
{"false", []string{"--cni=false"}, "", "", "", false},
|
||||
{"false", []string{"--cni=false"}, "", "", "", true},
|
||||
{"custom-flannel", []string{fmt.Sprintf("--cni=%s", filepath.Join(*testdataDir, "kube-flannel.yaml"))}, "cni", "", "kube-flannel", true},
|
||||
{"calico", []string{"--cni=calico"}, "cni", "k8s-app=calico-node", "kube-system", true},
|
||||
{"cilium", []string{"--cni=cilium"}, "cni", "k8s-app=cilium", "kube-system", true},
|
||||
|
|
@ -93,10 +92,19 @@ func TestNetworkPlugins(t *testing.T) {
|
|||
t.Skipf("Skipping the test as %s container runtimes requires CNI", ContainerRuntime())
|
||||
}
|
||||
|
||||
// (current) cilium is known to mess up the system when interfering with other network tests, so we disable it for now - probably needs updating?
|
||||
// hint: most probably the problem is in combination of: containerd + (outdated) cgroup_v1(cgroupfs) + (outdated) cilium, on systemd it should work
|
||||
// unfortunately, cilium changed how cni is deployed and does not provide manifests anymore (since v1.9) so that we can "just update" ours
|
||||
// ref: https://docs.cilium.io/en/stable/gettingstarted/k8s-install-default/
|
||||
// ref: https://docs.cilium.io/en/stable/gettingstarted/k8s-install-kubeadm/
|
||||
if tc.name == "cilium" {
|
||||
t.Skip("Skipping the test as it's interfering with other tests and is outdated")
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
MaybeParallel(t)
|
||||
|
||||
startArgs := append([]string{"start", "-p", profile, "--memory=3072", "--alsologtostderr", "--wait=true", "--wait-timeout=20m"}, tc.args...)
|
||||
startArgs := append([]string{"start", "-p", profile, "--memory=3072", "--alsologtostderr", "--wait=true", "--wait-timeout=15m"}, tc.args...)
|
||||
startArgs = append(startArgs, StartArgs()...)
|
||||
|
||||
t.Run("Start", func(t *testing.T) {
|
||||
|
|
@ -157,10 +165,6 @@ func TestNetworkPlugins(t *testing.T) {
|
|||
})
|
||||
}
|
||||
|
||||
if strings.Contains(tc.name, "weave") {
|
||||
t.Skipf("skipping remaining tests for weave, as results can be unpredictable")
|
||||
}
|
||||
|
||||
if !t.Failed() {
|
||||
t.Run("DNS", func(t *testing.T) {
|
||||
var rr *RunResult
|
||||
|
|
|
|||
Loading…
Reference in New Issue