diff --git a/deploy/kicbase/Dockerfile b/deploy/kicbase/Dockerfile index c240d6ca0c..7983889b8b 100644 --- a/deploy/kicbase/Dockerfile +++ b/deploy/kicbase/Dockerfile @@ -126,9 +126,7 @@ RUN clean-install \ openssh-server \ dnsutils \ # libglib2.0-0 is required for conmon, which is required for podman - libglib2.0-0 \ - # fuse3 is required for fuse-overlayfs - fuse3 + libglib2.0-0 # install docker # use the bionic packages for arm32 @@ -139,19 +137,6 @@ RUN export ARCH=$(dpkg --print-architecture | sed 's/armhf/arm-v7/') && \ apt-key add - < docker.key && \ clean-install docker-ce docker-ce-cli containerd.io -# install fuse-overlayfs (used by rootless; apt-get version is old) -RUN curl -sSL --retry 5 --output /usr/local/bin/fuse-overlayfs https://github.com/containers/fuse-overlayfs/releases/download/${FUSE_OVERLAYFS_VERSION}/fuse-overlayfs-$(uname -m) \ - && chmod +x /usr/local/bin/fuse-overlayfs - -# install containerd-fuse-overlayfs (used by rootless) -RUN export ARCH=$(dpkg --print-architecture | sed 's/ppc64el/ppc64le/' | sed 's/armhf/arm-v7/') \ - && echo "Installing containerd-fuse-overlayfs..." \ - && export CONTAINERD_FUSE_OVERLAYFS_BASE_URL="https://github.com/containerd/fuse-overlayfs-snapshotter/releases/download/v${CONTAINERD_FUSE_OVERLAYFS_VERSION}" \ - && curl -sSL --retry 5 --output /tmp/containerd-fuse-overlayfs.tgz "${CONTAINERD_FUSE_OVERLAYFS_BASE_URL}/containerd-fuse-overlayfs-${CONTAINERD_FUSE_OVERLAYFS_VERSION}-linux-${ARCH}.tar.gz" \ - && tar -C /usr/local/bin -xzvf /tmp/containerd-fuse-overlayfs.tgz \ - && rm -rf /tmp/containerd-fuse-overlayfs.tgz -COPY deploy/kicbase/containerd-fuse-overlayfs.service /etc/systemd/system/containerd-fuse-overlayfs.service - # install buildkit RUN export ARCH=$(dpkg --print-architecture | sed 's/ppc64el/ppc64le/' | sed 's/armhf/arm-v7/') \ && echo "Installing buildkit ..." \ diff --git a/deploy/kicbase/containerd-fuse-overlayfs.service b/deploy/kicbase/containerd-fuse-overlayfs.service deleted file mode 100644 index a3d12bd567..0000000000 --- a/deploy/kicbase/containerd-fuse-overlayfs.service +++ /dev/null @@ -1,13 +0,0 @@ -# From https://github.com/kubernetes-sigs/kind/blob/0d3780371091b2dc9ff6eea1b6054f14ff5d970a/images/base/files/etc/systemd/system/containerd-fuse-overlayfs.service -[Unit] -Description=containerd fuse-overlayfs snapshotter -PartOf=containerd.service - -[Service] -ExecStart=/usr/local/bin/containerd-fuse-overlayfs-grpc /run/containerd-fuse-overlayfs.sock /var/lib/containerd-fuse-overlayfs -Type=notify -Restart=always -RestartSec=1 - -[Install] -WantedBy=multi-user.target diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go index 5f5a84ce8d..93b3bd8e24 100644 --- a/pkg/drivers/kic/oci/oci.go +++ b/pkg/drivers/kic/oci/oci.go @@ -162,9 +162,6 @@ func CreateContainerNode(p CreateParams) error { // including some ones docker would otherwise do by default. // for now this is what we want. in the future we may revisit this. "--privileged", - // enable /dev/fuse explicitly for fuse-overlayfs - // (Rootless Docker does not automatically mount /dev/fuse with --privileged) - "--device", "/dev/fuse", "--security-opt", "seccomp=unconfined", // ignore seccomp "--tmpfs", "/tmp", // various things depend on working /tmp "--tmpfs", "/run", // systemd wants a writable /run diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go index e8bb1567fd..2b96ba21bd 100644 --- a/pkg/minikube/cruntime/containerd.go +++ b/pkg/minikube/cruntime/containerd.go @@ -70,12 +70,6 @@ oom_score = 0 [cgroup] path = "" -[proxy_plugins] -# fuse-overlayfs is used for rootless -[proxy_plugins."fuse-overlayfs"] - type = "snapshot" - address = "/run/containerd-fuse-overlayfs.sock" - [plugins] [plugins."io.containerd.monitor.v1.cgroups"] no_prometheus = false @@ -208,9 +202,6 @@ func generateContainerdConfig(cr CommandRunner, imageRepository string, kv semve } pauseImage := images.Pause(kv, imageRepository) snapshotter := "overlayfs" - if inUserNamespace { - snapshotter = "fuse-overlayfs" - } opts := struct { PodInfraContainerImage string SystemdCgroup bool @@ -239,6 +230,16 @@ func generateContainerdConfig(cr CommandRunner, imageRepository string, kv semve // Enable idempotently enables containerd on a host func (r *Containerd) Enable(disOthers, forceSystemd, inUserNamespace bool) error { + if inUserNamespace { + if err := CheckKernelCompatibility(r.Runner, 5, 11); err != nil { + // For using overlayfs + return fmt.Errorf("kernel >= 5.11 is required for rootless mode: %w", err) + } + if err := CheckKernelCompatibility(r.Runner, 5, 13); err != nil { + // For avoiding SELinux error with overlayfs + klog.Warningf("kernel >= 5.13 is recommended for rootless mode %v", err) + } + } if disOthers { if err := disableOthers(r, r.Runner); err != nil { klog.Warningf("disableOthers: %v", err) @@ -254,12 +255,6 @@ func (r *Containerd) Enable(disOthers, forceSystemd, inUserNamespace bool) error return err } - if inUserNamespace { - if err := r.Init.EnableNow("containerd-fuse-overlayfs"); err != nil { - return err - } - } - // Otherwise, containerd will fail API requests with 'Unimplemented' return r.Init.Restart("containerd") } diff --git a/pkg/minikube/cruntime/crio.go b/pkg/minikube/cruntime/crio.go index 9cbcc5238f..86af57ac97 100644 --- a/pkg/minikube/cruntime/crio.go +++ b/pkg/minikube/cruntime/crio.go @@ -150,18 +150,13 @@ func enableIPForwarding(cr CommandRunner) error { // enableRootless enables configurations for running CRI-O in Rootless Docker. // // 1. Create /etc/systemd/system/crio.service.d/10-rootless.conf to set _CRIO_ROOTLESS=1 -// 2. Create /etc/crio/crio.conf.d/10-fuse-overlayfs.conf to enable fuse-overlayfs -// 3. Reload systemd +// 2. Reload systemd // // See https://kubernetes.io/docs/tasks/administer-cluster/kubelet-in-userns/#configuring-cri func (r *CRIO) enableRootless() error { files := map[string]string{ "/etc/systemd/system/crio.service.d/10-rootless.conf": `[Service] Environment="_CRIO_ROOTLESS=1" -`, - "/etc/crio/crio.conf.d/10-fuse-overlayfs.conf": `[crio] -storage_driver = "overlay" -storage_option = ["overlay.mount_program=/usr/local/bin/fuse-overlayfs"] `, } for target, content := range files { @@ -211,6 +206,14 @@ func (r *CRIO) Enable(disOthers, forceSystemd, inUserNamespace bool) error { } } if inUserNamespace { + if err := CheckKernelCompatibility(r.Runner, 5, 11); err != nil { + // For using overlayfs + return fmt.Errorf("kernel >= 5.11 is required for rootless mode: %w", err) + } + if err := CheckKernelCompatibility(r.Runner, 5, 13); err != nil { + // For avoiding SELinux error with overlayfs + klog.Warningf("kernel >= 5.13 is recommended for rootless mode %v", err) + } if err := r.enableRootless(); err != nil { return err } diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go index 514ad97670..76833ed82d 100644 --- a/pkg/minikube/cruntime/cruntime.go +++ b/pkg/minikube/cruntime/cruntime.go @@ -20,6 +20,7 @@ package cruntime import ( "fmt" "os/exec" + "strings" "github.com/blang/semver/v4" "github.com/pkg/errors" @@ -316,3 +317,22 @@ func CheckCompatibility(cr Manager) error { } return compatibleWithVersion(cr.Name(), v) } + +// CheckKernelCompatibility returns an error when the kernel is older than the specified version. +func CheckKernelCompatibility(cr CommandRunner, major, minor int) error { + expected := fmt.Sprintf("%d.%d", major, minor) + unameRes, err := cr.RunCmd(exec.Command("uname", "-r")) + if err != nil { + return err + } + actual := strings.TrimSpace(unameRes.Stdout.String()) + sortRes, err := cr.RunCmd(exec.Command("sh", "-euc", fmt.Sprintf(`(echo %s; echo %s) | sort -V | head -n1`, actual, expected))) + if err != nil { + return err + } + comparison := strings.TrimSpace(sortRes.Stdout.String()) + if comparison != expected { + return NewErrServiceVersion("kernel", expected, actual) + } + return nil +} diff --git a/site/content/en/docs/drivers/docker.md b/site/content/en/docs/drivers/docker.md index 25121e945d..8f312a983a 100644 --- a/site/content/en/docs/drivers/docker.md +++ b/site/content/en/docs/drivers/docker.md @@ -34,6 +34,7 @@ minikube config set driver docker ## Requirements - Docker 20.10 or higher, see https://rootlesscontaine.rs/getting-started/docker/ - Cgroup v2 delegation, see https://rootlesscontaine.rs/getting-started/common/cgroup2/ +- Kernel 5.11 or later (5.13 or later is recommended when SELinux is enabled), see https://rootlesscontaine.rs/how-it-works/overlayfs/ ## Usage