From 87b6a4bf7bb1a76f8b2cda4cecf2c25d3b8dcd42 Mon Sep 17 00:00:00 2001 From: Bhavya Shah Date: Mon, 16 Feb 2026 16:19:26 +0530 Subject: [PATCH] docker: make storage driver selection version-aware and configurable - Determine storage driver based on Docker version - Preserve backward compatibility with overlay2 for older versions - removed docker-runtime to generate preload images, it will use containerd's Co-authored-by: Saurabh Deulkar --- hack/preload-images/generate.go | 18 +------ hack/preload-images/preload_images.go | 18 +------ pkg/minikube/cruntime/docker.go | 71 +++++++++++++++++++++++---- pkg/minikube/download/preload.go | 4 ++ 4 files changed, 68 insertions(+), 43 deletions(-) diff --git a/hack/preload-images/generate.go b/hack/preload-images/generate.go index c20325b916..a208319471 100644 --- a/hack/preload-images/generate.go +++ b/hack/preload-images/generate.go @@ -68,9 +68,8 @@ func generateTarball(kubernetesVersion, containerRuntime, tarballFilename string return fmt.Errorf("kubeadm images: %w", err) } - if containerRuntime != "docker" { // kic overlay image is only needed by containerd and cri-o https://github.com/kubernetes/minikube/issues/7428 - imgs = append(imgs, images.KindNet("")) - } + // kic overlay image is needed by containerd and cri-o https://github.com/kubernetes/minikube/issues/7428 + imgs = append(imgs, images.KindNet("")) runner := command.NewKICRunner(profile, driver.OCIBinary) @@ -137,11 +136,6 @@ func generateTarball(kubernetesVersion, containerRuntime, tarballFilename string } func verifyStorage(containerRuntime string) error { - if containerRuntime == "docker" { - if err := retry.Expo(verifyDockerStorage, 100*time.Microsecond, time.Minute*2); err != nil { - return fmt.Errorf("Docker storage type is incompatible: %w", err) - } - } if containerRuntime == "containerd" { if err := retry.Expo(verifyContainerdStorage, 100*time.Microsecond, time.Minute*2); err != nil { return fmt.Errorf("containerd storage type is incompatible: %w", err) @@ -157,10 +151,6 @@ func verifyStorage(containerRuntime string) error { // returns the right command to pull image for a specific runtime func imagePullCommand(containerRuntime, img string) *exec.Cmd { - if containerRuntime == "docker" { - return exec.Command("docker", "exec", profile, "docker", "pull", img) - } - if containerRuntime == "containerd" { return exec.Command("docker", "exec", profile, "sudo", "crictl", "pull", img) } @@ -177,10 +167,6 @@ func createImageTarball(tarballFilename, containerRuntime string) error { "./lib/minikube/binaries", } - if containerRuntime == "docker" { - dirs = append(dirs, fmt.Sprintf("./lib/docker/%s", dockerStorageDriver), "./lib/docker/image") - } - if containerRuntime == "containerd" { dirs = append(dirs, "./lib/containerd") } diff --git a/hack/preload-images/preload_images.go b/hack/preload-images/preload_images.go index 0a585d48f6..6acda0ac7b 100644 --- a/hack/preload-images/preload_images.go +++ b/hack/preload-images/preload_images.go @@ -41,10 +41,9 @@ const ( ) var ( - dockerStorageDriver = "overlay2" containerdSnapshotter = "overlayfs" podmanStorageDriver = "overlay" - containerRuntimes = []string{"docker", "containerd", "cri-o"} + containerRuntimes = []string{"containerd", "cri-o"} k8sVersions []string k8sVersion = flag.String("k8s-version", "", "desired Kubernetes version, for example `v1.17.2`") noUpload = flag.Bool("no-upload", false, "Do not upload tarballs to GCS") @@ -165,21 +164,6 @@ func makePreload(cfg preloadCfg) error { return nil } -var verifyDockerStorage = func() error { - cmd := exec.Command("docker", "exec", profile, "docker", "info", "-f", "{{.Info.Driver}}") - var stderr bytes.Buffer - cmd.Stderr = &stderr - output, err := cmd.Output() - if err != nil { - return fmt.Errorf("%v: %v:\n%s", cmd.Args, err, stderr.String()) - } - driver := strings.Trim(string(output), " \n") - if driver != dockerStorageDriver { - return fmt.Errorf("docker storage driver %s does not match requested %s", driver, dockerStorageDriver) - } - return nil -} - var verifyContainerdStorage = func() error { cmd := exec.Command("docker", "exec", profile, "sudo", "containerd", "config", "dump") var stderr bytes.Buffer diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go index efbad32695..d02f5c267c 100644 --- a/pkg/minikube/cruntime/docker.go +++ b/pkg/minikube/cruntime/docker.go @@ -132,6 +132,7 @@ func (r *Docker) Active() bool { // Enable idempotently enables Docker on a host func (r *Docker) Enable(disOthers bool, cgroupDriver string, inUserNamespace bool) error { + storageDriver := r.selectStorageDriver() if inUserNamespace { if err := CheckKernelCompatibility(r.Runner, 5, 11); err != nil { // For using overlayfs @@ -162,8 +163,9 @@ func (r *Docker) Enable(disOthers bool, cgroupDriver string, inUserNamespace boo if err := r.Init.Enable("docker.socket"); err != nil { klog.ErrorS(err, "Failed to enable", "service", "docker.socket") } + klog.Infof("Configuring docker service ...") - if err := r.configureDocker(cgroupDriver); err != nil { + if err := r.configureDocker(cgroupDriver, storageDriver); err != nil { return err } @@ -550,16 +552,22 @@ func (r *Docker) SystemLogCmd(length int) string { } type dockerDaemonConfig struct { - ExecOpts []string `json:"exec-opts"` - LogDriver string `json:"log-driver"` - LogOpts dockerDaemonLogOpts `json:"log-opts"` - StorageDriver string `json:"storage-driver"` - DefaultRuntime string `json:"default-runtime,omitempty"` - Runtimes *dockerDaemonRuntimes `json:"runtimes,omitempty"` + ExecOpts []string `json:"exec-opts"` + LogDriver string `json:"log-driver"` + LogOpts dockerDaemonLogOpts `json:"log-opts"` + StorageDriver string `json:"storage-driver"` + DefaultRuntime string `json:"default-runtime,omitempty"` + Runtimes *dockerDaemonRuntimes `json:"runtimes,omitempty"` + Features dockerDaemonFeatures `json:"features"` + ContainerdNamespace string `json:"containerd-namespace,omitempty"` } type dockerDaemonLogOpts struct { MaxSize string `json:"max-size"` } +type dockerDaemonFeatures struct { + ContainerdSnapshotter bool `json:"containerd-snapshotter"` + ContainerdMigration bool `json:"containerd-migration"` +} type dockerDaemonRuntimes struct { Nvidia struct { Path string `json:"path"` @@ -569,21 +577,40 @@ type dockerDaemonRuntimes struct { // configureDocker configures the docker daemon to use driver as cgroup manager // ref: https://docs.docker.com/engine/reference/commandline/dockerd/#options-for-the-runtime -func (r *Docker) configureDocker(driver string) error { +func (r *Docker) configureDocker(driver string, storageDriver string) error { if driver == constants.UnknownCgroupDriver { return fmt.Errorf("unable to configure docker to use unknown cgroup driver") } klog.Infof("configuring docker to use %q as cgroup driver...", driver) + + var isOverlayfs bool + if storageDriver == "overlayfs" { + isOverlayfs = true + } else { + isOverlayfs = false + } + daemonConfig := dockerDaemonConfig{ ExecOpts: []string{"native.cgroupdriver=" + driver}, LogDriver: "json-file", LogOpts: dockerDaemonLogOpts{ MaxSize: "100m", }, - StorageDriver: "overlay2", + StorageDriver: storageDriver, + Features: dockerDaemonFeatures{ + ContainerdSnapshotter: isOverlayfs, + ContainerdMigration: isOverlayfs, + }, } + if isOverlayfs { + // the docker uses containerd's preload images which are present in k8s.io namespace + daemonConfig.ContainerdNamespace = "k8s.io" + } + + klog.Infof("Configured docker with storageDriver: %s", storageDriver) + switch r.GPUs { case "all", "nvidia", "nvidia.com": assets.Addons["nvidia-device-plugin"].EnableByDefault() @@ -603,6 +630,26 @@ func (r *Docker) configureDocker(driver string) error { return r.Runner.Copy(ma) } +func (r *Docker) selectStorageDriver() string { + ver, err := r.Version() + + if err == nil { + if v, err := semver.Make(ver); err == nil && v.GTE(semver.Version{Major: 29}) { + return "overlayfs" + } + } + return "overlay2" +} + +func (r *Docker) getStorageDriver() string { + c := exec.Command("docker", "info", "-f", "{{.Info.Driver}}") + rr, err := r.Runner.RunCmd(c) + if err != nil { + return "" + } + return strings.Split(rr.Stdout.String(), "\n")[0] +} + // Preload preloads docker with k8s images: // 1. Copy over the preloaded tarball into the VM // 2. Extract the preloaded tarball to the correct directory @@ -629,6 +676,10 @@ func (r *Docker) Preload(cc config.ClusterConfig) error { klog.Infof("error saving reference store: %v", err) } + if r.getStorageDriver() == "overlayfs" { + cRuntime = "containerd" + } + tarballPath := download.TarballPath(k8sVersion, cRuntime) targetDir := "/" targetName := "preloaded.tar.lz4" @@ -821,7 +872,7 @@ func (r *Docker) restartServiceWithExpoRetry(service string) error { // try to restart service if stopped, restart until it works return retry.Expo( func() error { - if !r.Init.Active(service) { + if !r.Init.Active(service) && r.getStorageDriver() != r.selectStorageDriver() { r.Init.ResetFailed(service) r.Init.Restart(service) return fmt.Errorf("%s not running", service) diff --git a/pkg/minikube/download/preload.go b/pkg/minikube/download/preload.go index 26426ef473..3e3f79c275 100644 --- a/pkg/minikube/download/preload.go +++ b/pkg/minikube/download/preload.go @@ -79,6 +79,10 @@ func TarballName(k8sVersion, containerRuntime string) string { } else { storageDriver = "overlay2" } + + if containerRuntime == "docker" { + containerRuntime = "containerd" + } return fmt.Sprintf("preloaded-images-k8s-%s-%s-%s-%s-%s.tar.lz4", PreloadVersion, k8sVersion, containerRuntime, storageDriver, runtime.GOARCH) }