docker: make storage driver selection version-aware and configurable

- Determine storage driver based on Docker version
- Preserve backward compatibility with overlay2 for older versions
- removed docker-runtime to generate preload images, it will use containerd's

Co-authored-by: Saurabh Deulkar <saurabhdeulkar11@gmail.com>

hack/preload-images/generate.go

@@ -68,9 +68,8 @@ func generateTarball(kubernetesVersion, containerRuntime, tarballFilename string
 		return fmt.Errorf("kubeadm images: %w", err)
 	}
 
-	if containerRuntime != "docker" { // kic overlay image is only needed by containerd and cri-o https://github.com/kubernetes/minikube/issues/7428
-		imgs = append(imgs, images.KindNet(""))
-	}
+	// kic overlay image is needed by containerd and cri-o https://github.com/kubernetes/minikube/issues/7428
+	imgs = append(imgs, images.KindNet(""))
 
 	runner := command.NewKICRunner(profile, driver.OCIBinary)
 
@@ -137,11 +136,6 @@ func generateTarball(kubernetesVersion, containerRuntime, tarballFilename string
 }
 
 func verifyStorage(containerRuntime string) error {
-	if containerRuntime == "docker" {
-		if err := retry.Expo(verifyDockerStorage, 100*time.Microsecond, time.Minute*2); err != nil {
-			return fmt.Errorf("Docker storage type is incompatible: %w", err)
-		}
-	}
 	if containerRuntime == "containerd" {
 		if err := retry.Expo(verifyContainerdStorage, 100*time.Microsecond, time.Minute*2); err != nil {
 			return fmt.Errorf("containerd storage type is incompatible: %w", err)
@@ -157,10 +151,6 @@ func verifyStorage(containerRuntime string) error {
 
 // returns the right command to pull image for a specific runtime
 func imagePullCommand(containerRuntime, img string) *exec.Cmd {
-	if containerRuntime == "docker" {
-		return exec.Command("docker", "exec", profile, "docker", "pull", img)
-	}
-
 	if containerRuntime == "containerd" {
 		return exec.Command("docker", "exec", profile, "sudo", "crictl", "pull", img)
 	}
@@ -177,10 +167,6 @@ func createImageTarball(tarballFilename, containerRuntime string) error {
 		"./lib/minikube/binaries",
 	}
 
-	if containerRuntime == "docker" {
-		dirs = append(dirs, fmt.Sprintf("./lib/docker/%s", dockerStorageDriver), "./lib/docker/image")
-	}
-
 	if containerRuntime == "containerd" {
 		dirs = append(dirs, "./lib/containerd")
 	}

hack/preload-images/preload_images.go

@@ -41,10 +41,9 @@ const (
 )
 
 var (
-	dockerStorageDriver   = "overlay2"
 	containerdSnapshotter = "overlayfs"
 	podmanStorageDriver   = "overlay"
-	containerRuntimes     = []string{"docker", "containerd", "cri-o"}
+	containerRuntimes     = []string{"containerd", "cri-o"}
 	k8sVersions           []string
 	k8sVersion            = flag.String("k8s-version", "", "desired Kubernetes version, for example `v1.17.2`")
 	noUpload              = flag.Bool("no-upload", false, "Do not upload tarballs to GCS")
@@ -165,21 +164,6 @@ func makePreload(cfg preloadCfg) error {
 	return nil
 }
 
-var verifyDockerStorage = func() error {
-	cmd := exec.Command("docker", "exec", profile, "docker", "info", "-f", "{{.Info.Driver}}")
-	var stderr bytes.Buffer
-	cmd.Stderr = &stderr
-	output, err := cmd.Output()
-	if err != nil {
-		return fmt.Errorf("%v: %v:\n%s", cmd.Args, err, stderr.String())
-	}
-	driver := strings.Trim(string(output), " \n")
-	if driver != dockerStorageDriver {
-		return fmt.Errorf("docker storage driver %s does not match requested %s", driver, dockerStorageDriver)
-	}
-	return nil
-}
-
 var verifyContainerdStorage = func() error {
 	cmd := exec.Command("docker", "exec", profile, "sudo", "containerd", "config", "dump")
 	var stderr bytes.Buffer

pkg/minikube/cruntime/docker.go

@@ -132,6 +132,7 @@ func (r *Docker) Active() bool {
 
 // Enable idempotently enables Docker on a host
 func (r *Docker) Enable(disOthers bool, cgroupDriver string, inUserNamespace bool) error {
+	storageDriver := r.selectStorageDriver()
 	if inUserNamespace {
 		if err := CheckKernelCompatibility(r.Runner, 5, 11); err != nil {
 			// For using overlayfs
@@ -162,8 +163,9 @@ func (r *Docker) Enable(disOthers bool, cgroupDriver string, inUserNamespace boo
 	if err := r.Init.Enable("docker.socket"); err != nil {
 		klog.ErrorS(err, "Failed to enable", "service", "docker.socket")
 	}
+	klog.Infof("Configuring docker service ...")
 
-	if err := r.configureDocker(cgroupDriver); err != nil {
+	if err := r.configureDocker(cgroupDriver, storageDriver); err != nil {
 		return err
 	}
 
@@ -550,16 +552,22 @@ func (r *Docker) SystemLogCmd(length int) string {
 }
 
 type dockerDaemonConfig struct {
-	ExecOpts       []string              `json:"exec-opts"`
-	LogDriver      string                `json:"log-driver"`
-	LogOpts        dockerDaemonLogOpts   `json:"log-opts"`
-	StorageDriver  string                `json:"storage-driver"`
-	DefaultRuntime string                `json:"default-runtime,omitempty"`
-	Runtimes       *dockerDaemonRuntimes `json:"runtimes,omitempty"`
+	ExecOpts            []string              `json:"exec-opts"`
+	LogDriver           string                `json:"log-driver"`
+	LogOpts             dockerDaemonLogOpts   `json:"log-opts"`
+	StorageDriver       string                `json:"storage-driver"`
+	DefaultRuntime      string                `json:"default-runtime,omitempty"`
+	Runtimes            *dockerDaemonRuntimes `json:"runtimes,omitempty"`
+	Features            dockerDaemonFeatures  `json:"features"`
+	ContainerdNamespace string                `json:"containerd-namespace,omitempty"`
 }
 type dockerDaemonLogOpts struct {
 	MaxSize string `json:"max-size"`
 }
+type dockerDaemonFeatures struct {
+	ContainerdSnapshotter bool `json:"containerd-snapshotter"`
+	ContainerdMigration   bool `json:"containerd-migration"`
+}
 type dockerDaemonRuntimes struct {
 	Nvidia struct {
 		Path        string        `json:"path"`
@@ -569,21 +577,40 @@ type dockerDaemonRuntimes struct {
 
 // configureDocker configures the docker daemon to use driver as cgroup manager
 // ref: https://docs.docker.com/engine/reference/commandline/dockerd/#options-for-the-runtime
-func (r *Docker) configureDocker(driver string) error {
+func (r *Docker) configureDocker(driver string, storageDriver string) error {
 	if driver == constants.UnknownCgroupDriver {
 		return fmt.Errorf("unable to configure docker to use unknown cgroup driver")
 	}
 
 	klog.Infof("configuring docker to use %q as cgroup driver...", driver)
+
+	var isOverlayfs bool
+	if storageDriver == "overlayfs" {
+		isOverlayfs = true
+	} else {
+		isOverlayfs = false
+	}
+
 	daemonConfig := dockerDaemonConfig{
 		ExecOpts:  []string{"native.cgroupdriver=" + driver},
 		LogDriver: "json-file",
 		LogOpts: dockerDaemonLogOpts{
 			MaxSize: "100m",
 		},
-		StorageDriver: "overlay2",
+		StorageDriver: storageDriver,
+		Features: dockerDaemonFeatures{
+			ContainerdSnapshotter: isOverlayfs,
+			ContainerdMigration:   isOverlayfs,
+		},
 	}
 
+	if isOverlayfs {
+		// the docker uses containerd's preload images which are present in k8s.io namespace
+		daemonConfig.ContainerdNamespace = "k8s.io"
+	}
+
+	klog.Infof("Configured docker with storageDriver: %s", storageDriver)
+
 	switch r.GPUs {
 	case "all", "nvidia", "nvidia.com":
 		assets.Addons["nvidia-device-plugin"].EnableByDefault()
@@ -603,6 +630,26 @@ func (r *Docker) configureDocker(driver string) error {
 	return r.Runner.Copy(ma)
 }
 
+func (r *Docker) selectStorageDriver() string {
+	ver, err := r.Version()
+
+	if err == nil {
+		if v, err := semver.Make(ver); err == nil && v.GTE(semver.Version{Major: 29}) {
+			return "overlayfs"
+		}
+	}
+	return "overlay2"
+}
+
+func (r *Docker) getStorageDriver() string {
+	c := exec.Command("docker", "info", "-f", "{{.Info.Driver}}")
+	rr, err := r.Runner.RunCmd(c)
+	if err != nil {
+		return ""
+	}
+	return strings.Split(rr.Stdout.String(), "\n")[0]
+}
+
 // Preload preloads docker with k8s images:
 // 1. Copy over the preloaded tarball into the VM
 // 2. Extract the preloaded tarball to the correct directory
@@ -629,6 +676,10 @@ func (r *Docker) Preload(cc config.ClusterConfig) error {
 		klog.Infof("error saving reference store: %v", err)
 	}
 
+	if r.getStorageDriver() == "overlayfs" {
+		cRuntime = "containerd"
+	}
+
 	tarballPath := download.TarballPath(k8sVersion, cRuntime)
 	targetDir := "/"
 	targetName := "preloaded.tar.lz4"
@@ -821,7 +872,7 @@ func (r *Docker) restartServiceWithExpoRetry(service string) error {
 	// try to restart service if stopped, restart until it works
 	return retry.Expo(
 		func() error {
-			if !r.Init.Active(service) {
+			if !r.Init.Active(service) && r.getStorageDriver() != r.selectStorageDriver() {
 				r.Init.ResetFailed(service)
 				r.Init.Restart(service)
 				return fmt.Errorf("%s not running", service)

pkg/minikube/download/preload.go

@@ -79,6 +79,10 @@ func TarballName(k8sVersion, containerRuntime string) string {
 	} else {
 		storageDriver = "overlay2"
 	}
+
+	if containerRuntime == "docker" {
+		containerRuntime = "containerd"
+	}
 	return fmt.Sprintf("preloaded-images-k8s-%s-%s-%s-%s-%s.tar.lz4", PreloadVersion, k8sVersion, containerRuntime, storageDriver, runtime.GOARCH)
 }