Merge pull request #15336 from prezha/fix-kubelet-localStorageCapacityIsolation-option

fix kubelet localStorageCapacityIsolation option

cmd/minikube/cmd/start_flags.go

@@ -620,9 +620,18 @@ func generateNewConfigFromFlags(cmd *cobra.Command, k8sVersion string, rtime str
 			}
 			out.Styled(style.Notice, "Using {{.driver_name}} driver with root privileges", out.V{"driver_name": driver.FullName(drvName)})
 		}
+		// for btrfs: if k8s < v1.25.0-beta.0 set kubelet's LocalStorageCapacityIsolation feature gate flag to false,
+		// and if k8s >= v1.25.0-beta.0 (when it went ga and removed as feature gate), set kubelet's localStorageCapacityIsolation option (via kubeadm config) to false.
+		// ref: https://github.com/kubernetes/minikube/issues/14728#issue-1327885840
 		if si.StorageDriver == "btrfs" {
-			klog.Info("auto-setting LocalStorageCapacityIsolation to false because using btrfs storage driver")
-			cc.KubernetesConfig.FeatureGates = addFeatureGate(cc.KubernetesConfig.FeatureGates, "LocalStorageCapacityIsolation=false")
+			if semver.MustParse(strings.TrimPrefix(k8sVersion, version.VersionPrefix)).LT(semver.MustParse("1.25.0-beta.0")) {
+				klog.Info("auto-setting LocalStorageCapacityIsolation to false because using btrfs storage driver")
+				cc.KubernetesConfig.FeatureGates = addFeatureGate(cc.KubernetesConfig.FeatureGates, "LocalStorageCapacityIsolation=false")
+			} else if !cc.KubernetesConfig.ExtraOptions.Exists("kubelet.localStorageCapacityIsolation=false") {
+				if err := cc.KubernetesConfig.ExtraOptions.Set("kubelet.localStorageCapacityIsolation=false"); err != nil {
+					exit.Error(reason.InternalConfigSet, "failed to set extra option", err)
+				}
+			}
 		}
 		if runtime.GOOS == "linux" && si.DockerOS == "Docker Desktop" {
 			out.WarningT("For an improved experience it's recommended to use Docker Engine instead of Docker Desktop.\nDocker Engine installation instructions: https://docs.docker.com/engine/install/#server")

pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta3.go

@@ -78,6 +78,9 @@ authentication:
   x509:
     clientCAFile: {{.ClientCAFile}}
 cgroupDriver: {{.CgroupDriver}}
+{{- range $key, $val := .KubeletConfigOpts}}
+{{$key}}: {{$val}}
+{{- end}}
 clusterDomain: "{{if .DNSDomain}}{{.DNSDomain}}{{else}}cluster.local{{end}}"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100

pkg/minikube/bootstrapper/bsutil/kubeadm.go

@@ -109,6 +109,7 @@ func GenerateKubeadmYAML(cc config.ClusterConfig, n config.Node, r cruntime.Mana
 		ControlPlaneAddress        string
 		KubeProxyOptions           map[string]string
 		ResolvConfSearchRegression bool
+		KubeletConfigOpts          map[string]string
 	}{
 		CertDir:           vmpath.GuestKubernetesCertsDir,
 		ServiceCIDR:       constants.DefaultServiceCIDR,
@@ -133,6 +134,7 @@ func GenerateKubeadmYAML(cc config.ClusterConfig, n config.Node, r cruntime.Mana
 		ControlPlaneAddress:        constants.ControlPlaneAlias,
 		KubeProxyOptions:           createKubeProxyOptions(k8s.ExtraOptions),
 		ResolvConfSearchRegression: HasResolvConfSearchRegression(k8s.KubernetesVersion),
+		KubeletConfigOpts:          kubeletConfigOpts(k8s.ExtraOptions),
 	}
 
 	if k8s.ServiceCIDR != "" {
@@ -215,3 +217,17 @@ func HasResolvConfSearchRegression(k8sVersion string) bool {
 	}
 	return versionSemver.EQ(semver.Version{Major: 1, Minor: 25})
 }
+
+// kubeletConfigOpts extracts only those kubelet extra options allowed by kubeletConfigParams.
+func kubeletConfigOpts(extraOpts config.ExtraOptionSlice) map[string]string {
+	args := map[string]string{}
+	for _, eo := range extraOpts {
+		if eo.Component != Kubelet {
+			continue
+		}
+		if config.ContainsParam(kubeletConfigParams, eo.Key) {
+			args[eo.Key] = eo.Value
+		}
+	}
+	return args
+}

pkg/minikube/bootstrapper/bsutil/kubeadm_test.go

@@ -307,3 +307,27 @@ func TestEtcdExtraArgs(t *testing.T) {
 		t.Errorf("machines mismatch (-want +got):\n%s", diff)
 	}
 }
+
+func TestKubeletConfig(t *testing.T) {
+	expected := map[string]string{
+		"localStorageCapacityIsolation": "false",
+	}
+	extraOpts := append(getExtraOpts(), []config.ExtraOption{
+		{
+			Component: Kubelet,
+			Key:       "unsupported-config-option",
+			Value:     "any",
+		}, {
+			Component: Kubelet,
+			Key:       "localStorageCapacityIsolation",
+			Value:     "false",
+		}, {
+			Component: Kubelet,
+			Key:       "kubelet.cgroups-per-qos",
+			Value:     "false",
+		}}...)
+	actual := kubeletConfigOpts(extraOpts)
+	if diff := cmp.Diff(expected, actual); diff != "" {
+		t.Errorf("machines mismatch (-want +got):\n%s", diff)
+	}
+}

pkg/minikube/bootstrapper/bsutil/kubelet.go

@@ -36,6 +36,11 @@ import (
 	"k8s.io/minikube/pkg/util"
 )
 
+// kubeletConfigParams are the only allowed kubelet parameters for kubeadmin config file and not to be used as kubelet flags
+var kubeletConfigParams = []string{
+	"localStorageCapacityIsolation",
+}
+
 func extraKubeletOpts(mc config.ClusterConfig, nc config.Node, r cruntime.Manager) (map[string]string, error) {
 	k8s := mc.KubernetesConfig
 	version, err := util.ParseKubernetesVersion(k8s.KubernetesVersion)
@@ -95,6 +100,11 @@ func extraKubeletOpts(mc config.ClusterConfig, nc config.Node, r cruntime.Manage
 		extraOpts["feature-gates"] = kubeletFeatureArgs
 	}
 
+	// filter out non-flag extra kubelet config options
+	for _, opt := range kubeletConfigParams {
+		delete(extraOpts, opt)
+	}
+
 	return extraOpts, nil
 }