Merge pull request #5032 from tstromberg/persistent

Move program data files onto persistent storage

pkg/drivers/none/none.go

@@ -38,8 +38,8 @@ const driverName = constants.DriverNone
 
 // cleanupPaths are paths to be removed by cleanup, and are used by both kubeadm and minikube.
 var cleanupPaths = []string{
-	"/data/minikube",
-	"/etc/kubernetes/manifests",
+	constants.GuestEphemeralDir,
+	constants.GuestManifestsDir,
 	"/var/lib/minikube",
 }
 
@@ -233,8 +233,8 @@ func stopKubelet(exec command.Runner) error {
 		if errStatus != nil {
 			glog.Errorf("temporary error: for %q : %v", cmdCheck, errStatus)
 		}
-		if !strings.Contains(out.String(), "dead") {
-			return fmt.Errorf("expected to kubelet to be dead but it got : %q", out)
+		if !strings.Contains(out.String(), "dead") && !strings.Contains(out.String(), "failed") {
+			return fmt.Errorf("unexpected kubelet state: %q", out)
 		}
 		return nil
 	}

pkg/minikube/assets/addons.go

@@ -70,7 +70,7 @@ var Addons = map[string]*Addon{
 	"addon-manager": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/addon-manager.yaml.tmpl",
-			"/etc/kubernetes/manifests/",
+			constants.GuestManifestsDir,
 			"addon-manager.yaml.tmpl",
 			"0640",
 			true),
@@ -78,13 +78,13 @@ var Addons = map[string]*Addon{
 	"dashboard": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/dashboard/dashboard-dp.yaml.tmpl",
-			constants.AddonsPath,
+			constants.GuestAddonsDir,
 			"dashboard-dp.yaml",
 			"0640",
 			true),
 		MustBinAsset(
 			"deploy/addons/dashboard/dashboard-svc.yaml.tmpl",
-			constants.AddonsPath,
+			constants.GuestAddonsDir,
 			"dashboard-svc.yaml",
 			"0640",
 			false),
@@ -92,7 +92,7 @@ var Addons = map[string]*Addon{
 	"default-storageclass": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/storageclass/storageclass.yaml.tmpl",
-			constants.AddonsPath,
+			constants.GuestAddonsDir,
 			"storageclass.yaml",
 			"0640",
 			false),
@@ -100,7 +100,7 @@ var Addons = map[string]*Addon{
 	"storage-provisioner": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/storage-provisioner/storage-provisioner.yaml.tmpl",
-			constants.AddonsPath,
+			constants.GuestAddonsDir,
 			"storage-provisioner.yaml",
 			"0640",
 			true),
@@ -108,25 +108,25 @@ var Addons = map[string]*Addon{
 	"storage-provisioner-gluster": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/storage-provisioner-gluster/storage-gluster-ns.yaml.tmpl",
-			constants.AddonsPath,
+			constants.GuestAddonsDir,
 			"storage-gluster-ns.yaml",
 			"0640",
 			false),
 		MustBinAsset(
 			"deploy/addons/storage-provisioner-gluster/glusterfs-daemonset.yaml.tmpl",
-			constants.AddonsPath,
+			constants.GuestAddonsDir,
 			"glusterfs-daemonset.yaml",
 			"0640",
 			false),
 		MustBinAsset(
 			"deploy/addons/storage-provisioner-gluster/heketi-deployment.yaml.tmpl",
-			constants.AddonsPath,
+			constants.GuestAddonsDir,
 			"heketi-deployment.yaml",
 			"0640",
 			false),
 		MustBinAsset(
 			"deploy/addons/storage-provisioner-gluster/storage-provisioner-glusterfile.yaml.tmpl",
-			constants.AddonsPath,
+			constants.GuestAddonsDir,
 			"storage-privisioner-glusterfile.yaml",
 			"0640",
 			false),
@@ -134,31 +134,31 @@ var Addons = map[string]*Addon{
 	"heapster": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/heapster/influx-grafana-rc.yaml.tmpl",
-			constants.AddonsPath,
+			constants.GuestAddonsDir,
 			"influxGrafana-rc.yaml",
 			"0640",
 			true),
 		MustBinAsset(
 			"deploy/addons/heapster/grafana-svc.yaml.tmpl",
-			constants.AddonsPath,
+			constants.GuestAddonsDir,
 			"grafana-svc.yaml",
 			"0640",
 			false),
 		MustBinAsset(
 			"deploy/addons/heapster/influxdb-svc.yaml.tmpl",
-			constants.AddonsPath,
+			constants.GuestAddonsDir,
 			"influxdb-svc.yaml",
 			"0640",
 			false),
 		MustBinAsset(
 			"deploy/addons/heapster/heapster-rc.yaml.tmpl",
-			constants.AddonsPath,
+			constants.GuestAddonsDir,
 			"heapster-rc.yaml",
 			"0640",
 			true),
 		MustBinAsset(
 			"deploy/addons/heapster/heapster-svc.yaml.tmpl",
-			constants.AddonsPath,
+			constants.GuestAddonsDir,
 			"heapster-svc.yaml",
 			"0640",
 			false),
@@ -166,37 +166,37 @@ var Addons = map[string]*Addon{
 	"efk": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/efk/elasticsearch-rc.yaml.tmpl",
-			constants.AddonsPath,
+			constants.GuestAddonsDir,
 			"elasticsearch-rc.yaml",
 			"0640",
 			true),
 		MustBinAsset(
 			"deploy/addons/efk/elasticsearch-svc.yaml.tmpl",
-			constants.AddonsPath,
+			constants.GuestAddonsDir,
 			"elasticsearch-svc.yaml",
 			"0640",
 			false),
 		MustBinAsset(
 			"deploy/addons/efk/fluentd-es-rc.yaml.tmpl",
-			constants.AddonsPath,
+			constants.GuestAddonsDir,
 			"fluentd-es-rc.yaml",
 			"0640",
 			true),
 		MustBinAsset(
 			"deploy/addons/efk/fluentd-es-configmap.yaml.tmpl",
-			constants.AddonsPath,
+			constants.GuestAddonsDir,
 			"fluentd-es-configmap.yaml",
 			"0640",
 			false),
 		MustBinAsset(
 			"deploy/addons/efk/kibana-rc.yaml.tmpl",
-			constants.AddonsPath,
+			constants.GuestAddonsDir,
 			"kibana-rc.yaml",
 			"0640",
 			false),
 		MustBinAsset(
 			"deploy/addons/efk/kibana-svc.yaml.tmpl",
-			constants.AddonsPath,
+			constants.GuestAddonsDir,
 			"kibana-svc.yaml",
 			"0640",
 			false),
@@ -204,19 +204,19 @@ var Addons = map[string]*Addon{
 	"ingress": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/ingress/ingress-configmap.yaml.tmpl",
-			constants.AddonsPath,
+			constants.GuestAddonsDir,
 			"ingress-configmap.yaml",
 			"0640",
 			false),
 		MustBinAsset(
 			"deploy/addons/ingress/ingress-rbac.yaml.tmpl",
-			constants.AddonsPath,
+			constants.GuestAddonsDir,
 			"ingress-rbac.yaml",
 			"0640",
 			false),
 		MustBinAsset(
 			"deploy/addons/ingress/ingress-dp.yaml.tmpl",
-			constants.AddonsPath,
+			constants.GuestAddonsDir,
 			"ingress-dp.yaml",
 			"0640",
 			true),
@@ -224,19 +224,19 @@ var Addons = map[string]*Addon{
 	"metrics-server": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/metrics-server/metrics-apiservice.yaml.tmpl",
-			constants.AddonsPath,
+			constants.GuestAddonsDir,
 			"metrics-apiservice.yaml",
 			"0640",
 			false),
 		MustBinAsset(
 			"deploy/addons/metrics-server/metrics-server-deployment.yaml.tmpl",
-			constants.AddonsPath,
+			constants.GuestAddonsDir,
 			"metrics-server-deployment.yaml",
 			"0640",
 			true),
 		MustBinAsset(
 			"deploy/addons/metrics-server/metrics-server-service.yaml.tmpl",
-			constants.AddonsPath,
+			constants.GuestAddonsDir,
 			"metrics-server-service.yaml",
 			"0640",
 			false),
@@ -244,19 +244,19 @@ var Addons = map[string]*Addon{
 	"registry": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/registry/registry-rc.yaml.tmpl",
-			constants.AddonsPath,
+			constants.GuestAddonsDir,
 			"registry-rc.yaml",
 			"0640",
 			false),
 		MustBinAsset(
 			"deploy/addons/registry/registry-svc.yaml.tmpl",
-			constants.AddonsPath,
+			constants.GuestAddonsDir,
 			"registry-svc.yaml",
 			"0640",
 			false),
 		MustBinAsset(
 			"deploy/addons/registry/registry-proxy.yaml.tmpl",
-			constants.AddonsPath,
+			constants.GuestAddonsDir,
 			"registry-proxy.yaml",
 			"0640",
 			false),
@@ -264,7 +264,7 @@ var Addons = map[string]*Addon{
 	"registry-creds": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/registry-creds/registry-creds-rc.yaml.tmpl",
-			constants.AddonsPath,
+			constants.GuestAddonsDir,
 			"registry-creds-rc.yaml",
 			"0640",
 			false),
@@ -272,7 +272,7 @@ var Addons = map[string]*Addon{
 	"freshpod": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/freshpod/freshpod-rc.yaml.tmpl",
-			constants.AddonsPath,
+			constants.GuestAddonsDir,
 			"freshpod-rc.yaml",
 			"0640",
 			true),
@@ -280,7 +280,7 @@ var Addons = map[string]*Addon{
 	"nvidia-driver-installer": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/gpu/nvidia-driver-installer.yaml.tmpl",
-			constants.AddonsPath,
+			constants.GuestAddonsDir,
 			"nvidia-driver-installer.yaml",
 			"0640",
 			true),
@@ -288,7 +288,7 @@ var Addons = map[string]*Addon{
 	"nvidia-gpu-device-plugin": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/gpu/nvidia-gpu-device-plugin.yaml.tmpl",
-			constants.AddonsPath,
+			constants.GuestAddonsDir,
 			"nvidia-gpu-device-plugin.yaml",
 			"0640",
 			true),
@@ -296,13 +296,13 @@ var Addons = map[string]*Addon{
 	"logviewer": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/logviewer/logviewer-dp-and-svc.yaml.tmpl",
-			constants.AddonsPath,
+			constants.GuestAddonsDir,
 			"logviewer-dp-and-svc.yaml",
 			"0640",
 			false),
 		MustBinAsset(
 			"deploy/addons/logviewer/logviewer-rbac.yaml.tmpl",
-			constants.AddonsPath,
+			constants.GuestAddonsDir,
 			"logviewer-rbac.yaml",
 			"0640",
 			false),
@@ -310,7 +310,7 @@ var Addons = map[string]*Addon{
 	"gvisor": NewAddon([]*BinAsset{
 		MustBinAsset(
 			"deploy/addons/gvisor/gvisor-pod.yaml.tmpl",
-			constants.AddonsPath,
+			constants.GuestAddonsDir,
 			"gvisor-pod.yaml",
 			"0640",
 			true),
@@ -332,7 +332,7 @@ var Addons = map[string]*Addon{
 // AddMinikubeDirAssets adds all addons and files to the list
 // of files to be copied to the vm.
 func AddMinikubeDirAssets(assets *[]CopyableFile) error {
-	if err := addMinikubeDirToAssets(constants.MakeMiniPath("addons"), constants.AddonsPath, assets); err != nil {
+	if err := addMinikubeDirToAssets(constants.MakeMiniPath("addons"), constants.GuestAddonsDir, assets); err != nil {
 		return errors.Wrap(err, "adding addons folder to assets")
 	}
 	if err := addMinikubeDirToAssets(constants.MakeMiniPath("files"), "", assets); err != nil {

pkg/minikube/assets/addons_test.go

@@ -56,18 +56,18 @@ func TestAddMinikubeDirAssets(t *testing.T) {
 			}{
 				{
 					relativePath: "/dir1/file1.txt",
-					expectedPath: constants.AddonsPath,
+					expectedPath: constants.GuestAddonsDir,
 				},
 				{
 					relativePath: "/dir1/file2.txt",
-					expectedPath: constants.AddonsPath,
+					expectedPath: constants.GuestAddonsDir,
 				},
 				{
 					relativePath: "/dir2/file1.txt",
-					expectedPath: constants.AddonsPath,
+					expectedPath: constants.GuestAddonsDir,
 				},
 			},
-			vmPath: constants.AddonsPath,
+			vmPath: constants.GuestAddonsDir,
 		},
 		{
 			description: "absolute path assets",

pkg/minikube/assets/vm_assets.go

@@ -23,6 +23,7 @@ import (
 	"io"
 	"os"
 	"path"
+	"path/filepath"
 
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
@@ -78,14 +79,15 @@ func NewMemoryAssetTarget(d []byte, targetPath, permissions string) *MemoryAsset
 }
 
 // NewFileAsset creates a new FileAsset
-func NewFileAsset(path, targetDir, targetName, permissions string) (*FileAsset, error) {
-	f, err := os.Open(path)
+func NewFileAsset(src, targetDir, targetName, permissions string) (*FileAsset, error) {
+	glog.Infof("NewFileAsset: %s -> %s", src, filepath.Join(targetDir, targetName))
+	f, err := os.Open(src)
 	if err != nil {
-		return nil, errors.Wrapf(err, "Error opening file asset: %s", path)
+		return nil, errors.Wrapf(err, "Error opening file asset: %s", src)
 	}
 	return &FileAsset{
 		BaseAsset: BaseAsset{
-			AssetName:   path,
+			AssetName:   src,
 			TargetDir:   targetDir,
 			TargetName:  targetName,
 			Permissions: permissions,

pkg/minikube/bootstrapper/bootstrapper.go

@@ -57,7 +57,7 @@ const (
 func GetCachedBinaryList(bootstrapper string) []string {
 	switch bootstrapper {
 	case BootstrapperTypeKubeadm:
-		return constants.GetKubeadmCachedBinaries()
+		return constants.KubeadmBinaries
 	default:
 		return []string{}
 	}

pkg/minikube/bootstrapper/certs.go

@@ -59,16 +59,14 @@ func SetupCerts(cmd command.Runner, k8s config.KubernetesConfig) error {
 	if err := generateCerts(k8s); err != nil {
 		return errors.Wrap(err, "Error generating certs")
 	}
-
 	copyableFiles := []assets.CopyableFile{}
-
 	for _, cert := range certs {
 		p := filepath.Join(localPath, cert)
 		perms := "0644"
 		if strings.HasSuffix(cert, ".key") {
 			perms = "0600"
 		}
-		certFile, err := assets.NewFileAsset(p, util.DefaultCertPath, cert, perms)
+		certFile, err := assets.NewFileAsset(p, constants.GuestCertsDir, cert, perms)
 		if err != nil {
 			return err
 		}
@@ -91,9 +89,9 @@ func SetupCerts(cmd command.Runner, k8s config.KubernetesConfig) error {
 	kcs := &kubeconfig.Settings{
 		ClusterName:          k8s.NodeName,
 		ClusterServerAddress: fmt.Sprintf("https://localhost:%d", k8s.NodePort),
-		ClientCertificate:    path.Join(util.DefaultCertPath, "apiserver.crt"),
-		ClientKey:            path.Join(util.DefaultCertPath, "apiserver.key"),
-		CertificateAuthority: path.Join(util.DefaultCertPath, "ca.crt"),
+		ClientCertificate:    path.Join(constants.GuestCertsDir, "apiserver.crt"),
+		ClientKey:            path.Join(constants.GuestCertsDir, "apiserver.key"),
+		CertificateAuthority: path.Join(constants.GuestCertsDir, "ca.crt"),
 		KeepContext:          false,
 	}
 
@@ -107,13 +105,12 @@ func SetupCerts(cmd command.Runner, k8s config.KubernetesConfig) error {
 		return errors.Wrap(err, "encoding kubeconfig")
 	}
 
-	kubeCfgFile := assets.NewMemoryAsset(data,
-		util.DefaultMinikubeDirectory, "kubeconfig", "0644")
+	kubeCfgFile := assets.NewMemoryAsset(data, constants.GuestPersistentDir, "kubeconfig", "0644")
 	copyableFiles = append(copyableFiles, kubeCfgFile)
 
 	for _, f := range copyableFiles {
 		if err := cmd.Copy(f); err != nil {
-			return err
+			return errors.Wrapf(err, "Copy %s", f.GetAssetName())
 		}
 	}
 

pkg/minikube/bootstrapper/certs_test.go

@@ -34,7 +34,6 @@ func TestSetupCerts(t *testing.T) {
 	tempDir := tests.MakeTempDir()
 	defer os.RemoveAll(tempDir)
 
-	f := command.NewFakeCommandRunner()
 	k8s := config.KubernetesConfig{
 		APIServerName: constants.APIServerName,
 		DNSDomain:     constants.ClusterDNSDomain,
@@ -53,7 +52,9 @@ func TestSetupCerts(t *testing.T) {
 		t.Fatalf("error generating certificate: %v", err)
 	}
 
-	cmdMap := map[string]string{}
+	cmdMap := map[string]string{
+		"sudo mkdir -p  /var/lib/minikube/certs": "",
+	}
 	certFilenames := map[string]string{"ca.crt": "minikubeCA.pem", "mycert.pem": "mycert.pem"}
 	for _, dst := range certFilenames {
 		certFile := path.Join(CACertificatesDir, dst)
@@ -64,6 +65,7 @@ func TestSetupCerts(t *testing.T) {
 		cmdMap[fmt.Sprintf("openssl x509 -hash -noout -in '%s'", certFile)] = certNameHash
 		cmdMap[fmt.Sprintf("sudo ln -s '%s' '%s'", certStorePath, remoteCertHashLink)] = "1"
 	}
+	f := command.NewFakeCommandRunner()
 	f.SetCommandToOutput(cmdMap)
 
 	var filesToBeTransferred []string

pkg/minikube/bootstrapper/kubeadm/kubeadm.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"net"
 	"net/http"
+	"path/filepath"
 	"runtime"
 	"strings"
 	"time"
@@ -93,6 +94,9 @@ var PodsByLayer = []pod{
 	{"dns", "k8s-app", "kube-dns"},
 }
 
+// yamlConfigPath is the path to the kubeadm configuration
+var yamlConfigPath = filepath.Join(constants.GuestEphemeralDir, "kubeadm.yaml")
+
 // SkipAdditionalPreflights are additional preflights we skip depending on the runtime in use.
 var SkipAdditionalPreflights = map[string][]string{}
 
@@ -194,6 +198,28 @@ func createFlagsFromExtraArgs(extraOptions config.ExtraOptionSlice) string {
 	return convertToFlags(kubeadmExtraOpts)
 }
 
+// etcdDataDir is where etcd data is stored.
+func etcdDataDir() string {
+	return filepath.Join(constants.GuestPersistentDir, "etcd")
+}
+
+// createCompatSymlinks creates compatibility symlinks to transition running services to new directory structures
+func (k *Bootstrapper) createCompatSymlinks() error {
+	legacyEtcd := "/data/minikube"
+	if err := k.c.Run(fmt.Sprintf("sudo test -d %s", legacyEtcd)); err != nil {
+		glog.Infof("%s check failed, skipping compat symlinks: %v", legacyEtcd, err)
+		return nil
+	}
+
+	glog.Infof("Found %s, creating compatibility symlinks ...", legacyEtcd)
+	cmd := fmt.Sprintf("sudo ln -s %s %s", legacyEtcd, etcdDataDir())
+	out, err := k.c.CombinedOutput(cmd)
+	if err != nil {
+		return errors.Wrapf(err, "cmd failed: %s\n%s\n", cmd, out)
+	}
+	return nil
+}
+
 // StartCluster starts the cluster
 func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error {
 	version, err := parseKubernetesVersion(k8s.KubernetesVersion)
@@ -202,15 +228,14 @@ func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error {
 	}
 
 	extraFlags := createFlagsFromExtraArgs(k8s.ExtraOptions)
-
 	r, err := cruntime.New(cruntime.Config{Type: k8s.ContainerRuntime})
 	if err != nil {
 		return err
 	}
 
 	ignore := []string{
-		"DirAvailable--etc-kubernetes-manifests", // Addons are stored in /etc/kubernetes/manifests
-		"DirAvailable--data-minikube",
+		fmt.Sprintf("DirAvailable-%s", strings.Replace(constants.GuestManifestsDir, "/", "-", -1)),
+		fmt.Sprintf("DirAvailable-%s", strings.Replace(constants.GuestPersistentDir, "/", "-", -1)),
 		"FileAvailable--etc-kubernetes-manifests-kube-scheduler.yaml",
 		"FileAvailable--etc-kubernetes-manifests-kube-apiserver.yaml",
 		"FileAvailable--etc-kubernetes-manifests-kube-controller-manager.yaml",
@@ -226,8 +251,8 @@ func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error {
 		ignore = append(ignore, "SystemVerification")
 	}
 
-	cmd := fmt.Sprintf("sudo /usr/bin/kubeadm init --config %s %s --ignore-preflight-errors=%s",
-		constants.KubeadmConfigFile, extraFlags, strings.Join(ignore, ","))
+	cmd := fmt.Sprintf("%s init --config %s %s --ignore-preflight-errors=%s",
+		invokeKubeadm(k8s.KubernetesVersion), yamlConfigPath, extraFlags, strings.Join(ignore, ","))
 	out, err := k.c.CombinedOutput(cmd)
 	if err != nil {
 		return errors.Wrapf(err, "cmd failed: %s\n%s\n", cmd, out)
@@ -339,6 +364,12 @@ func (k *Bootstrapper) WaitCluster(k8s config.KubernetesConfig, timeout time.Dur
 
 // RestartCluster restarts the Kubernetes cluster configured by kubeadm
 func (k *Bootstrapper) RestartCluster(k8s config.KubernetesConfig) error {
+	glog.Infof("RestartCluster start")
+	start := time.Now()
+	defer func() {
+		glog.Infof("RestartCluster took %s", time.Since(start))
+	}()
+
 	version, err := parseKubernetesVersion(k8s.KubernetesVersion)
 	if err != nil {
 		return errors.Wrap(err, "parsing kubernetes version")
@@ -351,13 +382,16 @@ func (k *Bootstrapper) RestartCluster(k8s config.KubernetesConfig) error {
 		controlPlane = "control-plane"
 	}
 
-	configPath := constants.KubeadmConfigFile
-	baseCmd := fmt.Sprintf("sudo kubeadm %s", phase)
+	if err := k.createCompatSymlinks(); err != nil {
+		glog.Errorf("failed to create compat symlinks: %v", err)
+	}
+
+	baseCmd := fmt.Sprintf("%s %s", invokeKubeadm(k8s.KubernetesVersion), phase)
 	cmds := []string{
-		fmt.Sprintf("%s phase certs all --config %s", baseCmd, configPath),
-		fmt.Sprintf("%s phase kubeconfig all --config %s", baseCmd, configPath),
-		fmt.Sprintf("%s phase %s all --config %s", baseCmd, controlPlane, configPath),
-		fmt.Sprintf("%s phase etcd local --config %s", baseCmd, configPath),
+		fmt.Sprintf("%s phase certs all --config %s", baseCmd, yamlConfigPath),
+		fmt.Sprintf("%s phase kubeconfig all --config %s", baseCmd, yamlConfigPath),
+		fmt.Sprintf("%s phase %s all --config %s", baseCmd, controlPlane, yamlConfigPath),
+		fmt.Sprintf("%s phase etcd local --config %s", baseCmd, yamlConfigPath),
 	}
 
 	// Run commands one at a time so that it is easier to root cause failures.
@@ -371,7 +405,7 @@ func (k *Bootstrapper) RestartCluster(k8s config.KubernetesConfig) error {
 		return errors.Wrap(err, "waiting for apiserver")
 	}
 	// restart the proxy and coredns
-	if err := k.c.Run(fmt.Sprintf("%s phase addon all --config %s", baseCmd, configPath)); err != nil {
+	if err := k.c.Run(fmt.Sprintf("%s phase addon all --config %s", baseCmd, yamlConfigPath)); err != nil {
 		return errors.Wrapf(err, "addon phase")
 	}
 
@@ -384,27 +418,53 @@ func (k *Bootstrapper) RestartCluster(k8s config.KubernetesConfig) error {
 // waitForAPIServer waits for the apiserver to start up
 func (k *Bootstrapper) waitForAPIServer(k8s config.KubernetesConfig) error {
 	start := time.Now()
-	glog.Infof("Waiting for apiserver ...")
+	defer func() {
+		glog.Infof("duration metric: took %s to wait for apiserver status ...", time.Since(start))
+	}()
 
+	glog.Infof("Waiting for apiserver process ...")
+	// To give a better error message, first check for process existence via ssh
+	// Needs minutes in case the image isn't cached (such as with v1.10.x)
+	err := wait.PollImmediate(time.Millisecond*300, time.Minute*3, func() (bool, error) {
+		ierr := k.c.Run(`sudo pgrep kube-apiserver`)
+		if ierr != nil {
+			glog.Warningf("pgrep apiserver: %v", ierr)
+			return false, nil
+		}
+		return true, nil
+	})
+	if err != nil {
+		return fmt.Errorf("apiserver process never appeared")
+	}
+
+	glog.Infof("Waiting for apiserver to port healthy status ...")
 	f := func() (bool, error) {
 		status, err := k.GetAPIServerStatus(net.ParseIP(k8s.NodeIP), k8s.NodePort)
 		glog.Infof("apiserver status: %s, err: %v", status, err)
 		if err != nil {
-			return false, err
+			glog.Warningf("status: %v", err)
+			return false, nil
 		}
 		if status != "Running" {
 			return false, nil
 		}
 		return true, nil
 	}
-	err := wait.PollImmediate(kconst.APICallRetryInterval, kconst.DefaultControlPlaneTimeout, f)
-	glog.Infof("duration metric: took %s to wait for apiserver status ...", time.Since(start))
+	err = wait.PollImmediate(kconst.APICallRetryInterval, kconst.DefaultControlPlaneTimeout, f)
 	return err
 }
 
 // DeleteCluster removes the components that were started earlier
 func (k *Bootstrapper) DeleteCluster(k8s config.KubernetesConfig) error {
-	cmd := fmt.Sprintf("sudo kubeadm reset --force")
+	version, err := parseKubernetesVersion(k8s.KubernetesVersion)
+	if err != nil {
+		return errors.Wrap(err, "parsing kubernetes version")
+	}
+
+	cmd := fmt.Sprintf("%s reset --force", invokeKubeadm(k8s.KubernetesVersion))
+	if version.LT(semver.MustParse("1.11.0")) {
+		cmd = fmt.Sprintf("%s reset", invokeKubeadm(k8s.KubernetesVersion))
+	}
 	out, err := k.c.CombinedOutput(cmd)
 	if err != nil {
 		return errors.Wrapf(err, "kubeadm reset: %s\n%s\n", cmd, out)
@@ -423,7 +483,7 @@ func (k *Bootstrapper) PullImages(k8s config.KubernetesConfig) error {
 		return fmt.Errorf("pull command is not supported by kubeadm v%s", version)
 	}
 
-	cmd := fmt.Sprintf("sudo kubeadm config images pull --config %s", constants.KubeadmConfigFile)
+	cmd := fmt.Sprintf("%s config images pull --config %s", invokeKubeadm(k8s.KubernetesVersion), yamlConfigPath)
 	if err := k.c.Run(cmd); err != nil {
 		return errors.Wrapf(err, "running cmd: %s", cmd)
 	}
@@ -437,15 +497,15 @@ func (k *Bootstrapper) SetupCerts(k8s config.KubernetesConfig) error {
 
 // NewKubeletConfig generates a new systemd unit containing a configured kubelet
 // based on the options present in the KubernetesConfig.
-func NewKubeletConfig(k8s config.KubernetesConfig, r cruntime.Manager) (string, error) {
+func NewKubeletConfig(k8s config.KubernetesConfig, r cruntime.Manager) ([]byte, error) {
 	version, err := parseKubernetesVersion(k8s.KubernetesVersion)
 	if err != nil {
-		return "", errors.Wrap(err, "parsing kubernetes version")
+		return nil, errors.Wrap(err, "parsing kubernetes version")
 	}
 
 	extraOpts, err := ExtraConfigForComponent(Kubelet, k8s.ExtraOptions, version)
 	if err != nil {
-		return "", errors.Wrap(err, "generating extra configuration for kubelet")
+		return nil, errors.Wrap(err, "generating extra configuration for kubelet")
 	}
 
 	for k, v := range r.KubeletOptions() {
@@ -463,28 +523,28 @@ func NewKubeletConfig(k8s config.KubernetesConfig, r cruntime.Manager) (string,
 	// parses a map of the feature gates for kubelet
 	_, kubeletFeatureArgs, err := ParseFeatureArgs(k8s.FeatureGates)
 	if err != nil {
-		return "", errors.Wrap(err, "parses feature gate config for kubelet")
+		return nil, errors.Wrap(err, "parses feature gate config for kubelet")
 	}
 
 	if kubeletFeatureArgs != "" {
 		extraOpts["feature-gates"] = kubeletFeatureArgs
 	}
 
-	extraFlags := convertToFlags(extraOpts)
-
 	b := bytes.Buffer{}
 	opts := struct {
 		ExtraOptions     string
 		ContainerRuntime string
+		KubeletPath      string
 	}{
-		ExtraOptions:     extraFlags,
+		ExtraOptions:     convertToFlags(extraOpts),
 		ContainerRuntime: k8s.ContainerRuntime,
+		KubeletPath:      filepath.Join(binRoot(k8s.KubernetesVersion), "kubelet"),
 	}
 	if err := kubeletSystemdTemplate.Execute(&b, opts); err != nil {
-		return "", err
+		return nil, err
 	}
 
-	return b.String(), nil
+	return b.Bytes(), nil
 }
 
 // UpdateCluster updates the cluster
@@ -508,32 +568,35 @@ func (k *Bootstrapper) UpdateCluster(cfg config.KubernetesConfig) error {
 	if err != nil {
 		return errors.Wrap(err, "generating kubelet config")
 	}
-	glog.Infof("kubelet %s config:\n%s", cfg.KubernetesVersion, kubeletCfg)
 
-	var files []assets.CopyableFile
-	files = copyConfig(cfg, files, kubeadmCfg, kubeletCfg)
-
-	if err := downloadBinaries(cfg, k.c); err != nil {
-		return errors.Wrap(err, "downloading binaries")
+	kubeletService, err := NewKubeletService(cfg)
+	if err != nil {
+		return errors.Wrap(err, "generating kubelet service")
 	}
 
+	glog.Infof("kubelet %s config:\n%s", cfg.KubernetesVersion, kubeletCfg)
+
+	// stop kubelet to avoid "Text File Busy" error
+	err = k.c.Run(`pgrep kubelet && sudo systemctl stop kubelet`)
+	if err != nil {
+		glog.Warningf("unable to stop kubelet: %s", err)
+	}
+	if err := transferBinaries(cfg, k.c); err != nil {
+		return errors.Wrap(err, "downloading binaries")
+	}
+	files := configFiles(cfg, kubeadmCfg, kubeletCfg, kubeletService)
 	if err := addAddons(&files, assets.GenerateTemplateData(cfg)); err != nil {
 		return errors.Wrap(err, "adding addons")
 	}
-
 	for _, f := range files {
 		if err := k.c.Copy(f); err != nil {
 			return errors.Wrapf(err, "copy")
 		}
 	}
-	err = k.c.Run(`
-sudo systemctl daemon-reload &&
-sudo systemctl start kubelet
-`)
-	if err != nil {
+
+	if err := k.c.Run(`sudo systemctl daemon-reload && sudo systemctl start kubelet`); err != nil {
 		return errors.Wrap(err, "starting kubelet")
 	}
-
 	return nil
 }
 
@@ -556,21 +619,22 @@ func createExtraComponentConfig(extraOptions config.ExtraOptionSlice, version se
 	return extraArgsSlice, nil
 }
 
-func generateConfig(k8s config.KubernetesConfig, r cruntime.Manager) (string, error) {
+// generateConfig generates the kubeadm.yaml file
+func generateConfig(k8s config.KubernetesConfig, r cruntime.Manager) ([]byte, error) {
 	version, err := parseKubernetesVersion(k8s.KubernetesVersion)
 	if err != nil {
-		return "", errors.Wrap(err, "parsing kubernetes version")
+		return nil, errors.Wrap(err, "parsing kubernetes version")
 	}
 
 	// parses a map of the feature gates for kubeadm and component
 	kubeadmFeatureArgs, componentFeatureArgs, err := ParseFeatureArgs(k8s.FeatureGates)
 	if err != nil {
-		return "", errors.Wrap(err, "parses feature gate config for kubeadm and component")
+		return nil, errors.Wrap(err, "parses feature gate config for kubeadm and component")
 	}
 
 	extraComponentConfig, err := createExtraComponentConfig(k8s.ExtraOptions, version, componentFeatureArgs)
 	if err != nil {
-		return "", errors.Wrap(err, "generating extra component config for kubeadm")
+		return nil, errors.Wrap(err, "generating extra component config for kubeadm")
 	}
 
 	// In case of no port assigned, use util.APIServerPort
@@ -594,13 +658,13 @@ func generateConfig(k8s config.KubernetesConfig, r cruntime.Manager) (string, er
 		FeatureArgs       map[string]bool
 		NoTaintMaster     bool
 	}{
-		CertDir:           util.DefaultCertPath,
+		CertDir:           constants.GuestCertsDir,
 		ServiceCIDR:       util.DefaultServiceCIDR,
 		PodSubnet:         k8s.ExtraOptions.Get("pod-network-cidr", Kubeadm),
 		AdvertiseAddress:  k8s.NodeIP,
 		APIServerPort:     nodePort,
 		KubernetesVersion: k8s.KubernetesVersion,
-		EtcdDataDir:       "/data/minikube", //TODO(r2d4): change to something else persisted
+		EtcdDataDir:       etcdDataDir(),
 		NodeName:          k8s.NodeName,
 		CRISocket:         r.SocketPath(),
 		ImageRepository:   k8s.ImageRepository,
@@ -627,41 +691,62 @@ func generateConfig(k8s config.KubernetesConfig, r cruntime.Manager) (string, er
 		configTmpl = configTmplV1Beta1
 	}
 	if err := configTmpl.Execute(&b, opts); err != nil {
-		return "", err
+		return nil, err
 	}
 
-	return b.String(), nil
+	return b.Bytes(), nil
 }
 
-func copyConfig(cfg config.KubernetesConfig, files []assets.CopyableFile, kubeadmCfg string, kubeletCfg string) []assets.CopyableFile {
-	files = append(files,
-		assets.NewMemoryAssetTarget([]byte(kubeletService), constants.KubeletServiceFile, "0640"),
-		assets.NewMemoryAssetTarget([]byte(kubeletCfg), constants.KubeletSystemdConfFile, "0640"),
-		assets.NewMemoryAssetTarget([]byte(kubeadmCfg), constants.KubeadmConfigFile, "0640"))
+// NewKubeletService returns a generated systemd unit file for the kubelet
+func NewKubeletService(cfg config.KubernetesConfig) ([]byte, error) {
+	var b bytes.Buffer
+	opts := struct{ KubeletPath string }{KubeletPath: filepath.Join(binRoot(cfg.KubernetesVersion), "kubelet")}
+	if err := kubeletServiceTemplate.Execute(&b, opts); err != nil {
+		return nil, errors.Wrap(err, "template execute")
+	}
+	return b.Bytes(), nil
+}
 
+// configFiles returns configuration file assets
+func configFiles(cfg config.KubernetesConfig, kubeadm []byte, kubelet []byte, kubeletSvc []byte) []assets.CopyableFile {
+	fs := []assets.CopyableFile{
+		assets.NewMemoryAssetTarget(kubeadm, yamlConfigPath, "0640"),
+		assets.NewMemoryAssetTarget(kubelet, constants.KubeletSystemdConfFile, "0640"),
+		assets.NewMemoryAssetTarget(kubeletSvc, constants.KubeletServiceFile, "0640"),
+	}
 	// Copy the default CNI config (k8s.conf), so that kubelet can successfully
 	// start a Pod in the case a user hasn't manually installed any CNI plugin
 	// and minikube was started with "--extra-config=kubelet.network-plugin=cni".
 	if cfg.EnableDefaultCNI {
-		files = append(files,
-			assets.NewMemoryAssetTarget([]byte(defaultCNIConfig), constants.DefaultCNIConfigPath, "0644"))
+		fs = append(fs, assets.NewMemoryAssetTarget([]byte(defaultCNIConfig), constants.DefaultCNIConfigPath, "0644"))
 	}
-
-	return files
+	return fs
 }
 
-func downloadBinaries(cfg config.KubernetesConfig, c command.Runner) error {
+// binDir returns the persistent path binaries are stored in
+func binRoot(version string) string {
+	return filepath.Join(constants.GuestPersistentDir, "binaries", version)
+}
+
+// invokeKubeadm returns the invocation command for Kubeadm
+func invokeKubeadm(version string) string {
+	return fmt.Sprintf("sudo env PATH=%s:$PATH kubeadm", binRoot(version))
+}
+
+// transferBinaries transfers all required Kubernetes binaries
+func transferBinaries(cfg config.KubernetesConfig, c command.Runner) error {
 	var g errgroup.Group
-	for _, bin := range constants.GetKubeadmCachedBinaries() {
-		bin := bin
+	for _, name := range constants.KubeadmBinaries {
+		name := name
 		g.Go(func() error {
-			path, err := machine.CacheBinary(bin, cfg.KubernetesVersion, "linux", runtime.GOARCH)
+			src, err := machine.CacheBinary(name, cfg.KubernetesVersion, "linux", runtime.GOARCH)
 			if err != nil {
-				return errors.Wrapf(err, "downloading %s", bin)
+				return errors.Wrapf(err, "downloading %s", name)
 			}
-			err = machine.CopyBinary(c, bin, path)
-			if err != nil {
-				return errors.Wrapf(err, "copying %s", bin)
+
+			dst := filepath.Join(binRoot(cfg.KubernetesVersion), name)
+			if err := machine.CopyBinary(c, src, dst); err != nil {
+				return errors.Wrapf(err, "copybinary %s -> %s", src, dst)
 			}
 			return nil
 		})

pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go

@@ -43,13 +43,12 @@ func TestGenerateKubeletConfig(t *testing.T) {
 				NodeName:          "minikube",
 				ContainerRuntime:  "docker",
 			},
-			expected: `
-[Unit]
+			expected: `[Unit]
 Wants=docker.socket
 
 [Service]
 ExecStart=
-ExecStart=/usr/bin/kubelet --allow-privileged=true --authorization-mode=Webhook --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --cadvisor-port=0 --cgroup-driver=cgroupfs --client-ca-file=/var/lib/minikube/certs/ca.crt --cluster-dns=10.96.0.10 --cluster-domain=cluster.local --container-runtime=docker --fail-swap-on=false --hostname-override=minikube --kubeconfig=/etc/kubernetes/kubelet.conf --pod-manifest-path=/etc/kubernetes/manifests
+ExecStart=/var/lib/minikube/binaries/v1.10.13/kubelet --allow-privileged=true --authorization-mode=Webhook --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --cadvisor-port=0 --cgroup-driver=cgroupfs --client-ca-file=/var/lib/minikube/certs/ca.crt --cluster-dns=10.96.0.10 --cluster-domain=cluster.local --container-runtime=docker --fail-swap-on=false --hostname-override=minikube --kubeconfig=/etc/kubernetes/kubelet.conf --pod-manifest-path=/etc/kubernetes/manifests
 
 [Install]
 `,
@@ -62,13 +61,12 @@ ExecStart=/usr/bin/kubelet --allow-privileged=true --authorization-mode=Webhook
 				NodeName:          "minikube",
 				ContainerRuntime:  "cri-o",
 			},
-			expected: `
-[Unit]
+			expected: `[Unit]
 Wants=crio.service
 
 [Service]
 ExecStart=
-ExecStart=/usr/bin/kubelet --authorization-mode=Webhook --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --cgroup-driver=cgroupfs --client-ca-file=/var/lib/minikube/certs/ca.crt --cluster-dns=10.96.0.10 --cluster-domain=cluster.local --container-runtime=remote --container-runtime-endpoint=/var/run/crio/crio.sock --fail-swap-on=false --hostname-override=minikube --image-service-endpoint=/var/run/crio/crio.sock --kubeconfig=/etc/kubernetes/kubelet.conf --pod-manifest-path=/etc/kubernetes/manifests --runtime-request-timeout=15m
+ExecStart=/var/lib/minikube/binaries/v1.16.0-beta.1/kubelet --authorization-mode=Webhook --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --cgroup-driver=cgroupfs --client-ca-file=/var/lib/minikube/certs/ca.crt --cluster-dns=10.96.0.10 --cluster-domain=cluster.local --container-runtime=remote --container-runtime-endpoint=/var/run/crio/crio.sock --fail-swap-on=false --hostname-override=minikube --image-service-endpoint=/var/run/crio/crio.sock --kubeconfig=/etc/kubernetes/kubelet.conf --pod-manifest-path=/etc/kubernetes/manifests --runtime-request-timeout=15m
 
 [Install]
 `,
@@ -81,13 +79,12 @@ ExecStart=/usr/bin/kubelet --authorization-mode=Webhook --bootstrap-kubeconfig=/
 				NodeName:          "minikube",
 				ContainerRuntime:  "containerd",
 			},
-			expected: `
-[Unit]
+			expected: `[Unit]
 Wants=containerd.service
 
 [Service]
 ExecStart=
-ExecStart=/usr/bin/kubelet --authorization-mode=Webhook --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --cgroup-driver=cgroupfs --client-ca-file=/var/lib/minikube/certs/ca.crt --cluster-dns=10.96.0.10 --cluster-domain=cluster.local --container-runtime=remote --container-runtime-endpoint=unix:///run/containerd/containerd.sock --fail-swap-on=false --hostname-override=minikube --image-service-endpoint=unix:///run/containerd/containerd.sock --kubeconfig=/etc/kubernetes/kubelet.conf --pod-manifest-path=/etc/kubernetes/manifests --runtime-request-timeout=15m
+ExecStart=/var/lib/minikube/binaries/v1.15.2/kubelet --authorization-mode=Webhook --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --cgroup-driver=cgroupfs --client-ca-file=/var/lib/minikube/certs/ca.crt --cluster-dns=10.96.0.10 --cluster-domain=cluster.local --container-runtime=remote --container-runtime-endpoint=unix:///run/containerd/containerd.sock --fail-swap-on=false --hostname-override=minikube --image-service-endpoint=unix:///run/containerd/containerd.sock --kubeconfig=/etc/kubernetes/kubelet.conf --pod-manifest-path=/etc/kubernetes/manifests --runtime-request-timeout=15m
 
 [Install]
 `,
@@ -101,13 +98,12 @@ ExecStart=/usr/bin/kubelet --authorization-mode=Webhook --bootstrap-kubeconfig=/
 				ContainerRuntime:  "docker",
 				ImageRepository:   "docker-proxy-image.io/google_containers",
 			},
-			expected: `
-[Unit]
+			expected: `[Unit]
 Wants=docker.socket
 
 [Service]
 ExecStart=
-ExecStart=/usr/bin/kubelet --authorization-mode=Webhook --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --cgroup-driver=cgroupfs --client-ca-file=/var/lib/minikube/certs/ca.crt --cluster-dns=10.96.0.10 --cluster-domain=cluster.local --container-runtime=docker --fail-swap-on=false --hostname-override=minikube --kubeconfig=/etc/kubernetes/kubelet.conf --pod-infra-container-image=docker-proxy-image.io/google_containers/pause:3.1 --pod-manifest-path=/etc/kubernetes/manifests
+ExecStart=/var/lib/minikube/binaries/v1.15.2/kubelet --authorization-mode=Webhook --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --cgroup-driver=cgroupfs --client-ca-file=/var/lib/minikube/certs/ca.crt --cluster-dns=10.96.0.10 --cluster-domain=cluster.local --container-runtime=docker --fail-swap-on=false --hostname-override=minikube --kubeconfig=/etc/kubernetes/kubelet.conf --pod-infra-container-image=docker-proxy-image.io/google_containers/pause:3.1 --pod-manifest-path=/etc/kubernetes/manifests
 
 [Install]
 `,
@@ -133,7 +129,7 @@ ExecStart=/usr/bin/kubelet --authorization-mode=Webhook --bootstrap-kubeconfig=/
 
 			diff, err := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{
 				A:        difflib.SplitLines(tc.expected),
-				B:        difflib.SplitLines(got),
+				B:        difflib.SplitLines(string(got)),
 				FromFile: "Expected",
 				ToFile:   "Got",
 				Context:  1,
@@ -266,7 +262,7 @@ func TestGenerateConfig(t *testing.T) {
 				}
 				diff, err := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{
 					A:        difflib.SplitLines(string(expected)),
-					B:        difflib.SplitLines(got),
+					B:        difflib.SplitLines(string(got)),
 					FromFile: "Expected",
 					ToFile:   "Got",
 					Context:  1,

pkg/minikube/bootstrapper/kubeadm/templates.go

@@ -151,24 +151,24 @@ evictionHard:
   imagefs.available: "0%"
 `))
 
-var kubeletSystemdTemplate = template.Must(template.New("kubeletSystemdTemplate").Parse(`
-[Unit]
+// kubeletSystemdTemplate hosts the override kubelet flags, written to constants.KubeletSystemdConfFile
+var kubeletSystemdTemplate = template.Must(template.New("kubeletSystemdTemplate").Parse(`[Unit]
 {{if or (eq .ContainerRuntime "cri-o") (eq .ContainerRuntime "cri")}}Wants=crio.service{{else if eq .ContainerRuntime "containerd"}}Wants=containerd.service{{else}}Wants=docker.socket{{end}}
 
 [Service]
 ExecStart=
-ExecStart=/usr/bin/kubelet{{if .ExtraOptions}} {{.ExtraOptions}}{{end}}
+ExecStart={{.KubeletPath}}{{if .ExtraOptions}} {{.ExtraOptions}}{{end}}
 
 [Install]
 `))
 
-const kubeletService = `
-[Unit]
+// kubeletServiceTemplate is the base kubelet systemd template, written to constanst.KubeletServiceFile
+var kubeletServiceTemplate = template.Must(template.New("kubeletServiceTemplate").Parse(`[Unit]
 Description=kubelet: The Kubernetes Node Agent
 Documentation=http://kubernetes.io/docs/
 
 [Service]
-ExecStart=/usr/bin/kubelet
+ExecStart={{.KubeletPath}}
 Restart=always
 StartLimitInterval=0
 # Tuned for local dev: faster than upstream default (10s), but slower than systemd default (100ms)
@@ -176,7 +176,7 @@ RestartSec=600ms
 
 [Install]
 WantedBy=multi-user.target
-`
+`))
 
 // printMapInOrder sorts the keys and prints the map in order, combining key
 // value pairs with the separator character

pkg/minikube/bootstrapper/kubeadm/testdata/v1.10/containerd-api-port.yaml

@@ -6,11 +6,11 @@ api:
   bindPort: 12345
   controlPlaneEndpoint: localhost
 kubernetesVersion: v1.10.0
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 networking:
   serviceSubnet: 10.96.0.0/12
 etcd:
-  dataDir: /data/minikube
+  dataDir: /var/lib/minikube/etcd
 nodeName: mk
 criSocket: /run/containerd/containerd.sock
 apiServerExtraArgs:

pkg/minikube/bootstrapper/kubeadm/testdata/v1.10/containerd-pod-network-cidr.yaml

@@ -6,11 +6,11 @@ api:
   bindPort: 8443
   controlPlaneEndpoint: localhost
 kubernetesVersion: v1.10.0
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 networking:
   serviceSubnet: 10.96.0.0/12
 etcd:
-  dataDir: /data/minikube
+  dataDir: /var/lib/minikube/etcd
 nodeName: mk
 criSocket: /run/containerd/containerd.sock
 apiServerExtraArgs:

pkg/minikube/bootstrapper/kubeadm/testdata/v1.10/containerd.yaml

@@ -6,11 +6,11 @@ api:
   bindPort: 8443
   controlPlaneEndpoint: localhost
 kubernetesVersion: v1.10.0
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 networking:
   serviceSubnet: 10.96.0.0/12
 etcd:
-  dataDir: /data/minikube
+  dataDir: /var/lib/minikube/etcd
 nodeName: mk
 criSocket: /run/containerd/containerd.sock
 apiServerExtraArgs:

pkg/minikube/bootstrapper/kubeadm/testdata/v1.10/crio-options-gates.yaml

@@ -6,11 +6,11 @@ api:
   bindPort: 8443
   controlPlaneEndpoint: localhost
 kubernetesVersion: v1.10.0
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 networking:
   serviceSubnet: 10.96.0.0/12
 etcd:
-  dataDir: /data/minikube
+  dataDir: /var/lib/minikube/etcd
 nodeName: mk
 criSocket: /var/run/crio/crio.sock
 apiServerExtraArgs:

pkg/minikube/bootstrapper/kubeadm/testdata/v1.10/crio.yaml

@@ -6,11 +6,11 @@ api:
   bindPort: 8443
   controlPlaneEndpoint: localhost
 kubernetesVersion: v1.10.0
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 networking:
   serviceSubnet: 10.96.0.0/12
 etcd:
-  dataDir: /data/minikube
+  dataDir: /var/lib/minikube/etcd
 nodeName: mk
 criSocket: /var/run/crio/crio.sock
 apiServerExtraArgs:

pkg/minikube/bootstrapper/kubeadm/testdata/v1.10/default.yaml

@@ -6,11 +6,11 @@ api:
   bindPort: 8443
   controlPlaneEndpoint: localhost
 kubernetesVersion: v1.10.0
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 networking:
   serviceSubnet: 10.96.0.0/12
 etcd:
-  dataDir: /data/minikube
+  dataDir: /var/lib/minikube/etcd
 nodeName: mk
 apiServerExtraArgs:
   admission-control: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"

pkg/minikube/bootstrapper/kubeadm/testdata/v1.10/image-repository.yaml

@@ -6,11 +6,11 @@ api:
   bindPort: 8443
   controlPlaneEndpoint: localhost
 kubernetesVersion: v1.10.0
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 networking:
   serviceSubnet: 10.96.0.0/12
 etcd:
-  dataDir: /data/minikube
+  dataDir: /var/lib/minikube/etcd
 nodeName: mk
 imageRepository: test/repo
 apiServerExtraArgs:

pkg/minikube/bootstrapper/kubeadm/testdata/v1.10/options.yaml

@@ -6,11 +6,11 @@ api:
   bindPort: 8443
   controlPlaneEndpoint: localhost
 kubernetesVersion: v1.10.0
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 networking:
   serviceSubnet: 10.96.0.0/12
 etcd:
-  dataDir: /data/minikube
+  dataDir: /var/lib/minikube/etcd
 nodeName: mk
 apiServerExtraArgs:
   admission-control: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"

pkg/minikube/bootstrapper/kubeadm/testdata/v1.11/containerd-api-port.yaml

@@ -6,11 +6,11 @@ api:
   bindPort: 12345
   controlPlaneEndpoint: localhost
 kubernetesVersion: v1.11.0
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 networking:
   serviceSubnet: 10.96.0.0/12
 etcd:
-  dataDir: /data/minikube
+  dataDir: /var/lib/minikube/etcd
 nodeName: mk
 criSocket: /run/containerd/containerd.sock
 apiServerExtraArgs:

pkg/minikube/bootstrapper/kubeadm/testdata/v1.11/containerd-pod-network-cidr.yaml

@@ -6,11 +6,11 @@ api:
   bindPort: 8443
   controlPlaneEndpoint: localhost
 kubernetesVersion: v1.11.0
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 networking:
   serviceSubnet: 10.96.0.0/12
 etcd:
-  dataDir: /data/minikube
+  dataDir: /var/lib/minikube/etcd
 nodeName: mk
 criSocket: /run/containerd/containerd.sock
 apiServerExtraArgs:

pkg/minikube/bootstrapper/kubeadm/testdata/v1.11/containerd.yaml

@@ -6,11 +6,11 @@ api:
   bindPort: 8443
   controlPlaneEndpoint: localhost
 kubernetesVersion: v1.11.0
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 networking:
   serviceSubnet: 10.96.0.0/12
 etcd:
-  dataDir: /data/minikube
+  dataDir: /var/lib/minikube/etcd
 nodeName: mk
 criSocket: /run/containerd/containerd.sock
 apiServerExtraArgs:

pkg/minikube/bootstrapper/kubeadm/testdata/v1.11/crio-options-gates.yaml

@@ -6,11 +6,11 @@ api:
   bindPort: 8443
   controlPlaneEndpoint: localhost
 kubernetesVersion: v1.11.0
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 networking:
   serviceSubnet: 10.96.0.0/12
 etcd:
-  dataDir: /data/minikube
+  dataDir: /var/lib/minikube/etcd
 nodeName: mk
 criSocket: /var/run/crio/crio.sock
 apiServerExtraArgs:

pkg/minikube/bootstrapper/kubeadm/testdata/v1.11/crio.yaml

@@ -6,11 +6,11 @@ api:
   bindPort: 8443
   controlPlaneEndpoint: localhost
 kubernetesVersion: v1.11.0
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 networking:
   serviceSubnet: 10.96.0.0/12
 etcd:
-  dataDir: /data/minikube
+  dataDir: /var/lib/minikube/etcd
 nodeName: mk
 criSocket: /var/run/crio/crio.sock
 apiServerExtraArgs:

pkg/minikube/bootstrapper/kubeadm/testdata/v1.11/default.yaml

@@ -6,11 +6,11 @@ api:
   bindPort: 8443
   controlPlaneEndpoint: localhost
 kubernetesVersion: v1.11.0
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 networking:
   serviceSubnet: 10.96.0.0/12
 etcd:
-  dataDir: /data/minikube
+  dataDir: /var/lib/minikube/etcd
 nodeName: mk
 apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"

pkg/minikube/bootstrapper/kubeadm/testdata/v1.11/image-repository.yaml

@@ -6,11 +6,11 @@ api:
   bindPort: 8443
   controlPlaneEndpoint: localhost
 kubernetesVersion: v1.11.0
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 networking:
   serviceSubnet: 10.96.0.0/12
 etcd:
-  dataDir: /data/minikube
+  dataDir: /var/lib/minikube/etcd
 nodeName: mk
 imageRepository: test/repo
 apiServerExtraArgs:

pkg/minikube/bootstrapper/kubeadm/testdata/v1.11/options.yaml

@@ -6,11 +6,11 @@ api:
   bindPort: 8443
   controlPlaneEndpoint: localhost
 kubernetesVersion: v1.11.0
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 networking:
   serviceSubnet: 10.96.0.0/12
 etcd:
-  dataDir: /data/minikube
+  dataDir: /var/lib/minikube/etcd
 nodeName: mk
 apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"

pkg/minikube/bootstrapper/kubeadm/testdata/v1.12/containerd-api-port.yaml

@@ -19,12 +19,12 @@ apiVersion: kubeadm.k8s.io/v1alpha3
 kind: ClusterConfiguration
 apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
 controlPlaneEndpoint: localhost:12345
 etcd:
   local:
-    dataDir: /data/minikube
+    dataDir: /var/lib/minikube/etcd
 kubernetesVersion: v1.12.0
 networking:
   dnsDomain: cluster.local

pkg/minikube/bootstrapper/kubeadm/testdata/v1.12/containerd-pod-network-cidr.yaml

@@ -19,12 +19,12 @@ apiVersion: kubeadm.k8s.io/v1alpha3
 kind: ClusterConfiguration
 apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
 controlPlaneEndpoint: localhost:8443
 etcd:
   local:
-    dataDir: /data/minikube
+    dataDir: /var/lib/minikube/etcd
 kubernetesVersion: v1.12.0
 networking:
   dnsDomain: cluster.local

pkg/minikube/bootstrapper/kubeadm/testdata/v1.12/containerd.yaml

@@ -19,12 +19,12 @@ apiVersion: kubeadm.k8s.io/v1alpha3
 kind: ClusterConfiguration
 apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
 controlPlaneEndpoint: localhost:8443
 etcd:
   local:
-    dataDir: /data/minikube
+    dataDir: /var/lib/minikube/etcd
 kubernetesVersion: v1.12.0
 networking:
   dnsDomain: cluster.local

pkg/minikube/bootstrapper/kubeadm/testdata/v1.12/crio-options-gates.yaml

@@ -27,12 +27,12 @@ controllerManagerExtraArgs:
 schedulerExtraArgs:
   feature-gates: "a=b"
   scheduler-name: "mini-scheduler"
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
 controlPlaneEndpoint: localhost:8443
 etcd:
   local:
-    dataDir: /data/minikube
+    dataDir: /var/lib/minikube/etcd
 kubernetesVersion: v1.12.0
 networking:
   dnsDomain: cluster.local

pkg/minikube/bootstrapper/kubeadm/testdata/v1.12/crio.yaml

@@ -19,12 +19,12 @@ apiVersion: kubeadm.k8s.io/v1alpha3
 kind: ClusterConfiguration
 apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
 controlPlaneEndpoint: localhost:8443
 etcd:
   local:
-    dataDir: /data/minikube
+    dataDir: /var/lib/minikube/etcd
 kubernetesVersion: v1.12.0
 networking:
   dnsDomain: cluster.local

pkg/minikube/bootstrapper/kubeadm/testdata/v1.12/default.yaml

@@ -19,12 +19,12 @@ apiVersion: kubeadm.k8s.io/v1alpha3
 kind: ClusterConfiguration
 apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
 controlPlaneEndpoint: localhost:8443
 etcd:
   local:
-    dataDir: /data/minikube
+    dataDir: /var/lib/minikube/etcd
 kubernetesVersion: v1.12.0
 networking:
   dnsDomain: cluster.local

pkg/minikube/bootstrapper/kubeadm/testdata/v1.12/image-repository.yaml

@@ -20,12 +20,12 @@ kind: ClusterConfiguration
 imageRepository: test/repo
 apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
 controlPlaneEndpoint: localhost:8443
 etcd:
   local:
-    dataDir: /data/minikube
+    dataDir: /var/lib/minikube/etcd
 kubernetesVersion: v1.12.0
 networking:
   dnsDomain: cluster.local

pkg/minikube/bootstrapper/kubeadm/testdata/v1.12/options.yaml

@@ -24,12 +24,12 @@ controllerManagerExtraArgs:
   kube-api-burst: "32"
 schedulerExtraArgs:
   scheduler-name: "mini-scheduler"
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
 controlPlaneEndpoint: localhost:8443
 etcd:
   local:
-    dataDir: /data/minikube
+    dataDir: /var/lib/minikube/etcd
 kubernetesVersion: v1.12.0
 networking:
   dnsDomain: cluster.local

pkg/minikube/bootstrapper/kubeadm/testdata/v1.13/containerd-api-port.yaml

@@ -19,12 +19,12 @@ apiVersion: kubeadm.k8s.io/v1alpha3
 kind: ClusterConfiguration
 apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
 controlPlaneEndpoint: localhost:12345
 etcd:
   local:
-    dataDir: /data/minikube
+    dataDir: /var/lib/minikube/etcd
 kubernetesVersion: v1.13.0
 networking:
   dnsDomain: cluster.local

pkg/minikube/bootstrapper/kubeadm/testdata/v1.13/containerd-pod-network-cidr.yaml

@@ -19,12 +19,12 @@ apiVersion: kubeadm.k8s.io/v1alpha3
 kind: ClusterConfiguration
 apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
 controlPlaneEndpoint: localhost:8443
 etcd:
   local:
-    dataDir: /data/minikube
+    dataDir: /var/lib/minikube/etcd
 kubernetesVersion: v1.13.0
 networking:
   dnsDomain: cluster.local

pkg/minikube/bootstrapper/kubeadm/testdata/v1.13/containerd.yaml

@@ -19,12 +19,12 @@ apiVersion: kubeadm.k8s.io/v1alpha3
 kind: ClusterConfiguration
 apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
 controlPlaneEndpoint: localhost:8443
 etcd:
   local:
-    dataDir: /data/minikube
+    dataDir: /var/lib/minikube/etcd
 kubernetesVersion: v1.13.0
 networking:
   dnsDomain: cluster.local

pkg/minikube/bootstrapper/kubeadm/testdata/v1.13/crio-options-gates.yaml

@@ -27,12 +27,12 @@ controllerManagerExtraArgs:
 schedulerExtraArgs:
   feature-gates: "a=b"
   scheduler-name: "mini-scheduler"
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
 controlPlaneEndpoint: localhost:8443
 etcd:
   local:
-    dataDir: /data/minikube
+    dataDir: /var/lib/minikube/etcd
 kubernetesVersion: v1.13.0
 networking:
   dnsDomain: cluster.local

pkg/minikube/bootstrapper/kubeadm/testdata/v1.13/crio.yaml

@@ -19,12 +19,12 @@ apiVersion: kubeadm.k8s.io/v1alpha3
 kind: ClusterConfiguration
 apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
 controlPlaneEndpoint: localhost:8443
 etcd:
   local:
-    dataDir: /data/minikube
+    dataDir: /var/lib/minikube/etcd
 kubernetesVersion: v1.13.0
 networking:
   dnsDomain: cluster.local

pkg/minikube/bootstrapper/kubeadm/testdata/v1.13/default.yaml

@@ -19,12 +19,12 @@ apiVersion: kubeadm.k8s.io/v1alpha3
 kind: ClusterConfiguration
 apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
 controlPlaneEndpoint: localhost:8443
 etcd:
   local:
-    dataDir: /data/minikube
+    dataDir: /var/lib/minikube/etcd
 kubernetesVersion: v1.13.0
 networking:
   dnsDomain: cluster.local

pkg/minikube/bootstrapper/kubeadm/testdata/v1.13/image-repository.yaml

@@ -20,12 +20,12 @@ kind: ClusterConfiguration
 imageRepository: test/repo
 apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
 controlPlaneEndpoint: localhost:8443
 etcd:
   local:
-    dataDir: /data/minikube
+    dataDir: /var/lib/minikube/etcd
 kubernetesVersion: v1.13.0
 networking:
   dnsDomain: cluster.local

pkg/minikube/bootstrapper/kubeadm/testdata/v1.13/options.yaml

@@ -24,12 +24,12 @@ controllerManagerExtraArgs:
   kube-api-burst: "32"
 schedulerExtraArgs:
   scheduler-name: "mini-scheduler"
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
 controlPlaneEndpoint: localhost:8443
 etcd:
   local:
-    dataDir: /data/minikube
+    dataDir: /var/lib/minikube/etcd
 kubernetesVersion: v1.13.0
 networking:
   dnsDomain: cluster.local

pkg/minikube/bootstrapper/kubeadm/testdata/v1.14/containerd-api-port.yaml

@@ -20,14 +20,14 @@ kind: ClusterConfiguration
 apiServer:
   extraArgs:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
 controlPlaneEndpoint: localhost:12345
 dns:
   type: CoreDNS
 etcd:
   local:
-    dataDir: /data/minikube
+    dataDir: /var/lib/minikube/etcd
 kubernetesVersion: v1.14.0
 networking:
   dnsDomain: cluster.local

pkg/minikube/bootstrapper/kubeadm/testdata/v1.14/containerd-pod-network-cidr.yaml

@@ -20,14 +20,14 @@ kind: ClusterConfiguration
 apiServer:
   extraArgs:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS
 etcd:
   local:
-    dataDir: /data/minikube
+    dataDir: /var/lib/minikube/etcd
 kubernetesVersion: v1.14.0
 networking:
   dnsDomain: cluster.local

pkg/minikube/bootstrapper/kubeadm/testdata/v1.14/containerd.yaml

@@ -20,14 +20,14 @@ kind: ClusterConfiguration
 apiServer:
   extraArgs:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS
 etcd:
   local:
-    dataDir: /data/minikube
+    dataDir: /var/lib/minikube/etcd
 kubernetesVersion: v1.14.0
 networking:
   dnsDomain: cluster.local

pkg/minikube/bootstrapper/kubeadm/testdata/v1.14/crio-options-gates.yaml

@@ -30,14 +30,14 @@ scheduler:
   extraArgs:
     feature-gates: "a=b"
     scheduler-name: "mini-scheduler"
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS
 etcd:
   local:
-    dataDir: /data/minikube
+    dataDir: /var/lib/minikube/etcd
 kubernetesVersion: v1.14.0
 networking:
   dnsDomain: cluster.local

pkg/minikube/bootstrapper/kubeadm/testdata/v1.14/crio.yaml

@@ -20,14 +20,14 @@ kind: ClusterConfiguration
 apiServer:
   extraArgs:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS
 etcd:
   local:
-    dataDir: /data/minikube
+    dataDir: /var/lib/minikube/etcd
 kubernetesVersion: v1.14.0
 networking:
   dnsDomain: cluster.local

pkg/minikube/bootstrapper/kubeadm/testdata/v1.14/default.yaml

@@ -20,14 +20,14 @@ kind: ClusterConfiguration
 apiServer:
   extraArgs:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS
 etcd:
   local:
-    dataDir: /data/minikube
+    dataDir: /var/lib/minikube/etcd
 kubernetesVersion: v1.14.0
 networking:
   dnsDomain: cluster.local

pkg/minikube/bootstrapper/kubeadm/testdata/v1.14/image-repository.yaml

@@ -21,14 +21,14 @@ imageRepository: test/repo
 apiServer:
   extraArgs:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS
 etcd:
   local:
-    dataDir: /data/minikube
+    dataDir: /var/lib/minikube/etcd
 kubernetesVersion: v1.14.0
 networking:
   dnsDomain: cluster.local

pkg/minikube/bootstrapper/kubeadm/testdata/v1.14/options.yaml

@@ -27,14 +27,14 @@ controllerManager:
 scheduler:
   extraArgs:
     scheduler-name: "mini-scheduler"
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS
 etcd:
   local:
-    dataDir: /data/minikube
+    dataDir: /var/lib/minikube/etcd
 kubernetesVersion: v1.14.0
 networking:
   dnsDomain: cluster.local

pkg/minikube/bootstrapper/kubeadm/testdata/v1.15/containerd-api-port.yaml

@@ -20,14 +20,14 @@ kind: ClusterConfiguration
 apiServer:
   extraArgs:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
 controlPlaneEndpoint: localhost:12345
 dns:
   type: CoreDNS
 etcd:
   local:
-    dataDir: /data/minikube
+    dataDir: /var/lib/minikube/etcd
 kubernetesVersion: v1.15.0
 networking:
   dnsDomain: cluster.local

pkg/minikube/bootstrapper/kubeadm/testdata/v1.15/containerd-pod-network-cidr.yaml

@@ -20,14 +20,14 @@ kind: ClusterConfiguration
 apiServer:
   extraArgs:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS
 etcd:
   local:
-    dataDir: /data/minikube
+    dataDir: /var/lib/minikube/etcd
 kubernetesVersion: v1.15.0
 networking:
   dnsDomain: cluster.local

pkg/minikube/bootstrapper/kubeadm/testdata/v1.15/containerd.yaml

@@ -20,14 +20,14 @@ kind: ClusterConfiguration
 apiServer:
   extraArgs:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS
 etcd:
   local:
-    dataDir: /data/minikube
+    dataDir: /var/lib/minikube/etcd
 kubernetesVersion: v1.15.0
 networking:
   dnsDomain: cluster.local

pkg/minikube/bootstrapper/kubeadm/testdata/v1.15/crio-options-gates.yaml

@@ -30,14 +30,14 @@ scheduler:
   extraArgs:
     feature-gates: "a=b"
     scheduler-name: "mini-scheduler"
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS
 etcd:
   local:
-    dataDir: /data/minikube
+    dataDir: /var/lib/minikube/etcd
 kubernetesVersion: v1.15.0
 networking:
   dnsDomain: cluster.local

pkg/minikube/bootstrapper/kubeadm/testdata/v1.15/crio.yaml

@@ -20,14 +20,14 @@ kind: ClusterConfiguration
 apiServer:
   extraArgs:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS
 etcd:
   local:
-    dataDir: /data/minikube
+    dataDir: /var/lib/minikube/etcd
 kubernetesVersion: v1.15.0
 networking:
   dnsDomain: cluster.local

pkg/minikube/bootstrapper/kubeadm/testdata/v1.15/default.yaml

@@ -20,14 +20,14 @@ kind: ClusterConfiguration
 apiServer:
   extraArgs:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS
 etcd:
   local:
-    dataDir: /data/minikube
+    dataDir: /var/lib/minikube/etcd
 kubernetesVersion: v1.15.0
 networking:
   dnsDomain: cluster.local

pkg/minikube/bootstrapper/kubeadm/testdata/v1.15/image-repository.yaml

@@ -21,14 +21,14 @@ imageRepository: test/repo
 apiServer:
   extraArgs:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS
 etcd:
   local:
-    dataDir: /data/minikube
+    dataDir: /var/lib/minikube/etcd
 kubernetesVersion: v1.15.0
 networking:
   dnsDomain: cluster.local

pkg/minikube/bootstrapper/kubeadm/testdata/v1.15/options.yaml

@@ -27,14 +27,14 @@ controllerManager:
 scheduler:
   extraArgs:
     scheduler-name: "mini-scheduler"
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS
 etcd:
   local:
-    dataDir: /data/minikube
+    dataDir: /var/lib/minikube/etcd
 kubernetesVersion: v1.15.0
 networking:
   dnsDomain: cluster.local

pkg/minikube/bootstrapper/kubeadm/testdata/v1.16/containerd-api-port.yaml

@@ -20,14 +20,14 @@ kind: ClusterConfiguration
 apiServer:
   extraArgs:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
 controlPlaneEndpoint: localhost:12345
 dns:
   type: CoreDNS
 etcd:
   local:
-    dataDir: /data/minikube
+    dataDir: /var/lib/minikube/etcd
 kubernetesVersion: v1.16.0
 networking:
   dnsDomain: cluster.local

pkg/minikube/bootstrapper/kubeadm/testdata/v1.16/containerd-pod-network-cidr.yaml

@@ -20,14 +20,14 @@ kind: ClusterConfiguration
 apiServer:
   extraArgs:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS
 etcd:
   local:
-    dataDir: /data/minikube
+    dataDir: /var/lib/minikube/etcd
 kubernetesVersion: v1.16.0
 networking:
   dnsDomain: cluster.local

pkg/minikube/bootstrapper/kubeadm/testdata/v1.16/containerd.yaml

@@ -20,14 +20,14 @@ kind: ClusterConfiguration
 apiServer:
   extraArgs:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS
 etcd:
   local:
-    dataDir: /data/minikube
+    dataDir: /var/lib/minikube/etcd
 kubernetesVersion: v1.16.0
 networking:
   dnsDomain: cluster.local

pkg/minikube/bootstrapper/kubeadm/testdata/v1.16/crio-options-gates.yaml

@@ -30,14 +30,14 @@ scheduler:
   extraArgs:
     feature-gates: "a=b"
     scheduler-name: "mini-scheduler"
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS
 etcd:
   local:
-    dataDir: /data/minikube
+    dataDir: /var/lib/minikube/etcd
 kubernetesVersion: v1.16.0
 networking:
   dnsDomain: cluster.local

pkg/minikube/bootstrapper/kubeadm/testdata/v1.16/crio.yaml

@@ -20,14 +20,14 @@ kind: ClusterConfiguration
 apiServer:
   extraArgs:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS
 etcd:
   local:
-    dataDir: /data/minikube
+    dataDir: /var/lib/minikube/etcd
 kubernetesVersion: v1.16.0
 networking:
   dnsDomain: cluster.local

pkg/minikube/bootstrapper/kubeadm/testdata/v1.16/default.yaml

@@ -20,14 +20,14 @@ kind: ClusterConfiguration
 apiServer:
   extraArgs:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS
 etcd:
   local:
-    dataDir: /data/minikube
+    dataDir: /var/lib/minikube/etcd
 kubernetesVersion: v1.16.0
 networking:
   dnsDomain: cluster.local

pkg/minikube/bootstrapper/kubeadm/testdata/v1.16/image-repository.yaml

@@ -21,14 +21,14 @@ imageRepository: test/repo
 apiServer:
   extraArgs:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS
 etcd:
   local:
-    dataDir: /data/minikube
+    dataDir: /var/lib/minikube/etcd
 kubernetesVersion: v1.16.0
 networking:
   dnsDomain: cluster.local

pkg/minikube/bootstrapper/kubeadm/testdata/v1.16/options.yaml

@@ -27,14 +27,14 @@ controllerManager:
 scheduler:
   extraArgs:
     scheduler-name: "mini-scheduler"
-certificatesDir: /var/lib/minikube/certs/
+certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS
 etcd:
   local:
-    dataDir: /data/minikube
+    dataDir: /var/lib/minikube/etcd
 kubernetesVersion: v1.16.0
 networking:
   dnsDomain: cluster.local

pkg/minikube/bootstrapper/kubeadm/versions.go

@@ -206,7 +206,7 @@ var versionSpecificOpts = []config.VersionedExtraOption{
 	config.NewUnversionedOption(Kubelet, "hostname-override", constants.DefaultNodeName),
 
 	// System pods args
-	config.NewUnversionedOption(Kubelet, "pod-manifest-path", "/etc/kubernetes/manifests"),
+	config.NewUnversionedOption(Kubelet, "pod-manifest-path", constants.GuestManifestsDir),
 	{
 		Option: config.ExtraOption{
 			Component: Kubelet,
@@ -222,7 +222,7 @@ var versionSpecificOpts = []config.VersionedExtraOption{
 
 	// Auth args
 	config.NewUnversionedOption(Kubelet, "authorization-mode", "Webhook"),
-	config.NewUnversionedOption(Kubelet, "client-ca-file", path.Join(util.DefaultCertPath, "ca.crt")),
+	config.NewUnversionedOption(Kubelet, "client-ca-file", path.Join(constants.GuestCertsDir, "ca.crt")),
 
 	// Cgroup args
 	config.NewUnversionedOption(Kubelet, "cgroup-driver", "cgroupfs"),

pkg/minikube/command/ssh_runner.go

@@ -21,11 +21,13 @@ import (
 	"fmt"
 	"io"
 	"path"
+	"path/filepath"
 	"sync"
 
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
 	"golang.org/x/crypto/ssh"
+	"golang.org/x/sync/errgroup"
 	"k8s.io/minikube/pkg/minikube/assets"
 	"k8s.io/minikube/pkg/util"
 )
@@ -152,14 +154,6 @@ func (s *SSHRunner) CombinedOutput(cmd string) (string, error) {
 
 // Copy copies a file to the remote over SSH.
 func (s *SSHRunner) Copy(f assets.CopyableFile) error {
-	deleteCmd := fmt.Sprintf("sudo rm -f %s", path.Join(f.GetTargetDir(), f.GetTargetName()))
-	mkdirCmd := fmt.Sprintf("sudo mkdir -p %s", f.GetTargetDir())
-	for _, cmd := range []string{deleteCmd, mkdirCmd} {
-		if err := s.Run(cmd); err != nil {
-			return errors.Wrapf(err, "pre-copy")
-		}
-	}
-
 	sess, err := s.c.NewSession()
 	if err != nil {
 		return errors.Wrap(err, "NewSession")
@@ -170,40 +164,38 @@ func (s *SSHRunner) Copy(f assets.CopyableFile) error {
 		return errors.Wrap(err, "StdinPipe")
 	}
 	// The scpcmd below *should not* return until all data is copied and the
-	// StdinPipe is closed. But let's use a WaitGroup to make it expicit.
-	var wg sync.WaitGroup
-	wg.Add(1)
-	var ierr error
+	// StdinPipe is closed. But let's use errgroup to make it expicit.
+	var g errgroup.Group
 	var copied int64
+	dst := filepath.Join(path.Join(f.GetTargetDir(), f.GetTargetName()))
+	glog.Infof("Transferring %d bytes to %s", f.GetLength(), dst)
 
-	go func() {
-		defer wg.Done()
+	g.Go(func() error {
 		defer w.Close()
-		glog.Infof("Transferring %d bytes to %s", f.GetLength(), f.GetTargetName())
 		header := fmt.Sprintf("C%s %d %s\n", f.GetPermissions(), f.GetLength(), f.GetTargetName())
 		fmt.Fprint(w, header)
 		if f.GetLength() == 0 {
 			glog.Warningf("%s is a 0 byte asset!", f.GetTargetName())
 			fmt.Fprint(w, "\x00")
-			return
+			return nil
 		}
 
-		copied, ierr = io.Copy(w, f)
+		copied, err = io.Copy(w, f)
+		if err != nil {
+			return errors.Wrap(err, "io.Copy")
+		}
 		if copied != int64(f.GetLength()) {
-			glog.Warningf("%s: expected to copy %d bytes, but copied %d instead", f.GetTargetName(), f.GetLength(), copied)
-		} else {
-			glog.Infof("%s: copied %d bytes", f.GetTargetName(), copied)
-		}
-		if ierr != nil {
-			glog.Errorf("io.Copy failed: %v", ierr)
+			return fmt.Errorf("%s: expected to copy %d bytes, but copied %d instead", f.GetTargetName(), f.GetLength(), copied)
 		}
+		glog.Infof("%s: copied %d bytes", f.GetTargetName(), copied)
 		fmt.Fprint(w, "\x00")
-	}()
+		return nil
+	})
 
-	_, err = sess.CombinedOutput(fmt.Sprintf("sudo scp -t %s", f.GetTargetDir()))
+	scp := fmt.Sprintf("sudo mkdir -p %s && sudo scp -t %s", f.GetTargetDir(), f.GetTargetDir())
+	out, err := sess.CombinedOutput(scp)
 	if err != nil {
-		return err
+		return fmt.Errorf("%s: %s\noutput: %s", scp, err, out)
 	}
-	wg.Wait()
-	return ierr
+	return g.Wait()
 }

pkg/minikube/constants/constants.go

@@ -186,30 +186,27 @@ var NewestKubernetesVersion = "v1.16.0-beta.1"
 // OldestKubernetesVersion is the oldest Kubernetes version to test against
 var OldestKubernetesVersion = "v1.10.13"
 
-// ConfigFilePath is the path of the config directory
-var ConfigFilePath = MakeMiniPath("config")
-
 // ConfigFile is the path of the config file
 var ConfigFile = MakeMiniPath("config", "config.json")
 
-// AddonsPath is the default path of the addons configuration
-const AddonsPath = "/etc/kubernetes/addons"
-
-// FilesPath is the default path of files
-const FilesPath = "/files"
-
 const (
 	// KubeletServiceFile is the path to the kubelet systemd service
 	KubeletServiceFile = "/lib/systemd/system/kubelet.service"
 	// KubeletSystemdConfFile is the path to the kubelet systemd configuration
 	KubeletSystemdConfFile = "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf"
-	// KubeadmConfigFile is the path to the kubeadm configuration
-	KubeadmConfigFile = "/var/lib/kubeadm.yaml"
 	// DefaultCNIConfigPath is the path to the CNI configuration
 	DefaultCNIConfigPath = "/etc/cni/net.d/k8s.conf"
-)
 
-const (
+	// GuestAddonsDir is the default path of the addons configuration
+	GuestAddonsDir = "/etc/kubernetes/addons"
+	// GuestManifestsDir is where the kubelet should look for static Pod manifests
+	GuestManifestsDir = "/etc/kubernetes/manifests"
+	// GuestEphemeralDir is the path where ephemeral data should be stored within the VM
+	GuestEphemeralDir = "/var/tmp/minikube"
+	// PersistentDir is the path where persistent data should be stored within the VM (not tmpfs)
+	GuestPersistentDir = "/var/lib/minikube"
+	// GuestCertsDir are where Kubernetes certificates are kept on the guest
+	GuestCertsDir = GuestPersistentDir + "/certs"
 	// DefaultUfsPort is the default port of UFS
 	DefaultUfsPort = "5640"
 	// DefaultUfsDebugLvl is the default debug level of UFS
@@ -220,6 +217,11 @@ const (
 	DefaultMsize = 262144
 	// DefaultMountVersion is the default 9p version to use for mount
 	DefaultMountVersion = "9p2000.L"
+
+	// IsMinikubeChildProcess is the name of "is minikube child process" variable
+	IsMinikubeChildProcess = "IS_MINIKUBE_CHILD_PROCESS"
+	// FileScheme is the file scheme
+	FileScheme = "file"
 )
 
 // ImageRepositories contains all known image repositories
@@ -238,16 +240,8 @@ func GetKubernetesReleaseURLSHA1(binaryName, version, osName, archName string) s
 	return fmt.Sprintf("%s.sha1", GetKubernetesReleaseURL(binaryName, version, osName, archName))
 }
 
-// IsMinikubeChildProcess is the name of "is minikube child process" variable
-const IsMinikubeChildProcess = "IS_MINIKUBE_CHILD_PROCESS"
-
-// FileScheme is the file scheme
-const FileScheme = "file"
-
-// GetKubeadmCachedBinaries gets the binaries to cache for kubeadm
-func GetKubeadmCachedBinaries() []string {
-	return []string{"kubelet", "kubeadm"}
-}
+// KubeadmBinaries are Kubernetes release binaries required for kubeadm
+var KubeadmBinaries = []string{"kubelet", "kubeadm"}
 
 // GetKubeadmCachedImages gets the images to cache for kubeadm for a version
 func GetKubeadmCachedImages(imageRepository string, kubernetesVersionStr string) (string, []string) {

pkg/minikube/logs/logs.go

@@ -34,7 +34,7 @@ import (
 )
 
 // rootCauseRe is a regular expression that matches known failure root causes
-var rootCauseRe = regexp.MustCompile(`^error: |eviction manager: pods.* evicted|unknown flag: --|forbidden.*no providers available|eviction manager:.*evicted|tls: bad certificate`)
+var rootCauseRe = regexp.MustCompile(`^error: |eviction manager: pods.* evicted|unknown flag: --|forbidden.*no providers available|eviction manager:.*evicted|tls: bad certificate|kubelet.*no API client|kubelet.*No api server|STDIN.*127.0.0.1:8080`)
 
 // ignoreRe is a regular expression that matches spurious errors to not surface
 var ignoreCauseRe = regexp.MustCompile("error: no objects passed to apply")

pkg/minikube/machine/cache_binaries.go

@@ -20,6 +20,7 @@ import (
 	"crypto"
 	"os"
 	"path"
+	"path/filepath"
 	"runtime"
 
 	"github.com/golang/glog"
@@ -90,9 +91,9 @@ func CacheBinary(binary, version, osName, archName string) (string, error) {
 	return targetFilepath, nil
 }
 
-// CopyBinary copies previously cached binaries into the path
-func CopyBinary(cr command.Runner, binary, path string) error {
-	f, err := assets.NewFileAsset(path, "/usr/bin", binary, "0755")
+// CopyBinary copies a locally cached binary to the guest VM
+func CopyBinary(cr command.Runner, src string, dest string) error {
+	f, err := assets.NewFileAsset(src, filepath.Dir(dest), filepath.Base(dest), "0755")
 	if err != nil {
 		return errors.Wrap(err, "new file asset")
 	}

pkg/minikube/machine/cache_images.go

@@ -43,7 +43,8 @@ import (
 	"k8s.io/minikube/pkg/minikube/cruntime"
 )
 
-const tempLoadDir = "/tmp"
+// loadRoot is where images should be loaded from within the guest VM
+var loadRoot = filepath.Join(constants.GuestPersistentDir, "images")
 
 var getWindowsVolumeName = getWindowsVolumeNameCmd
 
@@ -88,6 +89,9 @@ func CacheImages(images []string, cacheDir string) error {
 
 // LoadImages loads previously cached images into the container runtime
 func LoadImages(cmd command.Runner, images []string, cacheDir string) error {
+	glog.Infof("LoadImages start: %s", images)
+	defer glog.Infof("LoadImages end")
+
 	var g errgroup.Group
 	// Load profile cluster config from file
 	cc, err := config.Load()
@@ -99,7 +103,7 @@ func LoadImages(cmd command.Runner, images []string, cacheDir string) error {
 		g.Go(func() error {
 			src := filepath.Join(cacheDir, image)
 			src = sanitizeCacheDir(src)
-			if err := loadImageFromCache(cmd, cc.KubernetesConfig, src); err != nil {
+			if err := transferAndLoadImage(cmd, cc.KubernetesConfig, src); err != nil {
 				glog.Warningf("Failed to load %s: %v", src, err)
 				return errors.Wrapf(err, "loading image %s", src)
 			}
@@ -196,15 +200,15 @@ func getWindowsVolumeNameCmd(d string) (string, error) {
 	return vname, nil
 }
 
-// loadImageFromCache loads a single image from the cache
-func loadImageFromCache(cr command.Runner, k8s config.KubernetesConfig, src string) error {
+// transferAndLoadImage transfers and loads a single image from the cache
+func transferAndLoadImage(cr command.Runner, k8s config.KubernetesConfig, src string) error {
 	glog.Infof("Loading image from cache: %s", src)
 	filename := filepath.Base(src)
 	if _, err := os.Stat(src); err != nil {
 		return err
 	}
-	dst := path.Join(tempLoadDir, filename)
-	f, err := assets.NewFileAsset(src, tempLoadDir, filename, "0777")
+	dst := path.Join(loadRoot, filename)
+	f, err := assets.NewFileAsset(src, loadRoot, filename, "0644")
 	if err != nil {
 		return errors.Wrapf(err, "creating copyable file asset: %s", filename)
 	}
@@ -224,9 +228,6 @@ func loadImageFromCache(cr command.Runner, k8s config.KubernetesConfig, src stri
 		return errors.Wrapf(err, "%s load %s", r.Name(), dst)
 	}
 
-	if err := cr.Run("sudo rm -rf " + dst); err != nil {
-		return errors.Wrap(err, "deleting temp docker image location")
-	}
 	glog.Infof("Successfully loaded image %s from cache", src)
 	return nil
 }

pkg/util/constants.go

@@ -24,11 +24,9 @@ import (
 
 // These constants are used by both minikube
 const (
-	DefaultMinikubeDirectory = "/var/lib/minikube"
-	DefaultCertPath          = DefaultMinikubeDirectory + "/certs/"
-	DefaultKubeConfigPath    = DefaultMinikubeDirectory + "/kubeconfig"
-	DefaultDNSDomain         = "cluster.local"
-	DefaultServiceCIDR       = "10.96.0.0/12"
+	APIServerPort      = 8443
+	DefaultDNSDomain   = "cluster.local"
+	DefaultServiceCIDR = "10.96.0.0/12"
 )
 
 // DefaultV114AdmissionControllers are admission controllers we default to in v1.14.x