From 2f1a11d8d379bf971e44b228d642746d92b50547 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Fri, 9 Aug 2019 15:04:07 -0700 Subject: [PATCH 01/11] Move reusable files (images, binaries, certs) to /data --- pkg/drivers/none/none.go | 4 +- pkg/minikube/assets/addons.go | 74 ++++---- pkg/minikube/assets/addons_test.go | 8 +- pkg/minikube/assets/vm_assets.go | 10 +- pkg/minikube/bootstrapper/bootstrapper.go | 2 +- pkg/minikube/bootstrapper/certs.go | 17 +- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 165 +++++++++++------- .../bootstrapper/kubeadm/templates.go | 14 +- pkg/minikube/bootstrapper/kubeadm/versions.go | 4 +- pkg/minikube/command/ssh_runner.go | 48 +++-- pkg/minikube/constants/constants.go | 36 ++-- pkg/minikube/logs/logs.go | 2 +- pkg/minikube/machine/cache_binaries.go | 7 +- pkg/minikube/machine/cache_images.go | 19 +- pkg/util/constants.go | 1 - 15 files changed, 224 insertions(+), 187 deletions(-) diff --git a/pkg/drivers/none/none.go b/pkg/drivers/none/none.go index dd18634d4a..6c3a5cfef4 100644 --- a/pkg/drivers/none/none.go +++ b/pkg/drivers/none/none.go @@ -38,8 +38,8 @@ const driverName = constants.DriverNone // cleanupPaths are paths to be removed by cleanup, and are used by both kubeadm and minikube. var cleanupPaths = []string{ - "/data/minikube", - "/etc/kubernetes/manifests", + constants.GuestEphemeralDir, + constants.GuestManifestsDir, "/var/lib/minikube", } diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go index 969d5e30d9..ec72f813e3 100644 --- a/pkg/minikube/assets/addons.go +++ b/pkg/minikube/assets/addons.go @@ -70,7 +70,7 @@ var Addons = map[string]*Addon{ "addon-manager": NewAddon([]*BinAsset{ MustBinAsset( "deploy/addons/addon-manager.yaml.tmpl", - "/etc/kubernetes/manifests/", + constants.GuestManifestsDir, "addon-manager.yaml.tmpl", "0640", true), @@ -78,13 +78,13 @@ var Addons = map[string]*Addon{ "dashboard": NewAddon([]*BinAsset{ MustBinAsset( "deploy/addons/dashboard/dashboard-dp.yaml.tmpl", - constants.AddonsPath, + constants.GuestAddonsDir, "dashboard-dp.yaml", "0640", true), MustBinAsset( "deploy/addons/dashboard/dashboard-svc.yaml.tmpl", - constants.AddonsPath, + constants.GuestAddonsDir, "dashboard-svc.yaml", "0640", false), @@ -92,7 +92,7 @@ var Addons = map[string]*Addon{ "default-storageclass": NewAddon([]*BinAsset{ MustBinAsset( "deploy/addons/storageclass/storageclass.yaml.tmpl", - constants.AddonsPath, + constants.GuestAddonsDir, "storageclass.yaml", "0640", false), @@ -100,7 +100,7 @@ var Addons = map[string]*Addon{ "storage-provisioner": NewAddon([]*BinAsset{ MustBinAsset( "deploy/addons/storage-provisioner/storage-provisioner.yaml.tmpl", - constants.AddonsPath, + constants.GuestAddonsDir, "storage-provisioner.yaml", "0640", true), @@ -108,25 +108,25 @@ var Addons = map[string]*Addon{ "storage-provisioner-gluster": NewAddon([]*BinAsset{ MustBinAsset( "deploy/addons/storage-provisioner-gluster/storage-gluster-ns.yaml.tmpl", - constants.AddonsPath, + constants.GuestAddonsDir, "storage-gluster-ns.yaml", "0640", false), MustBinAsset( "deploy/addons/storage-provisioner-gluster/glusterfs-daemonset.yaml.tmpl", - constants.AddonsPath, + constants.GuestAddonsDir, "glusterfs-daemonset.yaml", "0640", false), MustBinAsset( "deploy/addons/storage-provisioner-gluster/heketi-deployment.yaml.tmpl", - constants.AddonsPath, + constants.GuestAddonsDir, "heketi-deployment.yaml", "0640", false), MustBinAsset( "deploy/addons/storage-provisioner-gluster/storage-provisioner-glusterfile.yaml.tmpl", - constants.AddonsPath, + constants.GuestAddonsDir, "storage-privisioner-glusterfile.yaml", "0640", false), @@ -134,31 +134,31 @@ var Addons = map[string]*Addon{ "heapster": NewAddon([]*BinAsset{ MustBinAsset( "deploy/addons/heapster/influx-grafana-rc.yaml.tmpl", - constants.AddonsPath, + constants.GuestAddonsDir, "influxGrafana-rc.yaml", "0640", true), MustBinAsset( "deploy/addons/heapster/grafana-svc.yaml.tmpl", - constants.AddonsPath, + constants.GuestAddonsDir, "grafana-svc.yaml", "0640", false), MustBinAsset( "deploy/addons/heapster/influxdb-svc.yaml.tmpl", - constants.AddonsPath, + constants.GuestAddonsDir, "influxdb-svc.yaml", "0640", false), MustBinAsset( "deploy/addons/heapster/heapster-rc.yaml.tmpl", - constants.AddonsPath, + constants.GuestAddonsDir, "heapster-rc.yaml", "0640", true), MustBinAsset( "deploy/addons/heapster/heapster-svc.yaml.tmpl", - constants.AddonsPath, + constants.GuestAddonsDir, "heapster-svc.yaml", "0640", false), @@ -166,37 +166,37 @@ var Addons = map[string]*Addon{ "efk": NewAddon([]*BinAsset{ MustBinAsset( "deploy/addons/efk/elasticsearch-rc.yaml.tmpl", - constants.AddonsPath, + constants.GuestAddonsDir, "elasticsearch-rc.yaml", "0640", true), MustBinAsset( "deploy/addons/efk/elasticsearch-svc.yaml.tmpl", - constants.AddonsPath, + constants.GuestAddonsDir, "elasticsearch-svc.yaml", "0640", false), MustBinAsset( "deploy/addons/efk/fluentd-es-rc.yaml.tmpl", - constants.AddonsPath, + constants.GuestAddonsDir, "fluentd-es-rc.yaml", "0640", true), MustBinAsset( "deploy/addons/efk/fluentd-es-configmap.yaml.tmpl", - constants.AddonsPath, + constants.GuestAddonsDir, "fluentd-es-configmap.yaml", "0640", false), MustBinAsset( "deploy/addons/efk/kibana-rc.yaml.tmpl", - constants.AddonsPath, + constants.GuestAddonsDir, "kibana-rc.yaml", "0640", false), MustBinAsset( "deploy/addons/efk/kibana-svc.yaml.tmpl", - constants.AddonsPath, + constants.GuestAddonsDir, "kibana-svc.yaml", "0640", false), @@ -204,19 +204,19 @@ var Addons = map[string]*Addon{ "ingress": NewAddon([]*BinAsset{ MustBinAsset( "deploy/addons/ingress/ingress-configmap.yaml.tmpl", - constants.AddonsPath, + constants.GuestAddonsDir, "ingress-configmap.yaml", "0640", false), MustBinAsset( "deploy/addons/ingress/ingress-rbac.yaml.tmpl", - constants.AddonsPath, + constants.GuestAddonsDir, "ingress-rbac.yaml", "0640", false), MustBinAsset( "deploy/addons/ingress/ingress-dp.yaml.tmpl", - constants.AddonsPath, + constants.GuestAddonsDir, "ingress-dp.yaml", "0640", true), @@ -224,19 +224,19 @@ var Addons = map[string]*Addon{ "metrics-server": NewAddon([]*BinAsset{ MustBinAsset( "deploy/addons/metrics-server/metrics-apiservice.yaml.tmpl", - constants.AddonsPath, + constants.GuestAddonsDir, "metrics-apiservice.yaml", "0640", false), MustBinAsset( "deploy/addons/metrics-server/metrics-server-deployment.yaml.tmpl", - constants.AddonsPath, + constants.GuestAddonsDir, "metrics-server-deployment.yaml", "0640", true), MustBinAsset( "deploy/addons/metrics-server/metrics-server-service.yaml.tmpl", - constants.AddonsPath, + constants.GuestAddonsDir, "metrics-server-service.yaml", "0640", false), @@ -244,19 +244,19 @@ var Addons = map[string]*Addon{ "registry": NewAddon([]*BinAsset{ MustBinAsset( "deploy/addons/registry/registry-rc.yaml.tmpl", - constants.AddonsPath, + constants.GuestAddonsDir, "registry-rc.yaml", "0640", false), MustBinAsset( "deploy/addons/registry/registry-svc.yaml.tmpl", - constants.AddonsPath, + constants.GuestAddonsDir, "registry-svc.yaml", "0640", false), MustBinAsset( "deploy/addons/registry/registry-proxy.yaml.tmpl", - constants.AddonsPath, + constants.GuestAddonsDir, "registry-proxy.yaml", "0640", false), @@ -264,7 +264,7 @@ var Addons = map[string]*Addon{ "registry-creds": NewAddon([]*BinAsset{ MustBinAsset( "deploy/addons/registry-creds/registry-creds-rc.yaml.tmpl", - constants.AddonsPath, + constants.GuestAddonsDir, "registry-creds-rc.yaml", "0640", false), @@ -272,7 +272,7 @@ var Addons = map[string]*Addon{ "freshpod": NewAddon([]*BinAsset{ MustBinAsset( "deploy/addons/freshpod/freshpod-rc.yaml.tmpl", - constants.AddonsPath, + constants.GuestAddonsDir, "freshpod-rc.yaml", "0640", true), @@ -280,7 +280,7 @@ var Addons = map[string]*Addon{ "nvidia-driver-installer": NewAddon([]*BinAsset{ MustBinAsset( "deploy/addons/gpu/nvidia-driver-installer.yaml.tmpl", - constants.AddonsPath, + constants.GuestAddonsDir, "nvidia-driver-installer.yaml", "0640", true), @@ -288,7 +288,7 @@ var Addons = map[string]*Addon{ "nvidia-gpu-device-plugin": NewAddon([]*BinAsset{ MustBinAsset( "deploy/addons/gpu/nvidia-gpu-device-plugin.yaml.tmpl", - constants.AddonsPath, + constants.GuestAddonsDir, "nvidia-gpu-device-plugin.yaml", "0640", true), @@ -296,13 +296,13 @@ var Addons = map[string]*Addon{ "logviewer": NewAddon([]*BinAsset{ MustBinAsset( "deploy/addons/logviewer/logviewer-dp-and-svc.yaml.tmpl", - constants.AddonsPath, + constants.GuestAddonsDir, "logviewer-dp-and-svc.yaml", "0640", false), MustBinAsset( "deploy/addons/logviewer/logviewer-rbac.yaml.tmpl", - constants.AddonsPath, + constants.GuestAddonsDir, "logviewer-rbac.yaml", "0640", false), @@ -310,7 +310,7 @@ var Addons = map[string]*Addon{ "gvisor": NewAddon([]*BinAsset{ MustBinAsset( "deploy/addons/gvisor/gvisor-pod.yaml.tmpl", - constants.AddonsPath, + constants.GuestAddonsDir, "gvisor-pod.yaml", "0640", true), @@ -332,7 +332,7 @@ var Addons = map[string]*Addon{ // AddMinikubeDirAssets adds all addons and files to the list // of files to be copied to the vm. func AddMinikubeDirAssets(assets *[]CopyableFile) error { - if err := addMinikubeDirToAssets(constants.MakeMiniPath("addons"), constants.AddonsPath, assets); err != nil { + if err := addMinikubeDirToAssets(constants.MakeMiniPath("addons"), constants.GuestAddonsDir, assets); err != nil { return errors.Wrap(err, "adding addons folder to assets") } if err := addMinikubeDirToAssets(constants.MakeMiniPath("files"), "", assets); err != nil { diff --git a/pkg/minikube/assets/addons_test.go b/pkg/minikube/assets/addons_test.go index eccdbf2ad8..55d073b10b 100644 --- a/pkg/minikube/assets/addons_test.go +++ b/pkg/minikube/assets/addons_test.go @@ -56,18 +56,18 @@ func TestAddMinikubeDirAssets(t *testing.T) { }{ { relativePath: "/dir1/file1.txt", - expectedPath: constants.AddonsPath, + expectedPath: constants.GuestAddonsDir, }, { relativePath: "/dir1/file2.txt", - expectedPath: constants.AddonsPath, + expectedPath: constants.GuestAddonsDir, }, { relativePath: "/dir2/file1.txt", - expectedPath: constants.AddonsPath, + expectedPath: constants.GuestAddonsDir, }, }, - vmPath: constants.AddonsPath, + vmPath: constants.GuestAddonsDir, }, { description: "absolute path assets", diff --git a/pkg/minikube/assets/vm_assets.go b/pkg/minikube/assets/vm_assets.go index 2dcec3d3ac..f73a86bf3d 100644 --- a/pkg/minikube/assets/vm_assets.go +++ b/pkg/minikube/assets/vm_assets.go @@ -23,6 +23,7 @@ import ( "io" "os" "path" + "path/filepath" "github.com/golang/glog" "github.com/pkg/errors" @@ -78,14 +79,15 @@ func NewMemoryAssetTarget(d []byte, targetPath, permissions string) *MemoryAsset } // NewFileAsset creates a new FileAsset -func NewFileAsset(path, targetDir, targetName, permissions string) (*FileAsset, error) { - f, err := os.Open(path) +func NewFileAsset(src, targetDir, targetName, permissions string) (*FileAsset, error) { + glog.Infof("NewFileAsset: %s -> %s", src, filepath.Join(targetDir, targetName)) + f, err := os.Open(src) if err != nil { - return nil, errors.Wrapf(err, "Error opening file asset: %s", path) + return nil, errors.Wrapf(err, "Error opening file asset: %s", src) } return &FileAsset{ BaseAsset: BaseAsset{ - AssetName: path, + AssetName: src, TargetDir: targetDir, TargetName: targetName, Permissions: permissions, diff --git a/pkg/minikube/bootstrapper/bootstrapper.go b/pkg/minikube/bootstrapper/bootstrapper.go index b3ff31095d..25a0b3a396 100644 --- a/pkg/minikube/bootstrapper/bootstrapper.go +++ b/pkg/minikube/bootstrapper/bootstrapper.go @@ -56,7 +56,7 @@ const ( func GetCachedBinaryList(bootstrapper string) []string { switch bootstrapper { case BootstrapperTypeKubeadm: - return constants.GetKubeadmCachedBinaries() + return constants.KubeadmBinaries default: return []string{} } diff --git a/pkg/minikube/bootstrapper/certs.go b/pkg/minikube/bootstrapper/certs.go index b24f2e7d83..64b2a912c3 100644 --- a/pkg/minikube/bootstrapper/certs.go +++ b/pkg/minikube/bootstrapper/certs.go @@ -50,7 +50,9 @@ func SetupCerts(cmd command.Runner, k8s config.KubernetesConfig) error { if err := generateCerts(k8s); err != nil { return errors.Wrap(err, "Error generating certs") } - + if err := cmd.Run(fmt.Sprintf("sudo mkdir -p %s", constants.GuestCertsDir)); err != nil { + return errors.Wrap(err, "mkdir") + } copyableFiles := []assets.CopyableFile{} for _, cert := range certs { @@ -59,7 +61,7 @@ func SetupCerts(cmd command.Runner, k8s config.KubernetesConfig) error { if strings.HasSuffix(cert, ".key") { perms = "0600" } - certFile, err := assets.NewFileAsset(p, util.DefaultCertPath, cert, perms) + certFile, err := assets.NewFileAsset(p, constants.GuestCertsDir, cert, perms) if err != nil { return err } @@ -69,9 +71,9 @@ func SetupCerts(cmd command.Runner, k8s config.KubernetesConfig) error { kubeCfgSetup := &util.KubeConfigSetup{ ClusterName: k8s.NodeName, ClusterServerAddress: fmt.Sprintf("https://localhost:%d", k8s.NodePort), - ClientCertificate: path.Join(util.DefaultCertPath, "apiserver.crt"), - ClientKey: path.Join(util.DefaultCertPath, "apiserver.key"), - CertificateAuthority: path.Join(util.DefaultCertPath, "ca.crt"), + ClientCertificate: path.Join(constants.GuestCertsDir, "apiserver.crt"), + ClientKey: path.Join(constants.GuestCertsDir, "apiserver.key"), + CertificateAuthority: path.Join(constants.GuestCertsDir, "ca.crt"), KeepContext: false, } @@ -85,13 +87,12 @@ func SetupCerts(cmd command.Runner, k8s config.KubernetesConfig) error { return errors.Wrap(err, "encoding kubeconfig") } - kubeCfgFile := assets.NewMemoryAsset(data, - util.DefaultMinikubeDirectory, "kubeconfig", "0644") + kubeCfgFile := assets.NewMemoryAsset(data, constants.GuestEphemeralDir, "kubeconfig", "0644") copyableFiles = append(copyableFiles, kubeCfgFile) for _, f := range copyableFiles { if err := cmd.Copy(f); err != nil { - return err + return errors.Wrapf(err, "Copy %s", f.GetAssetName()) } } return nil diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 9e45a9f963..2b7f15adb2 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -22,6 +22,7 @@ import ( "fmt" "net" "net/http" + "path/filepath" "runtime" "strings" "time" @@ -90,6 +91,9 @@ var PodsByLayer = []pod{ {"dns", "k8s-app", "kube-dns"}, } +// yamlConfigPath is the path to the kubeadm configuration +var yamlConfigPath = filepath.Join(constants.GuestEphemeralDir, "kubeadm.yaml") + // SkipAdditionalPreflights are additional preflights we skip depending on the runtime in use. var SkipAdditionalPreflights = map[string][]string{} @@ -206,8 +210,8 @@ func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error { } ignore := []string{ - "DirAvailable--etc-kubernetes-manifests", // Addons are stored in /etc/kubernetes/manifests - "DirAvailable--data-minikube", + fmt.Sprintf("DirAvailable-%s", strings.Replace(constants.GuestManifestsDir, "/", "-", -1)), + fmt.Sprintf("DirAvailable-%s", strings.Replace(constants.GuestPersistentDir, "/", "-", -1)), "FileAvailable--etc-kubernetes-manifests-kube-scheduler.yaml", "FileAvailable--etc-kubernetes-manifests-kube-apiserver.yaml", "FileAvailable--etc-kubernetes-manifests-kube-controller-manager.yaml", @@ -223,8 +227,8 @@ func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error { ignore = append(ignore, "SystemVerification") } - cmd := fmt.Sprintf("sudo /usr/bin/kubeadm init --config %s %s --ignore-preflight-errors=%s", - constants.KubeadmConfigFile, extraFlags, strings.Join(ignore, ",")) + cmd := fmt.Sprintf("%s init --config %s %s --ignore-preflight-errors=%s", + invokeKubeadm(k8s.KubernetesVersion), yamlConfigPath, extraFlags, strings.Join(ignore, ",")) out, err := k.c.CombinedOutput(cmd) if err != nil { return errors.Wrapf(err, "cmd failed: %s\n%s\n", cmd, out) @@ -347,13 +351,12 @@ func (k *Bootstrapper) RestartCluster(k8s config.KubernetesConfig) error { controlPlane = "control-plane" } - configPath := constants.KubeadmConfigFile - baseCmd := fmt.Sprintf("sudo kubeadm %s", phase) + baseCmd := fmt.Sprintf("%s %s", invokeKubeadm(k8s.KubernetesVersion), phase) cmds := []string{ - fmt.Sprintf("%s phase certs all --config %s", baseCmd, configPath), - fmt.Sprintf("%s phase kubeconfig all --config %s", baseCmd, configPath), - fmt.Sprintf("%s phase %s all --config %s", baseCmd, controlPlane, configPath), - fmt.Sprintf("%s phase etcd local --config %s", baseCmd, configPath), + fmt.Sprintf("%s phase certs all --config %s", baseCmd, yamlConfigPath), + fmt.Sprintf("%s phase kubeconfig all --config %s", baseCmd, yamlConfigPath), + fmt.Sprintf("%s phase %s all --config %s", baseCmd, controlPlane, yamlConfigPath), + fmt.Sprintf("%s phase etcd local --config %s", baseCmd, yamlConfigPath), } // Run commands one at a time so that it is easier to root cause failures. @@ -367,7 +370,7 @@ func (k *Bootstrapper) RestartCluster(k8s config.KubernetesConfig) error { return errors.Wrap(err, "waiting for apiserver") } // restart the proxy and coredns - if err := k.c.Run(fmt.Sprintf("%s phase addon all --config %s", baseCmd, configPath)); err != nil { + if err := k.c.Run(fmt.Sprintf("%s phase addon all --config %s", baseCmd, yamlConfigPath)); err != nil { return errors.Wrapf(err, "addon phase") } @@ -379,8 +382,21 @@ func (k *Bootstrapper) RestartCluster(k8s config.KubernetesConfig) error { // waitForAPIServer waits for the apiserver to start up func (k *Bootstrapper) waitForAPIServer(k8s config.KubernetesConfig) error { - glog.Infof("Waiting for apiserver ...") - return wait.PollImmediate(time.Millisecond*300, time.Minute*3, func() (bool, error) { + glog.Infof("Waiting for apiserver process ...") + // To give a better error message, first check for process existence via ssh + err := wait.PollImmediate(time.Millisecond*300, time.Minute*2, func() (bool, error) { + ierr := k.c.Run(`pgrep apiserver`) + if ierr != nil { + return false, ierr + } + return true, nil + }) + if err != nil { + return fmt.Errorf("apiserver process never appeared") + } + + glog.Infof("Waiting for apiserver status ...") + return wait.PollImmediate(time.Millisecond*300, time.Minute*1, func() (bool, error) { status, err := k.GetAPIServerStatus(net.ParseIP(k8s.NodeIP), k8s.NodePort) glog.Infof("apiserver status: %s, err: %v", status, err) if err != nil { @@ -395,7 +411,7 @@ func (k *Bootstrapper) waitForAPIServer(k8s config.KubernetesConfig) error { // DeleteCluster removes the components that were started earlier func (k *Bootstrapper) DeleteCluster(k8s config.KubernetesConfig) error { - cmd := fmt.Sprintf("sudo kubeadm reset --force") + cmd := fmt.Sprintf("%s reset --force", invokeKubeadm(k8s.KubernetesVersion)) out, err := k.c.CombinedOutput(cmd) if err != nil { return errors.Wrapf(err, "kubeadm reset: %s\n%s\n", cmd, out) @@ -414,7 +430,7 @@ func (k *Bootstrapper) PullImages(k8s config.KubernetesConfig) error { return fmt.Errorf("pull command is not supported by kubeadm v%s", version) } - cmd := fmt.Sprintf("sudo kubeadm config images pull --config %s", constants.KubeadmConfigFile) + cmd := fmt.Sprintf("%s config images pull --config %s", invokeKubeadm(k8s.KubernetesVersion), yamlConfigPath) if err := k.c.Run(cmd); err != nil { return errors.Wrapf(err, "running cmd: %s", cmd) } @@ -428,15 +444,15 @@ func (k *Bootstrapper) SetupCerts(k8s config.KubernetesConfig) error { // NewKubeletConfig generates a new systemd unit containing a configured kubelet // based on the options present in the KubernetesConfig. -func NewKubeletConfig(k8s config.KubernetesConfig, r cruntime.Manager) (string, error) { +func NewKubeletConfig(k8s config.KubernetesConfig, r cruntime.Manager) ([]byte, error) { version, err := ParseKubernetesVersion(k8s.KubernetesVersion) if err != nil { - return "", errors.Wrap(err, "parsing kubernetes version") + return nil, errors.Wrap(err, "parsing kubernetes version") } extraOpts, err := ExtraConfigForComponent(Kubelet, k8s.ExtraOptions, version) if err != nil { - return "", errors.Wrap(err, "generating extra configuration for kubelet") + return nil, errors.Wrap(err, "generating extra configuration for kubelet") } for k, v := range r.KubeletOptions() { @@ -454,28 +470,28 @@ func NewKubeletConfig(k8s config.KubernetesConfig, r cruntime.Manager) (string, // parses a map of the feature gates for kubelet _, kubeletFeatureArgs, err := ParseFeatureArgs(k8s.FeatureGates) if err != nil { - return "", errors.Wrap(err, "parses feature gate config for kubelet") + return nil, errors.Wrap(err, "parses feature gate config for kubelet") } if kubeletFeatureArgs != "" { extraOpts["feature-gates"] = kubeletFeatureArgs } - extraFlags := convertToFlags(extraOpts) - b := bytes.Buffer{} opts := struct { ExtraOptions string ContainerRuntime string + KubeletPath string }{ - ExtraOptions: extraFlags, + ExtraOptions: convertToFlags(extraOpts), ContainerRuntime: k8s.ContainerRuntime, + KubeletPath: filepath.Join(binRoot(k8s.KubernetesVersion), "kubelet"), } if err := kubeletSystemdTemplate.Execute(&b, opts); err != nil { - return "", err + return nil, err } - return b.String(), nil + return b.Bytes(), nil } // UpdateCluster updates the cluster @@ -499,32 +515,35 @@ func (k *Bootstrapper) UpdateCluster(cfg config.KubernetesConfig) error { if err != nil { return errors.Wrap(err, "generating kubelet config") } - glog.Infof("kubelet %s config:\n%s", cfg.KubernetesVersion, kubeletCfg) - var files []assets.CopyableFile - files = copyConfig(cfg, files, kubeadmCfg, kubeletCfg) - - if err := downloadBinaries(cfg, k.c); err != nil { - return errors.Wrap(err, "downloading binaries") + kubeletService, err := NewKubeletService(cfg) + if err != nil { + return errors.Wrap(err, "generating kubelet service") } + glog.Infof("kubelet %s config:\n%s", cfg.KubernetesVersion, kubeletCfg) + + // stop kubelet to avoid "Text File Busy" error + err = k.c.Run(`pgrep kubelet && sudo systemctl stop kubelet`) + if err != nil { + glog.Warningf("unable to stop kubelet: %s", err) + } + if err := transferBinaries(cfg, k.c); err != nil { + return errors.Wrap(err, "downloading binaries") + } + files := configFiles(cfg, kubeadmCfg, kubeletCfg, kubeletService) if err := addAddons(&files, assets.GenerateTemplateData(cfg)); err != nil { return errors.Wrap(err, "adding addons") } - for _, f := range files { if err := k.c.Copy(f); err != nil { return errors.Wrapf(err, "copy") } } - err = k.c.Run(` -sudo systemctl daemon-reload && -sudo systemctl start kubelet -`) - if err != nil { + + if err := k.c.Run(`sudo systemctl daemon-reload && sudo systemctl start kubelet`); err != nil { return errors.Wrap(err, "starting kubelet") } - return nil } @@ -547,21 +566,22 @@ func createExtraComponentConfig(extraOptions util.ExtraOptionSlice, version semv return extraArgsSlice, nil } -func generateConfig(k8s config.KubernetesConfig, r cruntime.Manager) (string, error) { +// generateConfig generates the kubeadm.yaml file +func generateConfig(k8s config.KubernetesConfig, r cruntime.Manager) ([]byte, error) { version, err := ParseKubernetesVersion(k8s.KubernetesVersion) if err != nil { - return "", errors.Wrap(err, "parsing kubernetes version") + return nil, errors.Wrap(err, "parsing kubernetes version") } // parses a map of the feature gates for kubeadm and component kubeadmFeatureArgs, componentFeatureArgs, err := ParseFeatureArgs(k8s.FeatureGates) if err != nil { - return "", errors.Wrap(err, "parses feature gate config for kubeadm and component") + return nil, errors.Wrap(err, "parses feature gate config for kubeadm and component") } extraComponentConfig, err := createExtraComponentConfig(k8s.ExtraOptions, version, componentFeatureArgs) if err != nil { - return "", errors.Wrap(err, "generating extra component config for kubeadm") + return nil, errors.Wrap(err, "generating extra component config for kubeadm") } // In case of no port assigned, use util.APIServerPort @@ -585,13 +605,13 @@ func generateConfig(k8s config.KubernetesConfig, r cruntime.Manager) (string, er FeatureArgs map[string]bool NoTaintMaster bool }{ - CertDir: util.DefaultCertPath, + CertDir: constants.GuestCertsDir, ServiceCIDR: util.DefaultServiceCIDR, PodSubnet: k8s.ExtraOptions.Get("pod-network-cidr", Kubeadm), AdvertiseAddress: k8s.NodeIP, APIServerPort: nodePort, KubernetesVersion: k8s.KubernetesVersion, - EtcdDataDir: "/data/minikube", //TODO(r2d4): change to something else persisted + EtcdDataDir: filepath.Join(constants.GuestPersistentDir, "etcd"), NodeName: k8s.NodeName, CRISocket: r.SocketPath(), ImageRepository: k8s.ImageRepository, @@ -618,41 +638,62 @@ func generateConfig(k8s config.KubernetesConfig, r cruntime.Manager) (string, er configTmpl = configTmplV1Beta1 } if err := configTmpl.Execute(&b, opts); err != nil { - return "", err + return nil, err } - return b.String(), nil + return b.Bytes(), nil } -func copyConfig(cfg config.KubernetesConfig, files []assets.CopyableFile, kubeadmCfg string, kubeletCfg string) []assets.CopyableFile { - files = append(files, - assets.NewMemoryAssetTarget([]byte(kubeletService), constants.KubeletServiceFile, "0640"), - assets.NewMemoryAssetTarget([]byte(kubeletCfg), constants.KubeletSystemdConfFile, "0640"), - assets.NewMemoryAssetTarget([]byte(kubeadmCfg), constants.KubeadmConfigFile, "0640")) +// NewKubeletService returns a generated systemd unit file for the kubelet +func NewKubeletService(cfg config.KubernetesConfig) ([]byte, error) { + var b bytes.Buffer + opts := struct{ KubeletPath string }{KubeletPath: filepath.Join(binRoot(cfg.KubernetesVersion), "kubelet")} + if err := kubeletServiceTemplate.Execute(&b, opts); err != nil { + return nil, errors.Wrap(err, "template execute") + } + return b.Bytes(), nil +} +// configFiles returns configuration file assets +func configFiles(cfg config.KubernetesConfig, kubeadm []byte, kubelet []byte, kubeletSvc []byte) []assets.CopyableFile { + fs := []assets.CopyableFile{ + assets.NewMemoryAssetTarget(kubeadm, yamlConfigPath, "0640"), + assets.NewMemoryAssetTarget(kubelet, constants.KubeletSystemdConfFile, "0640"), + assets.NewMemoryAssetTarget(kubeletSvc, constants.KubeletServiceFile, "0640"), + } // Copy the default CNI config (k8s.conf), so that kubelet can successfully // start a Pod in the case a user hasn't manually installed any CNI plugin // and minikube was started with "--extra-config=kubelet.network-plugin=cni". if cfg.EnableDefaultCNI { - files = append(files, - assets.NewMemoryAssetTarget([]byte(defaultCNIConfig), constants.DefaultCNIConfigPath, "0644")) + fs = append(fs, assets.NewMemoryAssetTarget([]byte(defaultCNIConfig), constants.DefaultCNIConfigPath, "0644")) } - - return files + return fs } -func downloadBinaries(cfg config.KubernetesConfig, c command.Runner) error { +// binDir returns the persistant path binaries are stored in +func binRoot(version string) string { + return filepath.Join(constants.GuestPersistentDir, "binaries", version) +} + +// invokeKubeadm returns the invocation command for Kubeadm +func invokeKubeadm(version string) string { + return fmt.Sprintf("sudo env PATH=%s:$PATH kubeadm", binRoot(version)) +} + +// transferBinaries transfers all required Kubernetes binaries +func transferBinaries(cfg config.KubernetesConfig, c command.Runner) error { var g errgroup.Group - for _, bin := range constants.GetKubeadmCachedBinaries() { - bin := bin + for _, name := range constants.KubeadmBinaries { + name := name g.Go(func() error { - path, err := machine.CacheBinary(bin, cfg.KubernetesVersion, "linux", runtime.GOARCH) + src, err := machine.CacheBinary(name, cfg.KubernetesVersion, "linux", runtime.GOARCH) if err != nil { - return errors.Wrapf(err, "downloading %s", bin) + return errors.Wrapf(err, "downloading %s", name) } - err = machine.CopyBinary(c, bin, path) - if err != nil { - return errors.Wrapf(err, "copying %s", bin) + + dst := filepath.Join(binRoot(cfg.KubernetesVersion), name) + if err := machine.CopyBinary(c, src, dst); err != nil { + return errors.Wrapf(err, "copybinary %s -> %s", src, dst) } return nil }) diff --git a/pkg/minikube/bootstrapper/kubeadm/templates.go b/pkg/minikube/bootstrapper/kubeadm/templates.go index 70ae1aad28..dd1bfe6590 100644 --- a/pkg/minikube/bootstrapper/kubeadm/templates.go +++ b/pkg/minikube/bootstrapper/kubeadm/templates.go @@ -151,24 +151,24 @@ evictionHard: imagefs.available: "0%" `)) -var kubeletSystemdTemplate = template.Must(template.New("kubeletSystemdTemplate").Parse(` -[Unit] +// kubeletSystemdTemplate hosts the override kubelet flags, written to constants.KubeletSystemdConfFile +var kubeletSystemdTemplate = template.Must(template.New("kubeletSystemdTemplate").Parse(`[Unit] {{if or (eq .ContainerRuntime "cri-o") (eq .ContainerRuntime "cri")}}Wants=crio.service{{else if eq .ContainerRuntime "containerd"}}Wants=containerd.service{{else}}Wants=docker.socket{{end}} [Service] ExecStart= -ExecStart=/usr/bin/kubelet{{if .ExtraOptions}} {{.ExtraOptions}}{{end}} +ExecStart={{.KubeletPath}}{{if .ExtraOptions}} {{.ExtraOptions}}{{end}} [Install] `)) -const kubeletService = ` -[Unit] +// kubeletServiceTemplate is the base kubelet systemd template, written to constanst.KubeletServiceFile +var kubeletServiceTemplate = template.Must(template.New("kubeletServiceTemplate").Parse(`[Unit] Description=kubelet: The Kubernetes Node Agent Documentation=http://kubernetes.io/docs/ [Service] -ExecStart=/usr/bin/kubelet +ExecStart={{.KubeletPath}} Restart=always StartLimitInterval=0 # Tuned for local dev: faster than upstream default (10s), but slower than systemd default (100ms) @@ -176,7 +176,7 @@ RestartSec=600ms [Install] WantedBy=multi-user.target -` +`)) // printMapInOrder sorts the keys and prints the map in order, combining key // value pairs with the separator character diff --git a/pkg/minikube/bootstrapper/kubeadm/versions.go b/pkg/minikube/bootstrapper/kubeadm/versions.go index 2c7c9a97f9..02d0de884f 100644 --- a/pkg/minikube/bootstrapper/kubeadm/versions.go +++ b/pkg/minikube/bootstrapper/kubeadm/versions.go @@ -241,7 +241,7 @@ var versionSpecificOpts = []VersionedExtraOption{ NewUnversionedOption(Kubelet, "hostname-override", constants.DefaultNodeName), // System pods args - NewUnversionedOption(Kubelet, "pod-manifest-path", "/etc/kubernetes/manifests"), + NewUnversionedOption(Kubelet, "pod-manifest-path", constants.GuestManifestsDir), { Option: util.ExtraOption{ Component: Kubelet, @@ -257,7 +257,7 @@ var versionSpecificOpts = []VersionedExtraOption{ // Auth args NewUnversionedOption(Kubelet, "authorization-mode", "Webhook"), - NewUnversionedOption(Kubelet, "client-ca-file", path.Join(util.DefaultCertPath, "ca.crt")), + NewUnversionedOption(Kubelet, "client-ca-file", path.Join(constants.GuestCertsDir, "ca.crt")), // Cgroup args NewUnversionedOption(Kubelet, "cgroup-driver", "cgroupfs"), diff --git a/pkg/minikube/command/ssh_runner.go b/pkg/minikube/command/ssh_runner.go index a07e3ad985..ad94c2036f 100644 --- a/pkg/minikube/command/ssh_runner.go +++ b/pkg/minikube/command/ssh_runner.go @@ -21,11 +21,13 @@ import ( "fmt" "io" "path" + "path/filepath" "sync" "github.com/golang/glog" "github.com/pkg/errors" "golang.org/x/crypto/ssh" + "golang.org/x/sync/errgroup" "k8s.io/minikube/pkg/minikube/assets" "k8s.io/minikube/pkg/util" ) @@ -152,14 +154,6 @@ func (s *SSHRunner) CombinedOutput(cmd string) (string, error) { // Copy copies a file to the remote over SSH. func (s *SSHRunner) Copy(f assets.CopyableFile) error { - deleteCmd := fmt.Sprintf("sudo rm -f %s", path.Join(f.GetTargetDir(), f.GetTargetName())) - mkdirCmd := fmt.Sprintf("sudo mkdir -p %s", f.GetTargetDir()) - for _, cmd := range []string{deleteCmd, mkdirCmd} { - if err := s.Run(cmd); err != nil { - return errors.Wrapf(err, "pre-copy") - } - } - sess, err := s.c.NewSession() if err != nil { return errors.Wrap(err, "NewSession") @@ -170,40 +164,38 @@ func (s *SSHRunner) Copy(f assets.CopyableFile) error { return errors.Wrap(err, "StdinPipe") } // The scpcmd below *should not* return until all data is copied and the - // StdinPipe is closed. But let's use a WaitGroup to make it expicit. - var wg sync.WaitGroup - wg.Add(1) - var ierr error + // StdinPipe is closed. But let's use errgroup to make it expicit. + var g errgroup.Group var copied int64 + dst := filepath.Join(path.Join(f.GetTargetDir(), f.GetTargetName())) + glog.Infof("Transferring %d bytes to %s", f.GetLength(), dst) - go func() { - defer wg.Done() + g.Go(func() error { defer w.Close() - glog.Infof("Transferring %d bytes to %s", f.GetLength(), f.GetTargetName()) header := fmt.Sprintf("C%s %d %s\n", f.GetPermissions(), f.GetLength(), f.GetTargetName()) fmt.Fprint(w, header) if f.GetLength() == 0 { glog.Warningf("%s is a 0 byte asset!", f.GetTargetName()) fmt.Fprint(w, "\x00") - return + return nil } - copied, ierr = io.Copy(w, f) + copied, err = io.Copy(w, f) + if err != nil { + return errors.Wrap(err, "io.Copy") + } if copied != int64(f.GetLength()) { - glog.Warningf("%s: expected to copy %d bytes, but copied %d instead", f.GetTargetName(), f.GetLength(), copied) - } else { - glog.Infof("%s: copied %d bytes", f.GetTargetName(), copied) - } - if ierr != nil { - glog.Errorf("io.Copy failed: %v", ierr) + return fmt.Errorf("%s: expected to copy %d bytes, but copied %d instead", f.GetTargetName(), f.GetLength(), copied) } + glog.Infof("%s: copied %d bytes", f.GetTargetName(), copied) fmt.Fprint(w, "\x00") - }() + return nil + }) - _, err = sess.CombinedOutput(fmt.Sprintf("sudo scp -t %s", f.GetTargetDir())) + scp := fmt.Sprintf("sudo mkdir -p %s && sudo scp -t %s", f.GetTargetDir(), f.GetTargetDir()) + out, err := sess.CombinedOutput(scp) if err != nil { - return err + return fmt.Errorf("%s: %s\noutput: %s", scp, err, out) } - wg.Wait() - return ierr + return g.Wait() } diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index b2cbd0680d..2a1ec7eef0 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -208,12 +208,6 @@ func GetProfilePath(profile string, miniHome ...string) string { return filepath.Join(miniPath, "profiles", profile) } -// AddonsPath is the default path of the addons configuration -const AddonsPath = "/etc/kubernetes/addons" - -// FilesPath is the default path of files -const FilesPath = "/files" - const ( // KubeletServiceFile is the path to the kubelet systemd service KubeletServiceFile = "/lib/systemd/system/kubelet.service" @@ -223,9 +217,17 @@ const ( KubeadmConfigFile = "/var/lib/kubeadm.yaml" // DefaultCNIConfigPath is the path to the CNI configuration DefaultCNIConfigPath = "/etc/cni/net.d/k8s.conf" -) + // GuestAddonsDir is the default path of the addons configuration + GuestAddonsDir = "/etc/kubernetes/addons" + // GuestManifestsDir is where the kubelet should look for static Pod manifests + GuestManifestsDir = "/etc/kubernetes/manifests" + // PersistentDir is the path where persistant data should be stored within the VM (not tmpfs) + GuestPersistentDir = "/data/minikube" + // GuestEphemeralDir is the path where ephemeral data should be stored within the VM + GuestEphemeralDir = "/var/tmp/minikube" + // GuestCertsDir are where Kubernetes certificates are kept on the guest + GuestCertsDir = "/data/minikube/certs" -const ( // DefaultUfsPort is the default port of UFS DefaultUfsPort = "5640" // DefaultUfsDebugLvl is the default debug level of UFS @@ -236,6 +238,12 @@ const ( DefaultMsize = 262144 // DefaultMountVersion is the default 9p version to use for mount DefaultMountVersion = "9p2000.L" + + // IsMinikubeChildProcess is the name of "is minikube child process" variable + IsMinikubeChildProcess = "IS_MINIKUBE_CHILD_PROCESS" + + // FileScheme is the file scheme + FileScheme = "file" ) // ImageRepositories contains all known image repositories @@ -254,16 +262,8 @@ func GetKubernetesReleaseURLSHA1(binaryName, version, osName, archName string) s return fmt.Sprintf("%s.sha1", GetKubernetesReleaseURL(binaryName, version, osName, archName)) } -// IsMinikubeChildProcess is the name of "is minikube child process" variable -const IsMinikubeChildProcess = "IS_MINIKUBE_CHILD_PROCESS" - -// FileScheme is the file scheme -const FileScheme = "file" - -// GetKubeadmCachedBinaries gets the binaries to cache for kubeadm -func GetKubeadmCachedBinaries() []string { - return []string{"kubelet", "kubeadm"} -} +// KubeadmBinaries are Kubernetes release binaries required for kubeadm +var KubeadmBinaries = []string{"kubelet", "kubeadm"} // GetKubeadmCachedImages gets the images to cache for kubeadm for a version func GetKubeadmCachedImages(imageRepository string, kubernetesVersionStr string) (string, []string) { diff --git a/pkg/minikube/logs/logs.go b/pkg/minikube/logs/logs.go index 6af57a34d4..05775f8779 100644 --- a/pkg/minikube/logs/logs.go +++ b/pkg/minikube/logs/logs.go @@ -34,7 +34,7 @@ import ( ) // rootCauseRe is a regular expression that matches known failure root causes -var rootCauseRe = regexp.MustCompile(`^error: |eviction manager: pods.* evicted|unknown flag: --|forbidden.*no providers available|eviction manager:.*evicted|tls: bad certificate`) +var rootCauseRe = regexp.MustCompile(`^error: |eviction manager: pods.* evicted|unknown flag: --|forbidden.*no providers available|eviction manager:.*evicted|tls: bad certificate|kubelet.*no API client|kubelet.*No api server`) // ignoreRe is a regular expression that matches spurious errors to not surface var ignoreCauseRe = regexp.MustCompile("error: no objects passed to apply") diff --git a/pkg/minikube/machine/cache_binaries.go b/pkg/minikube/machine/cache_binaries.go index a31cde810a..8dd2c03ff4 100644 --- a/pkg/minikube/machine/cache_binaries.go +++ b/pkg/minikube/machine/cache_binaries.go @@ -20,6 +20,7 @@ import ( "crypto" "os" "path" + "path/filepath" "runtime" "github.com/golang/glog" @@ -90,9 +91,9 @@ func CacheBinary(binary, version, osName, archName string) (string, error) { return targetFilepath, nil } -// CopyBinary copies previously cached binaries into the path -func CopyBinary(cr command.Runner, binary, path string) error { - f, err := assets.NewFileAsset(path, "/usr/bin", binary, "0755") +// CopyBinary copies a locally cached binary to the guest VM +func CopyBinary(cr command.Runner, src string, dest string) error { + f, err := assets.NewFileAsset(src, filepath.Dir(dest), filepath.Base(dest), "0755") if err != nil { return errors.Wrap(err, "new file asset") } diff --git a/pkg/minikube/machine/cache_images.go b/pkg/minikube/machine/cache_images.go index 3456e2270c..b3fcf45e1a 100644 --- a/pkg/minikube/machine/cache_images.go +++ b/pkg/minikube/machine/cache_images.go @@ -41,7 +41,8 @@ import ( "k8s.io/minikube/pkg/minikube/cruntime" ) -const tempLoadDir = "/tmp" +// loadRoot is where images should be loaded from within the guest VM +var loadRoot = filepath.Join(constants.GuestPersistentDir, "images") var getWindowsVolumeName = getWindowsVolumeNameCmd @@ -86,6 +87,9 @@ func CacheImages(images []string, cacheDir string) error { // LoadImages loads previously cached images into the container runtime func LoadImages(cmd command.Runner, images []string, cacheDir string) error { + glog.Infof("LoadImages start: %s", images) + defer glog.Infof("LoadImages end") + var g errgroup.Group // Load profile cluster config from file cc, err := config.Load() @@ -97,7 +101,7 @@ func LoadImages(cmd command.Runner, images []string, cacheDir string) error { g.Go(func() error { src := filepath.Join(cacheDir, image) src = sanitizeCacheDir(src) - if err := loadImageFromCache(cmd, cc.KubernetesConfig, src); err != nil { + if err := transferAndLoadImage(cmd, cc.KubernetesConfig, src); err != nil { glog.Warningf("Failed to load %s: %v", src, err) return errors.Wrapf(err, "loading image %s", src) } @@ -194,15 +198,15 @@ func getWindowsVolumeNameCmd(d string) (string, error) { return vname, nil } -// loadImageFromCache loads a single image from the cache -func loadImageFromCache(cr command.Runner, k8s config.KubernetesConfig, src string) error { +// transferAndLoadImage transfers and loads a single image from the cache +func transferAndLoadImage(cr command.Runner, k8s config.KubernetesConfig, src string) error { glog.Infof("Loading image from cache: %s", src) filename := filepath.Base(src) if _, err := os.Stat(src); err != nil { return err } - dst := path.Join(tempLoadDir, filename) - f, err := assets.NewFileAsset(src, tempLoadDir, filename, "0777") + dst := path.Join(loadRoot, filename) + f, err := assets.NewFileAsset(src, loadRoot, filename, "0644") if err != nil { return errors.Wrapf(err, "creating copyable file asset: %s", filename) } @@ -222,9 +226,6 @@ func loadImageFromCache(cr command.Runner, k8s config.KubernetesConfig, src stri return errors.Wrapf(err, "%s load %s", r.Name(), dst) } - if err := cr.Run("sudo rm -rf " + dst); err != nil { - return errors.Wrap(err, "deleting temp docker image location") - } glog.Infof("Successfully loaded image %s from cache", src) return nil } diff --git a/pkg/util/constants.go b/pkg/util/constants.go index 92f0c29bf5..9ca61ffb8b 100644 --- a/pkg/util/constants.go +++ b/pkg/util/constants.go @@ -26,7 +26,6 @@ import ( const ( APIServerPort = 8443 DefaultMinikubeDirectory = "/var/lib/minikube" - DefaultCertPath = DefaultMinikubeDirectory + "/certs/" DefaultKubeConfigPath = DefaultMinikubeDirectory + "/kubeconfig" DefaultDNSDomain = "cluster.local" DefaultServiceCIDR = "10.96.0.0/12" From 7c865c27cfc5cd06bab8ab00b9d009e0c7ef2cba Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Fri, 9 Aug 2019 15:19:23 -0700 Subject: [PATCH 02/11] Merge DefaultMinikubeDirectory with GuestPersistentDir --- pkg/minikube/constants/constants.go | 11 ++++------- pkg/util/constants.go | 8 +++----- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index 2a1ec7eef0..710e098b79 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -213,21 +213,19 @@ const ( KubeletServiceFile = "/lib/systemd/system/kubelet.service" // KubeletSystemdConfFile is the path to the kubelet systemd configuration KubeletSystemdConfFile = "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf" - // KubeadmConfigFile is the path to the kubeadm configuration - KubeadmConfigFile = "/var/lib/kubeadm.yaml" // DefaultCNIConfigPath is the path to the CNI configuration DefaultCNIConfigPath = "/etc/cni/net.d/k8s.conf" + // GuestAddonsDir is the default path of the addons configuration GuestAddonsDir = "/etc/kubernetes/addons" // GuestManifestsDir is where the kubelet should look for static Pod manifests GuestManifestsDir = "/etc/kubernetes/manifests" - // PersistentDir is the path where persistant data should be stored within the VM (not tmpfs) - GuestPersistentDir = "/data/minikube" // GuestEphemeralDir is the path where ephemeral data should be stored within the VM GuestEphemeralDir = "/var/tmp/minikube" + // PersistentDir is the path where persistant data should be stored within the VM (not tmpfs) + GuestPersistentDir = "/var/lib/minikube" // GuestCertsDir are where Kubernetes certificates are kept on the guest - GuestCertsDir = "/data/minikube/certs" - + GuestCertsDir = GuestPersistentDir + "/certs" // DefaultUfsPort is the default port of UFS DefaultUfsPort = "5640" // DefaultUfsDebugLvl is the default debug level of UFS @@ -241,7 +239,6 @@ const ( // IsMinikubeChildProcess is the name of "is minikube child process" variable IsMinikubeChildProcess = "IS_MINIKUBE_CHILD_PROCESS" - // FileScheme is the file scheme FileScheme = "file" ) diff --git a/pkg/util/constants.go b/pkg/util/constants.go index 9ca61ffb8b..f1132155c4 100644 --- a/pkg/util/constants.go +++ b/pkg/util/constants.go @@ -24,11 +24,9 @@ import ( // These constants are used by both minikube const ( - APIServerPort = 8443 - DefaultMinikubeDirectory = "/var/lib/minikube" - DefaultKubeConfigPath = DefaultMinikubeDirectory + "/kubeconfig" - DefaultDNSDomain = "cluster.local" - DefaultServiceCIDR = "10.96.0.0/12" + APIServerPort = 8443 + DefaultDNSDomain = "cluster.local" + DefaultServiceCIDR = "10.96.0.0/12" ) // DefaultV114AdmissionControllers are admission controllers we default to in v1.14.x From 3d1797606207c454c0d3653c6444ff0eb86a2e89 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Fri, 9 Aug 2019 16:24:49 -0700 Subject: [PATCH 03/11] lint and change detector test updates --- pkg/minikube/bootstrapper/certs.go | 4 ---- pkg/minikube/bootstrapper/certs_test.go | 3 ++- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 2 +- .../bootstrapper/kubeadm/kubeadm_test.go | 24 ++++++++----------- .../testdata/v1.10/containerd-api-port.yaml | 4 ++-- .../v1.10/containerd-pod-network-cidr.yaml | 4 ++-- .../kubeadm/testdata/v1.10/containerd.yaml | 4 ++-- .../testdata/v1.10/crio-options-gates.yaml | 4 ++-- .../kubeadm/testdata/v1.10/crio.yaml | 4 ++-- .../kubeadm/testdata/v1.10/default.yaml | 4 ++-- .../testdata/v1.10/image-repository.yaml | 4 ++-- .../kubeadm/testdata/v1.10/options.yaml | 4 ++-- .../testdata/v1.11/containerd-api-port.yaml | 4 ++-- .../v1.11/containerd-pod-network-cidr.yaml | 4 ++-- .../kubeadm/testdata/v1.11/containerd.yaml | 4 ++-- .../testdata/v1.11/crio-options-gates.yaml | 4 ++-- .../kubeadm/testdata/v1.11/crio.yaml | 4 ++-- .../kubeadm/testdata/v1.11/default.yaml | 4 ++-- .../testdata/v1.11/image-repository.yaml | 4 ++-- .../kubeadm/testdata/v1.11/options.yaml | 4 ++-- .../testdata/v1.12/containerd-api-port.yaml | 4 ++-- .../v1.12/containerd-pod-network-cidr.yaml | 4 ++-- .../kubeadm/testdata/v1.12/containerd.yaml | 4 ++-- .../testdata/v1.12/crio-options-gates.yaml | 4 ++-- .../kubeadm/testdata/v1.12/crio.yaml | 4 ++-- .../kubeadm/testdata/v1.12/default.yaml | 4 ++-- .../testdata/v1.12/image-repository.yaml | 4 ++-- .../kubeadm/testdata/v1.12/options.yaml | 4 ++-- .../testdata/v1.13/containerd-api-port.yaml | 4 ++-- .../v1.13/containerd-pod-network-cidr.yaml | 4 ++-- .../kubeadm/testdata/v1.13/containerd.yaml | 4 ++-- .../testdata/v1.13/crio-options-gates.yaml | 4 ++-- .../kubeadm/testdata/v1.13/crio.yaml | 4 ++-- .../kubeadm/testdata/v1.13/default.yaml | 4 ++-- .../testdata/v1.13/image-repository.yaml | 4 ++-- .../kubeadm/testdata/v1.13/options.yaml | 4 ++-- .../testdata/v1.14/containerd-api-port.yaml | 4 ++-- .../v1.14/containerd-pod-network-cidr.yaml | 4 ++-- .../kubeadm/testdata/v1.14/containerd.yaml | 4 ++-- .../testdata/v1.14/crio-options-gates.yaml | 4 ++-- .../kubeadm/testdata/v1.14/crio.yaml | 4 ++-- .../kubeadm/testdata/v1.14/default.yaml | 4 ++-- .../testdata/v1.14/image-repository.yaml | 4 ++-- .../kubeadm/testdata/v1.14/options.yaml | 4 ++-- .../testdata/v1.15/containerd-api-port.yaml | 4 ++-- .../v1.15/containerd-pod-network-cidr.yaml | 4 ++-- .../kubeadm/testdata/v1.15/containerd.yaml | 4 ++-- .../testdata/v1.15/crio-options-gates.yaml | 4 ++-- .../kubeadm/testdata/v1.15/crio.yaml | 4 ++-- .../kubeadm/testdata/v1.15/default.yaml | 4 ++-- .../testdata/v1.15/image-repository.yaml | 4 ++-- .../kubeadm/testdata/v1.15/options.yaml | 4 ++-- pkg/minikube/constants/constants.go | 2 +- 53 files changed, 110 insertions(+), 117 deletions(-) diff --git a/pkg/minikube/bootstrapper/certs.go b/pkg/minikube/bootstrapper/certs.go index 64b2a912c3..d235c1eadf 100644 --- a/pkg/minikube/bootstrapper/certs.go +++ b/pkg/minikube/bootstrapper/certs.go @@ -50,11 +50,7 @@ func SetupCerts(cmd command.Runner, k8s config.KubernetesConfig) error { if err := generateCerts(k8s); err != nil { return errors.Wrap(err, "Error generating certs") } - if err := cmd.Run(fmt.Sprintf("sudo mkdir -p %s", constants.GuestCertsDir)); err != nil { - return errors.Wrap(err, "mkdir") - } copyableFiles := []assets.CopyableFile{} - for _, cert := range certs { p := filepath.Join(localPath, cert) perms := "0644" diff --git a/pkg/minikube/bootstrapper/certs_test.go b/pkg/minikube/bootstrapper/certs_test.go index 893c7a0832..6cd39e5cf3 100644 --- a/pkg/minikube/bootstrapper/certs_test.go +++ b/pkg/minikube/bootstrapper/certs_test.go @@ -32,7 +32,6 @@ func TestSetupCerts(t *testing.T) { tempDir := tests.MakeTempDir() defer os.RemoveAll(tempDir) - f := command.NewFakeCommandRunner() k8s := config.KubernetesConfig{ APIServerName: constants.APIServerName, DNSDomain: constants.ClusterDNSDomain, @@ -44,6 +43,8 @@ func TestSetupCerts(t *testing.T) { filesToBeTransferred = append(filesToBeTransferred, filepath.Join(constants.GetMinipath(), cert)) } + f := command.NewFakeCommandRunner() + f.SetCommandToOutput(map[string]string{"sudo mkdir -p /var/lib/minikube/certs": ""}) if err := SetupCerts(f, k8s); err != nil { t.Fatalf("Error starting cluster: %v", err) } diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 2b7f15adb2..d96fe6ea99 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -670,7 +670,7 @@ func configFiles(cfg config.KubernetesConfig, kubeadm []byte, kubelet []byte, ku return fs } -// binDir returns the persistant path binaries are stored in +// binDir returns the persistent path binaries are stored in func binRoot(version string) string { return filepath.Join(constants.GuestPersistentDir, "binaries", version) } diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go index 5e6389f993..dd9e095eae 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go @@ -44,13 +44,12 @@ func TestGenerateKubeletConfig(t *testing.T) { NodeName: "minikube", ContainerRuntime: "docker", }, - expected: ` -[Unit] + expected: `[Unit] Wants=docker.socket [Service] ExecStart= -ExecStart=/usr/bin/kubelet --allow-privileged=true --authorization-mode=Webhook --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --cadvisor-port=0 --cgroup-driver=cgroupfs --client-ca-file=/var/lib/minikube/certs/ca.crt --cluster-dns=10.96.0.10 --cluster-domain=cluster.local --container-runtime=docker --fail-swap-on=false --hostname-override=minikube --kubeconfig=/etc/kubernetes/kubelet.conf --pod-manifest-path=/etc/kubernetes/manifests +ExecStart=/var/lib/minikube/binaries/v1.10.13/kubelet --allow-privileged=true --authorization-mode=Webhook --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --cadvisor-port=0 --cgroup-driver=cgroupfs --client-ca-file=/var/lib/minikube/certs/ca.crt --cluster-dns=10.96.0.10 --cluster-domain=cluster.local --container-runtime=docker --fail-swap-on=false --hostname-override=minikube --kubeconfig=/etc/kubernetes/kubelet.conf --pod-manifest-path=/etc/kubernetes/manifests [Install] `, @@ -63,13 +62,12 @@ ExecStart=/usr/bin/kubelet --allow-privileged=true --authorization-mode=Webhook NodeName: "minikube", ContainerRuntime: "cri-o", }, - expected: ` -[Unit] + expected: `[Unit] Wants=crio.service [Service] ExecStart= -ExecStart=/usr/bin/kubelet --authorization-mode=Webhook --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --cgroup-driver=cgroupfs --client-ca-file=/var/lib/minikube/certs/ca.crt --cluster-dns=10.96.0.10 --cluster-domain=cluster.local --container-runtime=remote --container-runtime-endpoint=/var/run/crio/crio.sock --fail-swap-on=false --hostname-override=minikube --image-service-endpoint=/var/run/crio/crio.sock --kubeconfig=/etc/kubernetes/kubelet.conf --pod-manifest-path=/etc/kubernetes/manifests --runtime-request-timeout=15m +ExecStart=/var/lib/minikube/binaries/v1.15.2/kubelet --authorization-mode=Webhook --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --cgroup-driver=cgroupfs --client-ca-file=/var/lib/minikube/certs/ca.crt --cluster-dns=10.96.0.10 --cluster-domain=cluster.local --container-runtime=remote --container-runtime-endpoint=/var/run/crio/crio.sock --fail-swap-on=false --hostname-override=minikube --image-service-endpoint=/var/run/crio/crio.sock --kubeconfig=/etc/kubernetes/kubelet.conf --pod-manifest-path=/etc/kubernetes/manifests --runtime-request-timeout=15m [Install] `, @@ -82,13 +80,12 @@ ExecStart=/usr/bin/kubelet --authorization-mode=Webhook --bootstrap-kubeconfig=/ NodeName: "minikube", ContainerRuntime: "containerd", }, - expected: ` -[Unit] + expected: `[Unit] Wants=containerd.service [Service] ExecStart= -ExecStart=/usr/bin/kubelet --authorization-mode=Webhook --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --cgroup-driver=cgroupfs --client-ca-file=/var/lib/minikube/certs/ca.crt --cluster-dns=10.96.0.10 --cluster-domain=cluster.local --container-runtime=remote --container-runtime-endpoint=unix:///run/containerd/containerd.sock --fail-swap-on=false --hostname-override=minikube --image-service-endpoint=unix:///run/containerd/containerd.sock --kubeconfig=/etc/kubernetes/kubelet.conf --pod-manifest-path=/etc/kubernetes/manifests --runtime-request-timeout=15m +ExecStart=/var/lib/minikube/binaries/v1.15.2/kubelet --authorization-mode=Webhook --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --cgroup-driver=cgroupfs --client-ca-file=/var/lib/minikube/certs/ca.crt --cluster-dns=10.96.0.10 --cluster-domain=cluster.local --container-runtime=remote --container-runtime-endpoint=unix:///run/containerd/containerd.sock --fail-swap-on=false --hostname-override=minikube --image-service-endpoint=unix:///run/containerd/containerd.sock --kubeconfig=/etc/kubernetes/kubelet.conf --pod-manifest-path=/etc/kubernetes/manifests --runtime-request-timeout=15m [Install] `, @@ -102,13 +99,12 @@ ExecStart=/usr/bin/kubelet --authorization-mode=Webhook --bootstrap-kubeconfig=/ ContainerRuntime: "docker", ImageRepository: "docker-proxy-image.io/google_containers", }, - expected: ` -[Unit] + expected: `[Unit] Wants=docker.socket [Service] ExecStart= -ExecStart=/usr/bin/kubelet --authorization-mode=Webhook --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --cgroup-driver=cgroupfs --client-ca-file=/var/lib/minikube/certs/ca.crt --cluster-dns=10.96.0.10 --cluster-domain=cluster.local --container-runtime=docker --fail-swap-on=false --hostname-override=minikube --kubeconfig=/etc/kubernetes/kubelet.conf --pod-infra-container-image=docker-proxy-image.io/google_containers/pause:3.1 --pod-manifest-path=/etc/kubernetes/manifests +ExecStart=/var/lib/minikube/binaries/v1.15.2/kubelet --authorization-mode=Webhook --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --cgroup-driver=cgroupfs --client-ca-file=/var/lib/minikube/certs/ca.crt --cluster-dns=10.96.0.10 --cluster-domain=cluster.local --container-runtime=docker --fail-swap-on=false --hostname-override=minikube --kubeconfig=/etc/kubernetes/kubelet.conf --pod-infra-container-image=docker-proxy-image.io/google_containers/pause:3.1 --pod-manifest-path=/etc/kubernetes/manifests [Install] `, @@ -134,7 +130,7 @@ ExecStart=/usr/bin/kubelet --authorization-mode=Webhook --bootstrap-kubeconfig=/ diff, err := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{ A: difflib.SplitLines(tc.expected), - B: difflib.SplitLines(got), + B: difflib.SplitLines(string(got)), FromFile: "Expected", ToFile: "Got", Context: 1, @@ -267,7 +263,7 @@ func TestGenerateConfig(t *testing.T) { } diff, err := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{ A: difflib.SplitLines(string(expected)), - B: difflib.SplitLines(got), + B: difflib.SplitLines(string(got)), FromFile: "Expected", ToFile: "Got", Context: 1, diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.10/containerd-api-port.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.10/containerd-api-port.yaml index 40c91637f3..877e831b4b 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.10/containerd-api-port.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.10/containerd-api-port.yaml @@ -6,11 +6,11 @@ api: bindPort: 12345 controlPlaneEndpoint: localhost kubernetesVersion: v1.10.0 -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs networking: serviceSubnet: 10.96.0.0/12 etcd: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd nodeName: mk criSocket: /run/containerd/containerd.sock apiServerExtraArgs: diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.10/containerd-pod-network-cidr.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.10/containerd-pod-network-cidr.yaml index b2d73f912b..2741f3dfbe 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.10/containerd-pod-network-cidr.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.10/containerd-pod-network-cidr.yaml @@ -6,11 +6,11 @@ api: bindPort: 8443 controlPlaneEndpoint: localhost kubernetesVersion: v1.10.0 -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs networking: serviceSubnet: 10.96.0.0/12 etcd: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd nodeName: mk criSocket: /run/containerd/containerd.sock apiServerExtraArgs: diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.10/containerd.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.10/containerd.yaml index b2d73f912b..2741f3dfbe 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.10/containerd.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.10/containerd.yaml @@ -6,11 +6,11 @@ api: bindPort: 8443 controlPlaneEndpoint: localhost kubernetesVersion: v1.10.0 -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs networking: serviceSubnet: 10.96.0.0/12 etcd: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd nodeName: mk criSocket: /run/containerd/containerd.sock apiServerExtraArgs: diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.10/crio-options-gates.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.10/crio-options-gates.yaml index e71f7dde0f..f4f766a146 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.10/crio-options-gates.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.10/crio-options-gates.yaml @@ -6,11 +6,11 @@ api: bindPort: 8443 controlPlaneEndpoint: localhost kubernetesVersion: v1.10.0 -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs networking: serviceSubnet: 10.96.0.0/12 etcd: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd nodeName: mk criSocket: /var/run/crio/crio.sock apiServerExtraArgs: diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.10/crio.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.10/crio.yaml index f68df3416f..4f689e42cd 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.10/crio.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.10/crio.yaml @@ -6,11 +6,11 @@ api: bindPort: 8443 controlPlaneEndpoint: localhost kubernetesVersion: v1.10.0 -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs networking: serviceSubnet: 10.96.0.0/12 etcd: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd nodeName: mk criSocket: /var/run/crio/crio.sock apiServerExtraArgs: diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.10/default.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.10/default.yaml index 9f0ea00701..1db5fb0711 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.10/default.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.10/default.yaml @@ -6,11 +6,11 @@ api: bindPort: 8443 controlPlaneEndpoint: localhost kubernetesVersion: v1.10.0 -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs networking: serviceSubnet: 10.96.0.0/12 etcd: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd nodeName: mk apiServerExtraArgs: admission-control: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.10/image-repository.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.10/image-repository.yaml index cebbc924ed..077385fcd6 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.10/image-repository.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.10/image-repository.yaml @@ -6,11 +6,11 @@ api: bindPort: 8443 controlPlaneEndpoint: localhost kubernetesVersion: v1.10.0 -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs networking: serviceSubnet: 10.96.0.0/12 etcd: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd nodeName: mk imageRepository: test/repo apiServerExtraArgs: diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.10/options.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.10/options.yaml index 97fcc2e56e..d52687fa94 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.10/options.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.10/options.yaml @@ -6,11 +6,11 @@ api: bindPort: 8443 controlPlaneEndpoint: localhost kubernetesVersion: v1.10.0 -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs networking: serviceSubnet: 10.96.0.0/12 etcd: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd nodeName: mk apiServerExtraArgs: admission-control: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.11/containerd-api-port.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.11/containerd-api-port.yaml index c7edeea5f2..cb5e0e62e5 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.11/containerd-api-port.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.11/containerd-api-port.yaml @@ -6,11 +6,11 @@ api: bindPort: 12345 controlPlaneEndpoint: localhost kubernetesVersion: v1.11.0 -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs networking: serviceSubnet: 10.96.0.0/12 etcd: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd nodeName: mk criSocket: /run/containerd/containerd.sock apiServerExtraArgs: diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.11/containerd-pod-network-cidr.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.11/containerd-pod-network-cidr.yaml index 320c885927..a78261c550 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.11/containerd-pod-network-cidr.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.11/containerd-pod-network-cidr.yaml @@ -6,11 +6,11 @@ api: bindPort: 8443 controlPlaneEndpoint: localhost kubernetesVersion: v1.11.0 -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs networking: serviceSubnet: 10.96.0.0/12 etcd: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd nodeName: mk criSocket: /run/containerd/containerd.sock apiServerExtraArgs: diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.11/containerd.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.11/containerd.yaml index 320c885927..a78261c550 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.11/containerd.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.11/containerd.yaml @@ -6,11 +6,11 @@ api: bindPort: 8443 controlPlaneEndpoint: localhost kubernetesVersion: v1.11.0 -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs networking: serviceSubnet: 10.96.0.0/12 etcd: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd nodeName: mk criSocket: /run/containerd/containerd.sock apiServerExtraArgs: diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.11/crio-options-gates.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.11/crio-options-gates.yaml index 37e2d4729a..15bb16aaa1 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.11/crio-options-gates.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.11/crio-options-gates.yaml @@ -6,11 +6,11 @@ api: bindPort: 8443 controlPlaneEndpoint: localhost kubernetesVersion: v1.11.0 -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs networking: serviceSubnet: 10.96.0.0/12 etcd: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd nodeName: mk criSocket: /var/run/crio/crio.sock apiServerExtraArgs: diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.11/crio.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.11/crio.yaml index 2b394b5335..7e33c3c861 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.11/crio.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.11/crio.yaml @@ -6,11 +6,11 @@ api: bindPort: 8443 controlPlaneEndpoint: localhost kubernetesVersion: v1.11.0 -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs networking: serviceSubnet: 10.96.0.0/12 etcd: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd nodeName: mk criSocket: /var/run/crio/crio.sock apiServerExtraArgs: diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.11/default.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.11/default.yaml index f831b4e46e..13ea3d1c53 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.11/default.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.11/default.yaml @@ -6,11 +6,11 @@ api: bindPort: 8443 controlPlaneEndpoint: localhost kubernetesVersion: v1.11.0 -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs networking: serviceSubnet: 10.96.0.0/12 etcd: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd nodeName: mk apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.11/image-repository.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.11/image-repository.yaml index 72ac00feb7..5e121b085d 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.11/image-repository.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.11/image-repository.yaml @@ -6,11 +6,11 @@ api: bindPort: 8443 controlPlaneEndpoint: localhost kubernetesVersion: v1.11.0 -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs networking: serviceSubnet: 10.96.0.0/12 etcd: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd nodeName: mk imageRepository: test/repo apiServerExtraArgs: diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.11/options.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.11/options.yaml index 653a3e4428..aac8924cbe 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.11/options.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.11/options.yaml @@ -6,11 +6,11 @@ api: bindPort: 8443 controlPlaneEndpoint: localhost kubernetesVersion: v1.11.0 -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs networking: serviceSubnet: 10.96.0.0/12 etcd: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd nodeName: mk apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.12/containerd-api-port.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.12/containerd-api-port.yaml index 7d84e36369..2c7ad8fc47 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.12/containerd-api-port.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.12/containerd-api-port.yaml @@ -19,12 +19,12 @@ apiVersion: kubeadm.k8s.io/v1alpha3 kind: ClusterConfiguration apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs clusterName: kubernetes controlPlaneEndpoint: localhost:12345 etcd: local: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd kubernetesVersion: v1.12.0 networking: dnsDomain: cluster.local diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.12/containerd-pod-network-cidr.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.12/containerd-pod-network-cidr.yaml index 64bf678f02..c13a251804 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.12/containerd-pod-network-cidr.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.12/containerd-pod-network-cidr.yaml @@ -19,12 +19,12 @@ apiVersion: kubeadm.k8s.io/v1alpha3 kind: ClusterConfiguration apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs clusterName: kubernetes controlPlaneEndpoint: localhost:8443 etcd: local: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd kubernetesVersion: v1.12.0 networking: dnsDomain: cluster.local diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.12/containerd.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.12/containerd.yaml index 4e4b1e4830..1398527e91 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.12/containerd.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.12/containerd.yaml @@ -19,12 +19,12 @@ apiVersion: kubeadm.k8s.io/v1alpha3 kind: ClusterConfiguration apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs clusterName: kubernetes controlPlaneEndpoint: localhost:8443 etcd: local: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd kubernetesVersion: v1.12.0 networking: dnsDomain: cluster.local diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.12/crio-options-gates.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.12/crio-options-gates.yaml index 6f933f84cc..d33f0e13dc 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.12/crio-options-gates.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.12/crio-options-gates.yaml @@ -27,12 +27,12 @@ controllerManagerExtraArgs: schedulerExtraArgs: feature-gates: "a=b" scheduler-name: "mini-scheduler" -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs clusterName: kubernetes controlPlaneEndpoint: localhost:8443 etcd: local: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd kubernetesVersion: v1.12.0 networking: dnsDomain: cluster.local diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.12/crio.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.12/crio.yaml index 033a7aefe2..c076b8ad9f 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.12/crio.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.12/crio.yaml @@ -19,12 +19,12 @@ apiVersion: kubeadm.k8s.io/v1alpha3 kind: ClusterConfiguration apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs clusterName: kubernetes controlPlaneEndpoint: localhost:8443 etcd: local: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd kubernetesVersion: v1.12.0 networking: dnsDomain: cluster.local diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.12/default.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.12/default.yaml index d9ba1066a5..36856509b9 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.12/default.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.12/default.yaml @@ -19,12 +19,12 @@ apiVersion: kubeadm.k8s.io/v1alpha3 kind: ClusterConfiguration apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs clusterName: kubernetes controlPlaneEndpoint: localhost:8443 etcd: local: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd kubernetesVersion: v1.12.0 networking: dnsDomain: cluster.local diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.12/image-repository.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.12/image-repository.yaml index bf00349785..dab9a37c23 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.12/image-repository.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.12/image-repository.yaml @@ -20,12 +20,12 @@ kind: ClusterConfiguration imageRepository: test/repo apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs clusterName: kubernetes controlPlaneEndpoint: localhost:8443 etcd: local: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd kubernetesVersion: v1.12.0 networking: dnsDomain: cluster.local diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.12/options.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.12/options.yaml index 3e9052efa8..3a6ff1aeed 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.12/options.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.12/options.yaml @@ -24,12 +24,12 @@ controllerManagerExtraArgs: kube-api-burst: "32" schedulerExtraArgs: scheduler-name: "mini-scheduler" -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs clusterName: kubernetes controlPlaneEndpoint: localhost:8443 etcd: local: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd kubernetesVersion: v1.12.0 networking: dnsDomain: cluster.local diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.13/containerd-api-port.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.13/containerd-api-port.yaml index e776755631..f365193c61 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.13/containerd-api-port.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.13/containerd-api-port.yaml @@ -19,12 +19,12 @@ apiVersion: kubeadm.k8s.io/v1alpha3 kind: ClusterConfiguration apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs clusterName: kubernetes controlPlaneEndpoint: localhost:12345 etcd: local: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd kubernetesVersion: v1.13.0 networking: dnsDomain: cluster.local diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.13/containerd-pod-network-cidr.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.13/containerd-pod-network-cidr.yaml index ecc9a14631..d5f799e56f 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.13/containerd-pod-network-cidr.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.13/containerd-pod-network-cidr.yaml @@ -19,12 +19,12 @@ apiVersion: kubeadm.k8s.io/v1alpha3 kind: ClusterConfiguration apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs clusterName: kubernetes controlPlaneEndpoint: localhost:8443 etcd: local: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd kubernetesVersion: v1.13.0 networking: dnsDomain: cluster.local diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.13/containerd.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.13/containerd.yaml index a852561fbe..806106b3b7 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.13/containerd.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.13/containerd.yaml @@ -19,12 +19,12 @@ apiVersion: kubeadm.k8s.io/v1alpha3 kind: ClusterConfiguration apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs clusterName: kubernetes controlPlaneEndpoint: localhost:8443 etcd: local: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd kubernetesVersion: v1.13.0 networking: dnsDomain: cluster.local diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.13/crio-options-gates.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.13/crio-options-gates.yaml index 9e5812dbda..e602810b0d 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.13/crio-options-gates.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.13/crio-options-gates.yaml @@ -27,12 +27,12 @@ controllerManagerExtraArgs: schedulerExtraArgs: feature-gates: "a=b" scheduler-name: "mini-scheduler" -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs clusterName: kubernetes controlPlaneEndpoint: localhost:8443 etcd: local: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd kubernetesVersion: v1.13.0 networking: dnsDomain: cluster.local diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.13/crio.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.13/crio.yaml index 57b9f1b122..b82b3bf69b 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.13/crio.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.13/crio.yaml @@ -19,12 +19,12 @@ apiVersion: kubeadm.k8s.io/v1alpha3 kind: ClusterConfiguration apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs clusterName: kubernetes controlPlaneEndpoint: localhost:8443 etcd: local: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd kubernetesVersion: v1.13.0 networking: dnsDomain: cluster.local diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.13/default.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.13/default.yaml index db611e2263..9995a46c14 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.13/default.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.13/default.yaml @@ -19,12 +19,12 @@ apiVersion: kubeadm.k8s.io/v1alpha3 kind: ClusterConfiguration apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs clusterName: kubernetes controlPlaneEndpoint: localhost:8443 etcd: local: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd kubernetesVersion: v1.13.0 networking: dnsDomain: cluster.local diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.13/image-repository.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.13/image-repository.yaml index 037dd6add2..b2b8b6e3fb 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.13/image-repository.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.13/image-repository.yaml @@ -20,12 +20,12 @@ kind: ClusterConfiguration imageRepository: test/repo apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs clusterName: kubernetes controlPlaneEndpoint: localhost:8443 etcd: local: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd kubernetesVersion: v1.13.0 networking: dnsDomain: cluster.local diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.13/options.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.13/options.yaml index f1df66954d..484dd5a352 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.13/options.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.13/options.yaml @@ -24,12 +24,12 @@ controllerManagerExtraArgs: kube-api-burst: "32" schedulerExtraArgs: scheduler-name: "mini-scheduler" -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs clusterName: kubernetes controlPlaneEndpoint: localhost:8443 etcd: local: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd kubernetesVersion: v1.13.0 networking: dnsDomain: cluster.local diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.14/containerd-api-port.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.14/containerd-api-port.yaml index cc96f136ac..5755f5641d 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.14/containerd-api-port.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.14/containerd-api-port.yaml @@ -20,14 +20,14 @@ kind: ClusterConfiguration apiServer: extraArgs: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs clusterName: kubernetes controlPlaneEndpoint: localhost:12345 dns: type: CoreDNS etcd: local: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd kubernetesVersion: v1.14.0 networking: dnsDomain: cluster.local diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.14/containerd-pod-network-cidr.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.14/containerd-pod-network-cidr.yaml index 147a6ccbec..f4ff177f67 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.14/containerd-pod-network-cidr.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.14/containerd-pod-network-cidr.yaml @@ -20,14 +20,14 @@ kind: ClusterConfiguration apiServer: extraArgs: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs clusterName: kubernetes controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS etcd: local: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd kubernetesVersion: v1.14.0 networking: dnsDomain: cluster.local diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.14/containerd.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.14/containerd.yaml index 147a6ccbec..f4ff177f67 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.14/containerd.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.14/containerd.yaml @@ -20,14 +20,14 @@ kind: ClusterConfiguration apiServer: extraArgs: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs clusterName: kubernetes controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS etcd: local: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd kubernetesVersion: v1.14.0 networking: dnsDomain: cluster.local diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.14/crio-options-gates.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.14/crio-options-gates.yaml index 1b35bce5c6..6f46996151 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.14/crio-options-gates.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.14/crio-options-gates.yaml @@ -30,14 +30,14 @@ scheduler: extraArgs: feature-gates: "a=b" scheduler-name: "mini-scheduler" -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs clusterName: kubernetes controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS etcd: local: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd kubernetesVersion: v1.14.0 networking: dnsDomain: cluster.local diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.14/crio.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.14/crio.yaml index 66dac1c8af..47cfd0ecbb 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.14/crio.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.14/crio.yaml @@ -20,14 +20,14 @@ kind: ClusterConfiguration apiServer: extraArgs: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs clusterName: kubernetes controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS etcd: local: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd kubernetesVersion: v1.14.0 networking: dnsDomain: cluster.local diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.14/default.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.14/default.yaml index d88c0efebf..57371bf704 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.14/default.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.14/default.yaml @@ -20,14 +20,14 @@ kind: ClusterConfiguration apiServer: extraArgs: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs clusterName: kubernetes controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS etcd: local: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd kubernetesVersion: v1.14.0 networking: dnsDomain: cluster.local diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.14/image-repository.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.14/image-repository.yaml index bf53791b71..2c1a895bca 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.14/image-repository.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.14/image-repository.yaml @@ -21,14 +21,14 @@ imageRepository: test/repo apiServer: extraArgs: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs clusterName: kubernetes controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS etcd: local: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd kubernetesVersion: v1.14.0 networking: dnsDomain: cluster.local diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.14/options.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.14/options.yaml index a85a2447b2..325265c2c6 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.14/options.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.14/options.yaml @@ -27,14 +27,14 @@ controllerManager: scheduler: extraArgs: scheduler-name: "mini-scheduler" -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs clusterName: kubernetes controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS etcd: local: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd kubernetesVersion: v1.14.0 networking: dnsDomain: cluster.local diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.15/containerd-api-port.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.15/containerd-api-port.yaml index 0fbf400ebf..9f08022e33 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.15/containerd-api-port.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.15/containerd-api-port.yaml @@ -20,14 +20,14 @@ kind: ClusterConfiguration apiServer: extraArgs: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs clusterName: kubernetes controlPlaneEndpoint: localhost:12345 dns: type: CoreDNS etcd: local: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd kubernetesVersion: v1.15.0 networking: dnsDomain: cluster.local diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.15/containerd-pod-network-cidr.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.15/containerd-pod-network-cidr.yaml index c6354e860b..30e3b200f1 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.15/containerd-pod-network-cidr.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.15/containerd-pod-network-cidr.yaml @@ -20,14 +20,14 @@ kind: ClusterConfiguration apiServer: extraArgs: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs clusterName: kubernetes controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS etcd: local: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd kubernetesVersion: v1.15.0 networking: dnsDomain: cluster.local diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.15/containerd.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.15/containerd.yaml index c6354e860b..30e3b200f1 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.15/containerd.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.15/containerd.yaml @@ -20,14 +20,14 @@ kind: ClusterConfiguration apiServer: extraArgs: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs clusterName: kubernetes controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS etcd: local: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd kubernetesVersion: v1.15.0 networking: dnsDomain: cluster.local diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.15/crio-options-gates.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.15/crio-options-gates.yaml index 78bbe09f1c..bcc3e64b00 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.15/crio-options-gates.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.15/crio-options-gates.yaml @@ -30,14 +30,14 @@ scheduler: extraArgs: feature-gates: "a=b" scheduler-name: "mini-scheduler" -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs clusterName: kubernetes controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS etcd: local: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd kubernetesVersion: v1.15.0 networking: dnsDomain: cluster.local diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.15/crio.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.15/crio.yaml index b956109b81..7cc895ca15 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.15/crio.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.15/crio.yaml @@ -20,14 +20,14 @@ kind: ClusterConfiguration apiServer: extraArgs: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs clusterName: kubernetes controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS etcd: local: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd kubernetesVersion: v1.15.0 networking: dnsDomain: cluster.local diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.15/default.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.15/default.yaml index 46d6e4e78c..f31d943013 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.15/default.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.15/default.yaml @@ -20,14 +20,14 @@ kind: ClusterConfiguration apiServer: extraArgs: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs clusterName: kubernetes controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS etcd: local: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd kubernetesVersion: v1.15.0 networking: dnsDomain: cluster.local diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.15/image-repository.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.15/image-repository.yaml index 8ab50ff902..aec5820a00 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.15/image-repository.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.15/image-repository.yaml @@ -21,14 +21,14 @@ imageRepository: test/repo apiServer: extraArgs: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs clusterName: kubernetes controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS etcd: local: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd kubernetesVersion: v1.15.0 networking: dnsDomain: cluster.local diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.15/options.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.15/options.yaml index 2e6f561dd4..eb354d47ee 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.15/options.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.15/options.yaml @@ -27,14 +27,14 @@ controllerManager: scheduler: extraArgs: scheduler-name: "mini-scheduler" -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs clusterName: kubernetes controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS etcd: local: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd kubernetesVersion: v1.15.0 networking: dnsDomain: cluster.local diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index 710e098b79..bc2085e8e0 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -222,7 +222,7 @@ const ( GuestManifestsDir = "/etc/kubernetes/manifests" // GuestEphemeralDir is the path where ephemeral data should be stored within the VM GuestEphemeralDir = "/var/tmp/minikube" - // PersistentDir is the path where persistant data should be stored within the VM (not tmpfs) + // PersistentDir is the path where persistent data should be stored within the VM (not tmpfs) GuestPersistentDir = "/var/lib/minikube" // GuestCertsDir are where Kubernetes certificates are kept on the guest GuestCertsDir = GuestPersistentDir + "/certs" From 03ebad25ab79e63ab85dd5385476ae9c9e23d648 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Wed, 14 Aug 2019 08:14:03 -0700 Subject: [PATCH 04/11] merge conflict --- pkg/minikube/constants/constants.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index d861681bc3..118a1eb1f5 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -210,12 +210,6 @@ func GetProfilePath(profile string, miniHome ...string) string { return filepath.Join(miniPath, "profiles", profile) } -// AddonsPath is the default path of the addons configuration -const AddonsPath = "/etc/kubernetes/addons" - -// FilesPath is the default path of files -const FilesPath = "/files" - const ( // KubeletServiceFile is the path to the kubelet systemd service KubeletServiceFile = "/lib/systemd/system/kubelet.service" From 1d6699f0dabae907477e170340a850a6e2c5daf1 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Fri, 16 Aug 2019 15:21:58 -0700 Subject: [PATCH 05/11] kubeconfig needs to live persistently --- pkg/minikube/bootstrapper/certs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/minikube/bootstrapper/certs.go b/pkg/minikube/bootstrapper/certs.go index 9a690f5b44..3ad2740bef 100644 --- a/pkg/minikube/bootstrapper/certs.go +++ b/pkg/minikube/bootstrapper/certs.go @@ -84,7 +84,7 @@ func SetupCerts(cmd command.Runner, k8s config.KubernetesConfig) error { return errors.Wrap(err, "encoding kubeconfig") } - kubeCfgFile := assets.NewMemoryAsset(data, constants.GuestEphemeralDir, "kubeconfig", "0644") + kubeCfgFile := assets.NewMemoryAsset(data, constants.GuestPersistentDir, "kubeconfig", "0644") copyableFiles = append(copyableFiles, kubeCfgFile) for _, f := range copyableFiles { From b17d377d68a0d96ebbb6723ea4045f9f66f97ca7 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Fri, 16 Aug 2019 15:22:27 -0700 Subject: [PATCH 06/11] Detect when the addon-manager isn't able to process --- pkg/minikube/logs/logs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/minikube/logs/logs.go b/pkg/minikube/logs/logs.go index 05775f8779..8897c4e0c2 100644 --- a/pkg/minikube/logs/logs.go +++ b/pkg/minikube/logs/logs.go @@ -34,7 +34,7 @@ import ( ) // rootCauseRe is a regular expression that matches known failure root causes -var rootCauseRe = regexp.MustCompile(`^error: |eviction manager: pods.* evicted|unknown flag: --|forbidden.*no providers available|eviction manager:.*evicted|tls: bad certificate|kubelet.*no API client|kubelet.*No api server`) +var rootCauseRe = regexp.MustCompile(`^error: |eviction manager: pods.* evicted|unknown flag: --|forbidden.*no providers available|eviction manager:.*evicted|tls: bad certificate|kubelet.*no API client|kubelet.*No api server|STDIN.*127.0.0.1:8080`) // ignoreRe is a regular expression that matches spurious errors to not surface var ignoreCauseRe = regexp.MustCompile("error: no objects passed to apply") From d11804e83f412672ec788ec101e699f0df422d8f Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Fri, 16 Aug 2019 16:45:59 -0700 Subject: [PATCH 07/11] Fix certs_test merge conflict --- pkg/minikube/bootstrapper/certs_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/minikube/bootstrapper/certs_test.go b/pkg/minikube/bootstrapper/certs_test.go index 58a0d97823..db4d2e4bec 100644 --- a/pkg/minikube/bootstrapper/certs_test.go +++ b/pkg/minikube/bootstrapper/certs_test.go @@ -52,7 +52,9 @@ func TestSetupCerts(t *testing.T) { t.Fatalf("error generating certificate: %v", err) } - cmdMap := map[string]string{} + cmdMap := map[string]string{ + "sudo mkdir -p /var/lib/minikube/certs": "", + } certFilenames := map[string]string{"ca.crt": "minikubeCA.pem", "mycert.pem": "mycert.pem"} for _, dst := range certFilenames { certFile := path.Join(CACertificatesDir, dst) @@ -63,6 +65,7 @@ func TestSetupCerts(t *testing.T) { cmdMap[fmt.Sprintf("openssl x509 -hash -noout -in '%s'", certFile)] = certNameHash cmdMap[fmt.Sprintf("sudo ln -s '%s' '%s'", certStorePath, remoteCertHashLink)] = "1" } + f := command.NewFakeCommandRunner() f.SetCommandToOutput(cmdMap) var filesToBeTransferred []string @@ -72,8 +75,6 @@ func TestSetupCerts(t *testing.T) { filesToBeTransferred = append(filesToBeTransferred, filepath.Join(constants.GetMinipath(), "ca.crt")) filesToBeTransferred = append(filesToBeTransferred, filepath.Join(constants.GetMinipath(), "certs", "mycert.pem")) - f := command.NewFakeCommandRunner() - f.SetCommandToOutput(map[string]string{"sudo mkdir -p /var/lib/minikube/certs": ""}) if err := SetupCerts(f, k8s); err != nil { t.Fatalf("Error starting cluster: %v", err) } From 5d325c745ca25e936ab8d26b424bad7520c3a1c2 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Wed, 21 Aug 2019 17:20:47 -0700 Subject: [PATCH 08/11] Create symlink for legacy etcd directory, make apiserver check more robust --- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 55 ++++++++++++++++---- 1 file changed, 46 insertions(+), 9 deletions(-) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 1c8e60c12b..a089e7c3fb 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -198,6 +198,28 @@ func createFlagsFromExtraArgs(extraOptions config.ExtraOptionSlice) string { return convertToFlags(kubeadmExtraOpts) } +// etcdDataDir is where etcd data is stored. +func etcdDataDir() string { + return filepath.Join(constants.GuestPersistentDir, "etcd") +} + +// createCompatSymlinks creates compatibility symlinks to transition running services to new directory structures +func (k *Bootstrapper) createCompatSymlinks() error { + legacyEtcd := "/data/minikube" + if err := k.c.Run(fmt.Sprintf("sudo test -d %s", legacyEtcd)); err != nil { + glog.Infof("%s check failed, skipping compat symlinks: %v", legacyEtcd, err) + return nil + } + + glog.Infof("Found %s, creating compatibility symlinks ...", legacyEtcd) + cmd := fmt.Sprintf("sudo ln -s %s %s", legacyEtcd, etcdDataDir()) + out, err := k.c.CombinedOutput(cmd) + if err != nil { + return errors.Wrapf(err, "cmd failed: %s\n%s\n", cmd, out) + } + return nil +} + // StartCluster starts the cluster func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error { version, err := parseKubernetesVersion(k8s.KubernetesVersion) @@ -206,7 +228,6 @@ func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error { } extraFlags := createFlagsFromExtraArgs(k8s.ExtraOptions) - r, err := cruntime.New(cruntime.Config{Type: k8s.ContainerRuntime}) if err != nil { return err @@ -343,6 +364,12 @@ func (k *Bootstrapper) WaitCluster(k8s config.KubernetesConfig, timeout time.Dur // RestartCluster restarts the Kubernetes cluster configured by kubeadm func (k *Bootstrapper) RestartCluster(k8s config.KubernetesConfig) error { + glog.Infof("RestartCluster start") + start := time.Now() + defer func() { + glog.Infof("RestartCluster took %s", time.Since(start)) + }() + version, err := parseKubernetesVersion(k8s.KubernetesVersion) if err != nil { return errors.Wrap(err, "parsing kubernetes version") @@ -355,6 +382,10 @@ func (k *Bootstrapper) RestartCluster(k8s config.KubernetesConfig) error { controlPlane = "control-plane" } + if err := k.createCompatSymlinks(); err != nil { + glog.Errorf("failed to create compat symlinks: %v", err) + } + baseCmd := fmt.Sprintf("%s %s", invokeKubeadm(k8s.KubernetesVersion), phase) cmds := []string{ fmt.Sprintf("%s phase certs all --config %s", baseCmd, yamlConfigPath), @@ -386,12 +417,19 @@ func (k *Bootstrapper) RestartCluster(k8s config.KubernetesConfig) error { // waitForAPIServer waits for the apiserver to start up func (k *Bootstrapper) waitForAPIServer(k8s config.KubernetesConfig) error { + start := time.Now() + defer func() { + glog.Infof("duration metric: took %s to wait for apiserver status ...", time.Since(start)) + }() + glog.Infof("Waiting for apiserver process ...") // To give a better error message, first check for process existence via ssh - err := wait.PollImmediate(time.Millisecond*300, time.Minute*2, func() (bool, error) { - ierr := k.c.Run(`pgrep apiserver`) + // Needs minutes in case the image isn't cached (such as with v1.10.x) + err := wait.PollImmediate(time.Millisecond*300, time.Minute*3, func() (bool, error) { + ierr := k.c.Run(`sudo pgrep kube-apiserver`) if ierr != nil { - return false, ierr + glog.Warningf("pgrep apiserver: %v", ierr) + return false, nil } return true, nil }) @@ -399,13 +437,13 @@ func (k *Bootstrapper) waitForAPIServer(k8s config.KubernetesConfig) error { return fmt.Errorf("apiserver process never appeared") } - start := time.Now() - glog.Infof("Waiting for apiserver ...") + glog.Infof("Waiting for apiserver to port healthy status ...") f := func() (bool, error) { status, err := k.GetAPIServerStatus(net.ParseIP(k8s.NodeIP), k8s.NodePort) glog.Infof("apiserver status: %s, err: %v", status, err) if err != nil { - return false, err + glog.Warningf("status: %v", err) + return false, nil } if status != "Running" { return false, nil @@ -413,7 +451,6 @@ func (k *Bootstrapper) waitForAPIServer(k8s config.KubernetesConfig) error { return true, nil } err = wait.PollImmediate(kconst.APICallRetryInterval, kconst.DefaultControlPlaneTimeout, f) - glog.Infof("duration metric: took %s to wait for apiserver status ...", time.Since(start)) return err } @@ -619,7 +656,7 @@ func generateConfig(k8s config.KubernetesConfig, r cruntime.Manager) ([]byte, er AdvertiseAddress: k8s.NodeIP, APIServerPort: nodePort, KubernetesVersion: k8s.KubernetesVersion, - EtcdDataDir: filepath.Join(constants.GuestPersistentDir, "etcd"), + EtcdDataDir: etcdDataDir(), NodeName: k8s.NodeName, CRISocket: r.SocketPath(), ImageRepository: k8s.ImageRepository, From 45c3219470299d0a77f56c0d31f169015df89814 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Wed, 21 Aug 2019 17:24:41 -0700 Subject: [PATCH 09/11] none reliability: --force does not work in v1.10, allow failed kubelet --- pkg/drivers/none/none.go | 4 ++-- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/pkg/drivers/none/none.go b/pkg/drivers/none/none.go index 18e48fcbbf..5731d22841 100644 --- a/pkg/drivers/none/none.go +++ b/pkg/drivers/none/none.go @@ -233,8 +233,8 @@ func stopKubelet(exec command.Runner) error { if errStatus != nil { glog.Errorf("temporary error: for %q : %v", cmdCheck, errStatus) } - if !strings.Contains(out.String(), "dead") { - return fmt.Errorf("expected to kubelet to be dead but it got : %q", out) + if !strings.Contains(out.String(), "dead") && !strings.Contains(out.String(), "failed") { + return fmt.Errorf("unexpected kubelet state: %q", out) } return nil } diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index a089e7c3fb..e061151022 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -456,7 +456,15 @@ func (k *Bootstrapper) waitForAPIServer(k8s config.KubernetesConfig) error { // DeleteCluster removes the components that were started earlier func (k *Bootstrapper) DeleteCluster(k8s config.KubernetesConfig) error { + version, err := parseKubernetesVersion(k8s.KubernetesVersion) + if err != nil { + return errors.Wrap(err, "parsing kubernetes version") + } + cmd := fmt.Sprintf("%s reset --force", invokeKubeadm(k8s.KubernetesVersion)) + if version.LT(semver.MustParse("1.11.0")) { + cmd = fmt.Sprintf("%s reset", invokeKubeadm(k8s.KubernetesVersion)) + } out, err := k.c.CombinedOutput(cmd) if err != nil { return errors.Wrapf(err, "kubeadm reset: %s\n%s\n", cmd, out) From 4e9d4bd00f7044573d8d36c60c46f4269ff3a806 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Wed, 21 Aug 2019 17:41:58 -0700 Subject: [PATCH 10/11] Remove unused constant function brought in by a bad merge --- pkg/minikube/constants/constants.go | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index 118a1eb1f5..82ae03a61f 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -186,30 +186,9 @@ var NewestKubernetesVersion = "v1.15.2" // OldestKubernetesVersion is the oldest Kubernetes version to test against var OldestKubernetesVersion = "v1.10.13" -// ConfigFilePath is the path of the config directory -var ConfigFilePath = MakeMiniPath("config") - // ConfigFile is the path of the config file var ConfigFile = MakeMiniPath("config", "config.json") -// GetProfileFile returns the Minikube profile config file -func GetProfileFile(profile string, miniHome ...string) string { - miniPath := GetMinipath() - if len(miniHome) > 0 { - miniPath = miniHome[0] - } - return filepath.Join(miniPath, "profiles", profile, "config.json") -} - -// GetProfilePath returns the Minikube profile path of config file -func GetProfilePath(profile string, miniHome ...string) string { - miniPath := GetMinipath() - if len(miniHome) > 0 { - miniPath = miniHome[0] - } - return filepath.Join(miniPath, "profiles", profile) -} - const ( // KubeletServiceFile is the path to the kubelet systemd service KubeletServiceFile = "/lib/systemd/system/kubelet.service" From a7af253eaae8c484fd8a5a8f13db88d56e53eb8c Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Wed, 21 Aug 2019 18:49:25 -0700 Subject: [PATCH 11/11] Merge in v1.16 test updates --- pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go | 2 +- .../kubeadm/testdata/v1.16/containerd-api-port.yaml | 4 ++-- .../kubeadm/testdata/v1.16/containerd-pod-network-cidr.yaml | 4 ++-- .../bootstrapper/kubeadm/testdata/v1.16/containerd.yaml | 4 ++-- .../kubeadm/testdata/v1.16/crio-options-gates.yaml | 4 ++-- pkg/minikube/bootstrapper/kubeadm/testdata/v1.16/crio.yaml | 4 ++-- pkg/minikube/bootstrapper/kubeadm/testdata/v1.16/default.yaml | 4 ++-- .../bootstrapper/kubeadm/testdata/v1.16/image-repository.yaml | 4 ++-- pkg/minikube/bootstrapper/kubeadm/testdata/v1.16/options.yaml | 4 ++-- 9 files changed, 17 insertions(+), 17 deletions(-) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go index 2ae3fe2e4a..d22f89cdb0 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go @@ -66,7 +66,7 @@ Wants=crio.service [Service] ExecStart= -ExecStart=/var/lib/minikube/binaries/v1.15.2/kubelet --authorization-mode=Webhook --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --cgroup-driver=cgroupfs --client-ca-file=/var/lib/minikube/certs/ca.crt --cluster-dns=10.96.0.10 --cluster-domain=cluster.local --container-runtime=remote --container-runtime-endpoint=/var/run/crio/crio.sock --fail-swap-on=false --hostname-override=minikube --image-service-endpoint=/var/run/crio/crio.sock --kubeconfig=/etc/kubernetes/kubelet.conf --pod-manifest-path=/etc/kubernetes/manifests --runtime-request-timeout=15m +ExecStart=/var/lib/minikube/binaries/v1.16.0-beta.1/kubelet --authorization-mode=Webhook --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --cgroup-driver=cgroupfs --client-ca-file=/var/lib/minikube/certs/ca.crt --cluster-dns=10.96.0.10 --cluster-domain=cluster.local --container-runtime=remote --container-runtime-endpoint=/var/run/crio/crio.sock --fail-swap-on=false --hostname-override=minikube --image-service-endpoint=/var/run/crio/crio.sock --kubeconfig=/etc/kubernetes/kubelet.conf --pod-manifest-path=/etc/kubernetes/manifests --runtime-request-timeout=15m [Install] `, diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.16/containerd-api-port.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.16/containerd-api-port.yaml index 7d0e44e47c..d7e9c9e975 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.16/containerd-api-port.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.16/containerd-api-port.yaml @@ -20,14 +20,14 @@ kind: ClusterConfiguration apiServer: extraArgs: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs clusterName: kubernetes controlPlaneEndpoint: localhost:12345 dns: type: CoreDNS etcd: local: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd kubernetesVersion: v1.16.0 networking: dnsDomain: cluster.local diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.16/containerd-pod-network-cidr.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.16/containerd-pod-network-cidr.yaml index 428bbfc3e3..d4de91b7eb 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.16/containerd-pod-network-cidr.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.16/containerd-pod-network-cidr.yaml @@ -20,14 +20,14 @@ kind: ClusterConfiguration apiServer: extraArgs: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs clusterName: kubernetes controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS etcd: local: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd kubernetesVersion: v1.16.0 networking: dnsDomain: cluster.local diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.16/containerd.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.16/containerd.yaml index 428bbfc3e3..d4de91b7eb 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.16/containerd.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.16/containerd.yaml @@ -20,14 +20,14 @@ kind: ClusterConfiguration apiServer: extraArgs: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs clusterName: kubernetes controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS etcd: local: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd kubernetesVersion: v1.16.0 networking: dnsDomain: cluster.local diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.16/crio-options-gates.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.16/crio-options-gates.yaml index dc661e66ed..5aa8ca8652 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.16/crio-options-gates.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.16/crio-options-gates.yaml @@ -30,14 +30,14 @@ scheduler: extraArgs: feature-gates: "a=b" scheduler-name: "mini-scheduler" -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs clusterName: kubernetes controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS etcd: local: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd kubernetesVersion: v1.16.0 networking: dnsDomain: cluster.local diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.16/crio.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.16/crio.yaml index 6fa2106724..57ac32b75e 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.16/crio.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.16/crio.yaml @@ -20,14 +20,14 @@ kind: ClusterConfiguration apiServer: extraArgs: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs clusterName: kubernetes controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS etcd: local: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd kubernetesVersion: v1.16.0 networking: dnsDomain: cluster.local diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.16/default.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.16/default.yaml index 74f7c79b92..4e1949e0f2 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.16/default.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.16/default.yaml @@ -20,14 +20,14 @@ kind: ClusterConfiguration apiServer: extraArgs: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs clusterName: kubernetes controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS etcd: local: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd kubernetesVersion: v1.16.0 networking: dnsDomain: cluster.local diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.16/image-repository.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.16/image-repository.yaml index caa1b2b456..f1b08545b9 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.16/image-repository.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.16/image-repository.yaml @@ -21,14 +21,14 @@ imageRepository: test/repo apiServer: extraArgs: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs clusterName: kubernetes controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS etcd: local: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd kubernetesVersion: v1.16.0 networking: dnsDomain: cluster.local diff --git a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.16/options.yaml b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.16/options.yaml index 348b4ab8c4..6c02b9c29e 100644 --- a/pkg/minikube/bootstrapper/kubeadm/testdata/v1.16/options.yaml +++ b/pkg/minikube/bootstrapper/kubeadm/testdata/v1.16/options.yaml @@ -27,14 +27,14 @@ controllerManager: scheduler: extraArgs: scheduler-name: "mini-scheduler" -certificatesDir: /var/lib/minikube/certs/ +certificatesDir: /var/lib/minikube/certs clusterName: kubernetes controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS etcd: local: - dataDir: /data/minikube + dataDir: /var/lib/minikube/etcd kubernetesVersion: v1.16.0 networking: dnsDomain: cluster.local