From 3074132aeb1dbdf0d2ce495555330d176ef8408d Mon Sep 17 00:00:00 2001 From: klaases Date: Thu, 2 Dec 2021 14:35:38 -0800 Subject: [PATCH 01/10] Stop K8s if running, when --no-kubernetes. --- cmd/minikube/cmd/start.go | 7 +++ pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 41 ++++++++----- pkg/minikube/node/start.go | 6 ++ test/integration/no_kubernetes_test.go | 61 +++++++++++++++++++- 4 files changed, 100 insertions(+), 15 deletions(-) diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index ef87ad02c9..544cda18a3 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -295,6 +295,12 @@ func provisionWithDriver(cmd *cobra.Command, ds registry.DriverState, existing * updateDriver(driverName) } + // If Kubernetes is running, and --no-kubernetes is specified, stop. + var stopk8s bool + if existing != nil && viper.GetBool(noKubernetes) { + stopk8s = true + } + k8sVersion := getKubernetesVersion(existing) cc, n, err := generateClusterConfig(cmd, existing, k8sVersion, driverName) if err != nil { @@ -337,6 +343,7 @@ func provisionWithDriver(cmd *cobra.Command, ds registry.DriverState, existing * return node.Starter{ Runner: mRunner, PreExists: preExists, + StopK8s: stopk8s, MachineAPI: mAPI, Host: host, ExistingAddons: existingAddons, diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 1b38c0d15e..e39ed6d2c0 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -801,6 +801,32 @@ func (k *Bootstrapper) GenerateToken(cc config.ClusterConfig) (string, error) { return joinCmd, nil } +// StopKubernetes stops existing kubernetes. +func StopKubernetes(k8s config.KubernetesConfig, runner command.Runner) error { + out.Infof("Stopping Kubernetes ...") + + cr, err := cruntime.New(cruntime.Config{Type: k8s.ContainerRuntime, Runner: runner, Socket: k8s.CRISocket}) + if err != nil { + return errors.Wrap(err, "runtime") + } + + if err := sysinit.New(runner).ForceStop("kubelet"); err != nil { + klog.Warningf("stop kubelet: %v", err) + } + + containers, err := cr.ListContainers(cruntime.ListContainersOptions{Namespaces: []string{"kube-system"}}) + if err != nil { + klog.Warningf("unable to list kube-system containers: %v", err) + } + if len(containers) > 0 { + klog.Warningf("found %d kube-system containers to stop", len(containers)) + if err := cr.StopContainers(containers); err != nil { + klog.Warningf("error stopping containers: %v", err) + } + } + return nil +} + // DeleteCluster removes the components that were started earlier func (k *Bootstrapper) DeleteCluster(k8s config.KubernetesConfig) error { cr, err := cruntime.New(cruntime.Config{Type: k8s.ContainerRuntime, Runner: k.c, Socket: k8s.CRISocket}) @@ -828,19 +854,8 @@ func (k *Bootstrapper) DeleteCluster(k8s config.KubernetesConfig) error { klog.Warningf("%s: %v", rr.Command(), err) } - if err := sysinit.New(k.c).ForceStop("kubelet"); err != nil { - klog.Warningf("stop kubelet: %v", err) - } - - containers, err := cr.ListContainers(cruntime.ListContainersOptions{Namespaces: []string{"kube-system"}}) - if err != nil { - klog.Warningf("unable to list kube-system containers: %v", err) - } - if len(containers) > 0 { - klog.Warningf("found %d kube-system containers to stop", len(containers)) - if err := cr.StopContainers(containers); err != nil { - klog.Warningf("error stopping containers: %v", err) - } + if err := StopKubernetes(k8s, k.c); err != nil { + return err } return derr diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index cc4e0bea09..0cd1ed89d1 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -44,6 +44,7 @@ import ( "k8s.io/minikube/pkg/minikube/bootstrapper" "k8s.io/minikube/pkg/minikube/bootstrapper/bsutil" "k8s.io/minikube/pkg/minikube/bootstrapper/images" + "k8s.io/minikube/pkg/minikube/bootstrapper/kubeadm" "k8s.io/minikube/pkg/minikube/cluster" "k8s.io/minikube/pkg/minikube/cni" "k8s.io/minikube/pkg/minikube/command" @@ -79,6 +80,7 @@ var ( type Starter struct { Runner command.Runner PreExists bool + StopK8s bool MachineAPI libmachine.API Host *host.Host Cfg *config.ClusterConfig @@ -89,6 +91,10 @@ type Starter struct { // Start spins up a guest and starts the Kubernetes node. func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) { var kcs *kubeconfig.Settings + // Stop existing Kubernetes node. + if starter.StopK8s { + kubeadm.StopKubernetes(starter.Cfg.KubernetesConfig, starter.Runner) + } if starter.Node.KubernetesVersion == constants.NoKubernetesVersion { // do not bootstrap cluster if --no-kubernetes return kcs, config.Write(viper.GetString(config.ProfileName), starter.Cfg) } diff --git a/test/integration/no_kubernetes_test.go b/test/integration/no_kubernetes_test.go index f88734b1f0..a6d18e83f2 100644 --- a/test/integration/no_kubernetes_test.go +++ b/test/integration/no_kubernetes_test.go @@ -21,6 +21,8 @@ package integration import ( "context" + "encoding/json" + "fmt" "os/exec" "strings" "testing" @@ -45,6 +47,8 @@ func TestNoKubernetes(t *testing.T) { name string validator validateFunc }{ + {"StartWithK8s", validateStartWithK8S}, + {"StartWithStopK8s", validateStartWithStopK8s}, {"Start", validateStartNoK8S}, {"VerifyK8sNotRunning", validateK8SNotRunning}, {"ProfileList", validateProfileListNoK8S}, @@ -70,6 +74,42 @@ func TestNoKubernetes(t *testing.T) { }) } +// validateStartWithK8S starts a minikube cluster with Kubernetes started/configured. +func validateStartWithK8S(ctx context.Context, t *testing.T, profile string) { + defer PostMortemLogs(t, profile) + + args := append([]string{"start", "-p", profile}, StartArgs()...) + rr, err := Run(t, exec.CommandContext(ctx, Target(), args...)) + if err != nil { + t.Fatalf("failed to start minikube with args: %q : %v", rr.Command(), err) + } + + if k8sStatus := getK8sStatus(ctx, t, profile); k8sStatus != "Running" { + t.Errorf("Kubernetes status, got: %s, want: Running", k8sStatus) + } +} + +// validateStartWithStopK8s starts a minikube cluster while stopping Kubernetes. +func validateStartWithStopK8s(ctx context.Context, t *testing.T, profile string) { + defer PostMortemLogs(t, profile) + + args := append([]string{"start", "-p", profile, "--no-kubernetes"}, StartArgs()...) + rr, err := Run(t, exec.CommandContext(ctx, Target(), args...)) + if err != nil { + t.Fatalf("failed to start minikube with args: %q : %v", rr.Command(), err) + } + + if k8sStatus := getK8sStatus(ctx, t, profile); k8sStatus != "Stopped" { + t.Errorf("Kubernetes status, got: %s, want: Stopped", k8sStatus) + } + + args = []string{"delete", "-p", profile} + rr, err = Run(t, exec.CommandContext(ctx, Target(), args...)) + if err != nil { + t.Fatalf("failed to delete minikube profile with args: %q : %v", rr.Command(), err) + } +} + // validateStartNoK8S starts a minikube cluster without kubernetes started/configured func validateStartNoK8S(ctx context.Context, t *testing.T, profile string) { defer PostMortemLogs(t, profile) @@ -125,7 +165,7 @@ func validateProfileListNoK8S(ctx context.Context, t *testing.T, profile string) } -// validateStartNoArgs valides that minikube start with no args works +// validateStartNoArgs validates that minikube start with no args works. func validateStartNoArgs(ctx context.Context, t *testing.T, profile string) { defer PostMortemLogs(t, profile) @@ -134,5 +174,22 @@ func validateStartNoArgs(ctx context.Context, t *testing.T, profile string) { if err != nil { t.Fatalf("failed to start minikube with args: %q : %v", rr.Command(), err) } - +} + +// getK8sStatus returns whether Kubernetes is running. 123 +func getK8sStatus(ctx context.Context, t *testing.T, profile string) string { + // Run `minikube status` as JSON output. + rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "status", "-o", "json")) + // We expect Kubernetes config to come back as configured, since we started Kubernetes in a previous test. + if err != nil && rr.ExitCode != 2 { + t.Errorf("failed to run minikube status with json output. args %q : %v", rr.Command(), err) + } + + // Unmarshal JSON output. + var jsonObject map[string]interface{} + err = json.Unmarshal(rr.Stdout.Bytes(), &jsonObject) + if err != nil { + t.Errorf("failed to decode json from minikube status. args %q. %v", rr.Command(), err) + } + return fmt.Sprintf("%s", jsonObject["Kubelet"]) } From 5b4e429d092515e944616d010855f27acbc1ed15 Mon Sep 17 00:00:00 2001 From: klaases Date: Thu, 2 Dec 2021 14:46:42 -0800 Subject: [PATCH 02/10] fix spacing --- test/integration/no_kubernetes_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/no_kubernetes_test.go b/test/integration/no_kubernetes_test.go index e882936f73..168688a6e3 100644 --- a/test/integration/no_kubernetes_test.go +++ b/test/integration/no_kubernetes_test.go @@ -48,7 +48,7 @@ func TestNoKubernetes(t *testing.T) { validator validateFunc }{ {"StartNoK8sWithVersion", validateStartNoK8sWithVersion}, - {"StartWithK8s", validateStartWithK8S}, + {"StartWithK8s", validateStartWithK8S}, {"StartWithStopK8s", validateStartWithStopK8s}, {"Start", validateStartNoK8S}, {"VerifyK8sNotRunning", validateK8SNotRunning}, @@ -83,7 +83,7 @@ func validateStartNoK8sWithVersion(ctx context.Context, t *testing.T, profile st rr, err := Run(t, exec.CommandContext(ctx, Target(), args...)) if err == nil { t.Fatalf("expected an error but none was thrown with args: %q", rr.Command()) - } + } } // validateStartWithK8S starts a minikube cluster with Kubernetes started/configured. From 31a0b6d51371bb78c62672b3c570ecf2913a59e4 Mon Sep 17 00:00:00 2001 From: klaases Date: Thu, 2 Dec 2021 14:49:08 -0800 Subject: [PATCH 03/10] remove 123 dev comment --- test/integration/no_kubernetes_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/no_kubernetes_test.go b/test/integration/no_kubernetes_test.go index 168688a6e3..fd6ae873c4 100644 --- a/test/integration/no_kubernetes_test.go +++ b/test/integration/no_kubernetes_test.go @@ -188,7 +188,7 @@ func validateStartNoArgs(ctx context.Context, t *testing.T, profile string) { } } -// getK8sStatus returns whether Kubernetes is running. 123 +// getK8sStatus returns whether Kubernetes is running. func getK8sStatus(ctx context.Context, t *testing.T, profile string) string { // Run `minikube status` as JSON output. rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "status", "-o", "json")) From a156f26fbb4a3017c53776248dbcdba5c912f9b5 Mon Sep 17 00:00:00 2001 From: klaases Date: Thu, 2 Dec 2021 15:54:31 -0800 Subject: [PATCH 04/10] resolve linter errors --- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 16 ++++++---------- pkg/minikube/node/start.go | 7 +++---- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index e39ed6d2c0..555f7909b8 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -802,13 +802,12 @@ func (k *Bootstrapper) GenerateToken(cc config.ClusterConfig) (string, error) { } // StopKubernetes stops existing kubernetes. -func StopKubernetes(k8s config.KubernetesConfig, runner command.Runner) error { - out.Infof("Stopping Kubernetes ...") - - cr, err := cruntime.New(cruntime.Config{Type: k8s.ContainerRuntime, Runner: runner, Socket: k8s.CRISocket}) - if err != nil { - return errors.Wrap(err, "runtime") +func StopKubernetes(stop bool, cr cruntime.Manager, k8s config.KubernetesConfig, runner command.Runner) { + // Minimize cyclomatic complexity for parent functions. + if !stop { + return } + out.Infof("Stopping Kubernetes ...") if err := sysinit.New(runner).ForceStop("kubelet"); err != nil { klog.Warningf("stop kubelet: %v", err) @@ -824,7 +823,6 @@ func StopKubernetes(k8s config.KubernetesConfig, runner command.Runner) error { klog.Warningf("error stopping containers: %v", err) } } - return nil } // DeleteCluster removes the components that were started earlier @@ -854,9 +852,7 @@ func (k *Bootstrapper) DeleteCluster(k8s config.KubernetesConfig) error { klog.Warningf("%s: %v", rr.Command(), err) } - if err := StopKubernetes(k8s, k.c); err != nil { - return err - } + StopKubernetes(true, cr, k8s, k.c) return derr } diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index 0cd1ed89d1..416d2f39f0 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -91,10 +91,6 @@ type Starter struct { // Start spins up a guest and starts the Kubernetes node. func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) { var kcs *kubeconfig.Settings - // Stop existing Kubernetes node. - if starter.StopK8s { - kubeadm.StopKubernetes(starter.Cfg.KubernetesConfig, starter.Runner) - } if starter.Node.KubernetesVersion == constants.NoKubernetesVersion { // do not bootstrap cluster if --no-kubernetes return kcs, config.Write(viper.GetString(config.ProfileName), starter.Cfg) } @@ -109,6 +105,9 @@ func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) { // configure the runtime (docker, containerd, crio) cr := configureRuntimes(starter.Runner, *starter.Cfg, sv) + // Stop existing Kubernetes node if applicable. + kubeadm.StopKubernetes(starter.StopK8s, cr, starter.Cfg.KubernetesConfig, starter.Runner) + // check if installed runtime is compatible with current minikube code if err = cruntime.CheckCompatibility(cr); err != nil { return nil, err From c6fc55aa98d85002426c50f6d5d47162ba248d24 Mon Sep 17 00:00:00 2001 From: klaases Date: Fri, 3 Dec 2021 11:22:58 -0800 Subject: [PATCH 05/10] Refactor Start() to reduce cyclomatic complexity. --- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 13 ++--- pkg/minikube/node/start.go | 58 +++++++++++++------- 2 files changed, 42 insertions(+), 29 deletions(-) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 555f7909b8..a1e55c0d4b 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -801,18 +801,16 @@ func (k *Bootstrapper) GenerateToken(cc config.ClusterConfig) (string, error) { return joinCmd, nil } -// StopKubernetes stops existing kubernetes. -func StopKubernetes(stop bool, cr cruntime.Manager, k8s config.KubernetesConfig, runner command.Runner) { - // Minimize cyclomatic complexity for parent functions. - if !stop { - return - } +// StopKubernetes attempts to stop existing kubernetes. +func StopKubernetes(runner command.Runner, cr cruntime.Manager) { out.Infof("Stopping Kubernetes ...") + // Force stop "Kubelet". if err := sysinit.New(runner).ForceStop("kubelet"); err != nil { klog.Warningf("stop kubelet: %v", err) } + // Stop each Kubernetes container. containers, err := cr.ListContainers(cruntime.ListContainersOptions{Namespaces: []string{"kube-system"}}) if err != nil { klog.Warningf("unable to list kube-system containers: %v", err) @@ -852,8 +850,7 @@ func (k *Bootstrapper) DeleteCluster(k8s config.KubernetesConfig) error { klog.Warningf("%s: %v", rr.Command(), err) } - StopKubernetes(true, cr, k8s, k.c) - + StopKubernetes(k.c, cr) return derr } diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index 416d2f39f0..3866457dd3 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -91,30 +91,11 @@ type Starter struct { // Start spins up a guest and starts the Kubernetes node. func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) { var kcs *kubeconfig.Settings - if starter.Node.KubernetesVersion == constants.NoKubernetesVersion { // do not bootstrap cluster if --no-kubernetes - return kcs, config.Write(viper.GetString(config.ProfileName), starter.Cfg) - } - // wait for preloaded tarball to finish downloading before configuring runtimes - waitCacheRequiredImages(&cacheGroup) - - sv, err := util.ParseKubernetesVersion(starter.Node.KubernetesVersion) + cr, err := handleNoKubernetes(starter) if err != nil { - return nil, errors.Wrap(err, "Failed to parse Kubernetes version") + return kcs, err } - // configure the runtime (docker, containerd, crio) - cr := configureRuntimes(starter.Runner, *starter.Cfg, sv) - - // Stop existing Kubernetes node if applicable. - kubeadm.StopKubernetes(starter.StopK8s, cr, starter.Cfg.KubernetesConfig, starter.Runner) - - // check if installed runtime is compatible with current minikube code - if err = cruntime.CheckCompatibility(cr); err != nil { - return nil, err - } - - showVersionInfo(starter.Node.KubernetesVersion, cr) - // Add "host.minikube.internal" DNS alias (intentionally non-fatal) hostIP, err := cluster.HostIP(starter.Host, starter.Cfg.Name) if err != nil { @@ -243,6 +224,41 @@ func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) { return kcs, config.Write(viper.GetString(config.ProfileName), starter.Cfg) } +// handleNoKubernetes handles starting minikube without Kubernetes. +func handleNoKubernetes(starter Starter) (cruntime.Manager, error) { + // Do not bootstrap cluster if --no-kubernetes. + if starter.Node.KubernetesVersion == constants.NoKubernetesVersion { + // Stop existing Kubernetes node if applicable. + if starter.StopK8s { + cr, err := cruntime.New(cruntime.Config{Type: starter.Cfg.KubernetesConfig.ContainerRuntime, Runner: starter.Runner, Socket: starter.Cfg.KubernetesConfig.CRISocket}) + if err != nil { + return nil, err + } + kubeadm.StopKubernetes(starter.Runner, cr) + } + return nil, config.Write(viper.GetString(config.ProfileName), starter.Cfg) + } + + // Wait for preloaded tarball to finish downloading before configuring runtimes. + waitCacheRequiredImages(&cacheGroup) + + sv, err := util.ParseKubernetesVersion(starter.Node.KubernetesVersion) + if err != nil { + return nil, errors.Wrap(err, "Failed to parse Kubernetes version") + } + + // Configure the runtime (docker, containerd, crio). + cr := configureRuntimes(starter.Runner, *starter.Cfg, sv) + + // Check if installed runtime is compatible with current minikube code. + if err = cruntime.CheckCompatibility(cr); err != nil { + return nil, err + } + + showVersionInfo(starter.Node.KubernetesVersion, cr) + return cr, nil +} + // joinCluster adds new or prepares and then adds existing node to the cluster. func joinCluster(starter Starter, cpBs bootstrapper.Bootstrapper, bs bootstrapper.Bootstrapper) error { start := time.Now() From f3186d07d8431669b57fecb1a915c7b10cac83dc Mon Sep 17 00:00:00 2001 From: klaases Date: Fri, 3 Dec 2021 12:35:34 -0800 Subject: [PATCH 06/10] Detect whether k8s needs to be stopped. --- cmd/minikube/cmd/start.go | 3 ++- pkg/minikube/node/start.go | 49 +++++++++++++++++++------------------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index 965a96ee4f..740d8fb824 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -297,7 +297,8 @@ func provisionWithDriver(cmd *cobra.Command, ds registry.DriverState, existing * // If Kubernetes is running, and --no-kubernetes is specified, stop. var stopk8s bool - if existing != nil && viper.GetBool(noKubernetes) { + if existing != nil && existing.KubernetesConfig.KubernetesVersion != constants.NoKubernetesVersion && viper.GetBool(noKubernetes) { + klog.Infof("ABC1 existing.KubernetesConfig.KubernetesVersion %s", existing.KubernetesConfig.KubernetesVersion) stopk8s = true } diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index 3866457dd3..6b080a447f 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -91,11 +91,28 @@ type Starter struct { // Start spins up a guest and starts the Kubernetes node. func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) { var kcs *kubeconfig.Settings - cr, err := handleNoKubernetes(starter) - if err != nil { - return kcs, err + if err := handleNoKubernetes(starter); err != nil { + return kcs, config.Write(viper.GetString(config.ProfileName), starter.Cfg) } + // wait for preloaded tarball to finish downloading before configuring runtimes + waitCacheRequiredImages(&cacheGroup) + + sv, err := util.ParseKubernetesVersion(starter.Node.KubernetesVersion) + if err != nil { + return nil, errors.Wrap(err, "Failed to parse Kubernetes version") + } + + // configure the runtime (docker, containerd, crio) + cr := configureRuntimes(starter.Runner, *starter.Cfg, sv) + + // check if installed runtime is compatible with current minikube code + if err = cruntime.CheckCompatibility(cr); err != nil { + return nil, err + } + + showVersionInfo(starter.Node.KubernetesVersion, cr) + // Add "host.minikube.internal" DNS alias (intentionally non-fatal) hostIP, err := cluster.HostIP(starter.Host, starter.Cfg.Name) if err != nil { @@ -225,38 +242,20 @@ func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) { } // handleNoKubernetes handles starting minikube without Kubernetes. -func handleNoKubernetes(starter Starter) (cruntime.Manager, error) { +func handleNoKubernetes(starter Starter) error { // Do not bootstrap cluster if --no-kubernetes. if starter.Node.KubernetesVersion == constants.NoKubernetesVersion { // Stop existing Kubernetes node if applicable. if starter.StopK8s { cr, err := cruntime.New(cruntime.Config{Type: starter.Cfg.KubernetesConfig.ContainerRuntime, Runner: starter.Runner, Socket: starter.Cfg.KubernetesConfig.CRISocket}) if err != nil { - return nil, err + return err } kubeadm.StopKubernetes(starter.Runner, cr) } - return nil, config.Write(viper.GetString(config.ProfileName), starter.Cfg) + return errors.New("stop kubernetes") } - - // Wait for preloaded tarball to finish downloading before configuring runtimes. - waitCacheRequiredImages(&cacheGroup) - - sv, err := util.ParseKubernetesVersion(starter.Node.KubernetesVersion) - if err != nil { - return nil, errors.Wrap(err, "Failed to parse Kubernetes version") - } - - // Configure the runtime (docker, containerd, crio). - cr := configureRuntimes(starter.Runner, *starter.Cfg, sv) - - // Check if installed runtime is compatible with current minikube code. - if err = cruntime.CheckCompatibility(cr); err != nil { - return nil, err - } - - showVersionInfo(starter.Node.KubernetesVersion, cr) - return cr, nil + return nil } // joinCluster adds new or prepares and then adds existing node to the cluster. From 47a280bdc791eeca7fa90a9c61344c7841ec00d9 Mon Sep 17 00:00:00 2001 From: klaases Date: Fri, 3 Dec 2021 14:15:00 -0800 Subject: [PATCH 07/10] Fix check for stopping k8s. --- cmd/minikube/cmd/start.go | 5 ++--- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 13 ++++++++++++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index 740d8fb824..3ce0c66fcc 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -295,10 +295,9 @@ func provisionWithDriver(cmd *cobra.Command, ds registry.DriverState, existing * updateDriver(driverName) } - // If Kubernetes is running, and --no-kubernetes is specified, stop. + // Check whether we may need to stop Kubernetes. var stopk8s bool - if existing != nil && existing.KubernetesConfig.KubernetesVersion != constants.NoKubernetesVersion && viper.GetBool(noKubernetes) { - klog.Infof("ABC1 existing.KubernetesConfig.KubernetesVersion %s", existing.KubernetesConfig.KubernetesVersion) + if existing != nil && viper.GetBool(noKubernetes) { stopk8s = true } diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index a1e55c0d4b..7e4cbb6b2b 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -803,7 +803,14 @@ func (k *Bootstrapper) GenerateToken(cc config.ClusterConfig) (string, error) { // StopKubernetes attempts to stop existing kubernetes. func StopKubernetes(runner command.Runner, cr cruntime.Manager) { - out.Infof("Stopping Kubernetes ...") + + // Verify that Kubernetes is still running. + stk := kverify.ServiceStatus(runner, "kubelet") + if stk.String() != "Running" { + return + } + + out.Infof("Kubernetes: Stopping ...") // Force stop "Kubelet". if err := sysinit.New(runner).ForceStop("kubelet"); err != nil { @@ -821,6 +828,10 @@ func StopKubernetes(runner command.Runner, cr cruntime.Manager) { klog.Warningf("error stopping containers: %v", err) } } + + // Verify that Kubernetes has stopped. + stk = kverify.ServiceStatus(runner, "kubelet") + out.Infof("Kubernetes: {{.status}}", out.V{"status": stk.String()}) } // DeleteCluster removes the components that were started earlier From 3ac94ae5e31ff5943acf5191e7bcb515937a5cb7 Mon Sep 17 00:00:00 2001 From: klaases Date: Fri, 3 Dec 2021 14:55:11 -0800 Subject: [PATCH 08/10] Remove extra space. --- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 7e4cbb6b2b..dd2e847d2d 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -803,7 +803,6 @@ func (k *Bootstrapper) GenerateToken(cc config.ClusterConfig) (string, error) { // StopKubernetes attempts to stop existing kubernetes. func StopKubernetes(runner command.Runner, cr cruntime.Manager) { - // Verify that Kubernetes is still running. stk := kverify.ServiceStatus(runner, "kubelet") if stk.String() != "Running" { From a37ea8e5c2fc2096441ca1bcf173ca5e1b38d9b1 Mon Sep 17 00:00:00 2001 From: klaases Date: Fri, 3 Dec 2021 15:42:38 -0800 Subject: [PATCH 09/10] Add tuple to make sure we handle error properly. --- pkg/minikube/node/start.go | 89 ++++++++++++++++++++++---------------- 1 file changed, 52 insertions(+), 37 deletions(-) diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index 6b080a447f..b78da0d926 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -90,9 +90,12 @@ type Starter struct { // Start spins up a guest and starts the Kubernetes node. func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) { - var kcs *kubeconfig.Settings - if err := handleNoKubernetes(starter); err != nil { - return kcs, config.Write(viper.GetString(config.ProfileName), starter.Cfg) + stop8ks, err := handleNoKubernetes(starter) + if err != nil { + return nil, err + } + if stop8ks { + return nil, config.Write(viper.GetString(config.ProfileName), starter.Cfg) } // wait for preloaded tarball to finish downloading before configuring runtimes @@ -121,41 +124,13 @@ func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) { klog.Errorf("Unable to add host alias: %v", err) } + var kcs *kubeconfig.Settings var bs bootstrapper.Bootstrapper if apiServer { - // Must be written before bootstrap, otherwise health checks may flake due to stale IP - kcs = setupKubeconfig(starter.Host, starter.Cfg, starter.Node, starter.Cfg.Name) + kcs, bs, err = handleAPIServer(starter, cr, hostIP) if err != nil { - return nil, errors.Wrap(err, "Failed to setup kubeconfig") - } - - // setup kubeadm (must come after setupKubeconfig) - bs = setupKubeAdm(starter.MachineAPI, *starter.Cfg, *starter.Node, starter.Runner) - err = bs.StartCluster(*starter.Cfg) - if err != nil { - ExitIfFatal(err) - out.LogEntries("Error starting cluster", err, logs.FindProblems(cr, bs, *starter.Cfg, starter.Runner)) return nil, err } - - // write the kubeconfig to the file system after everything required (like certs) are created by the bootstrapper - if err := kubeconfig.Update(kcs); err != nil { - return nil, errors.Wrap(err, "Failed kubeconfig update") - } - - // scale down CoreDNS from default 2 to 1 replica - if err := kapi.ScaleDeployment(starter.Cfg.Name, meta.NamespaceSystem, kconst.CoreDNSDeploymentName, 1); err != nil { - klog.Errorf("Unable to scale down deployment %q in namespace %q to 1 replica: %v", kconst.CoreDNSDeploymentName, meta.NamespaceSystem, err) - } - - // not running this in a Go func can result in DNS answering taking up to 38 seconds, with the Go func it takes 6-10 seconds - go func() { - // inject {"host.minikube.internal": hostIP} record into CoreDNS - if err := addCoreDNSEntry(starter.Runner, "host.minikube.internal", hostIP.String(), *starter.Cfg); err != nil { - klog.Warningf("Unable to inject {%q: %s} record into CoreDNS: %v", "host.minikube.internal", hostIP.String(), err) - out.Err("Failed to inject host.minikube.internal into CoreDNS, this will limit the pods access to the host IP") - } - }() } else { bs, err = cluster.Bootstrapper(starter.MachineAPI, viper.GetString(cmdcfg.Bootstrapper), *starter.Cfg, starter.Runner) if err != nil { @@ -242,20 +217,60 @@ func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) { } // handleNoKubernetes handles starting minikube without Kubernetes. -func handleNoKubernetes(starter Starter) error { +func handleNoKubernetes(starter Starter) (bool, error) { // Do not bootstrap cluster if --no-kubernetes. if starter.Node.KubernetesVersion == constants.NoKubernetesVersion { // Stop existing Kubernetes node if applicable. if starter.StopK8s { cr, err := cruntime.New(cruntime.Config{Type: starter.Cfg.KubernetesConfig.ContainerRuntime, Runner: starter.Runner, Socket: starter.Cfg.KubernetesConfig.CRISocket}) if err != nil { - return err + return false, err } kubeadm.StopKubernetes(starter.Runner, cr) } - return errors.New("stop kubernetes") + return true, config.Write(viper.GetString(config.ProfileName), starter.Cfg) } - return nil + return false, nil +} + +// handleAPIServer handles starting the API server. +func handleAPIServer(starter Starter, cr cruntime.Manager, hostIP net.IP) (*kubeconfig.Settings, bootstrapper.Bootstrapper, error) { + var err error + + // Must be written before bootstrap, otherwise health checks may flake due to stale IP. + kcs := setupKubeconfig(starter.Host, starter.Cfg, starter.Node, starter.Cfg.Name) + if err != nil { + return nil, nil, errors.Wrap(err, "Failed to setup kubeconfig") + } + + // Setup kubeadm (must come after setupKubeconfig). + bs := setupKubeAdm(starter.MachineAPI, *starter.Cfg, *starter.Node, starter.Runner) + err = bs.StartCluster(*starter.Cfg) + if err != nil { + ExitIfFatal(err) + out.LogEntries("Error starting cluster", err, logs.FindProblems(cr, bs, *starter.Cfg, starter.Runner)) + return nil, bs, err + } + + // Write the kubeconfig to the file system after everything required (like certs) are created by the bootstrapper. + if err := kubeconfig.Update(kcs); err != nil { + return nil, bs, errors.Wrap(err, "Failed kubeconfig update") + } + + // Scale down CoreDNS from default 2 to 1 replica. + if err := kapi.ScaleDeployment(starter.Cfg.Name, meta.NamespaceSystem, kconst.CoreDNSDeploymentName, 1); err != nil { + klog.Errorf("Unable to scale down deployment %q in namespace %q to 1 replica: %v", kconst.CoreDNSDeploymentName, meta.NamespaceSystem, err) + } + + // Not running this in a Go func can result in DNS answering taking up to 38 seconds, with the Go func it takes 6-10 seconds. + go func() { + // Inject {"host.minikube.internal": hostIP} record into CoreDNS. + if err := addCoreDNSEntry(starter.Runner, "host.minikube.internal", hostIP.String(), *starter.Cfg); err != nil { + klog.Warningf("Unable to inject {%q: %s} record into CoreDNS: %v", "host.minikube.internal", hostIP.String(), err) + out.Err("Failed to inject host.minikube.internal into CoreDNS, this will limit the pods access to the host IP") + } + }() + return kcs, bs, nil } // joinCluster adds new or prepares and then adds existing node to the cluster. From 1bae4f12b59b321518a75b10824e37e3be003513 Mon Sep 17 00:00:00 2001 From: klaases Date: Fri, 3 Dec 2021 15:57:24 -0800 Subject: [PATCH 10/10] Add test documentation. --- test/integration/no_kubernetes_test.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/integration/no_kubernetes_test.go b/test/integration/no_kubernetes_test.go index fd6ae873c4..369010e898 100644 --- a/test/integration/no_kubernetes_test.go +++ b/test/integration/no_kubernetes_test.go @@ -79,6 +79,7 @@ func TestNoKubernetes(t *testing.T) { func validateStartNoK8sWithVersion(ctx context.Context, t *testing.T, profile string) { defer PostMortemLogs(t, profile) + // docs: start minikube with no kubernetes. args := append([]string{"start", "-p", profile, "--no-kubernetes", "--kubernetes-version=1.20"}, StartArgs()...) rr, err := Run(t, exec.CommandContext(ctx, Target(), args...)) if err == nil { @@ -90,12 +91,14 @@ func validateStartNoK8sWithVersion(ctx context.Context, t *testing.T, profile st func validateStartWithK8S(ctx context.Context, t *testing.T, profile string) { defer PostMortemLogs(t, profile) + // docs: start minikube with Kubernetes. args := append([]string{"start", "-p", profile}, StartArgs()...) rr, err := Run(t, exec.CommandContext(ctx, Target(), args...)) if err != nil { t.Fatalf("failed to start minikube with args: %q : %v", rr.Command(), err) } + // docs: return an error if Kubernetes is not running. if k8sStatus := getK8sStatus(ctx, t, profile); k8sStatus != "Running" { t.Errorf("Kubernetes status, got: %s, want: Running", k8sStatus) } @@ -105,16 +108,19 @@ func validateStartWithK8S(ctx context.Context, t *testing.T, profile string) { func validateStartWithStopK8s(ctx context.Context, t *testing.T, profile string) { defer PostMortemLogs(t, profile) + // docs: start minikube with no Kubernetes. args := append([]string{"start", "-p", profile, "--no-kubernetes"}, StartArgs()...) rr, err := Run(t, exec.CommandContext(ctx, Target(), args...)) if err != nil { t.Fatalf("failed to start minikube with args: %q : %v", rr.Command(), err) } + // docs: return an error if Kubernetes is not stopped. if k8sStatus := getK8sStatus(ctx, t, profile); k8sStatus != "Stopped" { t.Errorf("Kubernetes status, got: %s, want: Stopped", k8sStatus) } + // docs: delete minikube profile. args = []string{"delete", "-p", profile} rr, err = Run(t, exec.CommandContext(ctx, Target(), args...)) if err != nil { @@ -126,6 +132,7 @@ func validateStartWithStopK8s(ctx context.Context, t *testing.T, profile string) func validateStartNoK8S(ctx context.Context, t *testing.T, profile string) { defer PostMortemLogs(t, profile) + // docs: start minikube with no Kubernetes. args := append([]string{"start", "-p", profile, "--no-kubernetes"}, StartArgs()...) rr, err := Run(t, exec.CommandContext(ctx, Target(), args...)) if err != nil {