From b602ae44bddf6052ed7680998824449455fc8192 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Mon, 27 Jul 2020 17:39:32 -0700 Subject: [PATCH 1/7] check kubectl for multinode test --- test/integration/multinode_test.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/integration/multinode_test.go b/test/integration/multinode_test.go index 4b67b09b06..f95aac4b98 100644 --- a/test/integration/multinode_test.go +++ b/test/integration/multinode_test.go @@ -291,4 +291,20 @@ func validateDeleteNodeFromMultiNode(ctx context.Context, t *testing.T, profile } } + // Make sure kubectl knows the node is gone + rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "get", "nodes")) + if err != nil { + t.Fatalf("failed to run kubectl get nodes. args %q : %v", rr.Command(), err) + } + if strings.Count(rr.Stdout.String(), "NotReady") > 0 { + t.Errorf("expected 2 nodes to be Ready, got %v", rr.Output()) + } + + rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "get", "nodes", "-o", `go-template='{{range .items}}{{range .status.conditions}}{{if eq .type "Ready"}} {{.status}}{{"\n"}}{{end}}{{end}}{{end}}'`)) + if err != nil { + t.Fatalf("failed to run kubectl get nodes. args %q : %v", rr.Command(), err) + } + if strings.Count(rr.Stdout.String(), "True") != 2 { + t.Errorf("expected 2 nodes Ready status to be True, got %v", rr.Output()) + } } From 2cc7445f1c9b125252eecaa4b5c7f560caf9da6d Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Tue, 28 Jul 2020 12:16:33 -0700 Subject: [PATCH 2/7] guard against joining the cluster twice --- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 9f20e61285..1fe2aa63eb 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -654,7 +654,11 @@ func (k *Bootstrapper) JoinCluster(cc config.ClusterConfig, n config.Node, joinC out, err := k.c.RunCmd(exec.Command("/bin/bash", "-c", joinCmd)) if err != nil { - return errors.Wrapf(err, "cmd failed: %s\n%+v\n", joinCmd, out.Output()) + if strings.Contains(err.Error(), "status \"Ready\" already exists in the cluster") { + glog.Infof("Node %s already joined the cluster, skip failure.", n.Name) + } else { + return errors.Wrapf(err, "cmd failed: %s\n%+v\n", joinCmd, out.Output()) + } } return nil } From de35861f8bd86ea6751b161e31cd5c8a39ed9088 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Tue, 28 Jul 2020 13:06:23 -0700 Subject: [PATCH 3/7] add verbosity to multinode start test --- test/integration/multinode_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/multinode_test.go b/test/integration/multinode_test.go index f95aac4b98..9e16d51d09 100644 --- a/test/integration/multinode_test.go +++ b/test/integration/multinode_test.go @@ -62,7 +62,7 @@ func TestMultiNode(t *testing.T) { func validateMultiNodeStart(ctx context.Context, t *testing.T, profile string) { // Start a 2 node cluster with the --nodes param - startArgs := append([]string{"start", "-p", profile, "--wait=true", "--memory=2200", "--nodes=2"}, StartArgs()...) + startArgs := append([]string{"start", "-p", profile, "--wait=true", "--memory=2200", "--nodes=2", "--alsologtostderr"}, StartArgs()...) rr, err := Run(t, exec.CommandContext(ctx, Target(), startArgs...)) if err != nil { t.Fatalf("failed to start cluster. args %q : %v", rr.Command(), err) From 174f90152f035e7bb059c71567fb969969b1ed3e Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Tue, 28 Jul 2020 14:42:08 -0700 Subject: [PATCH 4/7] add more verbosity --- test/integration/multinode_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/multinode_test.go b/test/integration/multinode_test.go index 9e16d51d09..b8caf8cd18 100644 --- a/test/integration/multinode_test.go +++ b/test/integration/multinode_test.go @@ -62,7 +62,7 @@ func TestMultiNode(t *testing.T) { func validateMultiNodeStart(ctx context.Context, t *testing.T, profile string) { // Start a 2 node cluster with the --nodes param - startArgs := append([]string{"start", "-p", profile, "--wait=true", "--memory=2200", "--nodes=2", "--alsologtostderr"}, StartArgs()...) + startArgs := append([]string{"start", "-p", profile, "--wait=true", "--memory=2200", "--nodes=2", "-v=8", "--alsologtostderr"}, StartArgs()...) rr, err := Run(t, exec.CommandContext(ctx, Target(), startArgs...)) if err != nil { t.Fatalf("failed to start cluster. args %q : %v", rr.Command(), err) From 916b795d4208ba5fd51f64989a0553863fa939fe Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Tue, 28 Jul 2020 15:51:04 -0700 Subject: [PATCH 5/7] add verbosity to restart multinode test too --- test/integration/multinode_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/multinode_test.go b/test/integration/multinode_test.go index b8caf8cd18..8d0d00b65c 100644 --- a/test/integration/multinode_test.go +++ b/test/integration/multinode_test.go @@ -219,7 +219,7 @@ func validateRestartMultiNodeCluster(ctx context.Context, t *testing.T, profile } } // Restart a full cluster with minikube start - startArgs := append([]string{"start", "-p", profile, "--wait=true"}, StartArgs()...) + startArgs := append([]string{"start", "-p", profile, "--wait=true", "-v=8", "--alsologtostderr"}, StartArgs()...) rr, err := Run(t, exec.CommandContext(ctx, Target(), startArgs...)) if err != nil { t.Fatalf("failed to start cluster. args %q : %v", rr.Command(), err) From 3730a4cca883a172bd513ef6fcfd97a501602f13 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Wed, 29 Jul 2020 10:54:02 -0700 Subject: [PATCH 6/7] move sleep around --- test/integration/multinode_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/multinode_test.go b/test/integration/multinode_test.go index 8d0d00b65c..6512e0db2c 100644 --- a/test/integration/multinode_test.go +++ b/test/integration/multinode_test.go @@ -225,6 +225,8 @@ func validateRestartMultiNodeCluster(ctx context.Context, t *testing.T, profile t.Fatalf("failed to start cluster. args %q : %v", rr.Command(), err) } + time.Sleep(Seconds(30)) + // Make sure minikube status shows 2 running nodes rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "status", "--alsologtostderr")) if err != nil { @@ -239,8 +241,6 @@ func validateRestartMultiNodeCluster(ctx context.Context, t *testing.T, profile t.Errorf("status says both kubelets are not running: args %q: %v", rr.Command(), rr.Output()) } - time.Sleep(Seconds(30)) - // Make sure kubectl reports that all nodes are ready rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "get", "nodes")) if err != nil { From 52b66b4b9adfbeb64413499d2a69de820f0d2055 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Wed, 29 Jul 2020 16:30:56 -0700 Subject: [PATCH 7/7] sleep longer because I have no idea --- test/integration/multinode_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/multinode_test.go b/test/integration/multinode_test.go index 6512e0db2c..920b2e9571 100644 --- a/test/integration/multinode_test.go +++ b/test/integration/multinode_test.go @@ -225,7 +225,7 @@ func validateRestartMultiNodeCluster(ctx context.Context, t *testing.T, profile t.Fatalf("failed to start cluster. args %q : %v", rr.Command(), err) } - time.Sleep(Seconds(30)) + time.Sleep(Seconds(45)) // Make sure minikube status shows 2 running nodes rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "status", "--alsologtostderr"))