From d19d55ce52fd163dad53d5f8cd39e8d4d2225709 Mon Sep 17 00:00:00 2001 From: Marcell Martini Date: Mon, 1 May 2023 22:14:35 -0400 Subject: [PATCH 1/5] Add public method ApplyNodeLabels. --- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index ad27427510..5fa8f04576 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -1035,8 +1035,11 @@ func kubectlPath(cfg config.ClusterConfig) string { return path.Join(vmpath.GuestPersistentDir, "binaries", cfg.KubernetesConfig.KubernetesVersion, "kubectl") } +func (k *Bootstrapper) ApplyNodeLabels(cfg config.ClusterConfig) error { + return k.applyNodeLabels(cfg) +} + // applyNodeLabels applies minikube labels to all the nodes -// but it's currently called only from kubeadm.StartCluster (via kubeadm.init) where there's only one - first node func (k *Bootstrapper) applyNodeLabels(cfg config.ClusterConfig) error { // time cluster was created. time format is based on ISO 8601 (RFC 3339) // converting - and : to _ because of Kubernetes label restriction @@ -1048,8 +1051,12 @@ func (k *Bootstrapper) applyNodeLabels(cfg config.ClusterConfig) error { // ensure that "primary" label is applied only to the 1st node in the cluster (used eg for placing ingress there) // this is used to uniquely distinguish that from other nodes in multi-master/multi-control-plane cluster config primaryLbl := "minikube.k8s.io/primary=false" + + // ensure that "primary" label is not removed when apply label to all others nodes + applyToNodes := "-l minikube.k8s.io/primary!=true" if len(cfg.Nodes) <= 1 { primaryLbl = "minikube.k8s.io/primary=true" + applyToNodes = "--all" } ctx, cancel := context.WithTimeout(context.Background(), applyTimeoutSeconds*time.Second) @@ -1057,7 +1064,7 @@ func (k *Bootstrapper) applyNodeLabels(cfg config.ClusterConfig) error { // example: // sudo /var/lib/minikube/binaries//kubectl label nodes minikube.k8s.io/version= minikube.k8s.io/commit=aa91f39ffbcf27dcbb93c4ff3f457c54e585cf4a-dirty minikube.k8s.io/name=p1 minikube.k8s.io/updated_at=2020_02_20T12_05_35_0700 --all --overwrite --kubeconfig=/var/lib/minikube/kubeconfig cmd := exec.CommandContext(ctx, "sudo", kubectlPath(cfg), - "label", "nodes", verLbl, commitLbl, nameLbl, createdAtLbl, primaryLbl, "--all", "--overwrite", + "label", "nodes", verLbl, commitLbl, nameLbl, createdAtLbl, primaryLbl, applyToNodes, "--overwrite", fmt.Sprintf("--kubeconfig=%s", path.Join(vmpath.GuestPersistentDir, "kubeconfig"))) if _, err := k.c.RunCmd(cmd); err != nil { From 94f72fe68201efb05a87342cb97e43f264d209e7 Mon Sep 17 00:00:00 2001 From: Marcell Martini Date: Mon, 1 May 2023 22:15:01 -0400 Subject: [PATCH 2/5] Add ApplyNodeLabels to Bootstrapper interface. --- pkg/minikube/bootstrapper/bootstrapper.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/minikube/bootstrapper/bootstrapper.go b/pkg/minikube/bootstrapper/bootstrapper.go index 43fc313a34..03b8364f91 100644 --- a/pkg/minikube/bootstrapper/bootstrapper.go +++ b/pkg/minikube/bootstrapper/bootstrapper.go @@ -35,6 +35,7 @@ type LogOptions struct { // Bootstrapper contains all the methods needed to bootstrap a Kubernetes cluster type Bootstrapper interface { + ApplyNodeLabels(config.ClusterConfig) error StartCluster(config.ClusterConfig) error UpdateCluster(config.ClusterConfig) error DeleteCluster(config.KubernetesConfig) error From cf74cf5a87467274f6d14c8d9e1a95957ed607e8 Mon Sep 17 00:00:00 2001 From: Marcell Martini Date: Mon, 1 May 2023 22:16:24 -0400 Subject: [PATCH 3/5] Apply labels to the node when joining the cluster. --- pkg/minikube/node/start.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index 877265cbe9..6e70c2cbc3 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -339,6 +339,9 @@ func joinCluster(starter Starter, cpBs bootstrapper.Bootstrapper, bs bootstrappe return fmt.Errorf("error joining worker node to cluster: %w", err) } + if err := cpBs.ApplyNodeLabels(*starter.Cfg); err != nil { + return fmt.Errorf("error applying node label: %w", err) + } return nil } From 38d59318b0a7a941727a36483b57c91b2661f0de Mon Sep 17 00:00:00 2001 From: Marcell Martini Date: Tue, 9 May 2023 14:20:27 -0400 Subject: [PATCH 4/5] Add test to MultiNodeLabels. --- test/integration/multinode_test.go | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/test/integration/multinode_test.go b/test/integration/multinode_test.go index 679b5027c2..e5a115b444 100644 --- a/test/integration/multinode_test.go +++ b/test/integration/multinode_test.go @@ -55,6 +55,7 @@ func TestMultiNode(t *testing.T) { {"DeployApp2Nodes", validateDeployAppToMultiNode}, {"PingHostFrom2Pods", validatePodsPingHost}, {"AddNode", validateAddNodeToMultiNode}, + {"MultiNodeLabels", validateMultiNodeLabels}, {"ProfileList", validateProfileListWithMultiNode}, {"CopyFile", validateCopyFileWithMultiNode}, {"StopNode", validateStopRunningNode}, @@ -204,6 +205,33 @@ func validateCopyFileWithMultiNode(ctx context.Context, t *testing.T, profile st } } +// validateMultiNodeLabels check if all node labels were configured correctly +func validateMultiNodeLabels(ctx context.Context, t *testing.T, profile string) { + // docs: Get the node labels from the cluster with `kubectl get nodes` + rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "get", "nodes", "-o", "jsonpath=[{range .items[*]}{.metadata.labels},{end}]")) + if err != nil { + t.Errorf("failed to 'kubectl get nodes' with args %q: %v", rr.Command(), err) + } + + nodeLabelsList := []map[string]string{} + fixedString := strings.Replace(rr.Stdout.String(), ",]", "]", 1) + err = json.Unmarshal([]byte(fixedString), &nodeLabelsList) + if err != nil { + t.Errorf("failed to decode json from label list: args %q: %v", rr.Command(), err) + } + + // docs: check if all node labels matches with the expected Minikube labels: `minikube.k8s.io/*` + expectedLabels := []string{"minikube.k8s.io/commit", "minikube.k8s.io/version", "minikube.k8s.io/updated_at", "minikube.k8s.io/name", "minikube.k8s.io/primary"} + + for _, nodeLabels := range nodeLabelsList { + for _, el := range expectedLabels { + if _, ok := nodeLabels[el]; !ok { + t.Errorf("expected to have label %q in node labels but got : %s", el, rr.Output()) + } + } + } +} + // validateStopRunningNode tests the minikube node stop command func validateStopRunningNode(ctx context.Context, t *testing.T, profile string) { // Run minikube node stop on that node From c6393b84fb847a01d6767afb0eb6da2cfc4d9edb Mon Sep 17 00:00:00 2001 From: Marcell Martini Date: Tue, 9 May 2023 14:20:46 -0400 Subject: [PATCH 5/5] Add on expectedLabels a missing label. --- test/integration/functional_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go index 12f040a705..e4392a980a 100644 --- a/test/integration/functional_test.go +++ b/test/integration/functional_test.go @@ -219,7 +219,7 @@ func validateNodeLabels(ctx context.Context, t *testing.T, profile string) { t.Errorf("failed to 'kubectl get nodes' with args %q: %v", rr.Command(), err) } // docs: check if the node labels matches with the expected Minikube labels: `minikube.k8s.io/*` - expectedLabels := []string{"minikube.k8s.io/commit", "minikube.k8s.io/version", "minikube.k8s.io/updated_at", "minikube.k8s.io/name"} + expectedLabels := []string{"minikube.k8s.io/commit", "minikube.k8s.io/version", "minikube.k8s.io/updated_at", "minikube.k8s.io/name", "minikube.k8s.io/primary"} for _, el := range expectedLabels { if !strings.Contains(rr.Output(), el) { t.Errorf("expected to have label %q in node labels but got : %s", el, rr.Output())