diff --git a/cmd/kubeadm/app/cmd/options/constant.go b/cmd/kubeadm/app/cmd/options/constant.go index 72bab51859..479533ee6e 100644 --- a/cmd/kubeadm/app/cmd/options/constant.go +++ b/cmd/kubeadm/app/cmd/options/constant.go @@ -59,6 +59,9 @@ const ( // KubernetesVersion flag sets the Kubernetes version for the control plane. KubernetesVersion = "kubernetes-version" + // KubeletVersion flag sets the version for the kubelet config. + KubeletVersion = "kubelet-version" + // NetworkingDNSDomain flag sets the domain for services, e.g. "myorg.internal". NetworkingDNSDomain = "service-dns-domain" diff --git a/cmd/kubeadm/app/cmd/phases/BUILD b/cmd/kubeadm/app/cmd/phases/BUILD index 09bd2bca8f..6b0aa73f06 100644 --- a/cmd/kubeadm/app/cmd/phases/BUILD +++ b/cmd/kubeadm/app/cmd/phases/BUILD @@ -35,6 +35,7 @@ filegroup( "//cmd/kubeadm/app/cmd/phases/init:all-srcs", "//cmd/kubeadm/app/cmd/phases/join:all-srcs", "//cmd/kubeadm/app/cmd/phases/reset:all-srcs", + "//cmd/kubeadm/app/cmd/phases/upgrade/node:all-srcs", "//cmd/kubeadm/app/cmd/phases/workflow:all-srcs", ], tags = ["automanaged"], diff --git a/cmd/kubeadm/app/cmd/phases/upgrade/node/BUILD b/cmd/kubeadm/app/cmd/phases/upgrade/node/BUILD new file mode 100644 index 0000000000..5744ffaf52 --- /dev/null +++ b/cmd/kubeadm/app/cmd/phases/upgrade/node/BUILD @@ -0,0 +1,40 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = [ + "controlplane.go", + "data.go", + "kubeletconfig.go", + ], + importpath = "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/upgrade/node", + visibility = ["//visibility:public"], + deps = [ + "//cmd/kubeadm/app/apis/kubeadm:go_default_library", + "//cmd/kubeadm/app/cmd/options:go_default_library", + "//cmd/kubeadm/app/cmd/phases/workflow:go_default_library", + "//cmd/kubeadm/app/constants:go_default_library", + "//cmd/kubeadm/app/phases/kubelet:go_default_library", + "//cmd/kubeadm/app/phases/upgrade:go_default_library", + "//cmd/kubeadm/app/util/apiclient:go_default_library", + "//cmd/kubeadm/app/util/dryrun:go_default_library", + "//pkg/util/normalizer:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", + "//vendor/github.com/pkg/errors:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/cmd/kubeadm/app/cmd/phases/upgrade/node/controlplane.go b/cmd/kubeadm/app/cmd/phases/upgrade/node/controlplane.go new file mode 100644 index 0000000000..0733c71acd --- /dev/null +++ b/cmd/kubeadm/app/cmd/phases/upgrade/node/controlplane.go @@ -0,0 +1,80 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package node + +import ( + "fmt" + "os" + + "github.com/pkg/errors" + + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" + "k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade" + "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" +) + +// NewControlPlane creates a kubeadm workflow phase that implements handling of control-plane upgrade. +func NewControlPlane() workflow.Phase { + phase := workflow.Phase{ + Name: "control-plane", + Short: "Upgrade the control plane instance deployed on this node, if any", + Run: runControlPlane(), + InheritFlags: []string{ + options.DryRun, + options.KubeconfigPath, + }, + } + return phase +} + +func runControlPlane() func(c workflow.RunData) error { + return func(c workflow.RunData) error { + data, ok := c.(Data) + if !ok { + return errors.New("control-plane phase invoked with an invalid data struct") + } + + // if this is not a control-plande node, this phase should not be executed + if !data.IsControlPlaneNode() { + fmt.Printf("[upgrade] Skipping phase. Not a control plane node") + } + + // otherwise, retrieve all the info required for control plane upgrade + cfg := data.Cfg() + client := data.Client() + dryRun := data.DryRun() + etcdUpgrade := data.EtcdUpgrade() + renewCerts := data.RenewCerts() + + // Upgrade the control plane and etcd if installed on this node + fmt.Printf("[upgrade] Upgrading your Static Pod-hosted control plane instance to version %q...\n", cfg.KubernetesVersion) + if dryRun { + return upgrade.DryRunStaticPodUpgrade(cfg) + } + + waiter := apiclient.NewKubeWaiter(data.Client(), upgrade.UpgradeManifestTimeout, os.Stdout) + + if err := upgrade.PerformStaticPodUpgrade(client, waiter, cfg, etcdUpgrade, renewCerts); err != nil { + return errors.Wrap(err, "couldn't complete the static pod upgrade") + } + + fmt.Println("[upgrade] The control plane instance for this node was successfully updated!") + + return nil + } +} diff --git a/cmd/kubeadm/app/cmd/phases/upgrade/node/data.go b/cmd/kubeadm/app/cmd/phases/upgrade/node/data.go new file mode 100644 index 0000000000..b280904825 --- /dev/null +++ b/cmd/kubeadm/app/cmd/phases/upgrade/node/data.go @@ -0,0 +1,34 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package node + +import ( + clientset "k8s.io/client-go/kubernetes" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" +) + +// Data is the interface to use for kubeadm upgrade node phases. +// The "nodeData" type from "cmd/upgrade/node.go" must satisfy this interface. +type Data interface { + EtcdUpgrade() bool + RenewCerts() bool + DryRun() bool + KubeletVersion() string + Cfg() *kubeadmapi.InitConfiguration + IsControlPlaneNode() bool + Client() clientset.Interface +} diff --git a/cmd/kubeadm/app/cmd/phases/upgrade/node/kubeletconfig.go b/cmd/kubeadm/app/cmd/phases/upgrade/node/kubeletconfig.go new file mode 100644 index 0000000000..086804ced2 --- /dev/null +++ b/cmd/kubeadm/app/cmd/phases/upgrade/node/kubeletconfig.go @@ -0,0 +1,126 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package node + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/pkg/errors" + + "k8s.io/apimachinery/pkg/util/version" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" + "k8s.io/kubernetes/cmd/kubeadm/app/constants" + kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet" + "k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade" + dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun" + "k8s.io/kubernetes/pkg/util/normalizer" +) + +var ( + kubeletConfigLongDesc = normalizer.LongDesc(` + Download the kubelet configuration from a ConfigMap of the form "kubelet-config-1.X" in the cluster, + where X is the minor version of the kubelet. kubeadm uses the KuberneteVersion field in the kubeadm-config + ConfigMap to determine what the _desired_ kubelet version is, but the user can override this by using the + --kubelet-version parameter. + `) +) + +// NewKubeletConfigPhase creates a kubeadm workflow phase that implements handling of kubelet-config upgrade. +func NewKubeletConfigPhase() workflow.Phase { + phase := workflow.Phase{ + Name: "kubelet-config", + Short: "Upgrade the kubelet configuration for this node", + Long: kubeletConfigLongDesc, + Run: runKubeletConfigPhase(), + InheritFlags: []string{ + options.DryRun, + options.KubeconfigPath, + options.KubeletVersion, + }, + } + return phase +} + +func runKubeletConfigPhase() func(c workflow.RunData) error { + return func(c workflow.RunData) error { + data, ok := c.(Data) + if !ok { + return errors.New("kubelet-config phase invoked with an invalid data struct") + } + + // otherwise, retrieve all the info required for kubelet config upgrade + cfg := data.Cfg() + client := data.Client() + dryRun := data.DryRun() + + // Set up the kubelet directory to use. If dry-running, this will return a fake directory + kubeletDir, err := upgrade.GetKubeletDir(dryRun) + if err != nil { + return err + } + + // Gets the target kubelet version. + // by default kubelet version is expected to be equal to ClusterConfiguration.KubernetesVersion, but + // users can specify a different kubelet version (this is a legacy of the original implementation + // of `kubeam upgrade node config` which we are preserving in order to don't break GA contract) + kubeletVersionStr := cfg.ClusterConfiguration.KubernetesVersion + if data.KubeletVersion() != "" && data.KubeletVersion() != kubeletVersionStr { + kubeletVersionStr = data.KubeletVersion() + fmt.Printf("[upgrade] Using kubelet config version %s, while kubernetes-version is %s\n", kubeletVersionStr, cfg.ClusterConfiguration.KubernetesVersion) + } + + // Parse the desired kubelet version + kubeletVersion, err := version.ParseSemantic(kubeletVersionStr) + if err != nil { + return err + } + + // TODO: Checkpoint the current configuration first so that if something goes wrong it can be recovered + if err := kubeletphase.DownloadConfig(client, kubeletVersion, kubeletDir); err != nil { + return err + } + + // If we're dry-running, print the generated manifests + if dryRun { + if err := printFilesIfDryRunning(dryRun, kubeletDir); err != nil { + return errors.Wrap(err, "error printing files on dryrun") + } + return nil + } + + fmt.Println("[upgrade] The configuration for this node was successfully updated!") + fmt.Println("[upgrade] Now you should go ahead and upgrade the kubelet package using your package manager.") + return nil + } +} + +// printFilesIfDryRunning prints the Static Pod manifests to stdout and informs about the temporary directory to go and lookup +func printFilesIfDryRunning(dryRun bool, kubeletDir string) error { + if !dryRun { + return nil + } + + // Print the contents of the upgraded file and pretend like they were in kubeadmconstants.KubeletRunDirectory + fileToPrint := dryrunutil.FileToPrint{ + RealPath: filepath.Join(kubeletDir, constants.KubeletConfigurationFileName), + PrintPath: filepath.Join(constants.KubeletRunDirectory, constants.KubeletConfigurationFileName), + } + return dryrunutil.PrintDryRunFiles([]dryrunutil.FileToPrint{fileToPrint}, os.Stdout) +} diff --git a/cmd/kubeadm/app/cmd/upgrade/BUILD b/cmd/kubeadm/app/cmd/upgrade/BUILD index 682b7683a3..78a1eb103f 100644 --- a/cmd/kubeadm/app/cmd/upgrade/BUILD +++ b/cmd/kubeadm/app/cmd/upgrade/BUILD @@ -16,11 +16,12 @@ go_library( "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/apis/kubeadm/validation:go_default_library", "//cmd/kubeadm/app/cmd/options:go_default_library", + "//cmd/kubeadm/app/cmd/phases/upgrade/node:go_default_library", + "//cmd/kubeadm/app/cmd/phases/workflow:go_default_library", "//cmd/kubeadm/app/cmd/util:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", "//cmd/kubeadm/app/features:go_default_library", "//cmd/kubeadm/app/phases/controlplane:go_default_library", - "//cmd/kubeadm/app/phases/kubelet:go_default_library", "//cmd/kubeadm/app/phases/upgrade:go_default_library", "//cmd/kubeadm/app/preflight:go_default_library", "//cmd/kubeadm/app/util:go_default_library", @@ -29,8 +30,6 @@ go_library( "//cmd/kubeadm/app/util/dryrun:go_default_library", "//cmd/kubeadm/app/util/etcd:go_default_library", "//cmd/kubeadm/app/util/kubeconfig:go_default_library", - "//pkg/util/node:go_default_library", - "//pkg/util/normalizer:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", diff --git a/cmd/kubeadm/app/cmd/upgrade/apply.go b/cmd/kubeadm/app/cmd/upgrade/apply.go index e56797686b..a3130dd8c3 100644 --- a/cmd/kubeadm/app/cmd/upgrade/apply.go +++ b/cmd/kubeadm/app/cmd/upgrade/apply.go @@ -18,7 +18,6 @@ package upgrade import ( "fmt" - "os" "time" "github.com/pkg/errors" @@ -31,13 +30,10 @@ import ( cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/features" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane" "k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" - dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun" - etcdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd" ) const ( @@ -230,50 +226,9 @@ func PerformControlPlaneUpgrade(flags *applyFlags, client clientset.Interface, w fmt.Printf("[upgrade/apply] Upgrading your Static Pod-hosted control plane to version %q...\n", internalcfg.KubernetesVersion) if flags.dryRun { - return DryRunStaticPodUpgrade(internalcfg) + return upgrade.DryRunStaticPodUpgrade(internalcfg) } // Don't save etcd backup directory if etcd is HA, as this could cause corruption - return PerformStaticPodUpgrade(client, waiter, internalcfg, flags.etcdUpgrade, flags.renewCerts) -} - -// GetPathManagerForUpgrade returns a path manager properly configured for the given InitConfiguration. -func GetPathManagerForUpgrade(kubernetesDir string, internalcfg *kubeadmapi.InitConfiguration, etcdUpgrade bool) (upgrade.StaticPodPathManager, error) { - isHAEtcd := etcdutil.CheckConfigurationIsHA(&internalcfg.Etcd) - return upgrade.NewKubeStaticPodPathManagerUsingTempDirs(kubernetesDir, true, etcdUpgrade && !isHAEtcd) -} - -// PerformStaticPodUpgrade performs the upgrade of the control plane components for a static pod hosted cluster -func PerformStaticPodUpgrade(client clientset.Interface, waiter apiclient.Waiter, internalcfg *kubeadmapi.InitConfiguration, etcdUpgrade, renewCerts bool) error { - pathManager, err := GetPathManagerForUpgrade(constants.KubernetesDir, internalcfg, etcdUpgrade) - if err != nil { - return err - } - - // The arguments oldEtcdClient and newEtdClient, are uninitialized because passing in the clients allow for mocking the client during testing - return upgrade.StaticPodControlPlane(client, waiter, pathManager, internalcfg, etcdUpgrade, renewCerts, nil, nil) -} - -// DryRunStaticPodUpgrade fakes an upgrade of the control plane -func DryRunStaticPodUpgrade(internalcfg *kubeadmapi.InitConfiguration) error { - - dryRunManifestDir, err := constants.CreateTempDirForKubeadm("", "kubeadm-upgrade-dryrun") - if err != nil { - return err - } - defer os.RemoveAll(dryRunManifestDir) - - if err := controlplane.CreateInitStaticPodManifestFiles(dryRunManifestDir, internalcfg); err != nil { - return err - } - - // Print the contents of the upgraded manifests and pretend like they were in /etc/kubernetes/manifests - files := []dryrunutil.FileToPrint{} - for _, component := range constants.ControlPlaneComponents { - realPath := constants.GetStaticPodFilepath(component, dryRunManifestDir) - outputPath := constants.GetStaticPodFilepath(component, constants.GetStaticPodDirectory()) - files = append(files, dryrunutil.NewFileToPrint(realPath, outputPath)) - } - - return dryrunutil.PrintDryRunFiles(files, os.Stdout) + return upgrade.PerformStaticPodUpgrade(client, waiter, internalcfg, flags.etcdUpgrade, flags.renewCerts) } diff --git a/cmd/kubeadm/app/cmd/upgrade/apply_test.go b/cmd/kubeadm/app/cmd/upgrade/apply_test.go index 4f95765132..5a19eaf858 100644 --- a/cmd/kubeadm/app/cmd/upgrade/apply_test.go +++ b/cmd/kubeadm/app/cmd/upgrade/apply_test.go @@ -17,11 +17,7 @@ limitations under the License. package upgrade import ( - "io/ioutil" - "os" "testing" - - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" ) func TestSessionIsInteractive(t *testing.T) { @@ -65,89 +61,3 @@ func TestSessionIsInteractive(t *testing.T) { }) } } - -func TestGetPathManagerForUpgrade(t *testing.T) { - - haEtcd := &kubeadmapi.InitConfiguration{ - ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - Etcd: kubeadmapi.Etcd{ - External: &kubeadmapi.ExternalEtcd{ - Endpoints: []string{"10.100.0.1:2379", "10.100.0.2:2379", "10.100.0.3:2379"}, - }, - }, - }, - } - - noHAEtcd := &kubeadmapi.InitConfiguration{} - - tests := []struct { - name string - cfg *kubeadmapi.InitConfiguration - etcdUpgrade bool - shouldDeleteEtcd bool - }{ - { - name: "ha etcd but no etcd upgrade", - cfg: haEtcd, - etcdUpgrade: false, - shouldDeleteEtcd: true, - }, - { - name: "non-ha etcd with etcd upgrade", - cfg: noHAEtcd, - etcdUpgrade: true, - shouldDeleteEtcd: false, - }, - { - name: "ha etcd and etcd upgrade", - cfg: haEtcd, - etcdUpgrade: true, - shouldDeleteEtcd: true, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - // Use a temporary directory - tmpdir, err := ioutil.TempDir("", "TestGetPathManagerForUpgrade") - if err != nil { - t.Fatalf("unexpected error making temporary directory: %v", err) - } - defer func() { - os.RemoveAll(tmpdir) - }() - - pathmgr, err := GetPathManagerForUpgrade(tmpdir, test.cfg, test.etcdUpgrade) - if err != nil { - t.Fatalf("unexpected error creating path manager: %v", err) - } - - if _, err := os.Stat(pathmgr.BackupManifestDir()); os.IsNotExist(err) { - t.Errorf("expected manifest dir %s to exist, but it did not (%v)", pathmgr.BackupManifestDir(), err) - } - - if _, err := os.Stat(pathmgr.BackupEtcdDir()); os.IsNotExist(err) { - t.Errorf("expected etcd dir %s to exist, but it did not (%v)", pathmgr.BackupEtcdDir(), err) - } - - if err := pathmgr.CleanupDirs(); err != nil { - t.Fatalf("unexpected error cleaning up directories: %v", err) - } - - if _, err := os.Stat(pathmgr.BackupManifestDir()); os.IsNotExist(err) { - t.Errorf("expected manifest dir %s to exist, but it did not (%v)", pathmgr.BackupManifestDir(), err) - } - - if test.shouldDeleteEtcd { - if _, err := os.Stat(pathmgr.BackupEtcdDir()); !os.IsNotExist(err) { - t.Errorf("expected etcd dir %s not to exist, but it did (%v)", pathmgr.BackupEtcdDir(), err) - } - } else { - if _, err := os.Stat(pathmgr.BackupEtcdDir()); os.IsNotExist(err) { - t.Errorf("expected etcd dir %s to exist, but it did not", pathmgr.BackupEtcdDir()) - } - } - }) - } - -} diff --git a/cmd/kubeadm/app/cmd/upgrade/node.go b/cmd/kubeadm/app/cmd/upgrade/node.go index ee754a5465..e1af80754b 100644 --- a/cmd/kubeadm/app/cmd/upgrade/node.go +++ b/cmd/kubeadm/app/cmd/upgrade/node.go @@ -17,52 +17,29 @@ limitations under the License. package upgrade import ( - "fmt" "os" - "path/filepath" "github.com/pkg/errors" "github.com/spf13/cobra" - "k8s.io/apimachinery/pkg/util/version" - "k8s.io/klog" + flag "github.com/spf13/pflag" + + clientset "k8s.io/client-go/kubernetes" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" - cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" + phases "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/upgrade/node" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" "k8s.io/kubernetes/cmd/kubeadm/app/constants" - kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" - "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" - dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun" - "k8s.io/kubernetes/pkg/util/node" - "k8s.io/kubernetes/pkg/util/normalizer" ) -var ( - upgradeNodeConfigLongDesc = normalizer.LongDesc(` - Download the kubelet configuration from a ConfigMap of the form "kubelet-config-1.X" in the cluster, - where X is the minor version of the kubelet. kubeadm uses the --kubelet-version parameter to determine - what the _desired_ kubelet version is. Give - `) - - upgradeNodeConfigExample = normalizer.Examples(fmt.Sprintf(` - # Download the kubelet configuration from the ConfigMap in the cluster. Use a specific desired kubelet version. - kubeadm upgrade node config --kubelet-version %s - - # Simulate the downloading of the kubelet configuration from the ConfigMap in the cluster with a specific desired - # version. Do not change any state locally on the node. - kubeadm upgrade node config --kubelet-version %[1]s --dry-run - `, constants.CurrentKubernetesVersion)) -) - -type nodeUpgradeFlags struct { - kubeConfigPath string - kubeletVersionStr string - dryRun bool -} - -type controlplaneUpgradeFlags struct { +// nodeOptions defines all the options exposed via flags by kubeadm upgrade node. +// Please note that this structure includes the public kubeadm config API, but only a subset of the options +// supported by this api will be exposed as a flag. +type nodeOptions struct { kubeConfigPath string + kubeletVersion string advertiseAddress string nodeName string etcdUpgrade bool @@ -70,170 +47,217 @@ type controlplaneUpgradeFlags struct { dryRun bool } +// compile-time assert that the local data object satisfies the phases data interface. +var _ phases.Data = &nodeData{} + +// nodeData defines all the runtime information used when running the kubeadm upgrade node worklow; +// this data is shared across all the phases that are included in the workflow. +type nodeData struct { + etcdUpgrade bool + renewCerts bool + dryRun bool + kubeletVersion string + cfg *kubeadmapi.InitConfiguration + isControlPlaneNode bool + client clientset.Interface +} + // NewCmdNode returns the cobra command for `kubeadm upgrade node` func NewCmdNode() *cobra.Command { + nodeOptions := newNodeOptions() + nodeRunner := workflow.NewRunner() + cmd := &cobra.Command{ Use: "node", - Short: "Upgrade commands for a node in the cluster. Currently only support upgrading the configuration, not the kubelet itself", - RunE: cmdutil.SubCmdRunE("node"), + Short: "Upgrade commands for a node in the cluster", + Run: func(cmd *cobra.Command, args []string) { + err := nodeRunner.Run(args) + kubeadmutil.CheckErr(err) + }, + Args: cobra.NoArgs, } + + // adds flags to the node command + // flags could be eventually inherited by the sub-commands automatically generated for phases + addUpgradeNodeFlags(cmd.Flags(), nodeOptions) + + // initialize the workflow runner with the list of phases + nodeRunner.AppendPhase(phases.NewControlPlane()) + nodeRunner.AppendPhase(phases.NewKubeletConfigPhase()) + + // sets the data builder function, that will be used by the runner + // both when running the entire workflow or single phases + nodeRunner.SetDataInitializer(func(cmd *cobra.Command, args []string) (workflow.RunData, error) { + return newNodeData(cmd, args, nodeOptions) + }) + + // binds the Runner to kubeadm upgrade node command by altering + // command help, adding --skip-phases flag and by adding phases subcommands + nodeRunner.BindToCommand(cmd) + + // upgrade node config command is subject to GA deprecation policy, so we should deprecate it + // and keep it here for one year or three releases - the longer of the two - starting from v1.15 included cmd.AddCommand(NewCmdUpgradeNodeConfig()) + // upgrade node experimental control plane can be removed, but we are keeping it for one more cycle cmd.AddCommand(NewCmdUpgradeControlPlane()) + return cmd } +// newNodeOptions returns a struct ready for being used for creating cmd kubeadm upgrade node flags. +func newNodeOptions() *nodeOptions { + return &nodeOptions{ + kubeConfigPath: constants.GetKubeletKubeConfigPath(), + dryRun: false, + } +} + +func addUpgradeNodeFlags(flagSet *flag.FlagSet, nodeOptions *nodeOptions) { + options.AddKubeConfigFlag(flagSet, &nodeOptions.kubeConfigPath) + flagSet.BoolVar(&nodeOptions.dryRun, options.DryRun, nodeOptions.dryRun, "Do not change any state, just output the actions that would be performed.") + flagSet.StringVar(&nodeOptions.kubeletVersion, options.KubeletVersion, nodeOptions.kubeletVersion, "The *desired* version for the kubelet config after the upgrade. If not specified, the KubernetesVersion from the kubeadm-config ConfigMap will be used") +} + +// newNodeData returns a new nodeData struct to be used for the execution of the kubeadm upgrade node workflow. +// This func takes care of validating nodeOptions passed to the command, and then it converts +// options into the internal InitConfiguration type that is used as input all the phases in the kubeadm upgrade node workflow +func newNodeData(cmd *cobra.Command, args []string, options *nodeOptions) (*nodeData, error) { + client, err := getClient(options.kubeConfigPath, options.dryRun) + if err != nil { + return nil, errors.Wrapf(err, "couldn't create a Kubernetes client from file %q", options.kubeConfigPath) + } + + // isControlPlane checks if a node is a control-plane node by looking up + // the kube-apiserver manifest file + isControlPlaneNode := true + filepath := kubeadmconstants.GetStaticPodFilepath(kubeadmconstants.KubeAPIServer, kubeadmconstants.GetStaticPodDirectory()) + if _, err := os.Stat(filepath); os.IsNotExist(err) { + isControlPlaneNode = false + } + + // Fetches the cluster configuration + // NB in case of control-plane node, we are reading all the info for the node; in case of NOT control-plane node + // (worker node), we are not reading local API address and the CRI socket from the node object + cfg, err := configutil.FetchInitConfigurationFromCluster(client, os.Stdout, "upgrade", !isControlPlaneNode) + if err != nil { + return nil, errors.Wrap(err, "unable to fetch the kubeadm-config ConfigMap") + } + + return &nodeData{ + etcdUpgrade: options.etcdUpgrade, + renewCerts: options.renewCerts, + dryRun: options.dryRun, + kubeletVersion: options.kubeletVersion, + cfg: cfg, + client: client, + isControlPlaneNode: isControlPlaneNode, + }, nil +} + +// DryRun returns the dryRun flag. +func (d *nodeData) DryRun() bool { + return d.dryRun +} + +// EtcdUpgrade returns the etcdUpgrade flag. +func (d *nodeData) EtcdUpgrade() bool { + return d.etcdUpgrade +} + +// RenewCerts returns the renewCerts flag. +func (d *nodeData) RenewCerts() bool { + return d.renewCerts +} + +// KubeletVersion returns the kubeletVersion flag. +func (d *nodeData) KubeletVersion() string { + return d.kubeletVersion +} + +// Cfg returns initConfiguration. +func (d *nodeData) Cfg() *kubeadmapi.InitConfiguration { + return d.cfg +} + +// IsControlPlaneNode returns the isControlPlaneNode flag. +func (d *nodeData) IsControlPlaneNode() bool { + return d.isControlPlaneNode +} + +// Client returns a Kubernetes client to be used by kubeadm. +func (d *nodeData) Client() clientset.Interface { + return d.client +} + // NewCmdUpgradeNodeConfig returns the cobra.Command for downloading the new/upgrading the kubelet configuration from the kubelet-config-1.X // ConfigMap in the cluster +// TODO: to remove when 1.18 is released func NewCmdUpgradeNodeConfig() *cobra.Command { - flags := &nodeUpgradeFlags{ - kubeConfigPath: constants.GetKubeletKubeConfigPath(), - kubeletVersionStr: "", - dryRun: false, - } + nodeOptions := newNodeOptions() + nodeRunner := workflow.NewRunner() cmd := &cobra.Command{ - Use: "config", - Short: "Download the kubelet configuration from the cluster ConfigMap kubelet-config-1.X, where X is the minor version of the kubelet", - Long: upgradeNodeConfigLongDesc, - Example: upgradeNodeConfigExample, + Use: "config", + Short: "Download the kubelet configuration from the cluster ConfigMap kubelet-config-1.X, where X is the minor version of the kubelet", + Deprecated: "use \"kubeadm upgrade node\" instead", Run: func(cmd *cobra.Command, args []string) { - err := RunUpgradeNodeConfig(flags) + // This is required for preserving the old behavior of `kubeadm upgrade node config`. + // The new implementation exposed as a phase under `kubeadm upgrade node` infers the target + // kubelet config version from the kubeadm-config ConfigMap + if len(nodeOptions.kubeletVersion) == 0 { + kubeadmutil.CheckErr(errors.New("the --kubelet-version argument is required")) + } + + err := nodeRunner.Run(args) kubeadmutil.CheckErr(err) }, } - options.AddKubeConfigFlag(cmd.Flags(), &flags.kubeConfigPath) - cmd.Flags().BoolVar(&flags.dryRun, options.DryRun, flags.dryRun, "Do not change any state, just output the actions that would be performed.") - cmd.Flags().StringVar(&flags.kubeletVersionStr, "kubelet-version", flags.kubeletVersionStr, "The *desired* version for the kubelet after the upgrade.") + // adds flags to the node command + addUpgradeNodeFlags(cmd.Flags(), nodeOptions) + + // initialize the workflow runner with the list of phases + nodeRunner.AppendPhase(phases.NewKubeletConfigPhase()) + + // sets the data builder function, that will be used by the runner + // both when running the entire workflow or single phases + nodeRunner.SetDataInitializer(func(cmd *cobra.Command, args []string) (workflow.RunData, error) { + return newNodeData(cmd, args, nodeOptions) + }) + return cmd } // NewCmdUpgradeControlPlane returns the cobra.Command for upgrading the controlplane instance on this node +// TODO: to remove when 1.16 is released func NewCmdUpgradeControlPlane() *cobra.Command { - - flags := &controlplaneUpgradeFlags{ - kubeConfigPath: constants.GetKubeletKubeConfigPath(), - advertiseAddress: "", - etcdUpgrade: true, - renewCerts: true, - dryRun: false, - } + nodeOptions := newNodeOptions() + nodeRunner := workflow.NewRunner() cmd := &cobra.Command{ - Use: "experimental-control-plane", - Short: "Upgrade the control plane instance deployed on this node. IMPORTANT. This command should be executed after executing `kubeadm upgrade apply` on another control plane instance", - Long: upgradeNodeConfigLongDesc, - Example: upgradeNodeConfigExample, + Use: "experimental-control-plane", + Short: "Upgrade the control plane instance deployed on this node. IMPORTANT. This command should be executed after executing `kubeadm upgrade apply` on another control plane instance", + Deprecated: "this command is deprecated. Use \"kubeadm upgrade node\" instead", Run: func(cmd *cobra.Command, args []string) { - - if flags.nodeName == "" { - klog.V(1).Infoln("[upgrade] found NodeName empty; considered OS hostname as NodeName") - } - nodeName, err := node.GetHostname(flags.nodeName) - if err != nil { - kubeadmutil.CheckErr(err) - } - flags.nodeName = nodeName - - if flags.advertiseAddress == "" { - ip, err := configutil.ChooseAPIServerBindAddress(nil) - if err != nil { - kubeadmutil.CheckErr(err) - return - } - - flags.advertiseAddress = ip.String() - } - - err = RunUpgradeControlPlane(flags) + err := nodeRunner.Run(args) kubeadmutil.CheckErr(err) }, } - options.AddKubeConfigFlag(cmd.Flags(), &flags.kubeConfigPath) - cmd.Flags().BoolVar(&flags.dryRun, options.DryRun, flags.dryRun, "Do not change any state, just output the actions that would be performed.") - cmd.Flags().BoolVar(&flags.etcdUpgrade, "etcd-upgrade", flags.etcdUpgrade, "Perform the upgrade of etcd.") - cmd.Flags().BoolVar(&flags.renewCerts, "certificate-renewal", flags.renewCerts, "Perform the renewal of certificates used by component changed during upgrades.") + // adds flags to the node command + options.AddKubeConfigFlag(cmd.Flags(), &nodeOptions.kubeConfigPath) + cmd.Flags().BoolVar(&nodeOptions.dryRun, options.DryRun, nodeOptions.dryRun, "Do not change any state, just output the actions that would be performed.") + cmd.Flags().BoolVar(&nodeOptions.etcdUpgrade, "etcd-upgrade", nodeOptions.etcdUpgrade, "Perform the upgrade of etcd.") + cmd.Flags().BoolVar(&nodeOptions.renewCerts, "certificate-renewal", nodeOptions.renewCerts, "Perform the renewal of certificates used by component changed during upgrades.") + + // initialize the workflow runner with the list of phases + nodeRunner.AppendPhase(phases.NewControlPlane()) + + // sets the data builder function, that will be used by the runner + // both when running the entire workflow or single phases + nodeRunner.SetDataInitializer(func(cmd *cobra.Command, args []string) (workflow.RunData, error) { + return newNodeData(cmd, args, nodeOptions) + }) + return cmd } - -// RunUpgradeNodeConfig is executed when `kubeadm upgrade node config` runs. -func RunUpgradeNodeConfig(flags *nodeUpgradeFlags) error { - if len(flags.kubeletVersionStr) == 0 { - return errors.New("the --kubelet-version argument is required") - } - - // Set up the kubelet directory to use. If dry-running, use a fake directory - kubeletDir, err := upgrade.GetKubeletDir(flags.dryRun) - if err != nil { - return err - } - - client, err := getClient(flags.kubeConfigPath, flags.dryRun) - if err != nil { - return errors.Wrapf(err, "couldn't create a Kubernetes client from file %q", flags.kubeConfigPath) - } - - // Parse the desired kubelet version - kubeletVersion, err := version.ParseSemantic(flags.kubeletVersionStr) - if err != nil { - return err - } - // TODO: Checkpoint the current configuration first so that if something goes wrong it can be recovered - if err := kubeletphase.DownloadConfig(client, kubeletVersion, kubeletDir); err != nil { - return err - } - - // If we're dry-running, print the generated manifests, otherwise do nothing - if err := printFilesIfDryRunning(flags.dryRun, kubeletDir); err != nil { - return errors.Wrap(err, "error printing files on dryrun") - } - - fmt.Println("[upgrade] The configuration for this node was successfully updated!") - fmt.Println("[upgrade] Now you should go ahead and upgrade the kubelet package using your package manager.") - return nil -} - -// printFilesIfDryRunning prints the Static Pod manifests to stdout and informs about the temporary directory to go and lookup -func printFilesIfDryRunning(dryRun bool, kubeletDir string) error { - if !dryRun { - return nil - } - - // Print the contents of the upgraded file and pretend like they were in kubeadmconstants.KubeletRunDirectory - fileToPrint := dryrunutil.FileToPrint{ - RealPath: filepath.Join(kubeletDir, constants.KubeletConfigurationFileName), - PrintPath: filepath.Join(constants.KubeletRunDirectory, constants.KubeletConfigurationFileName), - } - return dryrunutil.PrintDryRunFiles([]dryrunutil.FileToPrint{fileToPrint}, os.Stdout) -} - -// RunUpgradeControlPlane is executed when `kubeadm upgrade node controlplane` runs. -func RunUpgradeControlPlane(flags *controlplaneUpgradeFlags) error { - - client, err := getClient(flags.kubeConfigPath, flags.dryRun) - if err != nil { - return errors.Wrapf(err, "couldn't create a Kubernetes client from file %q", flags.kubeConfigPath) - } - - waiter := apiclient.NewKubeWaiter(client, upgrade.UpgradeManifestTimeout, os.Stdout) - - // Fetches the cluster configuration - cfg, err := configutil.FetchInitConfigurationFromCluster(client, os.Stdout, "upgrade", false) - if err != nil { - return errors.Wrap(err, "unable to fetch the kubeadm-config ConfigMap") - } - - // Upgrade the control plane and etcd if installed on this node - fmt.Printf("[upgrade] Upgrading your Static Pod-hosted control plane instance to version %q...\n", cfg.KubernetesVersion) - if flags.dryRun { - return DryRunStaticPodUpgrade(cfg) - } - - if err := PerformStaticPodUpgrade(client, waiter, cfg, flags.etcdUpgrade, flags.renewCerts); err != nil { - return errors.Wrap(err, "couldn't complete the static pod upgrade") - } - - fmt.Println("[upgrade] The control plane instance for this node was successfully updated!") - return nil -} diff --git a/cmd/kubeadm/app/phases/upgrade/staticpods.go b/cmd/kubeadm/app/phases/upgrade/staticpods.go index a50fb09101..2c3ac934a7 100644 --- a/cmd/kubeadm/app/phases/upgrade/staticpods.go +++ b/cmd/kubeadm/app/phases/upgrade/staticpods.go @@ -31,10 +31,12 @@ import ( "k8s.io/kubernetes/cmd/kubeadm/app/constants" certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/renewal" + "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane" controlplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane" etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd" "k8s.io/kubernetes/cmd/kubeadm/app/util" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" + dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun" etcdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd" "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod" ) @@ -583,3 +585,44 @@ func renewCertsByComponent(cfg *kubeadmapi.InitConfiguration, component string, return nil } + +// GetPathManagerForUpgrade returns a path manager properly configured for the given InitConfiguration. +func GetPathManagerForUpgrade(kubernetesDir string, internalcfg *kubeadmapi.InitConfiguration, etcdUpgrade bool) (StaticPodPathManager, error) { + isHAEtcd := etcdutil.CheckConfigurationIsHA(&internalcfg.Etcd) + return NewKubeStaticPodPathManagerUsingTempDirs(kubernetesDir, true, etcdUpgrade && !isHAEtcd) +} + +// PerformStaticPodUpgrade performs the upgrade of the control plane components for a static pod hosted cluster +func PerformStaticPodUpgrade(client clientset.Interface, waiter apiclient.Waiter, internalcfg *kubeadmapi.InitConfiguration, etcdUpgrade, renewCerts bool) error { + pathManager, err := GetPathManagerForUpgrade(constants.KubernetesDir, internalcfg, etcdUpgrade) + if err != nil { + return err + } + + // The arguments oldEtcdClient and newEtdClient, are uninitialized because passing in the clients allow for mocking the client during testing + return StaticPodControlPlane(client, waiter, pathManager, internalcfg, etcdUpgrade, renewCerts, nil, nil) +} + +// DryRunStaticPodUpgrade fakes an upgrade of the control plane +func DryRunStaticPodUpgrade(internalcfg *kubeadmapi.InitConfiguration) error { + + dryRunManifestDir, err := constants.CreateTempDirForKubeadm("", "kubeadm-upgrade-dryrun") + if err != nil { + return err + } + defer os.RemoveAll(dryRunManifestDir) + + if err := controlplane.CreateInitStaticPodManifestFiles(dryRunManifestDir, internalcfg); err != nil { + return err + } + + // Print the contents of the upgraded manifests and pretend like they were in /etc/kubernetes/manifests + files := []dryrunutil.FileToPrint{} + for _, component := range constants.ControlPlaneComponents { + realPath := constants.GetStaticPodFilepath(component, dryRunManifestDir) + outputPath := constants.GetStaticPodFilepath(component, constants.GetStaticPodDirectory()) + files = append(files, dryrunutil.NewFileToPrint(realPath, outputPath)) + } + + return dryrunutil.PrintDryRunFiles(files, os.Stdout) +} diff --git a/cmd/kubeadm/app/phases/upgrade/staticpods_test.go b/cmd/kubeadm/app/phases/upgrade/staticpods_test.go index 19400003ec..9b95a3c017 100644 --- a/cmd/kubeadm/app/phases/upgrade/staticpods_test.go +++ b/cmd/kubeadm/app/phases/upgrade/staticpods_test.go @@ -909,3 +909,89 @@ func getEmbeddedCerts(tmpDir, kubeConfig string) ([]*x509.Certificate, error) { return certutil.ParseCertsPEM(authInfo.ClientCertificateData) } + +func TestGetPathManagerForUpgrade(t *testing.T) { + + haEtcd := &kubeadmapi.InitConfiguration{ + ClusterConfiguration: kubeadmapi.ClusterConfiguration{ + Etcd: kubeadmapi.Etcd{ + External: &kubeadmapi.ExternalEtcd{ + Endpoints: []string{"10.100.0.1:2379", "10.100.0.2:2379", "10.100.0.3:2379"}, + }, + }, + }, + } + + noHAEtcd := &kubeadmapi.InitConfiguration{} + + tests := []struct { + name string + cfg *kubeadmapi.InitConfiguration + etcdUpgrade bool + shouldDeleteEtcd bool + }{ + { + name: "ha etcd but no etcd upgrade", + cfg: haEtcd, + etcdUpgrade: false, + shouldDeleteEtcd: true, + }, + { + name: "non-ha etcd with etcd upgrade", + cfg: noHAEtcd, + etcdUpgrade: true, + shouldDeleteEtcd: false, + }, + { + name: "ha etcd and etcd upgrade", + cfg: haEtcd, + etcdUpgrade: true, + shouldDeleteEtcd: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // Use a temporary directory + tmpdir, err := ioutil.TempDir("", "TestGetPathManagerForUpgrade") + if err != nil { + t.Fatalf("unexpected error making temporary directory: %v", err) + } + defer func() { + os.RemoveAll(tmpdir) + }() + + pathmgr, err := GetPathManagerForUpgrade(tmpdir, test.cfg, test.etcdUpgrade) + if err != nil { + t.Fatalf("unexpected error creating path manager: %v", err) + } + + if _, err := os.Stat(pathmgr.BackupManifestDir()); os.IsNotExist(err) { + t.Errorf("expected manifest dir %s to exist, but it did not (%v)", pathmgr.BackupManifestDir(), err) + } + + if _, err := os.Stat(pathmgr.BackupEtcdDir()); os.IsNotExist(err) { + t.Errorf("expected etcd dir %s to exist, but it did not (%v)", pathmgr.BackupEtcdDir(), err) + } + + if err := pathmgr.CleanupDirs(); err != nil { + t.Fatalf("unexpected error cleaning up directories: %v", err) + } + + if _, err := os.Stat(pathmgr.BackupManifestDir()); os.IsNotExist(err) { + t.Errorf("expected manifest dir %s to exist, but it did not (%v)", pathmgr.BackupManifestDir(), err) + } + + if test.shouldDeleteEtcd { + if _, err := os.Stat(pathmgr.BackupEtcdDir()); !os.IsNotExist(err) { + t.Errorf("expected etcd dir %s not to exist, but it did (%v)", pathmgr.BackupEtcdDir(), err) + } + } else { + if _, err := os.Stat(pathmgr.BackupEtcdDir()); os.IsNotExist(err) { + t.Errorf("expected etcd dir %s to exist, but it did not", pathmgr.BackupEtcdDir()) + } + } + }) + } + +}