From c306367734964fe4cc8fcd04d980a6a26ac78b1e Mon Sep 17 00:00:00 2001 From: Sohan Kunkerkar Date: Fri, 8 Mar 2024 10:41:48 -0500 Subject: [PATCH] docs: Update KubeletConfigDropinDir doc information Signed-off-by: Sohan Kunkerkar Signed-off-by: Peter Hunt --- .../node/kubelet-config-directory-merging.md | 155 +++++++++++++ .../administer-cluster/kubelet-config-file.md | 217 +++++++++++++++--- 2 files changed, 344 insertions(+), 28 deletions(-) create mode 100644 content/en/docs/reference/node/kubelet-config-directory-merging.md diff --git a/content/en/docs/reference/node/kubelet-config-directory-merging.md b/content/en/docs/reference/node/kubelet-config-directory-merging.md new file mode 100644 index 0000000000..99ed1bc631 --- /dev/null +++ b/content/en/docs/reference/node/kubelet-config-directory-merging.md @@ -0,0 +1,155 @@ +--- +content_type: "reference" +title: Kubelet Configuration Directory Merging +weight: 50 +--- + +When using the kubelet's `--config-dir` flag to specify a drop-in directory for +configuration, there is some specific behavior on how different types are +merged. + +Here are some examples of how different data types behave during configuration merging: + +### Structure Fields +There are two types of structure fields in a YAML structure: singular (or a +scalar type) and embedded (structures that contain scalar types). +The configuration merging process handles the overriding of singular and embedded struct fields to create a resulting kubelet configuration. + +For instance, you may want a baseline kubelet configuration for all nodes, but you may want to customize the `address` and `authorization` fields. +This can be done as follows: + +Main kubelet configuration file contents: +```yaml +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +port: 20250 +authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: "5m" + cacheUnauthorizedTTL: "30s" +serializeImagePulls: false +address: "192.168.0.1" +``` + +Contents of a file in `--config-dir` directory: +```yaml +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +authorization: + mode: AlwaysAllow + webhook: + cacheAuthorizedTTL: "8m" + cacheUnauthorizedTTL: "45s" +address: "192.168.0.8" +``` + +The resulting configuration will be as follows: +```yaml +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +port: 20250 +serializeImagePulls: false +authorization: + mode: AlwaysAllow + webhook: + cacheAuthorizedTTL: "8m" + cacheUnauthorizedTTL: "45s" +address: "192.168.0.8" +``` + +### Lists +You can overide the slices/lists values of the kubelet configuration. +However, the entire list gets overridden during the merging process. +For example, you can override the `clusterDNS` list as follows: + +Main kubelet configuration file contents: +```yaml +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +port: 20250 +serializeImagePulls: false +clusterDNS: + - "192.168.0.9" + - "192.168.0.8" +``` + +Contents of a file in `--config-dir` directory: +```yaml +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +clusterDNS: + - "192.168.0.2" + - "192.168.0.3" + - "192.168.0.5" +``` + +The resulting configuration will be as follows: +```yaml +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +port: 20250 +serializeImagePulls: false +clusterDNS: + - "192.168.0.2" + - "192.168.0.3" + - "192.168.0.5" +``` + +### Maps, including Nested Structures + +Individual fields in maps, regardless of their value types (boolean, string, etc.), can be selectively overridden. +However, for `map[string][]string`, the entire list associated with a specific field gets overridden. +Let's understand this better with an example, particularly on fields like `featureGates` and `staticPodURLHeader`: + +Main kubelet configuration file contents: +```yaml +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +port: 20250 +serializeImagePulls: false +featureGates: + AllAlpha: false + MemoryQoS: true +staticPodURLHeader: + kubelet-api-support: + - "Authorization: 234APSDFA" + - "X-Custom-Header: 123" + custom-static-pod: + - "Authorization: 223EWRWER" + - "X-Custom-Header: 456" +``` + +Contents of a file in `--config-dir` directory: +```yaml +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +featureGates: + MemoryQoS: false + KubeletTracing: true + DynamicResourceAllocation: true +staticPodURLHeader: + custom-static-pod: + - "Authorization: 223EWRWER" + - "X-Custom-Header: 345" +``` + +The resulting configuration will be as follows: +```yaml +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +port: 20250 +serializeImagePulls: false +featureGates: + AllAlpha: false + MemoryQoS: false + KubeletTracing: true + DynamicResourceAllocation: true +staticPodURLHeader: + kubelet-api-support: + - "Authorization: 234APSDFA" + - "X-Custom-Header: 123" + custom-static-pod: + - "Authorization: 223EWRWER" + - "X-Custom-Header: 345" +``` diff --git a/content/en/docs/tasks/administer-cluster/kubelet-config-file.md b/content/en/docs/tasks/administer-cluster/kubelet-config-file.md index 815f9f70c3..4b7da01209 100644 --- a/content/en/docs/tasks/administer-cluster/kubelet-config-file.md +++ b/content/en/docs/tasks/administer-cluster/kubelet-config-file.md @@ -7,6 +7,16 @@ content_type: task weight: 330 --- +## {{% heading "prerequisites" %}} + +Some steps in this page use the `jq` tool. If you don't have `jq`, you can +install it via your operating system's software sources, or fetch it from +[https://jqlang.github.io/jq/](https://jqlang.github.io/jq/). + +Some steps also involve installing `curl`, which can be installed via your +operating system's software sources. + + A subset of the kubelet's configuration parameters may be @@ -86,46 +96,195 @@ In the above example, this version is `kubelet.config.k8s.io/v1beta1`. ## Drop-in directory for kubelet configuration files {#kubelet-conf-d} -As of Kubernetes v1.28.0, the kubelet has been extended to support a drop-in configuration directory. The location of it can be specified with -`--config-dir` flag, and it defaults to `""`, or disabled, by default. +{{}} -You can only set `--config-dir` if you set the environment variable `KUBELET_CONFIG_DROPIN_DIR_ALPHA` for the kubelet process (the value of that variable does not matter). -For Kubernetes v{{< skew currentVersion >}}, the kubelet returns an error if you specify `--config-dir` without that variable set, and startup fails. -You cannot specify the drop-in configuration directory using the kubelet configuration file; only the CLI argument `--config-dir` can set it. +You can specify a drop-in configuration directory for the kubelet. By default, the kubelet does not look +for drop-in configuration files anywhere - you must specify a path. +For example: `--config-dir=/etc/kubernetes/kubelet.conf.d` + +For Kubernetes v1.28 to v1.29, you can only specify `--config-dir` if you also set +the environment variable `KUBELET_CONFIG_DROPIN_DIR_ALPHA` for the kubelet process (the value +of that variable does not matter). -One can use the kubelet configuration directory in a similar way to the kubelet config file. {{< note >}} -The suffix of a valid kubelet drop-in configuration file must be `.conf`. For instance: `99-kubelet-address.conf` +The suffix of a valid kubelet drop-in configuration file **must** be `.conf`. For instance: `99-kubelet-address.conf` {{< /note >}} -For instance, you may want a baseline kubelet configuration for all nodes, but you may want to customize the `address` field. This can be done as follows: +The kubelet processes files in its config drop-in directory by sorting the **entire file name** alphanumerically. +For instance, `00-kubelet.conf` is processed first, and then overridden with a file named `01-kubelet.conf`. -Main kubelet configuration file contents: -```yaml -apiVersion: kubelet.config.k8s.io/v1beta1 -kind: KubeletConfiguration -port: 20250 -serializeImagePulls: false -evictionHard: - memory.available: "200Mi" -``` +These files may contain partial configurations and might not be valid config files by themselves. +Validation is only performed on the final resulting configuration structure +stored internally in the kubelet. +This offers you flexibility in how you manage and combine kubelet configuration that comes from different sources. +However, it's important to note that the behavior varies based on the data type of the configuration fields. -Contents of a file in `--config-dir` directory: -```yaml -apiVersion: kubelet.config.k8s.io/v1beta1 -kind: KubeletConfiguration -address: "192.168.0.8" -``` +Different data types in the kubelet configuration structure merge differently. +See the [reference +document](/docs/reference/node/kubelet-config-directory-merging.md) for more +information. + +### Kubelet configuration merging order On startup, the kubelet merges configuration from: -* Command line arguments (lowest precedence). -* the kubelet configuration +* Feature gates specified over the command line (lowest precedence). +* The kubelet configuration. * Drop-in configuration files, according to sort order. -* Feature gates specified over the command line (highest precedence). +* Command line arguments excluding feature gates (highest precedence). -This produces the same outcome as if you used the [single configuration file](#create-the-config-file) used in the earlier example. +{{< note >}} +The config drop-in dir mechanism for the kubelet is similar but different from how the `kubeadm` tool allows you to patch configuration. +The `kubeadm` tool uses a specific [patching strategy](/docs/setup/production-environment/tools/kubeadm/control-plane-flags/#patches) for its configuration, +whereas the only patch strategy for kubelet configuration drop-in files is `replace`. The kubelet determines the order of merges based on sorting the **suffixes** alphanumerically, +and replaces every field present in a higher priority file. +{{< /note >}} +## Viewing the kubelet configuration + +Since the configuration could now be spread over multiple files with this feature, if someone wants to inspect the final actuated configuration, +they can follow these steps to inspect the kubelet configuration: + +1. Start a proxy server using [`kubectl proxy`](/docs/reference/kubectl/generated/kubectl-commands#proxy) in your terminal. + +```bash +kubectl proxy +``` + +Which gives output like: + +```bash +Starting to serve on 127.0.0.1:8001 + +``` +2. Open another terminal window and use `curl` to fetch the kubelet configuration. +Replace `` with the actual name of your node: + +```bash +curl -X GET http://127.0.0.1:8001/api/v1/nodes//proxy/configz | jq . +``` + +```bash +{ + "kubeletconfig": { + "enableServer": true, + "staticPodPath": "/var/run/kubernetes/static-pods", + "syncFrequency": "1m0s", + "fileCheckFrequency": "20s", + "httpCheckFrequency": "20s", + "address": "192.168.1.16", + "port": 10250, + "readOnlyPort": 10255, + "tlsCertFile": "/var/lib/kubelet/pki/kubelet.crt", + "tlsPrivateKeyFile": "/var/lib/kubelet/pki/kubelet.key", + "rotateCertificates": true, + "authentication": { + "x509": { + "clientCAFile": "/var/run/kubernetes/client-ca.crt" + }, + "webhook": { + "enabled": true, + "cacheTTL": "2m0s" + }, + "anonymous": { + "enabled": true + } + }, + "authorization": { + "mode": "AlwaysAllow", + "webhook": { + "cacheAuthorizedTTL": "5m0s", + "cacheUnauthorizedTTL": "30s" + } + }, + "registryPullQPS": 5, + "registryBurst": 10, + "eventRecordQPS": 50, + "eventBurst": 100, + "enableDebuggingHandlers": true, + "healthzPort": 10248, + "healthzBindAddress": "127.0.0.1", + "oomScoreAdj": -999, + "clusterDomain": "cluster.local", + "clusterDNS": [ + "10.0.0.10" + ], + "streamingConnectionIdleTimeout": "4h0m0s", + "nodeStatusUpdateFrequency": "10s", + "nodeStatusReportFrequency": "5m0s", + "nodeLeaseDurationSeconds": 40, + "imageMinimumGCAge": "2m0s", + "imageMaximumGCAge": "0s", + "imageGCHighThresholdPercent": 85, + "imageGCLowThresholdPercent": 80, + "volumeStatsAggPeriod": "1m0s", + "cgroupsPerQOS": true, + "cgroupDriver": "systemd", + "cpuManagerPolicy": "none", + "cpuManagerReconcilePeriod": "10s", + "memoryManagerPolicy": "None", + "topologyManagerPolicy": "none", + "topologyManagerScope": "container", + "runtimeRequestTimeout": "2m0s", + "hairpinMode": "promiscuous-bridge", + "maxPods": 110, + "podPidsLimit": -1, + "resolvConf": "/run/systemd/resolve/resolv.conf", + "cpuCFSQuota": true, + "cpuCFSQuotaPeriod": "100ms", + "nodeStatusMaxImages": 50, + "maxOpenFiles": 1000000, + "contentType": "application/vnd.kubernetes.protobuf", + "kubeAPIQPS": 50, + "kubeAPIBurst": 100, + "serializeImagePulls": true, + "evictionHard": { + "imagefs.available": "15%", + "memory.available": "100Mi", + "nodefs.available": "10%", + "nodefs.inodesFree": "5%" + }, + "evictionPressureTransitionPeriod": "1m0s", + "enableControllerAttachDetach": true, + "makeIPTablesUtilChains": true, + "iptablesMasqueradeBit": 14, + "iptablesDropBit": 15, + "featureGates": { + "AllAlpha": false + }, + "failSwapOn": false, + "memorySwap": {}, + "containerLogMaxSize": "10Mi", + "containerLogMaxFiles": 5, + "configMapAndSecretChangeDetectionStrategy": "Watch", + "enforceNodeAllocatable": [ + "pods" + ], + "volumePluginDir": "/usr/libexec/kubernetes/kubelet-plugins/volume/exec/", + "logging": { + "format": "text", + "flushFrequency": "5s", + "verbosity": 3, + "options": { + "json": { + "infoBufferSize": "0" + } + } + }, + "enableSystemLogHandler": true, + "enableSystemLogQuery": false, + "shutdownGracePeriod": "0s", + "shutdownGracePeriodCriticalPods": "0s", + "enableProfilingHandler": true, + "enableDebugFlagsHandler": true, + "seccompDefault": false, + "memoryThrottlingFactor": 0.9, + "registerNode": true, + "localStorageCapacityIsolation": true, + "containerRuntimeEndpoint": "unix:///var/run/crio/crio.sock" + } +} +``` @@ -133,4 +292,6 @@ This produces the same outcome as if you used the [single configuration file](#c - Learn more about kubelet configuration by checking the [`KubeletConfiguration`](/docs/reference/config-api/kubelet-config.v1beta1/) - reference. \ No newline at end of file + reference. +- Learn more about kubelet configuration merging in the + [reference document](/docs/reference/node/kubelet-config-directory-merging.md). \ No newline at end of file