diff --git a/third_party/kubeadm/app/constants/constants.go b/third_party/kubeadm/app/constants/constants.go new file mode 100644 index 0000000000..b3559734a3 --- /dev/null +++ b/third_party/kubeadm/app/constants/constants.go @@ -0,0 +1,651 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package constants + +import ( + "fmt" + "io/ioutil" + "net" + "os" + "path" + "path/filepath" + "strings" + "time" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/version" + "k8s.io/apimachinery/pkg/util/wait" + bootstrapapi "k8s.io/cluster-bootstrap/token/api" + utilnet "k8s.io/utils/net" + + "github.com/pkg/errors" +) + +const ( + // KubernetesDir is the directory Kubernetes owns for storing various configuration files + KubernetesDir = "/etc/kubernetes" + // ManifestsSubDirName defines directory name to store manifests + ManifestsSubDirName = "manifests" + // TempDirForKubeadm defines temporary directory for kubeadm + // should be joined with KubernetesDir. + TempDirForKubeadm = "tmp" + + // CertificateValidity defines the validity for all the signed certificates generated by kubeadm + CertificateValidity = time.Hour * 24 * 365 + + // CACertAndKeyBaseName defines certificate authority base name + CACertAndKeyBaseName = "ca" + // CACertName defines certificate name + CACertName = "ca.crt" + // CAKeyName defines certificate name + CAKeyName = "ca.key" + + // APIServerCertAndKeyBaseName defines API's server certificate and key base name + APIServerCertAndKeyBaseName = "apiserver" + // APIServerCertName defines API's server certificate name + APIServerCertName = "apiserver.crt" + // APIServerKeyName defines API's server key name + APIServerKeyName = "apiserver.key" + // APIServerCertCommonName defines API's server certificate common name (CN) + APIServerCertCommonName = "kube-apiserver" + + // APIServerKubeletClientCertAndKeyBaseName defines kubelet client certificate and key base name + APIServerKubeletClientCertAndKeyBaseName = "apiserver-kubelet-client" + // APIServerKubeletClientCertName defines kubelet client certificate name + APIServerKubeletClientCertName = "apiserver-kubelet-client.crt" + // APIServerKubeletClientKeyName defines kubelet client key name + APIServerKubeletClientKeyName = "apiserver-kubelet-client.key" + // APIServerKubeletClientCertCommonName defines kubelet client certificate common name (CN) + APIServerKubeletClientCertCommonName = "kube-apiserver-kubelet-client" + + // EtcdCACertAndKeyBaseName defines etcd's CA certificate and key base name + EtcdCACertAndKeyBaseName = "etcd/ca" + // EtcdCACertName defines etcd's CA certificate name + EtcdCACertName = "etcd/ca.crt" + // EtcdCAKeyName defines etcd's CA key name + EtcdCAKeyName = "etcd/ca.key" + + // EtcdServerCertAndKeyBaseName defines etcd's server certificate and key base name + EtcdServerCertAndKeyBaseName = "etcd/server" + // EtcdServerCertName defines etcd's server certificate name + EtcdServerCertName = "etcd/server.crt" + // EtcdServerKeyName defines etcd's server key name + EtcdServerKeyName = "etcd/server.key" + + // EtcdListenClientPort defines the port etcd listen on for client traffic + EtcdListenClientPort = 2379 + // EtcdMetricsPort is the port at which to obtain etcd metrics and health status + EtcdMetricsPort = 2381 + + // EtcdPeerCertAndKeyBaseName defines etcd's peer certificate and key base name + EtcdPeerCertAndKeyBaseName = "etcd/peer" + // EtcdPeerCertName defines etcd's peer certificate name + EtcdPeerCertName = "etcd/peer.crt" + // EtcdPeerKeyName defines etcd's peer key name + EtcdPeerKeyName = "etcd/peer.key" + + // EtcdListenPeerPort defines the port etcd listen on for peer traffic + EtcdListenPeerPort = 2380 + + // EtcdHealthcheckClientCertAndKeyBaseName defines etcd's healthcheck client certificate and key base name + EtcdHealthcheckClientCertAndKeyBaseName = "etcd/healthcheck-client" + // EtcdHealthcheckClientCertName defines etcd's healthcheck client certificate name + EtcdHealthcheckClientCertName = "etcd/healthcheck-client.crt" + // EtcdHealthcheckClientKeyName defines etcd's healthcheck client key name + EtcdHealthcheckClientKeyName = "etcd/healthcheck-client.key" + // EtcdHealthcheckClientCertCommonName defines etcd's healthcheck client certificate common name (CN) + EtcdHealthcheckClientCertCommonName = "kube-etcd-healthcheck-client" + + // APIServerEtcdClientCertAndKeyBaseName defines apiserver's etcd client certificate and key base name + APIServerEtcdClientCertAndKeyBaseName = "apiserver-etcd-client" + // APIServerEtcdClientCertName defines apiserver's etcd client certificate name + APIServerEtcdClientCertName = "apiserver-etcd-client.crt" + // APIServerEtcdClientKeyName defines apiserver's etcd client key name + APIServerEtcdClientKeyName = "apiserver-etcd-client.key" + // APIServerEtcdClientCertCommonName defines apiserver's etcd client certificate common name (CN) + APIServerEtcdClientCertCommonName = "kube-apiserver-etcd-client" + + // ServiceAccountKeyBaseName defines SA key base name + ServiceAccountKeyBaseName = "sa" + // ServiceAccountPublicKeyName defines SA public key base name + ServiceAccountPublicKeyName = "sa.pub" + // ServiceAccountPrivateKeyName defines SA private key base name + ServiceAccountPrivateKeyName = "sa.key" + + // FrontProxyCACertAndKeyBaseName defines front proxy CA certificate and key base name + FrontProxyCACertAndKeyBaseName = "front-proxy-ca" + // FrontProxyCACertName defines front proxy CA certificate name + FrontProxyCACertName = "front-proxy-ca.crt" + // FrontProxyCAKeyName defines front proxy CA key name + FrontProxyCAKeyName = "front-proxy-ca.key" + + // FrontProxyClientCertAndKeyBaseName defines front proxy certificate and key base name + FrontProxyClientCertAndKeyBaseName = "front-proxy-client" + // FrontProxyClientCertName defines front proxy certificate name + FrontProxyClientCertName = "front-proxy-client.crt" + // FrontProxyClientKeyName defines front proxy key name + FrontProxyClientKeyName = "front-proxy-client.key" + // FrontProxyClientCertCommonName defines front proxy certificate common name + FrontProxyClientCertCommonName = "front-proxy-client" //used as subject.commonname attribute (CN) + + // AdminKubeConfigFileName defines name for the kubeconfig aimed to be used by the superuser/admin of the cluster + AdminKubeConfigFileName = "admin.conf" + // KubeletBootstrapKubeConfigFileName defines the file name for the kubeconfig that the kubelet will use to do + // the TLS bootstrap to get itself an unique credential + KubeletBootstrapKubeConfigFileName = "bootstrap-kubelet.conf" + + // KubeletKubeConfigFileName defines the file name for the kubeconfig that the control-plane kubelet will use for talking + // to the API server + KubeletKubeConfigFileName = "kubelet.conf" + // ControllerManagerKubeConfigFileName defines the file name for the controller manager's kubeconfig file + ControllerManagerKubeConfigFileName = "controller-manager.conf" + // SchedulerKubeConfigFileName defines the file name for the scheduler's kubeconfig file + SchedulerKubeConfigFileName = "scheduler.conf" + + // Some well-known users and groups in the core Kubernetes authorization system + + // ControllerManagerUser defines the well-known user the controller-manager should be authenticated as + ControllerManagerUser = "system:kube-controller-manager" + // SchedulerUser defines the well-known user the scheduler should be authenticated as + SchedulerUser = "system:kube-scheduler" + // SystemPrivilegedGroup defines the well-known group for the apiservers. This group is also superuser by default + // (i.e. bound to the cluster-admin ClusterRole) + SystemPrivilegedGroup = "system:masters" + // NodesGroup defines the well-known group for all nodes. + NodesGroup = "system:nodes" + // NodesUserPrefix defines the user name prefix as requested by the Node authorizer. + NodesUserPrefix = "system:node:" + // NodesClusterRoleBinding defines the well-known ClusterRoleBinding which binds the too permissive system:node + // ClusterRole to the system:nodes group. Since kubeadm is using the Node Authorizer, this ClusterRoleBinding's + // system:nodes group subject is removed if present. + NodesClusterRoleBinding = "system:node" + + // APICallRetryInterval defines how long kubeadm should wait before retrying a failed API operation + APICallRetryInterval = 500 * time.Millisecond + // DiscoveryRetryInterval specifies how long kubeadm should wait before retrying to connect to the control-plane when doing discovery + DiscoveryRetryInterval = 5 * time.Second + // PatchNodeTimeout specifies how long kubeadm should wait for applying the label and taint on the control-plane before timing out + PatchNodeTimeout = 2 * time.Minute + // TLSBootstrapTimeout specifies how long kubeadm should wait for the kubelet to perform the TLS Bootstrap + TLSBootstrapTimeout = 5 * time.Minute + // TLSBootstrapRetryInterval specifies how long kubeadm should wait before retrying the TLS Bootstrap check + TLSBootstrapRetryInterval = 5 * time.Second + // APICallWithWriteTimeout specifies how long kubeadm should wait for api calls with at least one write + APICallWithWriteTimeout = 40 * time.Second + // APICallWithReadTimeout specifies how long kubeadm should wait for api calls with only reads + APICallWithReadTimeout = 15 * time.Second + // PullImageRetry specifies how many times ContainerRuntime retries when pulling image failed + PullImageRetry = 5 + + // DefaultControlPlaneTimeout specifies the default control plane (actually API Server) timeout for use by kubeadm + DefaultControlPlaneTimeout = 4 * time.Minute + + // MinimumAddressesInServiceSubnet defines minimum amount of nodes the Service subnet should allow. + // We need at least ten, because the DNS service is always at the tenth cluster clusterIP + MinimumAddressesInServiceSubnet = 10 + + // MaximumBitsForServiceSubnet defines maximum possible size of the service subnet in terms of bits. + // For example, if the value is 20, then the largest supported service subnet is /12 for IPv4 and /108 for IPv6. + // Note however that anything in between /108 and /112 will be clamped to /112 due to the limitations of the underlying allocation logic. + // TODO: https://github.com/kubernetes/enhancements/pull/1881 + MaximumBitsForServiceSubnet = 20 + + // MinimumAddressesInPodSubnet defines minimum amount of pods in the cluster. + // We need at least more than services, an IPv4 /28 or IPv6 /128 subnet means 14 util addresses + MinimumAddressesInPodSubnet = 14 + + // PodSubnetNodeMaskMaxDiff is limited to 16 due to an issue with uncompressed IP bitmap in core: + // xref: #44918 + // The node subnet mask size must be no more than the pod subnet mask size + 16 + PodSubnetNodeMaskMaxDiff = 16 + + // DefaultTokenDuration specifies the default amount of time that a bootstrap token will be valid + // Default behaviour is 24 hours + DefaultTokenDuration = 24 * time.Hour + + // DefaultCertTokenDuration specifies the default amount of time that the token used by upload certs will be valid + // Default behaviour is 2 hours + DefaultCertTokenDuration = 2 * time.Hour + + // CertificateKeySize specifies the size of the key used to encrypt certificates on uploadcerts phase + CertificateKeySize = 32 + + // LabelNodeRoleOldControlPlane specifies that a node hosts control-plane components + // DEPRECATED: https://github.com/kubernetes/kubeadm/issues/2200 + LabelNodeRoleOldControlPlane = "node-role.kubernetes.io/master" + + // LabelNodeRoleControlPlane specifies that a node hosts control-plane components + LabelNodeRoleControlPlane = "node-role.kubernetes.io/control-plane" + + // LabelExcludeFromExternalLB can be set on a node to exclude it from external load balancers. + // This is added to control plane nodes to preserve backwards compatibility with a legacy behavior. + LabelExcludeFromExternalLB = "node.kubernetes.io/exclude-from-external-load-balancers" + + // AnnotationKubeadmCRISocket specifies the annotation kubeadm uses to preserve the crisocket information given to kubeadm at + // init/join time for use later. kubeadm annotates the node object with this information + AnnotationKubeadmCRISocket = "kubeadm.alpha.kubernetes.io/cri-socket" + + // UnknownCRISocket defines the undetected or unknown CRI socket + UnknownCRISocket = "/var/run/unknown.sock" + + // KubeadmConfigConfigMap specifies in what ConfigMap in the kube-system namespace the `kubeadm init` configuration should be stored + KubeadmConfigConfigMap = "kubeadm-config" + + // ClusterConfigurationConfigMapKey specifies in what ConfigMap key the cluster configuration should be stored + ClusterConfigurationConfigMapKey = "ClusterConfiguration" + + // KubeProxyConfigMap specifies in what ConfigMap in the kube-system namespace the kube-proxy configuration should be stored + KubeProxyConfigMap = "kube-proxy" + + // KubeProxyConfigMapKey specifies in what ConfigMap key the component config of kube-proxy should be stored + KubeProxyConfigMapKey = "config.conf" + + // KubeletBaseConfigurationConfigMapPrefix specifies in what ConfigMap in the kube-system namespace the initial remote configuration of kubelet should be stored + KubeletBaseConfigurationConfigMapPrefix = "kubelet-config-" + + // KubeletBaseConfigurationConfigMapKey specifies in what ConfigMap key the initial remote configuration of kubelet should be stored + KubeletBaseConfigurationConfigMapKey = "kubelet" + + // KubeletBaseConfigMapRolePrefix defines the base kubelet configuration ConfigMap. + KubeletBaseConfigMapRolePrefix = "kubeadm:kubelet-config-" + + // KubeletRunDirectory specifies the directory where the kubelet runtime information is stored. + KubeletRunDirectory = "/var/lib/kubelet" + + // KubeletConfigurationFileName specifies the file name on the node which stores initial remote configuration of kubelet + // This file should exist under KubeletRunDirectory + KubeletConfigurationFileName = "config.yaml" + + // KubeletEnvFileName is a file "kubeadm init" writes at runtime. Using that interface, kubeadm can customize certain + // kubelet flags conditionally based on the environment at runtime. Also, parameters given to the configuration file + // might be passed through this file. "kubeadm init" writes one variable, with the name ${KubeletEnvFileVariableName}. + // This file should exist under KubeletRunDirectory + KubeletEnvFileName = "kubeadm-flags.env" + + // KubeletEnvFileVariableName specifies the shell script variable name "kubeadm init" should write a value to in KubeletEnvFile + KubeletEnvFileVariableName = "KUBELET_KUBEADM_ARGS" + + // KubeletHealthzPort is the port of the kubelet healthz endpoint + KubeletHealthzPort = 10248 + + // MinExternalEtcdVersion indicates minimum external etcd version which kubeadm supports + MinExternalEtcdVersion = "3.2.18" + + // DefaultEtcdVersion indicates the default etcd version that kubeadm uses + DefaultEtcdVersion = "3.5.0-0" + + // Etcd defines variable used internally when referring to etcd component + Etcd = "etcd" + // KubeAPIServer defines variable used internally when referring to kube-apiserver component + KubeAPIServer = "kube-apiserver" + // KubeControllerManager defines variable used internally when referring to kube-controller-manager component + KubeControllerManager = "kube-controller-manager" + // KubeScheduler defines variable used internally when referring to kube-scheduler component + KubeScheduler = "kube-scheduler" + // KubeProxy defines variable used internally when referring to kube-proxy component + KubeProxy = "kube-proxy" + // CoreDNS defines variable used internally when referring to the CoreDNS component + CoreDNS = "CoreDNS" + // Kubelet defines variable used internally when referring to the Kubelet + Kubelet = "kubelet" + + // KubeCertificatesVolumeName specifies the name for the Volume that is used for injecting certificates to control plane components (can be both a hostPath volume or a projected, all-in-one volume) + KubeCertificatesVolumeName = "k8s-certs" + + // KubeConfigVolumeName specifies the name for the Volume that is used for injecting the kubeconfig to talk securely to the api server for a control plane component if applicable + KubeConfigVolumeName = "kubeconfig" + + // NodeBootstrapTokenAuthGroup specifies which group a Node Bootstrap Token should be authenticated in + NodeBootstrapTokenAuthGroup = "system:bootstrappers:kubeadm:default-node-token" + + // DefaultCIImageRepository points to image registry where CI uploads images from ci-cross build job + DefaultCIImageRepository = "gcr.io/k8s-staging-ci-images" + + // CoreDNSConfigMap specifies in what ConfigMap in the kube-system namespace the CoreDNS config should be stored + CoreDNSConfigMap = "coredns" + + // CoreDNSDeploymentName specifies the name of the Deployment for CoreDNS add-on + CoreDNSDeploymentName = "coredns" + + // CoreDNSImageName specifies the name of the image for CoreDNS add-on + CoreDNSImageName = "coredns" + + // CoreDNSVersion is the version of CoreDNS to be deployed if it is used + CoreDNSVersion = "v1.8.4" + + // ClusterConfigurationKind is the string kind value for the ClusterConfiguration struct + ClusterConfigurationKind = "ClusterConfiguration" + + // InitConfigurationKind is the string kind value for the InitConfiguration struct + InitConfigurationKind = "InitConfiguration" + + // JoinConfigurationKind is the string kind value for the JoinConfiguration struct + JoinConfigurationKind = "JoinConfiguration" + + // YAMLDocumentSeparator is the separator for YAML documents + // TODO: Find a better place for this constant + YAMLDocumentSeparator = "---\n" + + // DefaultAPIServerBindAddress is the default bind address for the API Server + DefaultAPIServerBindAddress = "0.0.0.0" + + // ControlPlaneNumCPU is the number of CPUs required on control-plane + ControlPlaneNumCPU = 2 + + // ControlPlaneMem is the number of megabytes of memory required on the control-plane + // Below that amount of RAM running a stable control plane would be difficult. + ControlPlaneMem = 1700 + + // KubeadmCertsSecret specifies in what Secret in the kube-system namespace the certificates should be stored + KubeadmCertsSecret = "kubeadm-certs" + + // KubeletPort is the default port for the kubelet server on each host machine. + // May be overridden by a flag at startup. + KubeletPort = 10250 + // KubeSchedulerPort is the default port for the scheduler status server. + // May be overridden by a flag at startup. + KubeSchedulerPort = 10259 + // KubeControllerManagerPort is the default port for the controller manager status server. + // May be overridden by a flag at startup. + KubeControllerManagerPort = 10257 + + // EtcdAdvertiseClientUrlsAnnotationKey is the annotation key on every etcd pod, describing the + // advertise client URLs + EtcdAdvertiseClientUrlsAnnotationKey = "kubeadm.kubernetes.io/etcd.advertise-client-urls" + // KubeAPIServerAdvertiseAddressEndpointAnnotationKey is the annotation key on every apiserver pod, + // describing the API endpoint (advertise address and bind port of the api server) + KubeAPIServerAdvertiseAddressEndpointAnnotationKey = "kubeadm.kubernetes.io/kube-apiserver.advertise-address.endpoint" + // ComponentConfigHashAnnotationKey holds the config map annotation key that kubeadm uses to store + // a SHA256 sum to check for user changes + ComponentConfigHashAnnotationKey = "kubeadm.kubernetes.io/component-config.hash" + + // ControlPlaneTier is the value used in the tier label to identify control plane components + ControlPlaneTier = "control-plane" + + // Mode* constants were copied from pkg/kubeapiserver/authorizer/modes + // to avoid kubeadm dependency on the internal module + // TODO: share Mode* constants in component config + + // ModeAlwaysAllow is the mode to set all requests as authorized + ModeAlwaysAllow string = "AlwaysAllow" + // ModeAlwaysDeny is the mode to set no requests as authorized + ModeAlwaysDeny string = "AlwaysDeny" + // ModeABAC is the mode to use Attribute Based Access Control to authorize + ModeABAC string = "ABAC" + // ModeWebhook is the mode to make an external webhook call to authorize + ModeWebhook string = "Webhook" + // ModeRBAC is the mode to use Role Based Access Control to authorize + ModeRBAC string = "RBAC" + // ModeNode is an authorization mode that authorizes API requests made by kubelets. + ModeNode string = "Node" + + // PauseVersion indicates the default pause image version for kubeadm + PauseVersion = "3.5" + + // CgroupDriverSystemd holds the systemd driver type + CgroupDriverSystemd = "systemd" + + // The username of the user that kube-controller-manager runs as. + KubeControllerManagerUserName string = "kubeadm-kcm" + // The username of the user that kube-apiserver runs as. + KubeAPIServerUserName string = "kubeadm-kas" + // The username of the user that kube-scheduler runs as. + KubeSchedulerUserName string = "kubeadm-ks" + // The username of the user that etcd runs as. + EtcdUserName string = "kubeadm-etcd" + // The group of users that are allowed to read the service account private key. + ServiceAccountKeyReadersGroupName string = "kubeadm-sa-key-readers" +) + +var ( + // OldControlPlaneTaint is the taint to apply on the PodSpec for being able to run that Pod on the control-plane + // DEPRECATED: https://github.com/kubernetes/kubeadm/issues/2200 + OldControlPlaneTaint = v1.Taint{ + Key: LabelNodeRoleOldControlPlane, + Effect: v1.TaintEffectNoSchedule, + } + + // OldControlPlaneToleration is the toleration to apply on the PodSpec for being able to run that Pod on the control-plane + // DEPRECATED: https://github.com/kubernetes/kubeadm/issues/2200 + OldControlPlaneToleration = v1.Toleration{ + Key: LabelNodeRoleOldControlPlane, + Effect: v1.TaintEffectNoSchedule, + } + + // ControlPlaneTaint is the taint to apply on the PodSpec for being able to run that Pod on the control-plane + ControlPlaneTaint = v1.Taint{ + Key: LabelNodeRoleControlPlane, + Effect: v1.TaintEffectNoSchedule, + } + + // ControlPlaneToleration is the toleration to apply on the PodSpec for being able to run that Pod on the control-plane + ControlPlaneToleration = v1.Toleration{ + Key: LabelNodeRoleControlPlane, + Effect: v1.TaintEffectNoSchedule, + } + + // DefaultTokenUsages specifies the default functions a token will get + DefaultTokenUsages = bootstrapapi.KnownTokenUsages + + // DefaultTokenGroups specifies the default groups that this token will authenticate as when used for authentication + DefaultTokenGroups = []string{NodeBootstrapTokenAuthGroup} + + // ControlPlaneComponents defines the control-plane component names + ControlPlaneComponents = []string{KubeAPIServer, KubeControllerManager, KubeScheduler} + + // MinimumControlPlaneVersion specifies the minimum control plane version kubeadm can deploy + MinimumControlPlaneVersion = version.MustParseSemantic("v1.21.0") + + // MinimumKubeletVersion specifies the minimum version of kubelet which kubeadm supports + MinimumKubeletVersion = version.MustParseSemantic("v1.21.0") + + // CurrentKubernetesVersion specifies current Kubernetes version supported by kubeadm + CurrentKubernetesVersion = version.MustParseSemantic("v1.22.0") + + // SupportedEtcdVersion lists officially supported etcd versions with corresponding Kubernetes releases + SupportedEtcdVersion = map[uint8]string{ + 13: "3.2.24", + 14: "3.3.10", + 15: "3.3.10", + 16: "3.3.17-0", + 17: "3.4.3-0", + 18: "3.4.3-0", + 19: "3.4.13-0", + 20: "3.4.13-0", + 21: "3.4.13-0", + 22: "3.5.0-0", + 23: "3.5.0-0", + } + + // KubeadmCertsClusterRoleName sets the name for the ClusterRole that allows + // the bootstrap tokens to access the kubeadm-certs Secret during the join of a new control-plane + KubeadmCertsClusterRoleName = fmt.Sprintf("kubeadm:%s", KubeadmCertsSecret) + + // StaticPodMirroringDefaultRetry is used a backoff strategy for + // waiting for static pods to be mirrored to the apiserver. + StaticPodMirroringDefaultRetry = wait.Backoff{ + Steps: 30, + Duration: 1 * time.Second, + Factor: 1.0, + Jitter: 0.1, + } +) + +// EtcdSupportedVersion returns officially supported version of etcd for a specific Kubernetes release +// If passed version is not in the given list, the function returns the nearest version with a warning +func EtcdSupportedVersion(supportedEtcdVersion map[uint8]string, versionString string) (etcdVersion *version.Version, warning, err error) { + kubernetesVersion, err := version.ParseSemantic(versionString) + if err != nil { + return nil, nil, err + } + desiredVersion, etcdStringVersion := uint8(kubernetesVersion.Minor()), "" + + min, max := ^uint8(0), uint8(0) + for k, v := range supportedEtcdVersion { + if desiredVersion == k { + etcdStringVersion = v + break + } + if k < min { + min = k + } + if k > max { + max = k + } + } + + if len(etcdStringVersion) == 0 { + if desiredVersion < min { + etcdStringVersion = supportedEtcdVersion[min] + } + if desiredVersion > max { + etcdStringVersion = supportedEtcdVersion[max] + } + warning = fmt.Errorf("could not find officially supported version of etcd for Kubernetes %s, falling back to the nearest etcd version (%s)", + versionString, etcdStringVersion) + } + + etcdVersion, err = version.ParseSemantic(etcdStringVersion) + if err != nil { + return nil, nil, err + } + + return etcdVersion, warning, nil +} + +// GetStaticPodDirectory returns the location on the disk where the Static Pod should be present +func GetStaticPodDirectory() string { + return filepath.Join(KubernetesDir, ManifestsSubDirName) +} + +// GetStaticPodFilepath returns the location on the disk where the Static Pod should be present +func GetStaticPodFilepath(componentName, manifestsDir string) string { + return filepath.Join(manifestsDir, componentName+".yaml") +} + +// GetAdminKubeConfigPath returns the location on the disk where admin kubeconfig is located by default +func GetAdminKubeConfigPath() string { + return filepath.Join(KubernetesDir, AdminKubeConfigFileName) +} + +// GetBootstrapKubeletKubeConfigPath returns the location on the disk where bootstrap kubelet kubeconfig is located by default +func GetBootstrapKubeletKubeConfigPath() string { + return filepath.Join(KubernetesDir, KubeletBootstrapKubeConfigFileName) +} + +// GetKubeletKubeConfigPath returns the location on the disk where kubelet kubeconfig is located by default +func GetKubeletKubeConfigPath() string { + return filepath.Join(KubernetesDir, KubeletKubeConfigFileName) +} + +// CreateTempDirForKubeadm is a function that creates a temporary directory under /etc/kubernetes/tmp (not using /tmp as that would potentially be dangerous) +func CreateTempDirForKubeadm(kubernetesDir, dirName string) (string, error) { + tempDir := path.Join(KubernetesDir, TempDirForKubeadm) + if len(kubernetesDir) != 0 { + tempDir = path.Join(kubernetesDir, TempDirForKubeadm) + } + + // creates target folder if not already exists + if err := os.MkdirAll(tempDir, 0700); err != nil { + return "", errors.Wrapf(err, "failed to create directory %q", tempDir) + } + + tempDir, err := ioutil.TempDir(tempDir, dirName) + if err != nil { + return "", errors.Wrap(err, "couldn't create a temporary directory") + } + return tempDir, nil +} + +// CreateTimestampDirForKubeadm is a function that creates a temporary directory under /etc/kubernetes/tmp formatted with the current date +func CreateTimestampDirForKubeadm(kubernetesDir, dirName string) (string, error) { + tempDir := path.Join(KubernetesDir, TempDirForKubeadm) + if len(kubernetesDir) != 0 { + tempDir = path.Join(kubernetesDir, TempDirForKubeadm) + } + + // creates target folder if not already exists + if err := os.MkdirAll(tempDir, 0700); err != nil { + return "", errors.Wrapf(err, "failed to create directory %q", tempDir) + } + + timestampDirName := fmt.Sprintf("%s-%s", dirName, time.Now().Format("2006-01-02-15-04-05")) + timestampDir := path.Join(tempDir, timestampDirName) + if err := os.Mkdir(timestampDir, 0700); err != nil { + return "", errors.Wrap(err, "could not create timestamp directory") + } + + return timestampDir, nil +} + +// GetDNSIP returns a dnsIP, which is 10th IP in svcSubnet CIDR range +func GetDNSIP(svcSubnetList string, isDualStack bool) (net.IP, error) { + // Get the service subnet CIDR + svcSubnetCIDR, err := GetKubernetesServiceCIDR(svcSubnetList, isDualStack) + if err != nil { + return nil, errors.Wrapf(err, "unable to get internal Kubernetes Service IP from the given service CIDR (%s)", svcSubnetList) + } + + // Selects the 10th IP in service subnet CIDR range as dnsIP + dnsIP, err := utilnet.GetIndexedIP(svcSubnetCIDR, 10) + if err != nil { + return nil, errors.Wrap(err, "unable to get internal Kubernetes Service IP from the given service CIDR") + } + + return dnsIP, nil +} + +// GetKubernetesServiceCIDR returns the default Service CIDR for the Kubernetes internal service +func GetKubernetesServiceCIDR(svcSubnetList string, isDualStack bool) (*net.IPNet, error) { + if isDualStack { + // The default service address family for the cluster is the address family of the first + // service cluster IP range configured via the `--service-cluster-ip-range` flag + // of the kube-controller-manager and kube-apiserver. + svcSubnets, err := utilnet.ParseCIDRs(strings.Split(svcSubnetList, ",")) + if err != nil { + return nil, errors.Wrapf(err, "unable to parse ServiceSubnet %v", svcSubnetList) + } + if len(svcSubnets) == 0 { + return nil, errors.New("received empty ServiceSubnet for dual-stack") + } + return svcSubnets[0], nil + } + // internal IP address for the API server + _, svcSubnet, err := net.ParseCIDR(svcSubnetList) + if err != nil { + return nil, errors.Wrapf(err, "unable to parse ServiceSubnet %v", svcSubnetList) + } + return svcSubnet, nil +} + +// GetAPIServerVirtualIP returns the IP of the internal Kubernetes API service +func GetAPIServerVirtualIP(svcSubnetList string, isDualStack bool) (net.IP, error) { + svcSubnet, err := GetKubernetesServiceCIDR(svcSubnetList, isDualStack) + if err != nil { + return nil, errors.Wrap(err, "unable to get internal Kubernetes Service IP from the given service CIDR") + } + internalAPIServerVirtualIP, err := utilnet.GetIndexedIP(svcSubnet, 1) + if err != nil { + return nil, errors.Wrapf(err, "unable to get the first IP address from the given CIDR: %s", svcSubnet.String()) + } + return internalAPIServerVirtualIP, nil +} + +// GetKubeletConfigMapName returns the right ConfigMap name for the right branch of k8s +func GetKubeletConfigMapName(k8sVersion *version.Version) string { + return fmt.Sprintf("%s%d.%d", KubeletBaseConfigurationConfigMapPrefix, k8sVersion.Major(), k8sVersion.Minor()) +} diff --git a/third_party/kubeadm/app/constants/constants_test.go b/third_party/kubeadm/app/constants/constants_test.go new file mode 100644 index 0000000000..30872c4d12 --- /dev/null +++ b/third_party/kubeadm/app/constants/constants_test.go @@ -0,0 +1,239 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package constants + +import ( + "path/filepath" + "testing" + + "k8s.io/apimachinery/pkg/util/version" +) + +func TestGetStaticPodDirectory(t *testing.T) { + expected := "/etc/kubernetes/manifests" + actual := GetStaticPodDirectory() + + if actual != expected { + t.Errorf( + "failed GetStaticPodDirectory:\n\texpected: %s\n\t actual: %s", + expected, + actual, + ) + } +} + +func TestGetAdminKubeConfigPath(t *testing.T) { + expected := filepath.Join(KubernetesDir, AdminKubeConfigFileName) + actual := GetAdminKubeConfigPath() + + if actual != expected { + t.Errorf( + "failed GetAdminKubeConfigPath:\n\texpected: %s\n\t actual: %s", + expected, + actual, + ) + } +} + +func TestGetBootstrapKubeletKubeConfigPath(t *testing.T) { + expected := "/etc/kubernetes/bootstrap-kubelet.conf" + actual := GetBootstrapKubeletKubeConfigPath() + + if actual != expected { + t.Errorf( + "failed GetBootstrapKubeletKubeConfigPath:\n\texpected: %s\n\t actual: %s", + expected, + actual, + ) + } +} + +func TestGetKubeletKubeConfigPath(t *testing.T) { + expected := "/etc/kubernetes/kubelet.conf" + actual := GetKubeletKubeConfigPath() + + if actual != expected { + t.Errorf( + "failed GetKubeletKubeConfigPath:\n\texpected: %s\n\t actual: %s", + expected, + actual, + ) + } +} + +func TestGetStaticPodFilepath(t *testing.T) { + var tests = []struct { + componentName, manifestsDir, expected string + }{ + { + componentName: "kube-apiserver", + manifestsDir: "/etc/kubernetes/manifests", + expected: "/etc/kubernetes/manifests/kube-apiserver.yaml", + }, + { + componentName: "kube-controller-manager", + manifestsDir: "/etc/kubernetes/manifests/", + expected: "/etc/kubernetes/manifests/kube-controller-manager.yaml", + }, + { + componentName: "foo", + manifestsDir: "/etc/bar/", + expected: "/etc/bar/foo.yaml", + }, + } + for _, rt := range tests { + t.Run(rt.componentName, func(t *testing.T) { + actual := GetStaticPodFilepath(rt.componentName, rt.manifestsDir) + if actual != rt.expected { + t.Errorf( + "failed GetStaticPodFilepath:\n\texpected: %s\n\t actual: %s", + rt.expected, + actual, + ) + } + }) + } +} + +func TestEtcdSupportedVersion(t *testing.T) { + var supportedEtcdVersion = map[uint8]string{ + 13: "3.2.24", + 14: "3.3.10", + 15: "3.3.10", + 16: "3.3.17-0", + 17: "3.4.3-0", + 18: "3.4.3-0", + } + var tests = []struct { + kubernetesVersion string + expectedVersion *version.Version + expectedWarning bool + expectedError bool + }{ + { + kubernetesVersion: "1.x.1", + expectedVersion: nil, + expectedWarning: false, + expectedError: true, + }, + { + kubernetesVersion: "1.10.1", + expectedVersion: version.MustParseSemantic("3.2.24"), + expectedWarning: true, + expectedError: false, + }, + { + kubernetesVersion: "1.99.0", + expectedVersion: version.MustParseSemantic("3.4.3-0"), + expectedWarning: true, + expectedError: false, + }, + { + kubernetesVersion: "v1.16.0", + expectedVersion: version.MustParseSemantic("3.3.17-0"), + expectedWarning: false, + expectedError: false, + }, + { + kubernetesVersion: "1.17.2", + expectedVersion: version.MustParseSemantic("3.4.3-0"), + expectedWarning: false, + expectedError: false, + }, + } + for _, rt := range tests { + t.Run(rt.kubernetesVersion, func(t *testing.T) { + actualVersion, actualWarning, actualError := EtcdSupportedVersion(supportedEtcdVersion, rt.kubernetesVersion) + if (actualError != nil) != rt.expectedError { + t.Fatalf("expected error %v, got %v", rt.expectedError, actualError != nil) + } + if (actualWarning != nil) != rt.expectedWarning { + t.Fatalf("expected warning %v, got %v", rt.expectedWarning, actualWarning != nil) + } + if actualError == nil && actualVersion.String() != rt.expectedVersion.String() { + t.Errorf("expected version %s, got %s", rt.expectedVersion.String(), actualVersion.String()) + } + }) + } +} + +func TestGetKubernetesServiceCIDR(t *testing.T) { + var tests = []struct { + svcSubnetList string + isDualStack bool + expected string + expectedError bool + name string + }{ + { + svcSubnetList: "192.168.10.0/24", + isDualStack: false, + expected: "192.168.10.0/24", + expectedError: false, + name: "valid: valid IPv4 range from single-stack", + }, + { + svcSubnetList: "fd03::/112", + isDualStack: false, + expected: "fd03::/112", + expectedError: false, + name: "valid: valid IPv6 range from single-stack", + }, + { + svcSubnetList: "192.168.10.0/24,fd03::/112", + isDualStack: true, + expected: "192.168.10.0/24", + expectedError: false, + name: "valid: valid ranges from dual-stack", + }, + { + svcSubnetList: "fd03::/112,192.168.10.0/24", + isDualStack: true, + expected: "fd03::/112", + expectedError: false, + name: "valid: valid ranges from dual-stack", + }, + { + svcSubnetList: "192.168.10.0/24,fd03:x::/112", + isDualStack: true, + expected: "", + expectedError: true, + name: "invalid: failed to parse subnet range for dual-stack", + }, + } + + for _, rt := range tests { + t.Run(rt.name, func(t *testing.T) { + actual, actualError := GetKubernetesServiceCIDR(rt.svcSubnetList, rt.isDualStack) + if rt.expectedError { + if actualError == nil { + t.Errorf("failed GetKubernetesServiceCIDR:\n\texpected error, but got no error") + } + } else if !rt.expectedError && actualError != nil { + t.Errorf("failed GetKubernetesServiceCIDR:\n\texpected no error, but got: %v", actualError) + } else { + if actual.String() != rt.expected { + t.Errorf( + "failed GetKubernetesServiceCIDR:\n\texpected: %s\n\t actual: %s", + rt.expected, + actual.String(), + ) + } + } + }) + } +} diff --git a/third_party/kubeadm/app/constants/constants_unix.go b/third_party/kubeadm/app/constants/constants_unix.go new file mode 100644 index 0000000000..16ff72d555 --- /dev/null +++ b/third_party/kubeadm/app/constants/constants_unix.go @@ -0,0 +1,24 @@ +// +build !windows + +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package constants + +const ( + // DefaultDockerCRISocket defines the default Docker CRI socket + DefaultDockerCRISocket = "/var/run/dockershim.sock" +) diff --git a/third_party/kubeadm/app/constants/constants_windows.go b/third_party/kubeadm/app/constants/constants_windows.go new file mode 100644 index 0000000000..6daae0a1ff --- /dev/null +++ b/third_party/kubeadm/app/constants/constants_windows.go @@ -0,0 +1,24 @@ +// +build windows + +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package constants + +const ( + // DefaultDockerCRISocket defines the default Docker CRI socket + DefaultDockerCRISocket = "npipe:////./pipe/docker_engine" +) diff --git a/third_party/kubeadm/app/features/features.go b/third_party/kubeadm/app/features/features.go new file mode 100644 index 0000000000..a392f40faa --- /dev/null +++ b/third_party/kubeadm/app/features/features.go @@ -0,0 +1,182 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package features + +import ( + "fmt" + "sort" + "strconv" + "strings" + + "k8s.io/apimachinery/pkg/util/version" + "k8s.io/component-base/featuregate" + + "github.com/pkg/errors" +) + +const ( + // IPv6DualStack is expected to be beta in v1.21 + IPv6DualStack = "IPv6DualStack" + // PublicKeysECDSA is expected to be alpha in v1.19 + PublicKeysECDSA = "PublicKeysECDSA" + // RootlessControlPlane is expected to be in alpha in v1.22 + RootlessControlPlane = "RootlessControlPlane" +) + +// InitFeatureGates are the default feature gates for the init command +var InitFeatureGates = FeatureList{ + IPv6DualStack: {FeatureSpec: featuregate.FeatureSpec{Default: true, PreRelease: featuregate.Beta}}, + PublicKeysECDSA: {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Alpha}}, + RootlessControlPlane: {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Alpha}}, +} + +// Feature represents a feature being gated +type Feature struct { + featuregate.FeatureSpec + MinimumVersion *version.Version + HiddenInHelpText bool + DeprecationMessage string +} + +// FeatureList represents a list of feature gates +type FeatureList map[string]Feature + +// ValidateVersion ensures that a feature gate list is compatible with the chosen Kubernetes version +func ValidateVersion(allFeatures FeatureList, requestedFeatures map[string]bool, requestedVersion string) error { + if requestedVersion == "" { + return nil + } + parsedExpVersion, err := version.ParseSemantic(requestedVersion) + if err != nil { + return errors.Wrapf(err, "error parsing version %s", requestedVersion) + } + for k := range requestedFeatures { + if minVersion := allFeatures[k].MinimumVersion; minVersion != nil { + if !parsedExpVersion.AtLeast(minVersion) { + return errors.Errorf( + "the requested Kubernetes version (%s) is incompatible with the %s feature gate, which needs %s as a minimum", + requestedVersion, k, minVersion) + } + } + } + return nil +} + +// Enabled indicates whether a feature name has been enabled +func Enabled(featureList map[string]bool, featureName string) bool { + if enabled, ok := featureList[string(featureName)]; ok { + return enabled + } + return InitFeatureGates[string(featureName)].Default +} + +// Supports indicates whether a feature name is supported on the given +// feature set +func Supports(featureList FeatureList, featureName string) bool { + for k, v := range featureList { + if featureName == string(k) { + return v.PreRelease != featuregate.Deprecated + } + } + return false +} + +// Keys returns a slice of feature names for a given feature set +func Keys(featureList FeatureList) []string { + var list []string + for k := range featureList { + list = append(list, string(k)) + } + return list +} + +// KnownFeatures returns a slice of strings describing the FeatureList features. +func KnownFeatures(f *FeatureList) []string { + var known []string + for k, v := range *f { + if v.HiddenInHelpText { + continue + } + + pre := "" + if v.PreRelease != featuregate.GA { + pre = fmt.Sprintf("%s - ", v.PreRelease) + } + known = append(known, fmt.Sprintf("%s=true|false (%sdefault=%t)", k, pre, v.Default)) + } + sort.Strings(known) + return known +} + +// NewFeatureGate parses a string of the form "key1=value1,key2=value2,..." into a +// map[string]bool of known keys or returns an error. +func NewFeatureGate(f *FeatureList, value string) (map[string]bool, error) { + featureGate := map[string]bool{} + for _, s := range strings.Split(value, ",") { + if len(s) == 0 { + continue + } + + arr := strings.SplitN(s, "=", 2) + if len(arr) != 2 { + return nil, errors.Errorf("missing bool value for feature-gate key:%s", s) + } + + k := strings.TrimSpace(arr[0]) + v := strings.TrimSpace(arr[1]) + + featureSpec, ok := (*f)[k] + if !ok { + return nil, errors.Errorf("unrecognized feature-gate key: %s", k) + } + + if featureSpec.PreRelease == featuregate.Deprecated { + return nil, errors.Errorf("feature-gate key is deprecated: %s", k) + } + + boolValue, err := strconv.ParseBool(v) + if err != nil { + return nil, errors.Errorf("invalid value %v for feature-gate key: %s, use true|false instead", v, k) + } + featureGate[k] = boolValue + } + + return featureGate, nil +} + +// CheckDeprecatedFlags takes a list of existing feature gate flags and validates against the current feature flag set. +// It used during upgrades for ensuring consistency of feature gates used in an existing cluster, that might +// be created with a previous version of kubeadm, with the set of features currently supported by kubeadm +func CheckDeprecatedFlags(f *FeatureList, features map[string]bool) map[string]string { + deprecatedMsg := map[string]string{} + for k := range features { + featureSpec, ok := (*f)[k] + if !ok { + // This case should never happen, it is implemented only as a sentinel + // for removal of flags executed when flags are still in use (always before deprecate, then after one cycle remove) + deprecatedMsg[k] = fmt.Sprintf("Unknown feature gate flag: %s", k) + } + + if featureSpec.PreRelease == featuregate.Deprecated { + if _, ok := deprecatedMsg[k]; !ok { + deprecatedMsg[k] = featureSpec.DeprecationMessage + } + } + } + + return deprecatedMsg +} diff --git a/third_party/kubeadm/app/features/features_test.go b/third_party/kubeadm/app/features/features_test.go new file mode 100644 index 0000000000..81d29f3e6a --- /dev/null +++ b/third_party/kubeadm/app/features/features_test.go @@ -0,0 +1,219 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package features + +import ( + "reflect" + "testing" + + "k8s.io/apimachinery/pkg/util/version" + "k8s.io/component-base/featuregate" +) + +func TestKnownFeatures(t *testing.T) { + var someFeatures = FeatureList{ + "feature2": {FeatureSpec: featuregate.FeatureSpec{Default: true, PreRelease: featuregate.Alpha}}, + "feature1": {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Beta}}, + "feature3": {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.GA}}, + "hidden": {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.GA}, HiddenInHelpText: true}, + } + + r := KnownFeatures(&someFeatures) + + if len(r) != 3 { + t.Errorf("KnownFeatures returned %d values, expected 3", len(r)) + } + + // check the first value is feature1 (the list should be sorted); prerelease and default should be present + f1 := "feature1=true|false (BETA - default=false)" + if r[0] != f1 { + t.Errorf("KnownFeatures returned %s values, expected %s", r[0], f1) + } + // check the second value is feature2; prerelease and default should be present + f2 := "feature2=true|false (ALPHA - default=true)" + if r[1] != f2 { + t.Errorf("KnownFeatures returned %s values, expected %s", r[1], f2) + } + // check the second value is feature3; prerelease should not be shown for GA features; default should be present + f3 := "feature3=true|false (default=false)" + if r[2] != f3 { + t.Errorf("KnownFeatures returned %s values, expected %s", r[2], f3) + } +} + +func TestNewFeatureGate(t *testing.T) { + var someFeatures = FeatureList{ + "feature1": {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Beta}}, + "feature2": {FeatureSpec: featuregate.FeatureSpec{Default: true, PreRelease: featuregate.Alpha}}, + "deprecated": {FeatureSpec: featuregate.FeatureSpec{Default: true, PreRelease: featuregate.Deprecated}}, + } + + var tests = []struct { + value string + expectedError bool + expectedFeaturesGate map[string]bool + }{ + { //invalid value (missing =) + value: "invalidValue", + expectedError: true, + }, + { //invalid value (missing =) + value: "feature1=true,invalidValue", + expectedError: true, + }, + { //invalid value (not a boolean) + value: "feature1=notABoolean", + expectedError: true, + }, + { //invalid value (not a boolean) + value: "feature1=true,feature2=notABoolean", + expectedError: true, + }, + { //unrecognized feature-gate key + value: "unknownFeature=false", + expectedError: true, + }, + { //unrecognized feature-gate key + value: "feature1=true,unknownFeature=false", + expectedError: true, + }, + { //deprecated feature-gate key + value: "deprecated=true", + expectedError: true, + }, + { //one feature + value: "feature1=true", + expectedError: false, + expectedFeaturesGate: map[string]bool{"feature1": true}, + }, + { //two features + value: "feature1=true,feature2=false", + expectedError: false, + expectedFeaturesGate: map[string]bool{"feature1": true, "feature2": false}, + }, + } + + for _, test := range tests { + t.Run(test.value, func(t *testing.T) { + r, err := NewFeatureGate(&someFeatures, test.value) + + if !test.expectedError && err != nil { + t.Errorf("NewFeatureGate failed when not expected: %v", err) + return + } else if test.expectedError && err == nil { + t.Error("NewFeatureGate didn't failed when expected") + return + } + + if !reflect.DeepEqual(r, test.expectedFeaturesGate) { + t.Errorf("NewFeatureGate returned a unexpected value") + } + }) + } +} + +func TestValidateVersion(t *testing.T) { + var someFeatures = FeatureList{ + "feature1": {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Beta}}, + "feature2": {FeatureSpec: featuregate.FeatureSpec{Default: true, PreRelease: featuregate.Alpha}, MinimumVersion: version.MustParseSemantic("v1.17.0").WithPreRelease("alpha.1")}, + } + + var tests = []struct { + name string + requestedVersion string + requestedFeatures map[string]bool + expectedError bool + }{ + { + name: "no min version", + requestedFeatures: map[string]bool{"feature1": true}, + expectedError: false, + }, + { + name: "min version but correct value given", + requestedFeatures: map[string]bool{"feature2": true}, + requestedVersion: "v1.17.0", + expectedError: false, + }, + { + name: "min version and incorrect value given", + requestedFeatures: map[string]bool{"feature2": true}, + requestedVersion: "v1.11.2", + expectedError: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := ValidateVersion(someFeatures, test.requestedFeatures, test.requestedVersion) + if !test.expectedError && err != nil { + t.Errorf("ValidateVersion failed when not expected: %v", err) + return + } else if test.expectedError && err == nil { + t.Error("ValidateVersion didn't failed when expected") + return + } + }) + } +} + +// TestEnabledDefaults tests that Enabled returns the default values for +// each feature gate when no feature gates are specified. +func TestEnabledDefaults(t *testing.T) { + for featureName, feature := range InitFeatureGates { + featureList := make(map[string]bool) + + enabled := Enabled(featureList, featureName) + if enabled != feature.Default { + t.Errorf("Enabled returned %v instead of default value %v for feature %s", enabled, feature.Default, featureName) + } + } +} + +func TestCheckDeprecatedFlags(t *testing.T) { + dummyMessage := "dummy message" + var someFeatures = FeatureList{ + "feature1": {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Beta}}, + "deprecated": {FeatureSpec: featuregate.FeatureSpec{Default: true, PreRelease: featuregate.Deprecated}, DeprecationMessage: dummyMessage}, + } + + var tests = []struct { + name string + features map[string]bool + expectedMsg map[string]string + }{ + { + name: "deprecated feature", + features: map[string]bool{"deprecated": true}, + expectedMsg: map[string]string{"deprecated": dummyMessage}, + }, + { + name: "valid feature", + features: map[string]bool{"feature1": true}, + expectedMsg: map[string]string{}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + msg := CheckDeprecatedFlags(&someFeatures, test.features) + if !reflect.DeepEqual(test.expectedMsg, msg) { + t.Error("CheckDeprecatedFlags didn't returned expected message") + } + }) + } +}