diff --git a/deploy/iso/minikube-iso/package/crio-bin/crio-bin.hash b/deploy/iso/minikube-iso/package/crio-bin/crio-bin.hash index 93ddf49dab..4fb4c559b8 100644 --- a/deploy/iso/minikube-iso/package/crio-bin/crio-bin.hash +++ b/deploy/iso/minikube-iso/package/crio-bin/crio-bin.hash @@ -19,3 +19,5 @@ sha256 25dc558fbabc987bd58c7eab5230121b258a7b0eb34a49dc6595f1c6f3969116 v1.18.2. sha256 d5c6442e3990938badc966cdd1eb9ebe2fc11345452c233aa0d87ca38fbeed81 v1.18.3.tar.gz sha256 74a4e916acddc6cf47ab5752bdebb6732ce2c028505ef57b7edc21d2da9039b6 v1.18.4.tar.gz sha256 fc8a8e61375e3ce30563eeb0fd6534c4f48fc20300a72e6ff51cc99cb2703516 v1.19.0.tar.gz +sha256 6165c5b8212ea03be2a465403177318bfe25a54c3e8d66d720344643913a0223 v1.19.1.tar.gz +sha256 76fd7543bc92d4364a11060f43a5131893a76c6e6e9d6de3a6bb6292c110b631 v1.20.0.tar.gz diff --git a/deploy/iso/minikube-iso/package/crio-bin/crio-bin.mk b/deploy/iso/minikube-iso/package/crio-bin/crio-bin.mk index 0be5317687..41ec5044d2 100644 --- a/deploy/iso/minikube-iso/package/crio-bin/crio-bin.mk +++ b/deploy/iso/minikube-iso/package/crio-bin/crio-bin.mk @@ -4,8 +4,8 @@ # ################################################################################ -CRIO_BIN_VERSION = v1.19.0 -CRIO_BIN_COMMIT = 99c925bebdd9e392f2d575e25f2e6a1082e6c232 +CRIO_BIN_VERSION = v1.20.0 +CRIO_BIN_COMMIT = d388528dbed26b93c5bc1c89623607a1e597aa57 CRIO_BIN_SITE = https://github.com/cri-o/cri-o/archive CRIO_BIN_SOURCE = $(CRIO_BIN_VERSION).tar.gz CRIO_BIN_DEPENDENCIES = host-go libgpgme diff --git a/deploy/iso/minikube-iso/package/crio-bin/crio.conf b/deploy/iso/minikube-iso/package/crio-bin/crio.conf index a7e010c9ea..fafaed67bc 100644 --- a/deploy/iso/minikube-iso/package/crio-bin/crio.conf +++ b/deploy/iso/minikube-iso/package/crio-bin/crio.conf @@ -29,6 +29,7 @@ storage_driver = "overlay" # List to pass options to the storage driver. Please refer to # containers-storage.conf(5) to see all available storage options. #storage_option = [ +# "overlay.mountopt=nodev,metacopy=on", #] # The default log directory where all logs will go unless directly specified by @@ -92,11 +93,6 @@ grpc_max_recv_msg_size = 16777216 #default_ulimits = [ #] -# default_runtime is the _name_ of the OCI runtime to be used as the default. -# The name is matched against the runtimes map below. If this value is changed, -# the corresponding existing entry from the runtimes map below will be ignored. -default_runtime = "runc" - # If true, the runtime will not use pivot_root, but instead use MS_MOVE. no_pivot = false @@ -131,6 +127,12 @@ selinux = false # will be used. This option supports live configuration reload. seccomp_profile = "" +# Changes the meaning of an empty seccomp profile. By default +# (and according to CRI spec), an empty profile means unconfined. +# This option tells CRI-O to treat an empty profile as the default profile, +# which might increase security. +seccomp_use_default_when_empty = false + # Used to change the name of the default AppArmor profile of CRI-O. The default # profile name is "crio-default". This profile only takes effect if the user # does not specify a profile via the Kubernetes Pod's metadata annotation. If @@ -141,6 +143,9 @@ apparmor_profile = "crio-default" # Cgroup management implementation used for the runtime. cgroup_manager = "systemd" +# Specify whether the image pull must be performed in a separate cgroup. +separate_pull_cgroup = "" + # List of default capabilities for containers. If it is empty or commented out, # only the capabilities defined in the containers json file by the user/kube # will be added. @@ -174,11 +179,6 @@ hooks_dir = [ "/usr/share/containers/oci/hooks.d", ] -# List of default mounts for each container. **Deprecated:** this option will -# be removed in future versions in favor of default_mounts_file. -default_mounts = [ -] - # Path to the file specifying the defaults mounts for each container. The # format of the config is /SRC:/DST, one mount per line. Notice that CRI-O reads # its default mounts from the following two files: @@ -243,7 +243,8 @@ gid_mappings = "" ctr_stop_timeout = 30 # manage_ns_lifecycle determines whether we pin and remove namespaces -# and manage their lifecycle +# and manage their lifecycle. +# This option is being deprecated, and will be unconditionally true in the future. manage_ns_lifecycle = true # drop_infra_ctr determines whether CRI-O drops the infra container @@ -259,6 +260,11 @@ namespaces_dir = "/var/run" # pinns_path is the path to find the pinns binary, which is needed to manage namespace lifecycle pinns_path = "/usr/bin/pinns" +# default_runtime is the _name_ of the OCI runtime to be used as the default. +# The name is matched against the runtimes map below. If this value is changed, +# the corresponding existing entry from the runtimes map below will be ignored. +default_runtime = "runc" + # The "crio.runtime.runtimes" table defines a list of OCI compatible runtimes. # The runtime to use is picked based on the runtime_handler provided by the CRI. # If no runtime_handler is provided, the runtime will be picked based on the level @@ -268,7 +274,8 @@ pinns_path = "/usr/bin/pinns" # runtime_path = "/path/to/the/executable" # runtime_type = "oci" # runtime_root = "/path/to/the/root" -# +# privileged_without_host_devices = false +# allowed_annotations = [] # Where: # - runtime-handler: name used to identify the runtime # - runtime_path (optional, string): absolute path to the runtime executable in @@ -279,6 +286,14 @@ pinns_path = "/usr/bin/pinns" # omitted, an "oci" runtime is assumed. # - runtime_root (optional, string): root directory for storage of containers # state. +# - privileged_without_host_devices (optional, bool): an option for restricting +# host devices from being passed to privileged containers. +# - allowed_annotations (optional, array of strings): an option for specifying +# a list of experimental annotations that this runtime handler is allowed to process. +# The currently recognized values are: +# "io.kubernetes.cri-o.userns-mode" for configuring a user namespace for the pod. +# "io.kubernetes.cri-o.Devices" for configuring devices for the pod. +# "io.kubernetes.cri-o.ShmSize" for configuring the size of /dev/shm. [crio.runtime.runtimes.runc] @@ -287,6 +302,8 @@ runtime_type = "oci" runtime_root = "/run/runc" + + # crun is a fast and lightweight fully featured OCI runtime and C library for # running containers #[crio.runtime.runtimes.crun] diff --git a/deploy/iso/minikube-iso/package/crio-bin/crio.conf.default b/deploy/iso/minikube-iso/package/crio-bin/crio.conf.default index 9c22500d0b..25debfab9f 100644 --- a/deploy/iso/minikube-iso/package/crio-bin/crio.conf.default +++ b/deploy/iso/minikube-iso/package/crio-bin/crio.conf.default @@ -29,6 +29,7 @@ # List to pass options to the storage driver. Please refer to # containers-storage.conf(5) to see all available storage options. #storage_option = [ +# "overlay.mountopt=nodev,metacopy=on", #] # The default log directory where all logs will go unless directly specified by @@ -92,11 +93,6 @@ grpc_max_recv_msg_size = 16777216 #default_ulimits = [ #] -# default_runtime is the _name_ of the OCI runtime to be used as the default. -# The name is matched against the runtimes map below. If this value is changed, -# the corresponding existing entry from the runtimes map below will be ignored. -default_runtime = "runc" - # If true, the runtime will not use pivot_root, but instead use MS_MOVE. no_pivot = false @@ -131,6 +127,12 @@ selinux = false # will be used. This option supports live configuration reload. seccomp_profile = "" +# Changes the meaning of an empty seccomp profile. By default +# (and according to CRI spec), an empty profile means unconfined. +# This option tells CRI-O to treat an empty profile as the default profile, +# which might increase security. +seccomp_use_default_when_empty = false + # Used to change the name of the default AppArmor profile of CRI-O. The default # profile name is "crio-default". This profile only takes effect if the user # does not specify a profile via the Kubernetes Pod's metadata annotation. If @@ -141,6 +143,9 @@ apparmor_profile = "crio-default" # Cgroup management implementation used for the runtime. cgroup_manager = "systemd" +# Specify whether the image pull must be performed in a separate cgroup. +separate_pull_cgroup = "" + # List of default capabilities for containers. If it is empty or commented out, # only the capabilities defined in the containers json file by the user/kube # will be added. @@ -174,11 +179,6 @@ hooks_dir = [ "/usr/share/containers/oci/hooks.d", ] -# List of default mounts for each container. **Deprecated:** this option will -# be removed in future versions in favor of default_mounts_file. -default_mounts = [ -] - # Path to the file specifying the defaults mounts for each container. The # format of the config is /SRC:/DST, one mount per line. Notice that CRI-O reads # its default mounts from the following two files: @@ -243,7 +243,8 @@ gid_mappings = "" ctr_stop_timeout = 30 # manage_ns_lifecycle determines whether we pin and remove namespaces -# and manage their lifecycle +# and manage their lifecycle. +# This option is being deprecated, and will be unconditionally true in the future. manage_ns_lifecycle = true # drop_infra_ctr determines whether CRI-O drops the infra container @@ -259,6 +260,11 @@ namespaces_dir = "/var/run" # pinns_path is the path to find the pinns binary, which is needed to manage namespace lifecycle pinns_path = "" +# default_runtime is the _name_ of the OCI runtime to be used as the default. +# The name is matched against the runtimes map below. If this value is changed, +# the corresponding existing entry from the runtimes map below will be ignored. +default_runtime = "runc" + # The "crio.runtime.runtimes" table defines a list of OCI compatible runtimes. # The runtime to use is picked based on the runtime_handler provided by the CRI. # If no runtime_handler is provided, the runtime will be picked based on the level @@ -268,7 +274,8 @@ pinns_path = "" # runtime_path = "/path/to/the/executable" # runtime_type = "oci" # runtime_root = "/path/to/the/root" -# +# privileged_without_host_devices = false +# allowed_annotations = [] # Where: # - runtime-handler: name used to identify the runtime # - runtime_path (optional, string): absolute path to the runtime executable in @@ -279,6 +286,14 @@ pinns_path = "" # omitted, an "oci" runtime is assumed. # - runtime_root (optional, string): root directory for storage of containers # state. +# - privileged_without_host_devices (optional, bool): an option for restricting +# host devices from being passed to privileged containers. +# - allowed_annotations (optional, array of strings): an option for specifying +# a list of experimental annotations that this runtime handler is allowed to process. +# The currently recognized values are: +# "io.kubernetes.cri-o.userns-mode" for configuring a user namespace for the pod. +# "io.kubernetes.cri-o.Devices" for configuring devices for the pod. +# "io.kubernetes.cri-o.ShmSize" for configuring the size of /dev/shm. [crio.runtime.runtimes.runc] @@ -287,6 +302,8 @@ runtime_type = "oci" runtime_root = "/run/runc" + + # crun is a fast and lightweight fully featured OCI runtime and C library for # running containers #[crio.runtime.runtimes.crun] diff --git a/deploy/iso/minikube-iso/package/podman/podman.hash b/deploy/iso/minikube-iso/package/podman/podman.hash index 7f0122541b..4f5158b977 100644 --- a/deploy/iso/minikube-iso/package/podman/podman.hash +++ b/deploy/iso/minikube-iso/package/podman/podman.hash @@ -1,4 +1,4 @@ sha256 a16846fe076aaf2c9ea2e854c3baba9fb838d916be7fb4b5be332e6c92d907d4 v1.9.3.tar.gz sha256 5ebaa6e0dbd7fd1863f70d2bc71dc8a94e195c3339c17e3cac4560c9ec5747f8 v2.1.1.tar.gz sha256 ec5473e51fa28f29af323473fc484f742dc7df23d06d8ba9f217f13382893a71 v2.2.0.tar.gz -sha256 bd86b181251e2308cb52f18410fb52d89df7f130cecf0298bbf9a848fe7daf60 v2.2.1.tar.gz +sha256 3212bad60d945c1169b27da03959f36d92d1d8964645c701a5a82a89118e96d1 v2.2.1.tar.gz diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go index 4b4db916ab..41050358ab 100644 --- a/pkg/minikube/assets/addons.go +++ b/pkg/minikube/assets/addons.go @@ -689,19 +689,18 @@ func GenerateTemplateData(addon *Addon, cfg config.KubernetesConfig) interface{} opts.Registries[name] = "" // Avoid nil access when rendering } - // Send messages to stderr due to some tests rely on stdout if override, ok := opts.CustomRegistries[name]; ok { - out.ErrT(style.Option, "Using image {{.registry}}{{.image}}", out.V{ + out.Step(style.Option, "Using image {{.registry}}{{.image}}", out.V{ "registry": override, "image": image, }) } else if opts.ImageRepository != "" { - out.ErrT(style.Option, "Using image {{.registry}}{{.image}} (global image repository)", out.V{ + out.Step(style.Option, "Using image {{.registry}}{{.image}} (global image repository)", out.V{ "registry": opts.ImageRepository, "image": image, }) } else { - out.ErrT(style.Option, "Using image {{.registry}}{{.image}}", out.V{ + out.Step(style.Option, "Using image {{.registry}}{{.image}}", out.V{ "registry": opts.Registries[name], "image": image, }) diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go index e12f2d7074..a9f9ec6638 100644 --- a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go +++ b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go @@ -37,6 +37,8 @@ const ( NodeReadyKey = "node_ready" // KubeletKey is the name used in the flags for waiting for the kubelet status to be ready KubeletKey = "kubelet" + // ExtraKey is the name used for extra waiting for pods in CorePodsList to be Ready + ExtraKey = "extra" ) // vars related to the --wait flag @@ -44,9 +46,9 @@ var ( // DefaultComponents is map of the the default components to wait for DefaultComponents = map[string]bool{APIServerWaitKey: true, SystemPodsWaitKey: true} // NoWaitComponents is map of componets to wait for if specified 'none' or 'false' - NoComponents = map[string]bool{APIServerWaitKey: false, SystemPodsWaitKey: false, DefaultSAWaitKey: false, AppsRunningKey: false, NodeReadyKey: false, KubeletKey: false} + NoComponents = map[string]bool{APIServerWaitKey: false, SystemPodsWaitKey: false, DefaultSAWaitKey: false, AppsRunningKey: false, NodeReadyKey: false, KubeletKey: false, ExtraKey: false} // AllComponents is map for waiting for all components. - AllComponents = map[string]bool{APIServerWaitKey: true, SystemPodsWaitKey: true, DefaultSAWaitKey: true, AppsRunningKey: true, NodeReadyKey: true, KubeletKey: true} + AllComponents = map[string]bool{APIServerWaitKey: true, SystemPodsWaitKey: true, DefaultSAWaitKey: true, AppsRunningKey: true, NodeReadyKey: true, KubeletKey: true, ExtraKey: true} // DefaultWaitList is list of all default components to wait for. only names to be used for start flags. DefaultWaitList = []string{APIServerWaitKey, SystemPodsWaitKey} // AllComponentsList list of all valid components keys to wait for. only names to be used used for start flags. @@ -60,6 +62,15 @@ var ( "kube-proxy", "kube-scheduler", } + // CorePodsList is a list of essential pods for running kurnetes to extra wait for them to be Ready + CorePodsList = []string{ + "kube-dns", // coredns + "etcd", + "kube-apiserver", + "kube-controller-manager", + "kube-proxy", + "kube-scheduler", + } ) // ShouldWait will return true if the config says need to wait diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/pod_ready.go b/pkg/minikube/bootstrapper/bsutil/kverify/pod_ready.go new file mode 100644 index 0000000000..982cee5461 --- /dev/null +++ b/pkg/minikube/bootstrapper/bsutil/kverify/pod_ready.go @@ -0,0 +1,133 @@ +/* +Copyright 2021 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package kverify verifies a running Kubernetes cluster is healthy +package kverify + +import ( + "fmt" + "strings" + "time" + + "github.com/pkg/errors" + core "k8s.io/api/core/v1" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" + kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants" +) + +// WaitExtra calls WaitForPodReadyByLabel for each pod in labels list and returns any errors occurred. +func WaitExtra(cs *kubernetes.Clientset, labels []string, timeout time.Duration) error { + klog.Infof("extra waiting for kube-system core pods %s to be Ready ...", labels) + start := time.Now() + defer func() { + klog.Infof("duration metric: took %s for extra waiting for kube-system core pods to be Ready ...", time.Since(start)) + }() + + var errs []string + for _, label := range labels { + if err := waitForPodReadyByLabel(cs, label, "kube-system", timeout); err != nil { + errs = append(errs, fmt.Sprintf("%q: %q", label, err.Error())) + } + } + if errs != nil { + return fmt.Errorf(strings.Join(errs, ", ")) + } + + return nil +} + +// waitForPodReadyByLabel waits for pod with label ([key:]val) in a namespace to be in Ready condition. +// If namespace is not provided, it defaults to "kube-system". +// If label key is not provided, it will try with "component" and "k8s-app". +func waitForPodReadyByLabel(cs *kubernetes.Clientset, label, namespace string, timeout time.Duration) error { + klog.Infof("waiting %v for pod with %q label in %q namespace to be Ready ...", timeout, label, namespace) + start := time.Now() + defer func() { + klog.Infof("duration metric: took %v to run WaitForPodReadyByLabel for pod with %q label in %q namespace ...", time.Since(start), label, namespace) + }() + + if namespace == "" { + namespace = "kube-system" + } + + lkey := "" + lval := "" + l := strings.Split(label, ":") + switch len(l) { + case 1: // treat as no label key provided, just val + lval = strings.TrimSpace(l[0]) + case 2: + lkey = strings.TrimSpace(l[0]) + lval = strings.TrimSpace(l[1]) + default: + return fmt.Errorf("pod label %q is malformed", label) + } + + lap := time.Now() + checkReady := func() (bool, error) { + if time.Since(start) > timeout { + return false, fmt.Errorf("wait for pod with %q label in %q namespace to be Ready timed out", label, namespace) + } + pods, err := cs.CoreV1().Pods(namespace).List(meta.ListOptions{}) + if err != nil { + klog.Infof("error listing pods in %q namespace, will retry: %v", namespace, err) + return false, nil + } + for _, pod := range pods.Items { + for k, v := range pod.ObjectMeta.Labels { + if ((lkey == "" && (k == "component" || k == "k8s-app")) || lkey == k) && v == lval { + ready, reason := IsPodReady(&pod) + if ready { + klog.Info(reason) + return true, nil + } + // reduce log spam + if time.Since(lap) > (1 * time.Second) { + klog.Info(reason) + lap = time.Now() + } + return false, nil + } + } + } + klog.Infof("pod with %q label in %q namespace was not found, will retry", label, namespace) + return false, nil + } + if err := wait.PollImmediate(kconst.APICallRetryInterval, kconst.DefaultControlPlaneTimeout, checkReady); err != nil { + return errors.Wrapf(err, "wait pod Ready") + } + + return nil +} + +// IsPodReady returns if pod is Ready and verbose reason. +func IsPodReady(pod *core.Pod) (ready bool, reason string) { + if pod.Status.Phase != core.PodRunning { + return false, fmt.Sprintf("pod %q in %q namespace is not Running: %+v", pod.Name, pod.Namespace, pod.Status) + } + for _, c := range pod.Status.Conditions { + if c.Type == core.PodReady { + if c.Status != core.ConditionTrue { + return false, fmt.Sprintf("pod %q in %q namespace is not Ready: %+v", pod.Name, pod.Namespace, c) + } + return true, fmt.Sprintf("pod %q in %q namespace is Ready: %+v", pod.Name, pod.Namespace, c) + } + } + return false, fmt.Sprintf("pod %q in %q namespace does not have %q status: %+v", pod.Name, pod.Namespace, core.PodReady, pod.Status) +} diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index ad0348aa40..34194b9540 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -36,6 +36,7 @@ import ( "github.com/docker/machine/libmachine" "github.com/docker/machine/libmachine/state" "github.com/pkg/errors" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" @@ -470,6 +471,12 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time return nil } + if cfg.VerifyComponents[kverify.ExtraKey] { + if err := kverify.WaitExtra(client, kverify.CorePodsList, timeout); err != nil { + return errors.Wrap(err, "extra waiting") + } + } + cr, err := cruntime.New(cruntime.Config{Type: cfg.KubernetesConfig.ContainerRuntime, Runner: k.c}) if err != nil { return errors.Wrapf(err, "create runtme-manager %s", cfg.KubernetesConfig.ContainerRuntime) @@ -504,11 +511,11 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time } } } + if cfg.VerifyComponents[kverify.KubeletKey] { if err := kverify.WaitForService(k.c, "kubelet", timeout); err != nil { return errors.Wrap(err, "waiting for kubelet") } - } if cfg.VerifyComponents[kverify.NodeReadyKey] { @@ -658,6 +665,35 @@ func (k *Bootstrapper) restartControlPlane(cfg config.ClusterConfig) error { } } + if cfg.VerifyComponents[kverify.ExtraKey] { + // after kubelet is restarted (with 'kubeadm init phase kubelet-start' above), + // it appears as to be immediately Ready as well as all kube-system pods, + // then (after ~10sec) it realises it has some changes to apply, implying also pods restarts, + // and by that time we would exit completely, so we wait until kubelet begins restarting pods + klog.Info("waiting for restarted kubelet to initialise ...") + start := time.Now() + wait := func() error { + pods, err := client.CoreV1().Pods("kube-system").List(meta.ListOptions{}) + if err != nil { + return err + } + for _, pod := range pods.Items { + if pod.Labels["tier"] == "control-plane" { + if ready, _ := kverify.IsPodReady(&pod); !ready { + return nil + } + } + } + return fmt.Errorf("kubelet not initialised") + } + _ = retry.Expo(wait, 250*time.Millisecond, 1*time.Minute) + klog.Infof("kubelet initialised") + klog.Infof("duration metric: took %s waiting for restarted kubelet to initialise ...", time.Since(start)) + if err := kverify.WaitExtra(client, kverify.CorePodsList, kconst.DefaultControlPlaneTimeout); err != nil { + return errors.Wrap(err, "extra") + } + } + cr, err := cruntime.New(cruntime.Config{Type: cfg.KubernetesConfig.ContainerRuntime, Runner: k.c}) if err != nil { return errors.Wrap(err, "runtime") @@ -698,6 +734,7 @@ func (k *Bootstrapper) restartControlPlane(cfg config.ClusterConfig) error { if err := bsutil.AdjustResourceLimits(k.c); err != nil { klog.Warningf("unable to adjust resource limits: %v", err) } + return nil } diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go index ba9075051d..a201aaffb0 100644 --- a/pkg/minikube/cruntime/cruntime.go +++ b/pkg/minikube/cruntime/cruntime.go @@ -151,9 +151,12 @@ func New(c Config) (Manager, error) { switch c.Type { case "", "docker": return &Docker{ - Socket: c.Socket, - Runner: c.Runner, - Init: sm, + Socket: c.Socket, + Runner: c.Runner, + ImageRepository: c.ImageRepository, + KubernetesVersion: c.KubernetesVersion, + Init: sm, + UseCRI: (c.Socket != ""), // !dockershim }, nil case "crio", "cri-o": return &CRIO{ diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go index dddb1a0e72..6ce546e9fc 100644 --- a/pkg/minikube/cruntime/docker.go +++ b/pkg/minikube/cruntime/docker.go @@ -23,6 +23,7 @@ import ( "strings" "time" + "github.com/blang/semver" "github.com/pkg/errors" "k8s.io/klog/v2" "k8s.io/minikube/pkg/minikube/assets" @@ -56,9 +57,12 @@ func (e *ErrISOFeature) Error() string { // Docker contains Docker runtime state type Docker struct { - Socket string - Runner CommandRunner - Init sysinit.Manager + Socket string + Runner CommandRunner + ImageRepository string + KubernetesVersion semver.Version + Init sysinit.Manager + UseCRI bool } // Name is a human readable name for Docker @@ -181,6 +185,14 @@ func (r *Docker) CGroupDriver() (string, error) { // KubeletOptions returns kubelet options for a runtime. func (r *Docker) KubeletOptions() map[string]string { + if r.UseCRI { + return map[string]string{ + "container-runtime": "remote", + "container-runtime-endpoint": r.SocketPath(), + "image-service-endpoint": r.SocketPath(), + "runtime-request-timeout": "15m", + } + } return map[string]string{ "container-runtime": "docker", } @@ -188,6 +200,9 @@ func (r *Docker) KubeletOptions() map[string]string { // ListContainers returns a list of containers func (r *Docker) ListContainers(o ListOptions) ([]string, error) { + if r.UseCRI { + return listCRIContainers(r.Runner, "", o) + } args := []string{"ps"} switch o.State { case All: @@ -220,6 +235,9 @@ func (r *Docker) ListContainers(o ListOptions) ([]string, error) { // KillContainers forcibly removes a running container based on ID func (r *Docker) KillContainers(ids []string) error { + if r.UseCRI { + return killCRIContainers(r.Runner, ids) + } if len(ids) == 0 { return nil } @@ -234,6 +252,9 @@ func (r *Docker) KillContainers(ids []string) error { // StopContainers stops a running container based on ID func (r *Docker) StopContainers(ids []string) error { + if r.UseCRI { + return stopCRIContainers(r.Runner, ids) + } if len(ids) == 0 { return nil } @@ -248,6 +269,9 @@ func (r *Docker) StopContainers(ids []string) error { // PauseContainers pauses a running container based on ID func (r *Docker) PauseContainers(ids []string) error { + if r.UseCRI { + return pauseCRIContainers(r.Runner, "", ids) + } if len(ids) == 0 { return nil } @@ -262,6 +286,9 @@ func (r *Docker) PauseContainers(ids []string) error { // UnpauseContainers unpauses a container based on ID func (r *Docker) UnpauseContainers(ids []string) error { + if r.UseCRI { + return unpauseCRIContainers(r.Runner, "", ids) + } if len(ids) == 0 { return nil } @@ -276,6 +303,9 @@ func (r *Docker) UnpauseContainers(ids []string) error { // ContainerLogCmd returns the command to retrieve the log for a container based on ID func (r *Docker) ContainerLogCmd(id string, len int, follow bool) string { + if r.UseCRI { + return criContainerLogCmd(r.Runner, id, len, follow) + } var cmd strings.Builder cmd.WriteString("docker logs ") if len > 0 { diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index 42f9395e69..85e1d28488 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -245,6 +245,7 @@ func Provision(cc *config.ClusterConfig, n *config.Node, apiServer bool, delOnFa func configureRuntimes(runner cruntime.CommandRunner, cc config.ClusterConfig, kv semver.Version) cruntime.Manager { co := cruntime.Config{ Type: cc.KubernetesConfig.ContainerRuntime, + Socket: cc.KubernetesConfig.CRISocket, Runner: runner, ImageRepository: cc.KubernetesConfig.ImageRepository, KubernetesVersion: kv, diff --git a/site/content/en/docs/contrib/triage.md b/site/content/en/docs/contrib/triage.md index bc35a25bcd..e8627516de 100644 --- a/site/content/en/docs/contrib/triage.md +++ b/site/content/en/docs/contrib/triage.md @@ -8,7 +8,9 @@ description: > --- Community triage takes place **every Wednesday** from **11AM-12PM PST**. -Hangouts link: https://meet.google.com/ikf-fvrs-eer + +- Hangouts link: https://meet.google.com/ikf-fvrs-eer +- Google Group: https://groups.google.com/forum/#!forum/minikube-dev All community members are welcome and encouraged to join and help us triage minikube! diff --git a/site/content/en/docs/handbook/accessing.md b/site/content/en/docs/handbook/accessing.md index cb50e6f8a7..09a77f9a2c 100644 --- a/site/content/en/docs/handbook/accessing.md +++ b/site/content/en/docs/handbook/accessing.md @@ -25,7 +25,7 @@ A NodePort service is the most basic way to get external traffic directly to you We also have a shortcut for fetching the minikube IP and a service's `NodePort`: ```shell -minikube service --url $SERVICE +minikube service --url ``` ## Getting the NodePort using kubectl @@ -35,7 +35,7 @@ The minikube VM is exposed to the host system via a host-only IP address, that c To determine the NodePort for your service, you can use a `kubectl` command like this (note that `nodePort` begins with lowercase `n` in JSON output): ```shell -kubectl get service $SERVICE --output='jsonpath="{.spec.ports[0].nodePort}"' +kubectl get service --output='jsonpath="{.spec.ports[0].nodePort}"' ``` ### Increasing the NodePort range diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go index c63c6f94a8..d29a8f7916 100644 --- a/test/integration/functional_test.go +++ b/test/integration/functional_test.go @@ -256,7 +256,6 @@ func validateDockerEnv(ctx context.Context, t *testing.T, profile string) { if !strings.Contains(rr.Output(), expectedImgInside) { t.Fatalf("expected 'docker images' to have %q inside minikube. but the output is: *%s*", expectedImgInside, rr.Output()) } - } func validateStartWithProxy(ctx context.Context, t *testing.T, profile string) { @@ -269,7 +268,7 @@ func validateStartWithProxy(ctx context.Context, t *testing.T, profile string) { // Use more memory so that we may reliably fit MySQL and nginx // changing api server so later in soft start we verify it didn't change - startArgs := append([]string{"start", "-p", profile, "--memory=4000", fmt.Sprintf("--apiserver-port=%d", apiPortTest), "--wait=true"}, StartArgs()...) + startArgs := append([]string{"start", "-p", profile, "--memory=4000", fmt.Sprintf("--apiserver-port=%d", apiPortTest), "--wait=all"}, StartArgs()...) c := exec.CommandContext(ctx, Target(), startArgs...) env := os.Environ() env = append(env, fmt.Sprintf("HTTP_PROXY=%s", srv.Addr)) @@ -401,7 +400,6 @@ func validateMinikubeKubectlDirectCall(ctx context.Context, t *testing.T, profil if err != nil { t.Fatalf("failed to run kubectl directly. args %q: %v", rr.Command(), err) } - } func validateExtraConfig(ctx context.Context, t *testing.T, profile string) { @@ -409,7 +407,7 @@ func validateExtraConfig(ctx context.Context, t *testing.T, profile string) { start := time.Now() // The tests before this already created a profile, starting minikube with different --extra-config cmdline option. - startArgs := []string{"start", "-p", profile, "--extra-config=apiserver.enable-admission-plugins=NamespaceAutoProvision"} + startArgs := []string{"start", "-p", profile, "--extra-config=apiserver.enable-admission-plugins=NamespaceAutoProvision", "--wait=all"} c := exec.CommandContext(ctx, Target(), startArgs...) rr, err := Run(t, c) if err != nil { @@ -427,7 +425,6 @@ func validateExtraConfig(ctx context.Context, t *testing.T, profile string) { if !strings.Contains(afterCfg.Config.KubernetesConfig.ExtraOptions.String(), expectedExtraOptions) { t.Errorf("expected ExtraOptions to contain %s but got %s", expectedExtraOptions, afterCfg.Config.KubernetesConfig.ExtraOptions.String()) } - } // imageID returns a docker image id for image `image` and current architecture @@ -451,6 +448,7 @@ func imageID(image string) string { } // validateComponentHealth asserts that all Kubernetes components are healthy +// note: it expects all components to be Ready, so it makes sense to run it close after only those tests that include '--wait=all' start flag (ie, with extra wait) func validateComponentHealth(ctx context.Context, t *testing.T, profile string) { defer PostMortemLogs(t, profile) @@ -474,12 +472,22 @@ func validateComponentHealth(ctx context.Context, t *testing.T, profile string) for _, i := range cs.Items { for _, l := range i.Labels { - t.Logf("%s phase: %s", l, i.Status.Phase) - _, ok := found[l] - if ok { + if _, ok := found[l]; ok { // skip irrelevant (eg, repeating/redundant '"tier": "control-plane"') labels found[l] = true - if i.Status.Phase != "Running" { + t.Logf("%s phase: %s", l, i.Status.Phase) + if i.Status.Phase != api.PodRunning { t.Errorf("%s is not Running: %+v", l, i.Status) + continue + } + for _, c := range i.Status.Conditions { + if c.Type == api.PodReady { + if c.Status != api.ConditionTrue { + t.Errorf("%s is not Ready: %+v", l, i.Status) + } else { + t.Logf("%s status: %s", l, c.Type) + } + break + } } } } diff --git a/test/integration/json_output_test.go b/test/integration/json_output_test.go index e91147f231..f120f64279 100644 --- a/test/integration/json_output_test.go +++ b/test/integration/json_output_test.go @@ -155,7 +155,7 @@ func TestErrorJSONOutput(t *testing.T) { t.Fatalf("last cloud event is not of type error: %v", last) } last.validateData(t, "exitcode", fmt.Sprintf("%v", reason.ExDriverUnsupported)) - last.validateData(t, "message", fmt.Sprintf("The driver 'fail' is not supported on %s", runtime.GOOS)) + last.validateData(t, "message", fmt.Sprintf("The driver 'fail' is not supported on %s/%s", runtime.GOOS, runtime.GOARCH)) } type cloudEvent struct { diff --git a/test/integration/skaffold_test.go b/test/integration/skaffold_test.go index 7695f9dc77..0365b33177 100644 --- a/test/integration/skaffold_test.go +++ b/test/integration/skaffold_test.go @@ -95,7 +95,7 @@ func TestSkaffold(t *testing.T) { }() // make sure "skaffold run" exits without failure - cmd := exec.CommandContext(ctx, tf.Name(), "run", "--minikube-profile", profile, "--kube-context", profile, "--status-check=true", "--port-forward=false") + cmd := exec.CommandContext(ctx, tf.Name(), "run", "--minikube-profile", profile, "--kube-context", profile, "--status-check=true", "--port-forward=false", "--interactive=false") cmd.Dir = "testdata/skaffold" rr, err = Run(t, cmd) if err != nil { diff --git a/test/integration/start_stop_delete_test.go b/test/integration/start_stop_delete_test.go index 8154b53449..fec0c8d014 100644 --- a/test/integration/start_stop_delete_test.go +++ b/test/integration/start_stop_delete_test.go @@ -79,7 +79,7 @@ func TestStartStop(t *testing.T) { t.Run(tc.name, func(t *testing.T) { MaybeParallel(t) profile := UniqueProfileName(tc.name) - ctx, cancel := context.WithTimeout(context.Background(), Minutes(40)) + ctx, cancel := context.WithTimeout(context.Background(), Minutes(30)) defer Cleanup(t, profile, cancel) type validateStartStopFunc func(context.Context, *testing.T, string, string, string, []string) if !strings.Contains(tc.name, "docker") && NoneDriver() {