From d3bdac0379531b92dea8934fd93eb92ec27dfcb1 Mon Sep 17 00:00:00 2001 From: Predrag Rogic Date: Tue, 13 Dec 2022 21:16:25 +0000 Subject: [PATCH] fix containerd when bound to docker --- .../containerd-bin-aarch64/config.toml | 2 +- .../x86_64/package/containerd-bin/config.toml | 2 +- deploy/kicbase/containerd.toml | 2 +- pkg/minikube/cruntime/containerd.go | 3 +- pkg/minikube/cruntime/docker.go | 8 ++++++ test/integration/net_test.go | 28 ++++++++++++++++--- 6 files changed, 37 insertions(+), 8 deletions(-) diff --git a/deploy/iso/minikube-iso/arch/aarch64/package/containerd-bin-aarch64/config.toml b/deploy/iso/minikube-iso/arch/aarch64/package/containerd-bin-aarch64/config.toml index d5de73eae4..487a5a0a21 100644 --- a/deploy/iso/minikube-iso/arch/aarch64/package/containerd-bin-aarch64/config.toml +++ b/deploy/iso/minikube-iso/arch/aarch64/package/containerd-bin-aarch64/config.toml @@ -54,7 +54,7 @@ oom_score = 0 [plugins."io.containerd.grpc.v1.cri".cni] bin_dir = "/opt/cni/bin" - conf_dir = "/etc/cni/net.mk" + conf_dir = "/etc/cni/net.d" conf_template = "" [plugins."io.containerd.grpc.v1.cri".registry] config_path = "/etc/containerd/certs.d" diff --git a/deploy/iso/minikube-iso/arch/x86_64/package/containerd-bin/config.toml b/deploy/iso/minikube-iso/arch/x86_64/package/containerd-bin/config.toml index e63ad23c34..20abf3e760 100644 --- a/deploy/iso/minikube-iso/arch/x86_64/package/containerd-bin/config.toml +++ b/deploy/iso/minikube-iso/arch/x86_64/package/containerd-bin/config.toml @@ -54,7 +54,7 @@ oom_score = 0 [plugins."io.containerd.grpc.v1.cri".cni] bin_dir = "/opt/cni/bin" - conf_dir = "/etc/cni/net.mk" + conf_dir = "/etc/cni/net.d" conf_template = "" [plugins."io.containerd.grpc.v1.cri".registry] config_path = "/etc/containerd/certs.d" diff --git a/deploy/kicbase/containerd.toml b/deploy/kicbase/containerd.toml index 98b902d7a1..477645de9c 100644 --- a/deploy/kicbase/containerd.toml +++ b/deploy/kicbase/containerd.toml @@ -54,7 +54,7 @@ oom_score = 0 [plugins."io.containerd.grpc.v1.cri".cni] bin_dir = "/opt/cni/bin" - conf_dir = "/etc/cni/net.mk" + conf_dir = "/etc/cni/net.d" conf_template = "" [plugins."io.containerd.grpc.v1.cri".registry] config_path = "/etc/containerd/certs.d" diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go index 476eaaa4a1..c3035192aa 100644 --- a/pkg/minikube/cruntime/containerd.go +++ b/pkg/minikube/cruntime/containerd.go @@ -175,6 +175,7 @@ func generateContainerdConfig(cr CommandRunner, imageRepository string, kv semve } // Enable idempotently enables containerd on a host +// It is also called by docker.Enable() - if bound to containerd, to enforce proper containerd configuration completed by service restart. func (r *Containerd) Enable(disOthers, forceSystemd, inUserNamespace bool) error { if inUserNamespace { if err := CheckKernelCompatibility(r.Runner, 5, 11); err != nil { @@ -509,7 +510,7 @@ func (r *Containerd) Preload(cc config.ClusterConfig) error { return r.Restart() } -// Restart restarts Docker on a host +// Restart restarts this container runtime on a host func (r *Containerd) Restart() error { return r.Init.Restart("containerd") } diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go index 81aaa65fb5..33a861d559 100644 --- a/pkg/minikube/cruntime/docker.go +++ b/pkg/minikube/cruntime/docker.go @@ -137,6 +137,14 @@ func (r *Docker) Enable(disOthers, forceSystemd, inUserNamespace bool) error { klog.Warningf("disableOthers: %v", err) } } + // check if containerd is still alive (hopefully because being being bound to docker), and if so: ensure it's configured properly by calling its Enable() method + cdr, errCdR := New(Config{Type: "containerd", Runner: r.Runner}) + if errCdR == nil && cdr.Active() { + errCdR = cdr.Enable(false, forceSystemd, inUserNamespace) + } + if errCdR != nil { + klog.Warningf("cannot ensure containerd (as bound to docker) is configured properly and reloaded - cluster might be unstable: %v", errCdR) + } if err := populateCRIConfig(r.Runner, r.SocketPath()); err != nil { return err diff --git a/test/integration/net_test.go b/test/integration/net_test.go index 03f7f12de4..e24087c6d8 100644 --- a/test/integration/net_test.go +++ b/test/integration/net_test.go @@ -357,6 +357,10 @@ func debugLogs(profile string) string { out, _ = cmd.CombinedOutput() output.WriteString(fmt.Sprintf("\n>>> k8s: cms:\n%s\n", out)) + cmd = exec.Command("kubectl", "--context", profile, "get", "node", "-A", "-owide") + out, _ = cmd.CombinedOutput() + output.WriteString(fmt.Sprintf("\n>>> k8s: nodes:\n%s\n", out)) + cmd = exec.Command("kubectl", "--context", profile, "get", "svc", "-A", "-owide") out, _ = cmd.CombinedOutput() output.WriteString(fmt.Sprintf("\n>>> k8s: svcs:\n%s\n", out)) @@ -369,10 +373,18 @@ func debugLogs(profile string) string { out, _ = cmd.CombinedOutput() output.WriteString(fmt.Sprintf("\n>>> k8s: pods:\n%s\n", out)) + cmd = exec.Command(Target(), "ssh", "-p", profile, "sudo crictl pods") + out, _ = cmd.CombinedOutput() + output.WriteString(fmt.Sprintf("\n>>> host: crictl pods:\n%s\n", out)) + cmd = exec.Command(Target(), "ssh", "-p", profile, "sudo cat /etc/systemd/system/kubelet.service.d/10-kubeadm.conf") out, _ = cmd.CombinedOutput() output.WriteString(fmt.Sprintf("\n>>> host: /etc/systemd/system/kubelet.service.d/10-kubeadm.conf:\n%s\n", out)) + cmd = exec.Command(Target(), "ssh", "-p", profile, "sudo cat /etc/kubernetes/kubelet.conf") + out, _ = cmd.CombinedOutput() + output.WriteString(fmt.Sprintf("\n>>> host: /etc/kubernetes/kubelet.conf:\n%s\n", out)) + cmd = exec.Command(Target(), "ssh", "-p", profile, "sudo cat /var/lib/kubelet/config.yaml") out, _ = cmd.CombinedOutput() output.WriteString(fmt.Sprintf("\n>>> host: /var/lib/kubelet/config.yaml:\n%s\n", out)) @@ -397,14 +409,14 @@ func debugLogs(profile string) string { out, _ = cmd.CombinedOutput() output.WriteString(fmt.Sprintf("\n>>> host: cri daemon:\n%s\n", out)) - cmd = exec.Command(Target(), "ssh", "-p", profile, "sudo cat /usr/lib/systemd/system/cri-docker.service") - out, _ = cmd.CombinedOutput() - output.WriteString(fmt.Sprintf("\n>>> host: /usr/lib/systemd/system/cri-docker.service:\n%s\n", out)) - cmd = exec.Command(Target(), "ssh", "-p", profile, "sudo cat /etc/systemd/system/cri-docker.service.d/10-cni.conf") out, _ = cmd.CombinedOutput() output.WriteString(fmt.Sprintf("\n>>> host: /etc/systemd/system/cri-docker.service.d/10-cni.conf:\n%s\n", out)) + cmd = exec.Command(Target(), "ssh", "-p", profile, "sudo cat /usr/lib/systemd/system/cri-docker.service") + out, _ = cmd.CombinedOutput() + output.WriteString(fmt.Sprintf("\n>>> host: /usr/lib/systemd/system/cri-docker.service:\n%s\n", out)) + cmd = exec.Command(Target(), "ssh", "-p", profile, "sudo cri-dockerd --version") out, _ = cmd.CombinedOutput() output.WriteString(fmt.Sprintf("\n>>> host: cri-dockerd version:\n%s\n", out)) @@ -413,6 +425,10 @@ func debugLogs(profile string) string { out, _ = cmd.CombinedOutput() output.WriteString(fmt.Sprintf("\n>>> host: containerd daemon:\n%s\n", out)) + cmd = exec.Command(Target(), "ssh", "-p", profile, "sudo cat /lib/systemd/system/containerd.service") + out, _ = cmd.CombinedOutput() + output.WriteString(fmt.Sprintf("\n>>> host: /lib/systemd/system/containerd.service:\n%s\n", out)) + cmd = exec.Command(Target(), "ssh", "-p", profile, "sudo cat /etc/containerd/config.toml") out, _ = cmd.CombinedOutput() output.WriteString(fmt.Sprintf("\n>>> host: /etc/containerd/config.toml:\n%s\n", out)) @@ -461,5 +477,9 @@ func debugLogs(profile string) string { out, _ = cmd.CombinedOutput() output.WriteString(fmt.Sprintf("\n>>> k8s: kubelet logs:\n%s\n", out)) + cmd = exec.Command(Target(), "ssh", "-p", profile, "sudo cat /run/flannel/subnet.env") + out, _ = cmd.CombinedOutput() + output.WriteString(fmt.Sprintf("\n>>> host: /run/flannel/subnet.env:\n%s\n", out)) + return output.String() }