From e4a8e59cfa9d1ab87da7b76658a559c3d3884f26 Mon Sep 17 00:00:00 2001 From: Dean Coakley Date: Wed, 5 Aug 2020 02:06:06 +0100 Subject: [PATCH 1/8] Add prune docker images instructions --- pkg/minikube/machine/advice.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/minikube/machine/advice.go b/pkg/minikube/machine/advice.go index dc08d9a667..a7d4cd7787 100644 --- a/pkg/minikube/machine/advice.go +++ b/pkg/minikube/machine/advice.go @@ -40,7 +40,8 @@ func MaybeDisplayAdvice(err error, driver string) { if errors.Is(err, oci.ErrExitedUnexpectedly) || errors.Is(err, oci.ErrDaemonInfo) { out.T(style.Tip, "If you are still interested to make {{.driver_name}} driver work. The following suggestions might help you get passed this issue:", out.V{"driver_name": driver}) out.T(style.Empty, ` - - Prune unused {{.driver_name}} images, volumes and abandoned containers.`, out.V{"driver_name": driver}) + - Prune unused {{.driver_name}} images, volumes, networks and abandoned containers. + docker system prune --volumes`, out.V{"driver_name": driver}) out.T(style.Empty, ` - Restart your {{.driver_name}} service`, out.V{"driver_name": driver}) if runtime.GOOS != "linux" { From 481cb50f7492ab108cad8ce003075844d5462ad0 Mon Sep 17 00:00:00 2001 From: Dean Coakley Date: Tue, 1 Sep 2020 15:05:10 +0100 Subject: [PATCH 2/8] Add leading newline for clarity --- pkg/minikube/machine/advice.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/minikube/machine/advice.go b/pkg/minikube/machine/advice.go index a7d4cd7787..0f802567ae 100644 --- a/pkg/minikube/machine/advice.go +++ b/pkg/minikube/machine/advice.go @@ -41,6 +41,7 @@ func MaybeDisplayAdvice(err error, driver string) { out.T(style.Tip, "If you are still interested to make {{.driver_name}} driver work. The following suggestions might help you get passed this issue:", out.V{"driver_name": driver}) out.T(style.Empty, ` - Prune unused {{.driver_name}} images, volumes, networks and abandoned containers. + docker system prune --volumes`, out.V{"driver_name": driver}) out.T(style.Empty, ` - Restart your {{.driver_name}} service`, out.V{"driver_name": driver}) From dcd00b2150f3555c3a52bd6250af880790185a90 Mon Sep 17 00:00:00 2001 From: Dean Coakley Date: Tue, 8 Sep 2020 23:37:25 +0100 Subject: [PATCH 3/8] Make prune conditional. Add podman to prune instruction --- pkg/minikube/machine/advice.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/minikube/machine/advice.go b/pkg/minikube/machine/advice.go index 0f802567ae..268ab56788 100644 --- a/pkg/minikube/machine/advice.go +++ b/pkg/minikube/machine/advice.go @@ -39,10 +39,12 @@ func MaybeDisplayAdvice(err error, driver string) { if errors.Is(err, oci.ErrExitedUnexpectedly) || errors.Is(err, oci.ErrDaemonInfo) { out.T(style.Tip, "If you are still interested to make {{.driver_name}} driver work. The following suggestions might help you get passed this issue:", out.V{"driver_name": driver}) - out.T(style.Empty, ` + if driver == oci.Docker || driver == oci.Podman { + out.T(style.Empty, ` - Prune unused {{.driver_name}} images, volumes, networks and abandoned containers. - docker system prune --volumes`, out.V{"driver_name": driver}) + {{.driver_name}} system prune --volumes`, out.V{"driver_name": driver}) + } out.T(style.Empty, ` - Restart your {{.driver_name}} service`, out.V{"driver_name": driver}) if runtime.GOOS != "linux" { From bc182ce207a1f148182d4c15d009f257dcf348d4 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Thu, 10 Sep 2020 12:58:14 -0700 Subject: [PATCH 4/8] remove docker hub images --- test/integration/functional_test.go | 34 ++++++++++++++--------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go index 3fbbfdb26f..a419f25e08 100644 --- a/test/integration/functional_test.go +++ b/test/integration/functional_test.go @@ -496,11 +496,11 @@ func validateCacheCmd(ctx context.Context, t *testing.T, profile string) { } t.Run("cache", func(t *testing.T) { - t.Run("add", func(t *testing.T) { - for _, img := range []string{"busybox:latest", "busybox:1.28.4-glibc", "k8s.gcr.io/pause:latest"} { + t.Run("add_remote", func(t *testing.T) { + for _, img := range []string{"k8s.gcr.io/pause:3.0", "k8s.gcr.io/pause:3.3", "k8s.gcr.io/pause:latest"} { rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "cache", "add", img)) if err != nil { - t.Errorf("failed to cache add image %q. args %q err %v", img, rr.Command(), err) + t.Errorf("failed to 'cache add' remote image %q. args %q err %v", img, rr.Command(), err) } } }) @@ -514,7 +514,7 @@ func validateCacheCmd(ctx context.Context, t *testing.T, profile string) { message := []byte("FROM scratch\nADD Dockerfile /x") err = ioutil.WriteFile(filepath.Join(dname, "Dockerfile"), message, 0644) if err != nil { - t.Fatalf("unable to writefile: %v", err) + t.Fatalf("unable to write Dockerfile: %v", err) } img := "minikube-local-cache-test:" + profile @@ -525,14 +525,14 @@ func validateCacheCmd(ctx context.Context, t *testing.T, profile string) { rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "cache", "add", img)) if err != nil { - t.Errorf("failed to add local image %q. args %q err %v", img, rr.Command(), err) + t.Errorf("failed to 'cache add' local image %q. args %q err %v", img, rr.Command(), err) } }) - t.Run("delete_busybox:1.28.4-glibc", func(t *testing.T) { - rr, err := Run(t, exec.CommandContext(ctx, Target(), "cache", "delete", "busybox:1.28.4-glibc")) + t.Run("delete_k8s.gcr.io/pause:3.3", func(t *testing.T) { + rr, err := Run(t, exec.CommandContext(ctx, Target(), "cache", "delete", "k8s.gcr.io/pause:3.3")) if err != nil { - t.Errorf("failed to delete image busybox:1.28.4-glibc from cache. args %q: %v", rr.Command(), err) + t.Errorf("failed to delete image k8s.gcr.io/pause:3.3 from cache. args %q: %v", rr.Command(), err) } }) @@ -542,10 +542,10 @@ func validateCacheCmd(ctx context.Context, t *testing.T, profile string) { t.Errorf("failed to do cache list. args %q: %v", rr.Command(), err) } if !strings.Contains(rr.Output(), "k8s.gcr.io/pause") { - t.Errorf("expected 'cache list' output to include 'k8s.gcr.io/pause' but got:\n ***%s***", rr.Output()) + t.Errorf("expected 'cache list' output to include 'k8s.gcr.io/pause' but got: ***%s***", rr.Output()) } - if strings.Contains(rr.Output(), "busybox:1.28.4-glibc") { - t.Errorf("expected 'cache list' output not to include busybox:1.28.4-glibc but got:\n ***%s***", rr.Output()) + if strings.Contains(rr.Output(), "k8s.gcr.io/pause:3.3") { + t.Errorf("expected 'cache list' output not to include k8s.gcr.io/pause:3.3 but got: ***%s***", rr.Output()) } }) @@ -554,24 +554,24 @@ func validateCacheCmd(ctx context.Context, t *testing.T, profile string) { if err != nil { t.Errorf("failed to get images by %q ssh %v", rr.Command(), err) } - if !strings.Contains(rr.Output(), "1.28.4-glibc") { - t.Errorf("expected '1.28.4-glibc' to be in the output but got *%s*", rr.Output()) + if !strings.Contains(rr.Output(), "0184c1613d929") { + t.Errorf("expected sha for pause:3.3 '0184c1613d929' to be in the output but got *%s*", rr.Output()) } }) t.Run("cache_reload", func(t *testing.T) { // deleting image inside minikube node manually and expecting reload to bring it back - img := "busybox:latest" + img := "k8s.gcr.io/pause:latest" // deleting image inside minikube node manually rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", "sudo", "docker", "rmi", img)) if err != nil { - t.Errorf("failed to delete inside the node %q : %v", rr.Command(), err) + t.Errorf("failed to manualy delete image %q : %v", rr.Command(), err) } // make sure the image is deleted. rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", "sudo", "crictl", "inspecti", img)) if err == nil { - t.Errorf("expected an error. because image should not exist. but got *nil error* ! cmd: %q", rr.Command()) + t.Errorf("expected an error but got no error. image should not exist. ! cmd: %q", rr.Command()) } // minikube cache reload. rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "cache", "reload")) @@ -587,7 +587,7 @@ func validateCacheCmd(ctx context.Context, t *testing.T, profile string) { // delete will clean up the cached images since they are global and all other tests will load it for no reason t.Run("delete", func(t *testing.T) { - for _, img := range []string{"busybox:latest", "k8s.gcr.io/pause:latest"} { + for _, img := range []string{"k8s.gcr.io/pause:3.0", "k8s.gcr.io/pause:latest"} { rr, err := Run(t, exec.CommandContext(ctx, Target(), "cache", "delete", img)) if err != nil { t.Errorf("failed to delete %s from cache. args %q: %v", img, rr.Command(), err) From b4d4be8e23e48518bbdfdff26ff8d0ad160c6ded Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sun, 13 Sep 2020 23:03:36 +0200 Subject: [PATCH 5/8] Add podman storage warning to match docker --- pkg/minikube/machine/start.go | 20 ++++++++++++++++++-- pkg/minikube/reason/reason.go | 9 +++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/pkg/minikube/machine/start.go b/pkg/minikube/machine/start.go index ae0bebc0b8..81461f4880 100644 --- a/pkg/minikube/machine/start.go +++ b/pkg/minikube/machine/start.go @@ -36,6 +36,7 @@ import ( "github.com/juju/mutex" "github.com/pkg/errors" "github.com/spf13/viper" + "k8s.io/minikube/pkg/drivers/kic/oci" "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/constants" @@ -218,17 +219,32 @@ func postStartValidations(h *host.Host, drvName string) { glog.Warningf("error getting command runner: %v", err) } + var kind reason.Kind + var name string + if drvName == oci.Docker { + kind = reason.RsrcInsufficientDockerStorage + name = "Docker" + } + if drvName == oci.Podman { + kind = reason.RsrcInsufficientPodmanStorage + name = "Podman" + } + if name == "" { + glog.Warningf("unknown KIC driver: %v", drvName) + return + } + // make sure /var isn't full, as pod deployments will fail if it is percentageFull, err := DiskUsed(r, "/var") if err != nil { glog.Warningf("error getting percentage of /var that is free: %v", err) } if percentageFull >= 99 { - exit.Message(reason.RsrcInsufficientDockerStorage, `Docker is out of disk space! (/var is at {{.p}}% of capacity)`, out.V{"p": percentageFull}) + exit.Message(kind, `{{.n}} is out of disk space! (/var is at {{.p}}% of capacity)`, out.V{"n": name, "p": percentageFull}) } if percentageFull >= 85 { - out.WarnReason(reason.RsrcInsufficientDockerStorage, `Docker is nearly out of disk space, which may cause deployments to fail! ({{.p}}% of capacity)`, out.V{"p": percentageFull}) + out.WarnReason(kind, `{{.n}} is nearly out of disk space, which may cause deployments to fail! ({{.p}}% of capacity)`, out.V{"n": name, "p": percentageFull}) } } diff --git a/pkg/minikube/reason/reason.go b/pkg/minikube/reason/reason.go index 21b4bb8b95..ee9a3faf6b 100644 --- a/pkg/minikube/reason/reason.go +++ b/pkg/minikube/reason/reason.go @@ -176,6 +176,15 @@ var ( 3. Run "minikube ssh -- docker system prune" if using the docker container runtime`, Issues: []int{9024}, } + RsrcInsufficientPodmanStorage = Kind{ + ID: "RSRC_PODMAN_STORAGE", + ExitCode: ExInsufficientStorage, + Advice: `Try at least one of the following to free up space on the device: + + 1. Run "sudo podman system prune" to remove unused podman data + 2. Run "minikube ssh -- docker system prune" if using the docker container runtime`, + Issues: []int{9024}, + } RsrcInsufficientStorage = Kind{ID: "RSRC_INSUFFICIENT_STORAGE", ExitCode: ExInsufficientStorage, Style: style.UnmetRequirement} From f04a7c5c16ad488940da5400c379907540484980 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Wed, 16 Sep 2020 15:01:15 -0700 Subject: [PATCH 6/8] spell --- test/integration/functional_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go index a419f25e08..baedabd59e 100644 --- a/test/integration/functional_test.go +++ b/test/integration/functional_test.go @@ -566,7 +566,7 @@ func validateCacheCmd(ctx context.Context, t *testing.T, profile string) { rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", "sudo", "docker", "rmi", img)) if err != nil { - t.Errorf("failed to manualy delete image %q : %v", rr.Command(), err) + t.Errorf("failed to manually delete image %q : %v", rr.Command(), err) } // make sure the image is deleted. rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", "sudo", "crictl", "inspecti", img)) From eab92dc4557fdea5ec4c091e876d6122d0d59220 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Thu, 17 Sep 2020 10:22:28 -0700 Subject: [PATCH 7/8] Reduce cyclotomic complexity of CreateContainerNode to fix lint --- pkg/drivers/kic/oci/oci.go | 46 ++++++++++++++------------------------ 1 file changed, 17 insertions(+), 29 deletions(-) diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go index ebc32b6af4..b5a98d877d 100644 --- a/pkg/drivers/kic/oci/oci.go +++ b/pkg/drivers/kic/oci/oci.go @@ -146,14 +146,31 @@ func CreateContainerNode(p CreateParams) error { "--label", p.NodeLabel, } + // https://www.freedesktop.org/wiki/Software/systemd/ContainerInterface/ + var virtualization string if p.OCIBinary == Podman { // enable execing in /var // podman mounts var/lib with no-exec by default https://github.com/containers/libpod/issues/5103 runArgs = append(runArgs, "--volume", fmt.Sprintf("%s:/var:exec", p.Name)) + if _, err := os.Stat("/sys/fs/cgroup/memory/memsw.limit_in_bytes"); runtime.GOOS == "linux" && os.IsNotExist(err) { + // requires CONFIG_MEMCG_SWAP_ENABLED or cgroup_enable=memory in grub + glog.Warning("Your kernel does not support swap limit capabilities or the cgroup is not mounted.") + } else { + runArgs = append(runArgs, fmt.Sprintf("--memory=%s", p.Memory)) + // Disable swap by setting the value to match + runArgs = append(runArgs, fmt.Sprintf("--memory-swap=%s", p.Memory)) + } + virtualization = "podman" // VIRTUALIZATION_PODMAN } if p.OCIBinary == Docker { runArgs = append(runArgs, "--volume", fmt.Sprintf("%s:/var", p.Name)) // ignore apparmore github actions docker: https://github.com/kubernetes/minikube/issues/7624 runArgs = append(runArgs, "--security-opt", "apparmor=unconfined") + + runArgs = append(runArgs, fmt.Sprintf("--memory=%s", p.Memory)) + // Disable swap by setting the value to match + runArgs = append(runArgs, fmt.Sprintf("--memory-swap=%s", p.Memory)) + + virtualization = "docker" // VIRTUALIZATION_DOCKER } cpuCfsPeriod := true @@ -175,35 +192,6 @@ func CreateContainerNode(p CreateParams) error { runArgs = append(runArgs, fmt.Sprintf("--cpus=%s", p.CPUs)) } - memcgSwap := true - if runtime.GOOS == "linux" { - if _, err := os.Stat("/sys/fs/cgroup/memory/memsw.limit_in_bytes"); os.IsNotExist(err) { - // requires CONFIG_MEMCG_SWAP_ENABLED or cgroup_enable=memory in grub - glog.Warning("Your kernel does not support swap limit capabilities or the cgroup is not mounted.") - memcgSwap = false - } - } - - if p.OCIBinary == Podman && memcgSwap { // swap is required for memory - runArgs = append(runArgs, fmt.Sprintf("--memory=%s", p.Memory)) - // Disable swap by setting the value to match - runArgs = append(runArgs, fmt.Sprintf("--memory-swap=%s", p.Memory)) - } - - if p.OCIBinary == Docker { - runArgs = append(runArgs, fmt.Sprintf("--memory=%s", p.Memory)) - // Disable swap by setting the value to match - runArgs = append(runArgs, fmt.Sprintf("--memory-swap=%s", p.Memory)) - } - - // https://www.freedesktop.org/wiki/Software/systemd/ContainerInterface/ - var virtualization string - if p.OCIBinary == Podman { - virtualization = "podman" // VIRTUALIZATION_PODMAN - } - if p.OCIBinary == Docker { - virtualization = "docker" // VIRTUALIZATION_DOCKER - } runArgs = append(runArgs, "-e", fmt.Sprintf("%s=%s", "container", virtualization)) for key, val := range p.Envs { From b98580c820125573a113d3a485bc76f558e87908 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Thu, 17 Sep 2020 10:31:10 -0700 Subject: [PATCH 8/8] fix up if statement --- pkg/drivers/kic/oci/oci.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go index b5a98d877d..878189e387 100644 --- a/pkg/drivers/kic/oci/oci.go +++ b/pkg/drivers/kic/oci/oci.go @@ -145,20 +145,27 @@ func CreateContainerNode(p CreateParams) error { // label th enode wuth the node ID "--label", p.NodeLabel, } + memcgSwap := true + if runtime.GOOS == "linux" { + if _, err := os.Stat("/sys/fs/cgroup/memory/memsw.limit_in_bytes"); os.IsNotExist(err) { + // requires CONFIG_MEMCG_SWAP_ENABLED or cgroup_enable=memory in grub + glog.Warning("Your kernel does not support swap limit capabilities or the cgroup is not mounted.") + memcgSwap = false + } + } // https://www.freedesktop.org/wiki/Software/systemd/ContainerInterface/ var virtualization string if p.OCIBinary == Podman { // enable execing in /var // podman mounts var/lib with no-exec by default https://github.com/containers/libpod/issues/5103 runArgs = append(runArgs, "--volume", fmt.Sprintf("%s:/var:exec", p.Name)) - if _, err := os.Stat("/sys/fs/cgroup/memory/memsw.limit_in_bytes"); runtime.GOOS == "linux" && os.IsNotExist(err) { - // requires CONFIG_MEMCG_SWAP_ENABLED or cgroup_enable=memory in grub - glog.Warning("Your kernel does not support swap limit capabilities or the cgroup is not mounted.") - } else { + + if memcgSwap { runArgs = append(runArgs, fmt.Sprintf("--memory=%s", p.Memory)) // Disable swap by setting the value to match runArgs = append(runArgs, fmt.Sprintf("--memory-swap=%s", p.Memory)) } + virtualization = "podman" // VIRTUALIZATION_PODMAN } if p.OCIBinary == Docker {