diff --git a/Makefile b/Makefile index bc8e254a2c..8f0ba120cb 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ KUBERNETES_VERSION ?= $(shell egrep "DefaultKubernetesVersion =" pkg/minikube/co KIC_VERSION ?= $(shell egrep "Version =" pkg/drivers/kic/types.go | cut -d \" -f2) # Default to .0 for higher cache hit rates, as build increments typically don't require new ISO versions -ISO_VERSION ?= v1.25.2-1648191624-11679 +ISO_VERSION ?= v1.25.2-1648142972-13800 # Dashes are valid in semver, but not Linux packaging. Use ~ to delimit alpha/beta DEB_VERSION ?= $(subst -,~,$(RAW_VERSION)) DEB_REVISION ?= 0 diff --git a/cmd/auto-pause/auto-pause.go b/cmd/auto-pause/auto-pause.go index b5d816a9bc..e877878e89 100644 --- a/cmd/auto-pause/auto-pause.go +++ b/cmd/auto-pause/auto-pause.go @@ -103,6 +103,7 @@ func runPause() { mu.Lock() defer mu.Unlock() if runtimePaused { + out.Styled(style.AddonEnable, "Auto-pause is already enabled.") return } diff --git a/deploy/iso/minikube-iso/package/crio-bin/crio-bin.hash b/deploy/iso/minikube-iso/package/crio-bin/crio-bin.hash index bd509b0e4e..82c62806a8 100644 --- a/deploy/iso/minikube-iso/package/crio-bin/crio-bin.hash +++ b/deploy/iso/minikube-iso/package/crio-bin/crio-bin.hash @@ -24,3 +24,5 @@ sha256 76fd7543bc92d4364a11060f43a5131893a76c6e6e9d6de3a6bb6292c110b631 v1.20.0. sha256 36d9f4cf4966342e2d4099e44d8156c55c6a10745c67ce4f856aa9f6dcc2d9ba v1.20.2.tar.gz sha256 bc53ea8977e252bd9812974c33ff654ee22076598e901464468c5c105a5ef773 v1.22.0.tar.gz sha256 6e1c0e393cd16af907fabb24e4cc068e27c606c5f1071060d46efdcd29cb5c0d v1.22.1.tar.gz +sha256 34097a0f535aa79cf990aaee5d3ff6226663587b188cbee11089f120e7f869e4 v1.22.2.tar.gz +sha256 52836549cfa27a688659576be9266f4837357a6fa162b1d0a05fa8da62c724b3 v1.22.3.tar.gz diff --git a/deploy/iso/minikube-iso/package/crio-bin/crio-bin.mk b/deploy/iso/minikube-iso/package/crio-bin/crio-bin.mk index 3d87e44ae0..8d6bad04d6 100644 --- a/deploy/iso/minikube-iso/package/crio-bin/crio-bin.mk +++ b/deploy/iso/minikube-iso/package/crio-bin/crio-bin.mk @@ -4,8 +4,8 @@ # ################################################################################ -CRIO_BIN_VERSION = v1.22.1 -CRIO_BIN_COMMIT = 63ca93845d5fe05cdca826367afcb601ece8d7ad +CRIO_BIN_VERSION = v1.22.3 +CRIO_BIN_COMMIT = d93b2dfb8d0f2ad0f8b9061d941e3b216baa5814 CRIO_BIN_SITE = https://github.com/cri-o/cri-o/archive CRIO_BIN_SOURCE = $(CRIO_BIN_VERSION).tar.gz CRIO_BIN_DEPENDENCIES = host-go libgpgme diff --git a/deploy/kicbase/Dockerfile b/deploy/kicbase/Dockerfile index c240d6ca0c..7983889b8b 100644 --- a/deploy/kicbase/Dockerfile +++ b/deploy/kicbase/Dockerfile @@ -126,9 +126,7 @@ RUN clean-install \ openssh-server \ dnsutils \ # libglib2.0-0 is required for conmon, which is required for podman - libglib2.0-0 \ - # fuse3 is required for fuse-overlayfs - fuse3 + libglib2.0-0 # install docker # use the bionic packages for arm32 @@ -139,19 +137,6 @@ RUN export ARCH=$(dpkg --print-architecture | sed 's/armhf/arm-v7/') && \ apt-key add - < docker.key && \ clean-install docker-ce docker-ce-cli containerd.io -# install fuse-overlayfs (used by rootless; apt-get version is old) -RUN curl -sSL --retry 5 --output /usr/local/bin/fuse-overlayfs https://github.com/containers/fuse-overlayfs/releases/download/${FUSE_OVERLAYFS_VERSION}/fuse-overlayfs-$(uname -m) \ - && chmod +x /usr/local/bin/fuse-overlayfs - -# install containerd-fuse-overlayfs (used by rootless) -RUN export ARCH=$(dpkg --print-architecture | sed 's/ppc64el/ppc64le/' | sed 's/armhf/arm-v7/') \ - && echo "Installing containerd-fuse-overlayfs..." \ - && export CONTAINERD_FUSE_OVERLAYFS_BASE_URL="https://github.com/containerd/fuse-overlayfs-snapshotter/releases/download/v${CONTAINERD_FUSE_OVERLAYFS_VERSION}" \ - && curl -sSL --retry 5 --output /tmp/containerd-fuse-overlayfs.tgz "${CONTAINERD_FUSE_OVERLAYFS_BASE_URL}/containerd-fuse-overlayfs-${CONTAINERD_FUSE_OVERLAYFS_VERSION}-linux-${ARCH}.tar.gz" \ - && tar -C /usr/local/bin -xzvf /tmp/containerd-fuse-overlayfs.tgz \ - && rm -rf /tmp/containerd-fuse-overlayfs.tgz -COPY deploy/kicbase/containerd-fuse-overlayfs.service /etc/systemd/system/containerd-fuse-overlayfs.service - # install buildkit RUN export ARCH=$(dpkg --print-architecture | sed 's/ppc64el/ppc64le/' | sed 's/armhf/arm-v7/') \ && echo "Installing buildkit ..." \ diff --git a/deploy/kicbase/containerd-fuse-overlayfs.service b/deploy/kicbase/containerd-fuse-overlayfs.service deleted file mode 100644 index a3d12bd567..0000000000 --- a/deploy/kicbase/containerd-fuse-overlayfs.service +++ /dev/null @@ -1,13 +0,0 @@ -# From https://github.com/kubernetes-sigs/kind/blob/0d3780371091b2dc9ff6eea1b6054f14ff5d970a/images/base/files/etc/systemd/system/containerd-fuse-overlayfs.service -[Unit] -Description=containerd fuse-overlayfs snapshotter -PartOf=containerd.service - -[Service] -ExecStart=/usr/local/bin/containerd-fuse-overlayfs-grpc /run/containerd-fuse-overlayfs.sock /var/lib/containerd-fuse-overlayfs -Type=notify -Restart=always -RestartSec=1 - -[Install] -WantedBy=multi-user.target diff --git a/go.mod b/go.mod index c9ca41f30f..a6c3d25bbe 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/cloudevents/sdk-go/v2 v2.8.0 github.com/cloudfoundry-attic/jibber_jabber v0.0.0-20151120183258-bcc4c8345a21 github.com/cloudfoundry/jibber_jabber v0.0.0-20151120183258-bcc4c8345a21 // indirect - github.com/docker/docker v20.10.13+incompatible + github.com/docker/docker v20.10.14+incompatible github.com/docker/go-units v0.4.0 github.com/docker/machine v0.16.2 github.com/elazarl/goproxy v0.0.0-20210110162100-a92cc753f88e @@ -70,9 +70,9 @@ require ( github.com/spf13/viper v1.10.1 github.com/zchee/go-vmnet v0.0.0-20161021174912-97ebf9174097 go.opencensus.io v0.23.0 - go.opentelemetry.io/otel v1.5.0 - go.opentelemetry.io/otel/sdk v1.5.0 - go.opentelemetry.io/otel/trace v1.5.0 + go.opentelemetry.io/otel v1.6.0 + go.opentelemetry.io/otel/sdk v1.6.0 + go.opentelemetry.io/otel/trace v1.6.0 golang.org/x/build v0.0.0-20190927031335-2835ba2e683f golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 golang.org/x/exp v0.0.0-20210220032938-85be41e4509f @@ -131,7 +131,7 @@ require ( github.com/fsnotify/fsnotify v1.5.1 // indirect github.com/go-fonts/liberation v0.2.0 // indirect github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81 // indirect - github.com/go-logr/logr v1.2.2 // indirect + github.com/go-logr/logr v1.2.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-pdf/fpdf v0.6.0 // indirect diff --git a/go.sum b/go.sum index 6b162f1ccf..72ea79366f 100644 --- a/go.sum +++ b/go.sum @@ -380,8 +380,8 @@ github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4Kfc github.com/docker/docker v17.12.0-ce-rc1.0.20181225093023-5ddb1d410a8b+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v17.12.0-ce-rc1.0.20190115220918-5ec31380a5d3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v20.10.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v20.10.13+incompatible h1:5s7uxnKZG+b8hYWlPYUi6x1Sjpq2MSt96d15eLZeHyw= -github.com/docker/docker v20.10.13+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v20.10.14+incompatible h1:+T9/PRYWNDo5SZl5qS1r9Mo/0Q8AwxKKPtu9S1yxM0w= +github.com/docker/docker v20.10.14+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.6.3 h1:zI2p9+1NQYdnG6sMU26EX4aVGlqbInSQxQXLvzJ4RPQ= github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= @@ -473,8 +473,9 @@ github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTg github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.1/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.2 h1:ahHml/yUpnlb96Rp8HCvtYVPY8ZYpxq3g7UYchIYwbs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.0/go.mod h1:YkVgnZu1ZjjL7xTxrfm/LLZBfkhTqSR1ydtm6jTKKwI= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= @@ -1106,8 +1107,9 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= @@ -1190,8 +1192,8 @@ go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzox go.opentelemetry.io/otel v1.0.1/go.mod h1:OPEOD4jIT2SlZPMmwT6FqZz2C0ZNdQqiWcoK6M0SNFU= go.opentelemetry.io/otel v1.1.0/go.mod h1:7cww0OW51jQ8IaZChIEdqLwgh+44+7uiTdWsAL0wQpA= go.opentelemetry.io/otel v1.3.0/go.mod h1:PWIKzi6JCp7sM0k9yZ43VX+T345uNbAkDKwHVjb2PTs= -go.opentelemetry.io/otel v1.5.0 h1:DhCU8oR2sJH9rfnwPdoV/+BJ7UIN5kXHL8DuSGrPU8E= -go.opentelemetry.io/otel v1.5.0/go.mod h1:Jm/m+rNp/z0eqJc74H7LPwQ3G87qkU/AnnAydAjSAHk= +go.opentelemetry.io/otel v1.6.0 h1:YV6GkGe/Ag2PKsm4rjlqdSNs0w0A5ZzxeGkxhx1T+t4= +go.opentelemetry.io/otel v1.6.0/go.mod h1:bfJD2DZVw0LBxghOTlgnlI0CV3hLDu9XF/QKOUXMTQQ= go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= go.opentelemetry.io/otel/internal/metric v0.24.0 h1:O5lFy6kAl0LMWBjzy3k//M8VjEaTDWL9DPJuqZmWIAA= go.opentelemetry.io/otel/internal/metric v0.24.0/go.mod h1:PSkQG+KuApZjBpC6ea6082ZrWUUy/w132tJ/LOU3TXk= @@ -1201,16 +1203,16 @@ go.opentelemetry.io/otel/metric v0.24.0/go.mod h1:tpMFnCD9t+BEGiWY2bWF5+AwjuAdM0 go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= go.opentelemetry.io/otel/sdk v1.3.0/go.mod h1:rIo4suHNhQwBIPg9axF8V9CA72Wz2mKF1teNrup8yzs= -go.opentelemetry.io/otel/sdk v1.5.0 h1:QKhWBbcOC9fDCZKCfPFjWTWpfIlJR+i9xiUDYrLVmZs= -go.opentelemetry.io/otel/sdk v1.5.0/go.mod h1:CU4J1v+7iEljnm1G14QjdFWOXUyYLHVh0Lh+/BTYyFg= +go.opentelemetry.io/otel/sdk v1.6.0 h1:JoriAoiNENuxxIQApR1O0k2h1Md5QegZhbentcRJpWk= +go.opentelemetry.io/otel/sdk v1.6.0/go.mod h1:PjLRUfDsoPy0zl7yrDGSUqjj43tL7rEtFdCEiGlxXRM= go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= go.opentelemetry.io/otel/trace v1.0.1/go.mod h1:5g4i4fKLaX2BQpSBsxw8YYcgKpMMSW3x7ZTuYBr3sUk= go.opentelemetry.io/otel/trace v1.1.0/go.mod h1:i47XtdcBQiktu5IsrPqOHe8w+sBmnLwwHt8wiUsWGTI= go.opentelemetry.io/otel/trace v1.3.0/go.mod h1:c/VDhno8888bvQYmbYLqe41/Ldmr/KKunbvWM4/fEjk= -go.opentelemetry.io/otel/trace v1.5.0 h1:AKQZ9zJsBRFAp7zLdyGNkqG2rToCDIt3i5tcLzQlbmU= -go.opentelemetry.io/otel/trace v1.5.0/go.mod h1:sq55kfhjXYr1zVSyexg0w1mpa03AYXR5eyTkB9NPPdE= +go.opentelemetry.io/otel/trace v1.6.0 h1:NDzPermp9ISkhxIaJXjBTi2O60xOSHDHP/EezjOL2wo= +go.opentelemetry.io/otel/trace v1.6.0/go.mod h1:qs7BrU5cZ8dXQHBGxHMOxwME/27YH2qEp4/+tZLLwJE= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= diff --git a/hack/benchmark/time-to-k8s/chart.go b/hack/benchmark/time-to-k8s/chart.go index ba03ecadfa..e0ebcbcc18 100644 --- a/hack/benchmark/time-to-k8s/chart.go +++ b/hack/benchmark/time-to-k8s/chart.go @@ -19,10 +19,8 @@ package main import ( "bytes" "encoding/csv" - "flag" "fmt" "io" - "log" "os" "strconv" @@ -45,27 +43,12 @@ type run struct { type runs struct { version string runs []run + cpus []cpu } -func main() { - csvPath := flag.String("csv", "", "path to the CSV file") - chartPath := flag.String("output", "", "path to output the chart to") - flag.Parse() - - // map of the apps (minikube, kind, k3d) and their runs - apps := make(map[string]runs) - - if err := readInCSV(*csvPath, apps); err != nil { - log.Fatal(err) - } - - values, totals, names := values(apps) - - outputMarkdownTable(values, totals, names) - - if err := createChart(*chartPath, values, totals, names); err != nil { - log.Fatal(err) - } +type cpu struct { + cpuPct float64 // percentage + cpuTime float64 // second } func readInCSV(csvPath string, apps map[string]runs) error { @@ -91,8 +74,8 @@ func readInCSV(csvPath string, apps map[string]runs) error { values := []float64{} - // 8-13 contain the run results - for i := 8; i <= 13; i++ { + // 8-16 contain the run results + for i := 8; i <= 16; i++ { v, err := strconv.ParseFloat(d[i], 64) if err != nil { return err @@ -100,6 +83,7 @@ func readInCSV(csvPath string, apps map[string]runs) error { values = append(values, v) } newRun := run{values[0], values[1], values[2], values[3], values[4], values[5]} + newCPU := cpu{values[6], values[8]} // get the app from the map and add the new run to it name := d[0] @@ -108,14 +92,18 @@ func readInCSV(csvPath string, apps map[string]runs) error { k = runs{version: d[5]} } k.runs = append(k.runs, newRun) + k.cpus = append(k.cpus, newCPU) apps[name] = k } return nil } -func values(apps map[string]runs) ([]plotter.Values, []float64, []string) { +func values(apps map[string]runs) ([]plotter.Values, []plotter.Values, []plotter.Values, []float64, []string) { var cmdValues, apiValues, k8sValues, dnsSvcValues, appValues, dnsAnsValues plotter.Values + var cpuPctValues, cpuTimeValues plotter.Values + var cpuMinikube, cpuKind, cpuK3d plotter.Values + names := []string{} totals := []float64{} @@ -123,6 +111,8 @@ func values(apps map[string]runs) ([]plotter.Values, []float64, []string) { for _, name := range []string{"minikube", "kind", "k3d"} { app := apps[name] var cmd, api, k8s, dnsSvc, appRun, dnsAns float64 + var cpuPct, cpuTime float64 + names = append(names, app.version) for _, l := range app.runs { @@ -134,6 +124,11 @@ func values(apps map[string]runs) ([]plotter.Values, []float64, []string) { dnsAns += l.dnsAns } + for _, l := range app.cpus { + cpuPct += l.cpuPct + cpuTime += l.cpuTime + } + c := float64(len(app.runs)) cmdAvg := cmd / c @@ -143,6 +138,9 @@ func values(apps map[string]runs) ([]plotter.Values, []float64, []string) { appAvg := appRun / c dnsAnsAvg := dnsAns / c + cpuPctAvg := cpuPct / c + cpuTimeAvg := cpuTime / c + cmdValues = append(cmdValues, cmdAvg) apiValues = append(apiValues, apiAvg) k8sValues = append(k8sValues, k8sAvg) @@ -152,11 +150,32 @@ func values(apps map[string]runs) ([]plotter.Values, []float64, []string) { total := cmdAvg + apiAvg + k8sAvg + dnsSvcAvg + appAvg + dnsAnsAvg totals = append(totals, total) + + cpuPctValues = append(cpuPctValues, cpuPctAvg) + cpuTimeValues = append(cpuTimeValues, cpuTimeAvg) + + cpuSummary := []float64{cpuPctAvg, cpuTimeAvg} + + switch name { + case "minikube": + cpuMinikube = cpuSummary + case "kind": + cpuKind = cpuSummary + case "k3d": + cpuK3d = cpuSummary + } + } - values := []plotter.Values{cmdValues, apiValues, k8sValues, dnsSvcValues, appValues, dnsAnsValues} + runningTime := []plotter.Values{cmdValues, apiValues, k8sValues, dnsSvcValues, appValues, dnsAnsValues} - return values, totals, names + // for markdown table, row is either cpu utilization or cpu time, col is process name + cpu := []plotter.Values{cpuPctValues, cpuTimeValues} + + // row is process name, col is either cpu utilization, or cpu time + cpureverse := []plotter.Values{cpuMinikube, cpuKind, cpuK3d} + + return runningTime, cpu, cpureverse, totals, names } func outputMarkdownTable(categories []plotter.Values, totals []float64, names []string) { @@ -184,7 +203,7 @@ func outputMarkdownTable(categories []plotter.Values, totals []float64, names [] t.SetCenterSeparator("|") t.AppendBulk(c) t.Render() - fmt.Println(b.String()) + data.TimeMarkdown = b.String() } func createChart(chartPath string, values []plotter.Values, totals []float64, names []string) error { @@ -252,7 +271,13 @@ func createChart(chartPath string, values []plotter.Values, totals []float64, na p.Add(l) - return p.Save(12*vg.Inch, 8*vg.Inch, chartPath) + if err := p.Save(12*vg.Inch, 8*vg.Inch, chartPath); err != nil { + return err + } + + data.TimeChart = chartPath + + return nil } func createBars(values plotter.Values, index int) (*plotter.BarChart, error) { diff --git a/hack/benchmark/time-to-k8s/cpu.go b/hack/benchmark/time-to-k8s/cpu.go new file mode 100644 index 0000000000..a69d106726 --- /dev/null +++ b/hack/benchmark/time-to-k8s/cpu.go @@ -0,0 +1,116 @@ +/* +Copyright 2021 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "bytes" + "fmt" + + "github.com/olekukonko/tablewriter" + "gonum.org/v1/plot" + "gonum.org/v1/plot/plotter" + "gonum.org/v1/plot/plotutil" + "gonum.org/v1/plot/vg" +) + +var fields = []string{"CPU Utilization(%)", "CPU Time(seconds)"} + +func cpuMarkdownTable(categories []plotter.Values, names []string) { + + // categories row is the either cpu pct or time, col is process name + headers := append([]string{""}, names...) + c := [][]string{} + for i, values := range categories { + row := []string{fields[i]} + for _, value := range values { + row = append(row, fmt.Sprintf("%.3f", value)) + } + c = append(c, row) + } + b := new(bytes.Buffer) + t := tablewriter.NewWriter(b) + t.SetAutoWrapText(false) + t.SetHeader(headers) + t.SetAutoFormatHeaders(false) + t.SetBorders(tablewriter.Border{Left: true, Top: false, Right: true, Bottom: false}) + t.SetCenterSeparator("|") + t.AppendBulk(c) + t.Render() + data.CPUMarkdown = b.String() +} + +func createCPUChart(chartPath string, values []plotter.Values, names []string) error { + p := plot.New() + p.Title.Text = "CPU utilization to go from 0 to successful Kubernetes deployment" + p.Y.Label.Text = "CPU utilization" + setYMax(p, values) + + w := vg.Points(20) + + barsA, err := plotter.NewBarChart(values[0], w) + if err != nil { + return err + } + barsA.LineStyle.Width = vg.Length(0) + barsA.Color = plotutil.Color(0) + barsA.Offset = -w + + barsB, err := plotter.NewBarChart(values[1], w) + if err != nil { + return err + } + barsB.LineStyle.Width = vg.Length(0) + barsB.Color = plotutil.Color(1) + barsB.Offset = 0 + + barsC, err := plotter.NewBarChart(values[2], w) + if err != nil { + return err + } + barsC.LineStyle.Width = vg.Length(0) + barsC.Color = plotutil.Color(2) + barsC.Offset = w + + p.Add(barsA, barsB, barsC) + p.Legend.Add(names[0], barsA) + p.Legend.Add(names[1], barsB) + p.Legend.Add(names[2], barsC) + + p.Legend.Top = true + p.NominalX(fields...) + + if err := p.Save(8*vg.Inch, 8*vg.Inch, chartPath); err != nil { + return err + } + + data.CPUChart = chartPath + + return nil +} + +func setYMax(p *plot.Plot, values []plotter.Values) { + ymax := 0.0 + for _, value := range values { + for _, v := range value { + if v > ymax { + ymax = v + } + } + } + + p.Y.Max = ymax + 5 +} diff --git a/hack/benchmark/time-to-k8s/page.go b/hack/benchmark/time-to-k8s/page.go new file mode 100644 index 0000000000..36cfb6f5b5 --- /dev/null +++ b/hack/benchmark/time-to-k8s/page.go @@ -0,0 +1,108 @@ +/* +Copyright 2021 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "fmt" + "log" + "os" + "text/template" + "time" +) + +var page = `--- +title: "{{.Version}} Benchmark" +linkTitle: "{{.Version}} Benchmark" +weight: -{{.Weight}} +--- + +![time-to-k8s]({{.TimeChart}}) + +{{.TimeMarkdown}} + + +![cpu-to-k8s]({{.CPUChart}}) + +{{.CPUMarkdown}} +` + +type Data struct { + Version string + Weight string + TimeChart string + TimeMarkdown string + CPUChart string + CPUMarkdown string +} + +var data Data + +func main() { + csvPath := flag.String("csv", "", "path to the CSV file") + imagePath := flag.String("image", "", "path to output the chart to") + pagePath := flag.String("page", "", "path to output the page to") + + flag.Parse() + + t := time.Now() + data.Weight = fmt.Sprintf("%d%d%d", t.Year(), t.Month(), t.Day()) + + // map of the apps (minikube, kind, k3d) and their runs + apps := make(map[string]runs) + + if err := readInCSV(*csvPath, apps); err != nil { + log.Fatalf("fail to readin cvs file with err %s", err) + } + + runningTime, cpuMdPlot, cpuChartPlot, totals, names := values(apps) + + // markdown table for running time + outputMarkdownTable(runningTime, totals, names) + + // chart for running time + if err := createChart(*imagePath+"-time.png", runningTime, totals, names); err != nil { + log.Fatalf("fail to create running time chart with err %s", err) + } + + // markdown table for cpu utilization + cpuMarkdownTable(cpuMdPlot, names) + + // chart for cpu utilization + if err := createCPUChart(*imagePath+"-cpu.png", cpuChartPlot, names); err != nil { + log.Fatalf("fail to create CPU chart with err %s", err) + } + + // generate page and save + tmpl, err := template.New("msg").Parse(page) + + if err != nil { + log.Fatal(err) + } + + f, err := os.Create(*pagePath) + if err != nil { + log.Fatalf("fail to create file under path %s, with err %s", *pagePath, err) + } + + if err = tmpl.Execute(f, data); err != nil { + log.Fatalf("fail to populate the page with err %s", err) + } + + f.Close() + +} diff --git a/hack/benchmark/time-to-k8s/time-to-k8s.sh b/hack/benchmark/time-to-k8s/time-to-k8s.sh index 213f538aea..348cd86a96 100755 --- a/hack/benchmark/time-to-k8s/time-to-k8s.sh +++ b/hack/benchmark/time-to-k8s/time-to-k8s.sh @@ -37,12 +37,10 @@ run_benchmark() { go run . --config local-kubernetes.yaml --iterations 10 --output output.csv ) } -generate_chart() { - go run ./hack/benchmark/time-to-k8s/chart.go --csv ./hack/benchmark/time-to-k8s/time-to-k8s-repo/output.csv --output ./site/static/images/benchmarks/timeToK8s/"$1".png >> ./site/content/en/docs/benchmarks/timeToK8s/"$1".md -} +# create page and generate chart inside the code create_page() { - printf -- "---\ntitle: \"%s Benchmark\"\nlinkTitle: \"%s Benchmark\"\nweight: -$(date +'%Y%m%d')\n---\n\n![time-to-k8s](/images/benchmarks/timeToK8s/%s.png)\n" "$1" "$1" "$1" > ./site/content/en/docs/benchmarks/timeToK8s/"$1".md + go run ./hack/benchmark/time-to-k8s/*.go --csv ./hack/benchmark/time-to-k8s/time-to-k8s-repo/output.csv --image ./site/static/images/benchmarks/timeToK8s/"$1" --page ./site/content/en/docs/benchmarks/timeToK8s/"$1".md } cleanup() { @@ -56,5 +54,4 @@ install_minikube VERSION=$(minikube version --short) run_benchmark create_page "$VERSION" -generate_chart "$VERSION" cleanup diff --git a/pkg/addons/addons_autopause.go b/pkg/addons/addons_autopause.go index ec8836fa4f..3b33b7f9bd 100644 --- a/pkg/addons/addons_autopause.go +++ b/pkg/addons/addons_autopause.go @@ -31,8 +31,8 @@ import ( "k8s.io/minikube/pkg/minikube/sysinit" ) -// enableOrDisableAutoPause enables the service after the config was copied by generic enble -func enableOrDisableAutoPause(cc *config.ClusterConfig, name string, val string) error { +// enableOrDisableAutoPause enables the service after the config was copied by generic enable. +func enableOrDisableAutoPause(cc *config.ClusterConfig, name, val string) error { enable, err := strconv.ParseBool(val) if err != nil { return errors.Wrapf(err, "parsing bool: %s", name) @@ -44,16 +44,18 @@ func enableOrDisableAutoPause(cc *config.ClusterConfig, name string, val string) if enable { if err := sysinit.New(co.CP.Runner).EnableNow("auto-pause"); err != nil { klog.ErrorS(err, "failed to enable", "service", "auto-pause") + return err } } - port := co.CP.Port // api server port - if enable { // if enable then need to calculate the forwarded port + port := co.CP.Port // API server port + if enable { // if enable, calculate the forwarded port port = constants.AutoPauseProxyPort if driver.NeedsPortForward(cc.Driver) { port, err = oci.ForwardedPort(cc.Driver, cc.Name, port) if err != nil { klog.ErrorS(err, "failed to get forwarded port for", "auto-pause port", port) + return err } } } @@ -64,7 +66,7 @@ func enableOrDisableAutoPause(cc *config.ClusterConfig, name string, val string) return err } if updated { - klog.Infof("%s context has been updated to point to auto-pause proxy %s:%s", cc.Name, co.CP.Hostname, co.CP.Port) + klog.Infof("%s context has been updated to point to auto-pause proxy %s:%d", cc.Name, co.CP.Hostname, port) } else { klog.Info("no need to update kube-context for auto-pause proxy") } diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go index 5f5a84ce8d..93b3bd8e24 100644 --- a/pkg/drivers/kic/oci/oci.go +++ b/pkg/drivers/kic/oci/oci.go @@ -162,9 +162,6 @@ func CreateContainerNode(p CreateParams) error { // including some ones docker would otherwise do by default. // for now this is what we want. in the future we may revisit this. "--privileged", - // enable /dev/fuse explicitly for fuse-overlayfs - // (Rootless Docker does not automatically mount /dev/fuse with --privileged) - "--device", "/dev/fuse", "--security-opt", "seccomp=unconfined", // ignore seccomp "--tmpfs", "/tmp", // various things depend on working /tmp "--tmpfs", "/run", // systemd wants a writable /run diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go index e8bb1567fd..2b96ba21bd 100644 --- a/pkg/minikube/cruntime/containerd.go +++ b/pkg/minikube/cruntime/containerd.go @@ -70,12 +70,6 @@ oom_score = 0 [cgroup] path = "" -[proxy_plugins] -# fuse-overlayfs is used for rootless -[proxy_plugins."fuse-overlayfs"] - type = "snapshot" - address = "/run/containerd-fuse-overlayfs.sock" - [plugins] [plugins."io.containerd.monitor.v1.cgroups"] no_prometheus = false @@ -208,9 +202,6 @@ func generateContainerdConfig(cr CommandRunner, imageRepository string, kv semve } pauseImage := images.Pause(kv, imageRepository) snapshotter := "overlayfs" - if inUserNamespace { - snapshotter = "fuse-overlayfs" - } opts := struct { PodInfraContainerImage string SystemdCgroup bool @@ -239,6 +230,16 @@ func generateContainerdConfig(cr CommandRunner, imageRepository string, kv semve // Enable idempotently enables containerd on a host func (r *Containerd) Enable(disOthers, forceSystemd, inUserNamespace bool) error { + if inUserNamespace { + if err := CheckKernelCompatibility(r.Runner, 5, 11); err != nil { + // For using overlayfs + return fmt.Errorf("kernel >= 5.11 is required for rootless mode: %w", err) + } + if err := CheckKernelCompatibility(r.Runner, 5, 13); err != nil { + // For avoiding SELinux error with overlayfs + klog.Warningf("kernel >= 5.13 is recommended for rootless mode %v", err) + } + } if disOthers { if err := disableOthers(r, r.Runner); err != nil { klog.Warningf("disableOthers: %v", err) @@ -254,12 +255,6 @@ func (r *Containerd) Enable(disOthers, forceSystemd, inUserNamespace bool) error return err } - if inUserNamespace { - if err := r.Init.EnableNow("containerd-fuse-overlayfs"); err != nil { - return err - } - } - // Otherwise, containerd will fail API requests with 'Unimplemented' return r.Init.Restart("containerd") } diff --git a/pkg/minikube/cruntime/crio.go b/pkg/minikube/cruntime/crio.go index 9cbcc5238f..86af57ac97 100644 --- a/pkg/minikube/cruntime/crio.go +++ b/pkg/minikube/cruntime/crio.go @@ -150,18 +150,13 @@ func enableIPForwarding(cr CommandRunner) error { // enableRootless enables configurations for running CRI-O in Rootless Docker. // // 1. Create /etc/systemd/system/crio.service.d/10-rootless.conf to set _CRIO_ROOTLESS=1 -// 2. Create /etc/crio/crio.conf.d/10-fuse-overlayfs.conf to enable fuse-overlayfs -// 3. Reload systemd +// 2. Reload systemd // // See https://kubernetes.io/docs/tasks/administer-cluster/kubelet-in-userns/#configuring-cri func (r *CRIO) enableRootless() error { files := map[string]string{ "/etc/systemd/system/crio.service.d/10-rootless.conf": `[Service] Environment="_CRIO_ROOTLESS=1" -`, - "/etc/crio/crio.conf.d/10-fuse-overlayfs.conf": `[crio] -storage_driver = "overlay" -storage_option = ["overlay.mount_program=/usr/local/bin/fuse-overlayfs"] `, } for target, content := range files { @@ -211,6 +206,14 @@ func (r *CRIO) Enable(disOthers, forceSystemd, inUserNamespace bool) error { } } if inUserNamespace { + if err := CheckKernelCompatibility(r.Runner, 5, 11); err != nil { + // For using overlayfs + return fmt.Errorf("kernel >= 5.11 is required for rootless mode: %w", err) + } + if err := CheckKernelCompatibility(r.Runner, 5, 13); err != nil { + // For avoiding SELinux error with overlayfs + klog.Warningf("kernel >= 5.13 is recommended for rootless mode %v", err) + } if err := r.enableRootless(); err != nil { return err } diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go index 514ad97670..76833ed82d 100644 --- a/pkg/minikube/cruntime/cruntime.go +++ b/pkg/minikube/cruntime/cruntime.go @@ -20,6 +20,7 @@ package cruntime import ( "fmt" "os/exec" + "strings" "github.com/blang/semver/v4" "github.com/pkg/errors" @@ -316,3 +317,22 @@ func CheckCompatibility(cr Manager) error { } return compatibleWithVersion(cr.Name(), v) } + +// CheckKernelCompatibility returns an error when the kernel is older than the specified version. +func CheckKernelCompatibility(cr CommandRunner, major, minor int) error { + expected := fmt.Sprintf("%d.%d", major, minor) + unameRes, err := cr.RunCmd(exec.Command("uname", "-r")) + if err != nil { + return err + } + actual := strings.TrimSpace(unameRes.Stdout.String()) + sortRes, err := cr.RunCmd(exec.Command("sh", "-euc", fmt.Sprintf(`(echo %s; echo %s) | sort -V | head -n1`, actual, expected))) + if err != nil { + return err + } + comparison := strings.TrimSpace(sortRes.Stdout.String()) + if comparison != expected { + return NewErrServiceVersion("kernel", expected, actual) + } + return nil +} diff --git a/pkg/minikube/download/iso.go b/pkg/minikube/download/iso.go index 7d4d462369..7227eae071 100644 --- a/pkg/minikube/download/iso.go +++ b/pkg/minikube/download/iso.go @@ -40,7 +40,7 @@ const fileScheme = "file" // DefaultISOURLs returns a list of ISO URL's to consult by default, in priority order func DefaultISOURLs() []string { v := version.GetISOVersion() - isoBucket := "minikube-builds/iso/11679" + isoBucket := "minikube-builds/iso/13800" return []string{ fmt.Sprintf("https://storage.googleapis.com/%s/minikube-%s.iso", isoBucket, v), fmt.Sprintf("https://github.com/kubernetes/minikube/releases/download/%s/minikube-%s.iso", v, v), diff --git a/site/content/en/docs/commands/start.md b/site/content/en/docs/commands/start.md index 954e60272d..93f4a36108 100644 --- a/site/content/en/docs/commands/start.md +++ b/site/content/en/docs/commands/start.md @@ -69,7 +69,7 @@ minikube start [flags] --insecure-registry strings Insecure Docker registries to pass to the Docker daemon. The default service CIDR range will automatically be added. --install-addons If set, install addons. Defaults to true. (default true) --interactive Allow user prompts for more information (default true) - --iso-url strings Locations to fetch the minikube ISO from. (default [https://storage.googleapis.com/minikube-builds/iso/11679/minikube-v1.25.2-1648191624-11679.iso,https://github.com/kubernetes/minikube/releases/download/v1.25.2-1648191624-11679/minikube-v1.25.2-1648191624-11679.iso,https://kubernetes.oss-cn-hangzhou.aliyuncs.com/minikube/iso/minikube-v1.25.2-1648191624-11679.iso]) + --iso-url strings Locations to fetch the minikube ISO from. (default [https://storage.googleapis.com/minikube-builds/iso/13800/minikube-v1.25.2-1648142972-13800.iso,https://github.com/kubernetes/minikube/releases/download/v1.25.2-1648142972-13800/minikube-v1.25.2-1648142972-13800.iso,https://kubernetes.oss-cn-hangzhou.aliyuncs.com/minikube/iso/minikube-v1.25.2-1648142972-13800.iso]) --keep-context This will keep the existing kubectl context and will create a minikube context. --kubernetes-version string The Kubernetes version that the minikube VM will use (ex: v1.2.3, 'stable' for v1.23.3, 'latest' for v1.23.4-rc.0). Defaults to 'stable'. --kvm-gpu Enable experimental NVIDIA GPU support in minikube diff --git a/site/content/en/docs/drivers/docker.md b/site/content/en/docs/drivers/docker.md index 25121e945d..8f312a983a 100644 --- a/site/content/en/docs/drivers/docker.md +++ b/site/content/en/docs/drivers/docker.md @@ -34,6 +34,7 @@ minikube config set driver docker ## Requirements - Docker 20.10 or higher, see https://rootlesscontaine.rs/getting-started/docker/ - Cgroup v2 delegation, see https://rootlesscontaine.rs/getting-started/common/cgroup2/ +- Kernel 5.11 or later (5.13 or later is recommended when SELinux is enabled), see https://rootlesscontaine.rs/how-it-works/overlayfs/ ## Usage diff --git a/translations/de.json b/translations/de.json index 71b66a8d43..b782c32376 100644 --- a/translations/de.json +++ b/translations/de.json @@ -64,6 +64,7 @@ "Another minikube instance is downloading dependencies... ": "Eine andere Minikube-Instanz lädt Abhängigkeiten herunter... ", "Another program is using a file required by minikube. If you are using Hyper-V, try stopping the minikube VM from within the Hyper-V manager": "Ein anderes Programm benutzt eine Datei, die Minikube benötigt. Wenn Sie Hyper-V verwenden, versuchen Sie die minikube VM aus dem Hyper-V Manager heraus zu stoppen", "At least needs control plane nodes to enable addon": "Benötige mindestens Control Plane Nodes um das Addon zu aktivieren", + "Auto-pause is already enabled.": "", "Automatically selected the {{.driver}} driver": "Treiber {{.driver}} wurde automatisch ausgewählt", "Automatically selected the {{.driver}} driver. Other choices: {{.alternates}}": "Treiber {{.driver}} wurde automatisch ausgewählt. Andere Möglichkeiten: {{.alternates}}", "Available Commands": "Verfügbare Befehle", diff --git a/translations/es.json b/translations/es.json index 12d0e9d3e0..cdcffc3cc4 100644 --- a/translations/es.json +++ b/translations/es.json @@ -66,6 +66,7 @@ "Another minikube instance is downloading dependencies... ": "Otra instancia de minikube esta descargando dependencias...", "Another program is using a file required by minikube. If you are using Hyper-V, try stopping the minikube VM from within the Hyper-V manager": "Otro programa está usando un archivo requerido por minikube. Si estas usando Hyper-V, intenta detener la máquina virtual de minikube desde el administrador de Hyper-V", "At least needs control plane nodes to enable addon": "Al menos se necesita un nodo de plano de control para habilitar el addon", + "Auto-pause is already enabled.": "", "Automatically selected the {{.driver}} driver": "Controlador {{.driver}} seleccionado automáticamente", "Automatically selected the {{.driver}} driver. Other choices: {{.alternates}}": "Controlador {{.driver}} seleccionado automáticamente. Otras opciones: {{.alternates}}", "Available Commands": "Comandos disponibles", diff --git a/translations/fr.json b/translations/fr.json index 5abdf8a4ca..12b981c0c3 100644 --- a/translations/fr.json +++ b/translations/fr.json @@ -65,6 +65,7 @@ "Another minikube instance is downloading dependencies... ": "Une autre instance minikube télécharge des dépendances", "Another program is using a file required by minikube. If you are using Hyper-V, try stopping the minikube VM from within the Hyper-V manager": "Un autre programme utilise un fichier requis par minikube. Si vous utilisez Hyper-V, essayez d'arrêter la machine virtuelle minikube à partir du gestionnaire Hyper-V", "At least needs control plane nodes to enable addon": "Nécessite au moins des nœuds de plan de contrôle pour activer le module", + "Auto-pause is already enabled.": "", "Automatically selected the {{.driver}} driver": "Choix automatique du pilote {{.driver}}", "Automatically selected the {{.driver}} driver. Other choices: {{.alternates}}": "Choix automatique du pilote {{.driver}}. Autres choix: {{.alternates}}", "Available Commands": "Commandes disponibles", diff --git a/translations/ja.json b/translations/ja.json index 1c41d05c9e..806a7a7605 100644 --- a/translations/ja.json +++ b/translations/ja.json @@ -67,6 +67,7 @@ "Another minikube instance is downloading dependencies... ": "別の minikube のインスタンスが、依存関係をダウンロードしています... ", "Another program is using a file required by minikube. If you are using Hyper-V, try stopping the minikube VM from within the Hyper-V manager": "別のプログラムが、minikube に必要なファイルを使用しています。Hyper-V を使用している場合は、Hyper-V マネージャー内から minikube VM を停止してみてください", "At least needs control plane nodes to enable addon": "アドオンを有効にするには、少なくともコントロールプレーンノードが必要です", + "Auto-pause is already enabled.": "", "Automatically selected the {{.driver}} driver": "{{.driver}} ドライバーが自動的に選択されました", "Automatically selected the {{.driver}} driver. Other choices: {{.alternates}}": "{{.driver}} ドライバーが自動的に選択されました。他の選択肢: {{.alternates}}", "Available Commands": "利用可能なコマンド", diff --git a/translations/ko.json b/translations/ko.json index 3b16f8902b..acc9d13afa 100644 --- a/translations/ko.json +++ b/translations/ko.json @@ -69,6 +69,7 @@ "Another minikube instance is downloading dependencies... ": "", "Another program is using a file required by minikube. If you are using Hyper-V, try stopping the minikube VM from within the Hyper-V manager": "", "At least needs control plane nodes to enable addon": "", + "Auto-pause is already enabled.": "", "Automatically selected the {{.driver}} driver": "자동적으로 {{.driver}} 드라이버가 선택되었습니다", "Automatically selected the {{.driver}} driver. Other choices: {{.alternates}}": "자동적으로 {{.driver}} 드라이버가 선택되었습니다. 다른 드라이버 목록: {{.alternates}}", "Available Commands": "사용 가능한 명령어", diff --git a/translations/pl.json b/translations/pl.json index a7223b57bc..78a3d7fa6e 100644 --- a/translations/pl.json +++ b/translations/pl.json @@ -67,6 +67,7 @@ "Another minikube instance is downloading dependencies... ": "Inny program minikube już pobiera zależności...", "Another program is using a file required by minikube. If you are using Hyper-V, try stopping the minikube VM from within the Hyper-V manager": "Inny program używa pliku wymaganego przez minikube. Jeśli używasz Hyper-V, spróbuj zatrzymać maszynę wirtualną minikube z poziomu managera Hyper-V", "At least needs control plane nodes to enable addon": "Wymaga węzłów z płaszczyzny kontrolnej do włączenia addona", + "Auto-pause is already enabled.": "", "Automatically selected the {{.driver}} driver": "Automatycznie wybrano sterownik {{.driver}}", "Automatically selected the {{.driver}} driver. Other choices: {{.alternates}}": "Automatycznie wybrano sterownik {{.driver}}. Inne możliwe sterowniki: {{.alternates}}", "Available Commands": "Dostępne polecenia", diff --git a/translations/ru.json b/translations/ru.json index 5cbbc79e26..b8c9e585d3 100644 --- a/translations/ru.json +++ b/translations/ru.json @@ -60,6 +60,7 @@ "Another minikube instance is downloading dependencies... ": "", "Another program is using a file required by minikube. If you are using Hyper-V, try stopping the minikube VM from within the Hyper-V manager": "", "At least needs control plane nodes to enable addon": "", + "Auto-pause is already enabled.": "", "Automatically selected the {{.driver}} driver": "", "Automatically selected the {{.driver}} driver. Other choices: {{.alternates}}": "", "Available Commands": "", diff --git a/translations/strings.txt b/translations/strings.txt index a634bd3f14..8dd453de3d 100644 --- a/translations/strings.txt +++ b/translations/strings.txt @@ -60,6 +60,7 @@ "Another minikube instance is downloading dependencies... ": "", "Another program is using a file required by minikube. If you are using Hyper-V, try stopping the minikube VM from within the Hyper-V manager": "", "At least needs control plane nodes to enable addon": "", + "Auto-pause is already enabled.": "", "Automatically selected the {{.driver}} driver": "", "Automatically selected the {{.driver}} driver. Other choices: {{.alternates}}": "", "Available Commands": "", diff --git a/translations/zh-CN.json b/translations/zh-CN.json index dc20948fa1..325f6dda19 100644 --- a/translations/zh-CN.json +++ b/translations/zh-CN.json @@ -78,6 +78,7 @@ "Another minikube instance is downloading dependencies... ": "", "Another program is using a file required by minikube. If you are using Hyper-V, try stopping the minikube VM from within the Hyper-V manager": "", "At least needs control plane nodes to enable addon": "", + "Auto-pause is already enabled.": "", "Automatically selected the '{{.driver}}' driver": "自动选择 '{{.driver}}' 驱动", "Automatically selected the '{{.driver}}' driver (alternates: {{.alternates}})": "自动选择 '{{.driver}}' 驱动（可选项：{{.alternates}}）", "Automatically selected the {{.driver}} driver": "自动选择 {{.driver}} 驱动",