From 505c54db33f833e7b7f8d33077c247f879d17644 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Mon, 17 Feb 2020 22:39:34 +0100 Subject: [PATCH 01/50] Build cni and cni-plugins from source instead And make sure to install cnitool, not noop (!) --- deploy/iso/minikube-iso/package/Config.in | 4 +- .../minikube-iso/package/cni-bin/Config.in | 4 -- .../minikube-iso/package/cni-bin/cni-bin.mk | 29 ----------- .../package/cni-plugins-bin/Config.in | 4 -- .../package/cni-plugins/Config.in | 5 ++ .../cni-plugins.hash} | 1 + .../cni-plugins.mk} | 48 ++++++++++++------- deploy/iso/minikube-iso/package/cni/Config.in | 5 ++ .../{cni-bin/cni-bin.hash => cni/cni.hash} | 1 + deploy/iso/minikube-iso/package/cni/cni.mk | 43 +++++++++++++++++ 10 files changed, 88 insertions(+), 56 deletions(-) delete mode 100644 deploy/iso/minikube-iso/package/cni-bin/Config.in delete mode 100644 deploy/iso/minikube-iso/package/cni-bin/cni-bin.mk delete mode 100644 deploy/iso/minikube-iso/package/cni-plugins-bin/Config.in create mode 100644 deploy/iso/minikube-iso/package/cni-plugins/Config.in rename deploy/iso/minikube-iso/package/{cni-plugins-bin/cni-plugins-bin.hash => cni-plugins/cni-plugins.hash} (53%) rename deploy/iso/minikube-iso/package/{cni-plugins-bin/cni-plugins-bin.mk => cni-plugins/cni-plugins.mk} (69%) create mode 100644 deploy/iso/minikube-iso/package/cni/Config.in rename deploy/iso/minikube-iso/package/{cni-bin/cni-bin.hash => cni/cni.hash} (75%) create mode 100644 deploy/iso/minikube-iso/package/cni/cni.mk diff --git a/deploy/iso/minikube-iso/package/Config.in b/deploy/iso/minikube-iso/package/Config.in index 75aaaf10c6..78b363e12c 100644 --- a/deploy/iso/minikube-iso/package/Config.in +++ b/deploy/iso/minikube-iso/package/Config.in @@ -7,8 +7,8 @@ menu "System tools" source "$BR2_EXTERNAL_MINIKUBE_PATH/package/crictl-bin/Config.in" source "$BR2_EXTERNAL_MINIKUBE_PATH/package/automount/Config.in" source "$BR2_EXTERNAL_MINIKUBE_PATH/package/docker-bin/Config.in" - source "$BR2_EXTERNAL_MINIKUBE_PATH/package/cni-bin/Config.in" - source "$BR2_EXTERNAL_MINIKUBE_PATH/package/cni-plugins-bin/Config.in" + source "$BR2_EXTERNAL_MINIKUBE_PATH/package/cni/Config.in" + source "$BR2_EXTERNAL_MINIKUBE_PATH/package/cni-plugins/Config.in" source "$BR2_EXTERNAL_MINIKUBE_PATH/package/hyperv-daemons/Config.in" source "$BR2_EXTERNAL_MINIKUBE_PATH/package/gluster/Config.in" source "$BR2_EXTERNAL_MINIKUBE_PATH/package/vbox-guest/Config.in" diff --git a/deploy/iso/minikube-iso/package/cni-bin/Config.in b/deploy/iso/minikube-iso/package/cni-bin/Config.in deleted file mode 100644 index 9a0f860c67..0000000000 --- a/deploy/iso/minikube-iso/package/cni-bin/Config.in +++ /dev/null @@ -1,4 +0,0 @@ -config BR2_PACKAGE_CNI_BIN - bool "cni-bin" - default y - depends on BR2_x86_64 diff --git a/deploy/iso/minikube-iso/package/cni-bin/cni-bin.mk b/deploy/iso/minikube-iso/package/cni-bin/cni-bin.mk deleted file mode 100644 index bbc3cbe7bf..0000000000 --- a/deploy/iso/minikube-iso/package/cni-bin/cni-bin.mk +++ /dev/null @@ -1,29 +0,0 @@ -################################################################################ -# -# cni-bin -# -################################################################################ - -CNI_BIN_VERSION = v0.6.0 -CNI_BIN_SITE = https://github.com/containernetworking/cni/releases/download/$(CNI_BIN_VERSION) -CNI_BIN_SOURCE = cni-amd64-$(CNI_BIN_VERSION).tgz - -define CNI_BIN_INSTALL_TARGET_CMDS - $(INSTALL) -D -m 0755 \ - $(@D)/noop \ - $(TARGET_DIR)/opt/cni/bin/noop - - ln -sf \ - ../../opt/cni/bin/noop \ - $(TARGET_DIR)/usr/bin/noop - - $(INSTALL) -D -m 0755 \ - $(@D)/noop \ - $(TARGET_DIR)/opt/cni/bin/cnitool - - ln -sf \ - ../../opt/cni/bin/cnitool \ - $(TARGET_DIR)/usr/bin/cnitool -endef - -$(eval $(generic-package)) diff --git a/deploy/iso/minikube-iso/package/cni-plugins-bin/Config.in b/deploy/iso/minikube-iso/package/cni-plugins-bin/Config.in deleted file mode 100644 index 4e2f7815fd..0000000000 --- a/deploy/iso/minikube-iso/package/cni-plugins-bin/Config.in +++ /dev/null @@ -1,4 +0,0 @@ -config BR2_PACKAGE_CNI_PLUGINS_BIN - bool "cni-plugins-bin" - default y - depends on BR2_x86_64 diff --git a/deploy/iso/minikube-iso/package/cni-plugins/Config.in b/deploy/iso/minikube-iso/package/cni-plugins/Config.in new file mode 100644 index 0000000000..9566518974 --- /dev/null +++ b/deploy/iso/minikube-iso/package/cni-plugins/Config.in @@ -0,0 +1,5 @@ +config BR2_PACKAGE_CNI_PLUGINS + bool "cni-plugins" + default y + depends on BR2_x86_64 + depends on BR2_PACKAGE_HOST_GO_ARCH_SUPPORTS diff --git a/deploy/iso/minikube-iso/package/cni-plugins-bin/cni-plugins-bin.hash b/deploy/iso/minikube-iso/package/cni-plugins/cni-plugins.hash similarity index 53% rename from deploy/iso/minikube-iso/package/cni-plugins-bin/cni-plugins-bin.hash rename to deploy/iso/minikube-iso/package/cni-plugins/cni-plugins.hash index b8ba3fda6f..6528caedc2 100644 --- a/deploy/iso/minikube-iso/package/cni-plugins-bin/cni-plugins-bin.hash +++ b/deploy/iso/minikube-iso/package/cni-plugins/cni-plugins.hash @@ -1 +1,2 @@ sha256 f04339a21b8edf76d415e7f17b620e63b8f37a76b2f706671587ab6464411f2d cni-plugins-amd64-v0.6.0.tgz +sha256 8589670f7f9b211a351dfcd211d4fe0b961d77283a7415443dc188f3dbf05668 v0.6.0.tar.gz diff --git a/deploy/iso/minikube-iso/package/cni-plugins-bin/cni-plugins-bin.mk b/deploy/iso/minikube-iso/package/cni-plugins/cni-plugins.mk similarity index 69% rename from deploy/iso/minikube-iso/package/cni-plugins-bin/cni-plugins-bin.mk rename to deploy/iso/minikube-iso/package/cni-plugins/cni-plugins.mk index 44239195b8..09902d11f2 100644 --- a/deploy/iso/minikube-iso/package/cni-plugins-bin/cni-plugins-bin.mk +++ b/deploy/iso/minikube-iso/package/cni-plugins/cni-plugins.mk @@ -1,16 +1,30 @@ ################################################################################ # -# cni-plugins-bin +# cni-plugins # ################################################################################ -CNI_PLUGINS_BIN_VERSION = v0.6.0 -CNI_PLUGINS_BIN_SITE = https://github.com/containernetworking/plugins/releases/download/$(CNI_PLUGINS_BIN_VERSION) -CNI_PLUGINS_BIN_SOURCE = cni-plugins-amd64-$(CNI_PLUGINS_BIN_VERSION).tgz +CNI_PLUGINS_VERSION = v0.6.0 +CNI_PLUGINS_SITE = https://github.com/containernetworking/plugins/archive +CNI_PLUGINS_SOURCE = $(CNI_PLUGINS_VERSION).tar.gz +CNI_PLUGINS_LICENSE = Apache-2.0 +CNI_PLUGINS_LICENSE_FILES = LICENSE -define CNI_PLUGINS_BIN_INSTALL_TARGET_CMDS +CNI_PLUGINS_DEPENDENCIES = host-go + +CNI_PLUGINS_MAKE_ENV = \ + CGO_ENABLED=0 \ + GO111MODULE=off + +CNI_PLUGINS_BUILDFLAGS = -a --ldflags '-extldflags \"-static\"' + +define CNI_PLUGINS_BUILD_CMDS + (cd $(@D); $(CNI_PLUGINS_MAKE_ENV) ./build.sh $(CNI_PLUGINS_BUILDFLAGS)) +endef + +define CNI_PLUGINS_INSTALL_TARGET_CMDS $(INSTALL) -D -m 0755 \ - $(@D)/bridge \ + $(@D)/bin/bridge \ $(TARGET_DIR)/opt/cni/bin/bridge ln -sf \ @@ -18,7 +32,7 @@ define CNI_PLUGINS_BIN_INSTALL_TARGET_CMDS $(TARGET_DIR)/usr/bin/bridge $(INSTALL) -D -m 0755 \ - $(@D)/vlan \ + $(@D)/bin/vlan \ $(TARGET_DIR)/opt/cni/bin/vlan ln -sf \ @@ -26,7 +40,7 @@ define CNI_PLUGINS_BIN_INSTALL_TARGET_CMDS $(TARGET_DIR)/usr/bin/vlan $(INSTALL) -D -m 0755 \ - $(@D)/tuning \ + $(@D)/bin/tuning \ $(TARGET_DIR)/opt/cni/bin/tuning ln -sf \ @@ -34,7 +48,7 @@ define CNI_PLUGINS_BIN_INSTALL_TARGET_CMDS $(TARGET_DIR)/usr/bin/tuning $(INSTALL) -D -m 0755 \ - $(@D)/sample \ + $(@D)/bin/sample \ $(TARGET_DIR)/opt/cni/bin/sample ln -sf \ @@ -42,7 +56,7 @@ define CNI_PLUGINS_BIN_INSTALL_TARGET_CMDS $(TARGET_DIR)/usr/bin/sample $(INSTALL) -D -m 0755 \ - $(@D)/ptp \ + $(@D)/bin/ptp \ $(TARGET_DIR)/opt/cni/bin/ptp ln -sf \ @@ -50,7 +64,7 @@ define CNI_PLUGINS_BIN_INSTALL_TARGET_CMDS $(TARGET_DIR)/usr/bin/ptp $(INSTALL) -D -m 0755 \ - $(@D)/portmap \ + $(@D)/bin/portmap \ $(TARGET_DIR)/opt/cni/bin/portmap ln -sf \ @@ -58,7 +72,7 @@ define CNI_PLUGINS_BIN_INSTALL_TARGET_CMDS $(TARGET_DIR)/usr/bin/portmap $(INSTALL) -D -m 0755 \ - $(@D)/macvlan \ + $(@D)/bin/macvlan \ $(TARGET_DIR)/opt/cni/bin/macvlan ln -sf \ @@ -66,7 +80,7 @@ define CNI_PLUGINS_BIN_INSTALL_TARGET_CMDS $(TARGET_DIR)/usr/bin/macvlan $(INSTALL) -D -m 0755 \ - $(@D)/loopback \ + $(@D)/bin/loopback \ $(TARGET_DIR)/opt/cni/bin/loopback ln -sf \ @@ -74,7 +88,7 @@ define CNI_PLUGINS_BIN_INSTALL_TARGET_CMDS $(TARGET_DIR)/usr/bin/loopback $(INSTALL) -D -m 0755 \ - $(@D)/ipvlan \ + $(@D)/bin/ipvlan \ $(TARGET_DIR)/opt/cni/bin/ipvlan ln -sf \ @@ -82,7 +96,7 @@ define CNI_PLUGINS_BIN_INSTALL_TARGET_CMDS $(TARGET_DIR)/usr/bin/ipvlan $(INSTALL) -D -m 0755 \ - $(@D)/host-local \ + $(@D)/bin/host-local \ $(TARGET_DIR)/opt/cni/bin/host-local ln -sf \ @@ -90,7 +104,7 @@ define CNI_PLUGINS_BIN_INSTALL_TARGET_CMDS $(TARGET_DIR)/usr/bin/host-local $(INSTALL) -D -m 0755 \ - $(@D)/flannel \ + $(@D)/bin/flannel \ $(TARGET_DIR)/opt/cni/bin/flannel ln -sf \ @@ -99,7 +113,7 @@ define CNI_PLUGINS_BIN_INSTALL_TARGET_CMDS $(INSTALL) -D -m 0755 \ - $(@D)/dhcp \ + $(@D)/bin/dhcp \ $(TARGET_DIR)/opt/cni/bin/dhcp ln -sf \ diff --git a/deploy/iso/minikube-iso/package/cni/Config.in b/deploy/iso/minikube-iso/package/cni/Config.in new file mode 100644 index 0000000000..ca60da49fc --- /dev/null +++ b/deploy/iso/minikube-iso/package/cni/Config.in @@ -0,0 +1,5 @@ +config BR2_PACKAGE_CNI + bool "cni" + default y + depends on BR2_x86_64 + depends on BR2_PACKAGE_HOST_GO_ARCH_SUPPORTS diff --git a/deploy/iso/minikube-iso/package/cni-bin/cni-bin.hash b/deploy/iso/minikube-iso/package/cni/cni.hash similarity index 75% rename from deploy/iso/minikube-iso/package/cni-bin/cni-bin.hash rename to deploy/iso/minikube-iso/package/cni/cni.hash index ba8d154f6d..ba69096d60 100644 --- a/deploy/iso/minikube-iso/package/cni-bin/cni-bin.hash +++ b/deploy/iso/minikube-iso/package/cni/cni.hash @@ -1,3 +1,4 @@ sha256 b1ae09833a238c51161918a8849031efdb46cf0068ea5b752e362d9836e2af7d cni-v0.3.0.tgz sha256 84c9a0a41b59211d560bef14bf3f53bb370156f9ac7762270b3848fed96e1be8 cni-v0.4.0.tgz sha256 a7f84a742c8f3a95843b3cc636444742554a4853835649ec371a07c841daebab cni-amd64-v0.6.0.tgz +sha256 802f4a002b4eb774624a9dc1c859d3c9926eb2d862e66a673fc99cfc8bcd7494 v0.6.0.tar.gz diff --git a/deploy/iso/minikube-iso/package/cni/cni.mk b/deploy/iso/minikube-iso/package/cni/cni.mk new file mode 100644 index 0000000000..3c04894813 --- /dev/null +++ b/deploy/iso/minikube-iso/package/cni/cni.mk @@ -0,0 +1,43 @@ +################################################################################ +# +# cni +# +################################################################################ + +CNI_VERSION = v0.6.0 +CNI_SITE = https://github.com/containernetworking/cni/archive +CNI_SOURCE = $(CNI_VERSION).tar.gz +CNI_LICENSE = Apache-2.0 +CNI_LICENSE_FILES = LICENSE + +CNI_DEPENDENCIES = host-go + +CNI_MAKE_ENV = \ + CGO_ENABLED=0 \ + GO111MODULE=off + +CNI_BUILDFLAGS = -a --ldflags '-extldflags \"-static\"' + +define CNI_BUILD_CMDS + (cd $(@D); $(CNI_MAKE_ENV) ./build.sh $(CNI_BUILDFLAGS)) +endef + +define CNI_INSTALL_TARGET_CMDS + $(INSTALL) -D -m 0755 \ + $(@D)/bin/noop \ + $(TARGET_DIR)/opt/cni/bin/noop + + ln -sf \ + ../../opt/cni/bin/noop \ + $(TARGET_DIR)/usr/bin/noop + + $(INSTALL) -D -m 0755 \ + $(@D)/bin/cnitool \ + $(TARGET_DIR)/opt/cni/bin/cnitool + + ln -sf \ + ../../opt/cni/bin/cnitool \ + $(TARGET_DIR)/usr/bin/cnitool +endef + +$(eval $(generic-package)) From 96d992c8ca7798fdb24fcd58a8648374417f4c98 Mon Sep 17 00:00:00 2001 From: Song Shukun Date: Thu, 20 Feb 2020 17:11:06 +0900 Subject: [PATCH 02/50] Increase detection timeout from 2s to 4s for hyperV --- pkg/minikube/registry/drvs/hyperv/hyperv.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/pkg/minikube/registry/drvs/hyperv/hyperv.go b/pkg/minikube/registry/drvs/hyperv/hyperv.go index 9f15d0c470..3b448990ec 100644 --- a/pkg/minikube/registry/drvs/hyperv/hyperv.go +++ b/pkg/minikube/registry/drvs/hyperv/hyperv.go @@ -85,14 +85,23 @@ func status() registry.State { return registry.State{Error: err} } - // Allow no more than 2 seconds for querying state - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + // Allow no more than 4 seconds for querying state + ctx, cancel := context.WithTimeout(context.Background(), 4*time.Second) defer cancel() cmd := exec.CommandContext(ctx, path, "Get-WindowsOptionalFeature", "-FeatureName", "Microsoft-Hyper-V-All", "-Online") out, err := cmd.CombinedOutput() if err != nil { - return registry.State{Installed: false, Error: fmt.Errorf("%s failed:\n%s", strings.Join(cmd.Args, " "), out), Fix: "Start PowerShell as Administrator, and run: 'Enable-WindowsOptionalFeature -Online -FeatureName Microsoft-Hyper-V -All'", Doc: docURL} + errorMessage := fmt.Errorf("%s failed:\n%s", strings.Join(cmd.Args, " "), out) + fixMessage := "Start PowerShell as Administrator, and run: 'Enable-WindowsOptionalFeature -Online -FeatureName Microsoft-Hyper-V -All'" + + // If timed out, prompt different error and suggestion messages + // See https://github.com/kubernetes/minikube/issues/6579 + if ctx.Err() != nil { + errorMessage = fmt.Errorf("%s exited unexpectedly:\n%s", strings.Join(cmd.Args, " "), ctx.Err()) + fixMessage = "If you have Hyper-V configured correctly, please try start again with `--force` specified" + } + return registry.State{Installed: false, Error: errorMessage, Fix: fixMessage, Doc: docURL} } return registry.State{Installed: true, Healthy: true} } From 38437b2fa46a3238a346b927886e0df9227d0ee4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Str=C3=B6mberg?= Date: Thu, 20 Feb 2020 11:16:58 -0800 Subject: [PATCH 03/50] Set timeout to an outrageous 8s, based on the bug --- pkg/minikube/registry/drvs/hyperv/hyperv.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/minikube/registry/drvs/hyperv/hyperv.go b/pkg/minikube/registry/drvs/hyperv/hyperv.go index 3b448990ec..0aa426e766 100644 --- a/pkg/minikube/registry/drvs/hyperv/hyperv.go +++ b/pkg/minikube/registry/drvs/hyperv/hyperv.go @@ -85,8 +85,7 @@ func status() registry.State { return registry.State{Error: err} } - // Allow no more than 4 seconds for querying state - ctx, cancel := context.WithTimeout(context.Background(), 4*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 8*time.Second) defer cancel() cmd := exec.CommandContext(ctx, path, "Get-WindowsOptionalFeature", "-FeatureName", "Microsoft-Hyper-V-All", "-Online") From 0b00ed7668bc3ee887c5463761d30728ac4c8ded Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Fri, 10 Jan 2020 21:48:24 +0100 Subject: [PATCH 04/50] Add platform option when downloading images For now, assume that all the containers will have the same architecture as the current platform (minikube binary arch) --- pkg/minikube/image/image.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/minikube/image/image.go b/pkg/minikube/image/image.go index 5a61f840e4..d2a45f37fa 100644 --- a/pkg/minikube/image/image.go +++ b/pkg/minikube/image/image.go @@ -21,6 +21,7 @@ import ( "io/ioutil" "os" "path/filepath" + "runtime" "time" "github.com/docker/docker/client" @@ -33,6 +34,11 @@ import ( "k8s.io/minikube/pkg/minikube/constants" ) +var defaultPlatform = v1.Platform{ + Architecture: runtime.GOARCH, + OS: "linux", +} + // DigestByDockerLib uses client by docker lib to return image digest // img.ID in as same as image digest func DigestByDockerLib(imgClient *client.Client, imgName string) string { @@ -81,7 +87,8 @@ func retrieveImage(ref name.Reference) (v1.Image, error) { glog.Infof("daemon lookup for %+v: %v", ref, err) } - img, err = remote.Image(ref, remote.WithAuthFromKeychain(authn.DefaultKeychain)) + platform := defaultPlatform + img, err = remote.Image(ref, remote.WithAuthFromKeychain(authn.DefaultKeychain), remote.WithPlatform(platform)) if err == nil { return img, nil } From 8cfc825641611ceb297d6760f07b70827fc4ee84 Mon Sep 17 00:00:00 2001 From: vikkyomkar Date: Mon, 24 Feb 2020 17:49:25 +0530 Subject: [PATCH 05/50] fixes profile switch to non-existing one --- cmd/minikube/cmd/config/profile.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/minikube/cmd/config/profile.go b/cmd/minikube/cmd/config/profile.go index 280bb1c237..3904d4c823 100644 --- a/cmd/minikube/cmd/config/profile.go +++ b/cmd/minikube/cmd/config/profile.go @@ -66,6 +66,7 @@ var ProfileCmd = &cobra.Command{ if !pkgConfig.ProfileExists(profile) { out.FailureT("if you want to create a profile you can by this command: minikube start -p {{.profile_name}}", out.V{"profile_name": profile}) + os.Exit(0) } err := Set(pkgConfig.MachineProfile, profile) From fc3a5656451ced57cc8dd68b685dbf12604545a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Mon, 24 Feb 2020 18:34:01 +0100 Subject: [PATCH 06/50] Upgrade falco-probe to fixed version 0.20.0 No changes to the actual driver, though... --- deploy/iso/minikube-iso/package/falco-probe/falco-probe.hash | 1 + deploy/iso/minikube-iso/package/falco-probe/falco-probe.mk | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/iso/minikube-iso/package/falco-probe/falco-probe.hash b/deploy/iso/minikube-iso/package/falco-probe/falco-probe.hash index efe14c5233..a8afdf53cc 100644 --- a/deploy/iso/minikube-iso/package/falco-probe/falco-probe.hash +++ b/deploy/iso/minikube-iso/package/falco-probe/falco-probe.hash @@ -1,4 +1,5 @@ # falco sha256 87c60273c35d544256e471b403497be33f24df662673338236ec92ba3fc1f8b7 0.19.0.tar.gz +sha256 b873e3590e56ead740ed905108221f98da6100da3c5b7acf2355ea1cf628d931 0.20.0.tar.gz # sysdig sha256 6e477ac5fe9d3110b870bd4495f01541373a008c375a1934a2d1c46798b6bad6 146a431edf95829ac11bfd9c85ba3ef08789bffe.tar.gz diff --git a/deploy/iso/minikube-iso/package/falco-probe/falco-probe.mk b/deploy/iso/minikube-iso/package/falco-probe/falco-probe.mk index b96bbec29b..0e5986f1fc 100644 --- a/deploy/iso/minikube-iso/package/falco-probe/falco-probe.mk +++ b/deploy/iso/minikube-iso/package/falco-probe/falco-probe.mk @@ -4,7 +4,7 @@ # ######################################################################## -FALCO_PROBE_VERSION = 0.19.0 +FALCO_PROBE_VERSION = 0.20.0 FALCO_PROBE_SITE = https://github.com/falcosecurity/falco/archive FALCO_PROBE_SOURCE = $(FALCO_PROBE_VERSION).tar.gz FALCO_PROBE_DEPENDENCIES += ncurses libyaml @@ -22,7 +22,6 @@ endef FALCO_PROBE_POST_EXTRACT_HOOKS += FALCO_PROBE_SYSDIG_SRC FALCO_PROBE_CONF_OPTS = -DFALCO_VERSION=$(FALCO_PROBE_VERSION) -FALCO_PROBE_CONF_OPTS += -DSYSDIG_VERSION=$(FALCO_PROBE_SYSDIG_VERSION) FALCO_PROBE_CONF_OPTS += -DUSE_BUNDLED_DEPS=ON FALCO_PROBE_MAKE_OPTS = driver KERNELDIR=$(LINUX_DIR) From ff17e26b5bac8aee8b7d932389160af8ba362044 Mon Sep 17 00:00:00 2001 From: sayboras Date: Tue, 25 Feb 2020 08:08:00 +1100 Subject: [PATCH 07/50] Fix typo detected by linter --- pkg/minikube/tunnel/kic/ssh_tunnel.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/minikube/tunnel/kic/ssh_tunnel.go b/pkg/minikube/tunnel/kic/ssh_tunnel.go index eba12ea34b..29c2994cb1 100644 --- a/pkg/minikube/tunnel/kic/ssh_tunnel.go +++ b/pkg/minikube/tunnel/kic/ssh_tunnel.go @@ -118,7 +118,7 @@ func (t *SSHTunnel) stopMarkedConnections() { // sshConnName creates a uniq name for the tunnel, using its name/clusterIP/ports. // This allows a new process to be created if an existing service was changed, -// the new process will support the IP/Ports change ocurred. +// the new process will support the IP/Ports change occurred. func sshConnUniqName(service v1.Service) string { n := []string{ service.Name, From 2584457f053f8cde8f805ad156fdf53ed284dcc7 Mon Sep 17 00:00:00 2001 From: sayboras Date: Tue, 25 Feb 2020 08:15:19 +1100 Subject: [PATCH 08/50] Fix boilerplate --- pkg/minikube/tunnel/kic/ssh_conn.go | 16 ++++++++++++++++ pkg/minikube/tunnel/kic/ssh_tunnel.go | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/pkg/minikube/tunnel/kic/ssh_conn.go b/pkg/minikube/tunnel/kic/ssh_conn.go index c8a85c1974..8e4b085fa9 100644 --- a/pkg/minikube/tunnel/kic/ssh_conn.go +++ b/pkg/minikube/tunnel/kic/ssh_conn.go @@ -1,3 +1,19 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package kic import ( diff --git a/pkg/minikube/tunnel/kic/ssh_tunnel.go b/pkg/minikube/tunnel/kic/ssh_tunnel.go index 29c2994cb1..8567fe7d61 100644 --- a/pkg/minikube/tunnel/kic/ssh_tunnel.go +++ b/pkg/minikube/tunnel/kic/ssh_tunnel.go @@ -1,3 +1,19 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package kic import ( From 2e261d3d0027d9c2dfd8b5053b9eb17fe8f5af15 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Mon, 24 Feb 2020 14:41:34 -0800 Subject: [PATCH 09/50] add not implemented messages for docker on mac service --- cmd/minikube/cmd/service.go | 14 ++++++++++++++ cmd/minikube/cmd/service_list.go | 22 ++++++++++++++++++++-- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/cmd/minikube/cmd/service.go b/cmd/minikube/cmd/service.go index 0fabfbc32b..0800c0a9b3 100644 --- a/cmd/minikube/cmd/service.go +++ b/cmd/minikube/cmd/service.go @@ -20,6 +20,7 @@ import ( "fmt" "net/url" "os" + "runtime" "text/template" "github.com/golang/glog" @@ -27,6 +28,8 @@ import ( "github.com/spf13/cobra" "github.com/spf13/viper" + "k8s.io/minikube/pkg/drivers/kic/oci" + "k8s.io/minikube/pkg/minikube/config" pkg_config "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/exit" "k8s.io/minikube/pkg/minikube/machine" @@ -77,11 +80,21 @@ var serviceCmd = &cobra.Command{ os.Exit(1) } + cfg, err := config.Load(profileName) + if err != nil { + exit.WithError("Error getting config", err) + } + urls, err := service.WaitForService(api, namespace, svc, serviceURLTemplate, serviceURLMode, https, wait, interval) if err != nil { exit.WithError("Error opening service", err) } + if runtime.GOOS == "darwin" && cfg.Driver == oci.Docker { + out.T(out.Sad, "openning service in browser is not yet implemented in docker driver on mac.\nPlease track the URL bellow to see updates for in progress work.\nhttps://github.com/kubernetes/minikube/issues/6778") + exit.WithCodeT(exit.Unavailable, "not implemented for docker driver on MacOs yet") + } + for _, u := range urls { _, err := url.Parse(u) if err != nil { @@ -94,6 +107,7 @@ var serviceCmd = &cobra.Command{ out.String(fmt.Sprintf("%s\n", u)) continue } + out.T(out.Celebrate, "Opening service {{.namespace_name}}/{{.service_name}} in default browser...", out.V{"namespace_name": namespace, "service_name": svc}) if err := browser.OpenURL(u); err != nil { exit.WithError(fmt.Sprintf("open url failed: %s", u), err) diff --git a/cmd/minikube/cmd/service_list.go b/cmd/minikube/cmd/service_list.go index ad1903a4b8..6b6dac18f1 100644 --- a/cmd/minikube/cmd/service_list.go +++ b/cmd/minikube/cmd/service_list.go @@ -18,10 +18,15 @@ package cmd import ( "os" + "runtime" "strings" "github.com/spf13/cobra" + "github.com/spf13/viper" core "k8s.io/api/core/v1" + "k8s.io/minikube/pkg/drivers/kic/oci" + "k8s.io/minikube/pkg/minikube/config" + pkg_config "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/exit" "k8s.io/minikube/pkg/minikube/machine" "k8s.io/minikube/pkg/minikube/out" @@ -41,24 +46,37 @@ var serviceListCmd = &cobra.Command{ exit.WithError("Error getting client", err) } defer api.Close() + profileName := viper.GetString(pkg_config.MachineProfile) + if !machine.IsHostRunning(api, profileName) { + exit.WithCodeT(exit.Unavailable, "profile {{.name}} is not running. ", out.V{"name": profileName}) + } serviceURLs, err := service.GetServiceURLs(api, serviceListNamespace, serviceURLTemplate) if err != nil { out.FatalT("Failed to get service URL: {{.error}}", out.V{"error": err}) out.ErrT(out.Notice, "Check that minikube is running and that you have specified the correct namespace (-n flag) if required.") os.Exit(exit.Unavailable) } + cfg, err := config.Load(viper.GetString(config.MachineProfile)) + if err != nil { + exit.WithError("Error getting config", err) + } var data [][]string for _, serviceURL := range serviceURLs { if len(serviceURL.URLs) == 0 { data = append(data, []string{serviceURL.Namespace, serviceURL.Name, "No node port"}) } else { - data = append(data, []string{serviceURL.Namespace, serviceURL.Name, strings.Join(serviceURL.URLs, "\n")}) + data = append(data, []string{serviceURL.Namespace, serviceURL.Name, "", strings.Join(serviceURL.URLs, "\n")}) + } } - service.PrintServiceList(os.Stdout, data) + if runtime.GOOS == "darwin" && cfg.Driver == oci.Docker { + out.FailureT("Accessing service on docker driver for mac is not implemented yet.Please follow the work in progress and the workarround to solve this issue:\n\thttps://github.com/kubernetes/minikube/issues/6778") + exit.WithCodeT(exit.Failure, "not implemented for docker on mac driver yet.") + } + }, } From 850169764cb062c9f8978f1ef296d1d9cdffc212 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Mon, 24 Feb 2020 14:51:45 -0800 Subject: [PATCH 10/50] improve message --- cmd/minikube/cmd/service.go | 2 +- cmd/minikube/cmd/service_list.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/minikube/cmd/service.go b/cmd/minikube/cmd/service.go index 0800c0a9b3..507b1fcfae 100644 --- a/cmd/minikube/cmd/service.go +++ b/cmd/minikube/cmd/service.go @@ -91,7 +91,7 @@ var serviceCmd = &cobra.Command{ } if runtime.GOOS == "darwin" && cfg.Driver == oci.Docker { - out.T(out.Sad, "openning service in browser is not yet implemented in docker driver on mac.\nPlease track the URL bellow to see updates for in progress work.\nhttps://github.com/kubernetes/minikube/issues/6778") + out.FailureT("openning service in browser is not implemented yet for \"docker driver on mac\".\nPlease track the URL bellow to see updates for in progress work:\nhttps://github.com/kubernetes/minikube/issues/6778") exit.WithCodeT(exit.Unavailable, "not implemented for docker driver on MacOs yet") } diff --git a/cmd/minikube/cmd/service_list.go b/cmd/minikube/cmd/service_list.go index 6b6dac18f1..7cf9dc7c65 100644 --- a/cmd/minikube/cmd/service_list.go +++ b/cmd/minikube/cmd/service_list.go @@ -73,7 +73,7 @@ var serviceListCmd = &cobra.Command{ } service.PrintServiceList(os.Stdout, data) if runtime.GOOS == "darwin" && cfg.Driver == oci.Docker { - out.FailureT("Accessing service on docker driver for mac is not implemented yet.Please follow the work in progress and the workarround to solve this issue:\n\thttps://github.com/kubernetes/minikube/issues/6778") + out.FailureT("Accessing service on docker driver for mac is not fully implemented yet.\nTo read about workarroudns on mac and update on the work in progress, please reffer to this link:\nhttps://github.com/kubernetes/minikube/issues/6778") exit.WithCodeT(exit.Failure, "not implemented for docker on mac driver yet.") } From 7596b3960718f2ee8db9057476e18c5799fc988a Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Mon, 24 Feb 2020 15:10:34 -0800 Subject: [PATCH 11/50] addres PR review --- cmd/minikube/cmd/service.go | 4 ++-- cmd/minikube/cmd/service_list.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/minikube/cmd/service.go b/cmd/minikube/cmd/service.go index 507b1fcfae..65e1f69297 100644 --- a/cmd/minikube/cmd/service.go +++ b/cmd/minikube/cmd/service.go @@ -91,8 +91,8 @@ var serviceCmd = &cobra.Command{ } if runtime.GOOS == "darwin" && cfg.Driver == oci.Docker { - out.FailureT("openning service in browser is not implemented yet for \"docker driver on mac\".\nPlease track the URL bellow to see updates for in progress work:\nhttps://github.com/kubernetes/minikube/issues/6778") - exit.WithCodeT(exit.Unavailable, "not implemented for docker driver on MacOs yet") + out.FailureT("Opening service in browser is not implemented yet for docker driver on Mac.\nThe following issue is tracking the in progress work:\nhttps://github.com/kubernetes/minikube/issues/6778") + exit.WithCodeT(exit.Unavailable, "Not yet implemented for docker driver on MacOS.") } for _, u := range urls { diff --git a/cmd/minikube/cmd/service_list.go b/cmd/minikube/cmd/service_list.go index 7cf9dc7c65..d2bc661c66 100644 --- a/cmd/minikube/cmd/service_list.go +++ b/cmd/minikube/cmd/service_list.go @@ -48,7 +48,7 @@ var serviceListCmd = &cobra.Command{ defer api.Close() profileName := viper.GetString(pkg_config.MachineProfile) if !machine.IsHostRunning(api, profileName) { - exit.WithCodeT(exit.Unavailable, "profile {{.name}} is not running. ", out.V{"name": profileName}) + exit.WithCodeT(exit.Unavailable, "profile {{.name}} is not running.", out.V{"name": profileName}) } serviceURLs, err := service.GetServiceURLs(api, serviceListNamespace, serviceURLTemplate) if err != nil { @@ -73,8 +73,8 @@ var serviceListCmd = &cobra.Command{ } service.PrintServiceList(os.Stdout, data) if runtime.GOOS == "darwin" && cfg.Driver == oci.Docker { - out.FailureT("Accessing service on docker driver for mac is not fully implemented yet.\nTo read about workarroudns on mac and update on the work in progress, please reffer to this link:\nhttps://github.com/kubernetes/minikube/issues/6778") - exit.WithCodeT(exit.Failure, "not implemented for docker on mac driver yet.") + out.FailureT("Accessing service is not implemented yet for docker driver on Mac.\nThe following issue is tracking the in progress work::\nhttps://github.com/kubernetes/minikube/issues/6778") + exit.WithCodeT(exit.Failure, "Not yet implemented for docker driver on MacOS.") } }, From b0ea2fd4d7ac9f370abdcf7f1961bac25a953bc9 Mon Sep 17 00:00:00 2001 From: Jose Donizetti Date: Mon, 24 Feb 2020 20:43:53 -0300 Subject: [PATCH 12/50] kic tunnel avoid printing exit as error --- pkg/minikube/tunnel/kic/ssh_conn.go | 26 ++++++++++++++++++-------- pkg/minikube/tunnel/kic/ssh_tunnel.go | 3 +-- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/pkg/minikube/tunnel/kic/ssh_conn.go b/pkg/minikube/tunnel/kic/ssh_conn.go index 8e4b085fa9..229bd0e5ec 100644 --- a/pkg/minikube/tunnel/kic/ssh_conn.go +++ b/pkg/minikube/tunnel/kic/ssh_conn.go @@ -24,8 +24,9 @@ import ( ) type sshConn struct { - name string - cmd *exec.Cmd + name string + service string + cmd *exec.Cmd } func createSSHConn(name, sshPort, sshKey string, svc v1.Service) *sshConn { @@ -51,17 +52,26 @@ func createSSHConn(name, sshPort, sshKey string, svc v1.Service) *sshConn { cmd := exec.Command("ssh", sshArgs...) return &sshConn{ - name: name, - cmd: cmd, + name: name, + service: svc.Name, + cmd: cmd, } } -func (c *sshConn) run() error { - fmt.Printf("starting tunnel for %s\n", c.name) - return c.cmd.Run() +func (c *sshConn) startAndWait() error { + fmt.Printf("starting tunnel for %s\n", c.service) + err := c.cmd.Start() + if err != nil { + return err + } + + // we ignore wait error because the process will be killed + c.cmd.Wait() + + return nil } func (c *sshConn) stop() error { - fmt.Printf("stopping tunnel for %s\n", c.name) + fmt.Printf("stopping tunnel for %s\n", c.service) return c.cmd.Process.Kill() } diff --git a/pkg/minikube/tunnel/kic/ssh_tunnel.go b/pkg/minikube/tunnel/kic/ssh_tunnel.go index 8567fe7d61..3dca0a1d34 100644 --- a/pkg/minikube/tunnel/kic/ssh_tunnel.go +++ b/pkg/minikube/tunnel/kic/ssh_tunnel.go @@ -108,11 +108,10 @@ func (t *SSHTunnel) startConnection(svc v1.Service) { t.conns[newSSHConn.name] = newSSHConn go func() { - err := newSSHConn.run() + err := newSSHConn.startAndWait() if err != nil { glog.Errorf("error starting ssh tunnel: %v", err) } - }() err := t.LoadBalancerEmulator.PatchServiceIP(t.v1Core.RESTClient(), svc, "127.0.0.1") From fc7b13aa0643fd30e3d79ba995fd71e1d1675791 Mon Sep 17 00:00:00 2001 From: Jose Donizetti Date: Mon, 24 Feb 2020 20:48:33 -0300 Subject: [PATCH 13/50] Fix kic tunnel to avoid asking ssh permission --- pkg/minikube/tunnel/kic/ssh_conn.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/minikube/tunnel/kic/ssh_conn.go b/pkg/minikube/tunnel/kic/ssh_conn.go index 8e4b085fa9..439c446921 100644 --- a/pkg/minikube/tunnel/kic/ssh_conn.go +++ b/pkg/minikube/tunnel/kic/ssh_conn.go @@ -31,6 +31,7 @@ type sshConn struct { func createSSHConn(name, sshPort, sshKey string, svc v1.Service) *sshConn { // extract sshArgs sshArgs := []string{ + "-o", "StrictHostKeyChecking no", "-N", "docker@127.0.0.1", "-p", sshPort, From c01ab3cd4c7d6a68e7e6f3398456653691002c62 Mon Sep 17 00:00:00 2001 From: Jose Donizetti Date: Mon, 24 Feb 2020 20:51:10 -0300 Subject: [PATCH 14/50] Add TODO kic tunnel --- pkg/minikube/tunnel/kic/ssh_conn.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/minikube/tunnel/kic/ssh_conn.go b/pkg/minikube/tunnel/kic/ssh_conn.go index 439c446921..bc02a2c1b1 100644 --- a/pkg/minikube/tunnel/kic/ssh_conn.go +++ b/pkg/minikube/tunnel/kic/ssh_conn.go @@ -31,6 +31,7 @@ type sshConn struct { func createSSHConn(name, sshPort, sshKey string, svc v1.Service) *sshConn { // extract sshArgs sshArgs := []string{ + // TODO: document the options here "-o", "StrictHostKeyChecking no", "-N", "docker@127.0.0.1", From 83a7bafbda0b5b2e810954ff8df89403b567ba51 Mon Sep 17 00:00:00 2001 From: Jose Donizetti Date: Mon, 24 Feb 2020 21:03:41 -0300 Subject: [PATCH 15/50] Fix linter --- pkg/minikube/tunnel/kic/ssh_conn.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/minikube/tunnel/kic/ssh_conn.go b/pkg/minikube/tunnel/kic/ssh_conn.go index 229bd0e5ec..339299ec9f 100644 --- a/pkg/minikube/tunnel/kic/ssh_conn.go +++ b/pkg/minikube/tunnel/kic/ssh_conn.go @@ -66,7 +66,7 @@ func (c *sshConn) startAndWait() error { } // we ignore wait error because the process will be killed - c.cmd.Wait() + _ = c.cmd.Wait() return nil } From 9d87d644412d3eb238d993f9c8878c041ebfd690 Mon Sep 17 00:00:00 2001 From: vikkyomkar Date: Tue, 25 Feb 2020 11:24:25 +0530 Subject: [PATCH 16/50] added suggestion message --- cmd/minikube/cmd/config/profile.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/minikube/cmd/config/profile.go b/cmd/minikube/cmd/config/profile.go index 3904d4c823..a7394e8435 100644 --- a/cmd/minikube/cmd/config/profile.go +++ b/cmd/minikube/cmd/config/profile.go @@ -65,7 +65,7 @@ var ProfileCmd = &cobra.Command{ } if !pkgConfig.ProfileExists(profile) { - out.FailureT("if you want to create a profile you can by this command: minikube start -p {{.profile_name}}", out.V{"profile_name": profile}) + out.ErrT(out.Tip, `if you want to create a profile you can by this command: minikube start -p {{.profile_name}}`, out.V{"profile_name": profile}) os.Exit(0) } From 72a355389dbec2b42cb14756175044042e64ef9a Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Tue, 25 Feb 2020 13:25:57 -0800 Subject: [PATCH 17/50] add mac os integration test for docker driver --- hack/jenkins/common.sh | 2 +- .../jenkins/cron/cleanup_and_reboot_Darwin.sh | 5 +++ hack/jenkins/cron/cleanup_and_reboot_Linux.sh | 5 +++ hack/jenkins/minikube_set_pending.sh | 1 + hack/jenkins/osx_integration_tests_docker.sh | 41 +++++++++++++++++++ 5 files changed, 53 insertions(+), 1 deletion(-) create mode 100755 hack/jenkins/osx_integration_tests_docker.sh diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh index 7153b5aead..eecf71fc99 100755 --- a/hack/jenkins/common.sh +++ b/hack/jenkins/common.sh @@ -33,7 +33,7 @@ export PATH=$PATH:"/usr/local/bin/:/usr/local/go/bin/:$GOPATH/bin" # installing golang so we could do go get for gopogh sudo ./installers/check_install_golang.sh "1.13.6" "/usr/local" || true -docker rm -f $(docker ps -aq) >/dev/null 2>&1 || true +docker rm -f -v $(docker ps -aq) >/dev/null 2>&1 || true docker volume prune -f || true docker system df || true diff --git a/hack/jenkins/cron/cleanup_and_reboot_Darwin.sh b/hack/jenkins/cron/cleanup_and_reboot_Darwin.sh index 993d6c82f6..e8f0c73415 100755 --- a/hack/jenkins/cron/cleanup_and_reboot_Darwin.sh +++ b/hack/jenkins/cron/cleanup_and_reboot_Darwin.sh @@ -41,6 +41,11 @@ logger "cleanup_and_reboot is happening!" # kill jenkins to avoid an incoming request killall java +# clean docker left overs +docker rm -f -v $(docker ps -aq) >/dev/null 2>&1 || true +docker volume prune -f || true +docker system df || true + # macOS specific cleanup sudo rm /var/db/dhcpd_leases || echo "could not clear dhcpd leases" sudo softwareupdate -i -a -R diff --git a/hack/jenkins/cron/cleanup_and_reboot_Linux.sh b/hack/jenkins/cron/cleanup_and_reboot_Linux.sh index b2e067c58e..c55742970b 100755 --- a/hack/jenkins/cron/cleanup_and_reboot_Linux.sh +++ b/hack/jenkins/cron/cleanup_and_reboot_Linux.sh @@ -36,6 +36,11 @@ logger "cleanup_and_reboot is happening!" # kill jenkins to avoid an incoming request killall java +# clean docker left overs +docker rm -f -v $(docker ps -aq) >/dev/null 2>&1 || true +docker volume prune -f || true +docker system df || true + # Linux-specific cleanup # disable localkube, kubelet diff --git a/hack/jenkins/minikube_set_pending.sh b/hack/jenkins/minikube_set_pending.sh index c26324635a..1d91deaa31 100755 --- a/hack/jenkins/minikube_set_pending.sh +++ b/hack/jenkins/minikube_set_pending.sh @@ -41,6 +41,7 @@ jobs=( 'KVM_Linux' 'none_Linux' 'Docker_Linux' + 'Docker_macOS' 'Podman_Linux' ) diff --git a/hack/jenkins/osx_integration_tests_docker.sh b/hack/jenkins/osx_integration_tests_docker.sh new file mode 100755 index 0000000000..648c00c636 --- /dev/null +++ b/hack/jenkins/osx_integration_tests_docker.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# This script runs the integration tests on an OSX machine for the Hyperkit Driver + +# The script expects the following env variables: +# MINIKUBE_LOCATION: GIT_COMMIT from upstream build. +# COMMIT: Actual commit ID from upstream build +# EXTRA_BUILD_ARGS (optional): Extra args to be passed into the minikube integrations tests +# access_token: The Github API access token. Injected by the Jenkins credential provider. + + +set -e + +OS_ARCH="darwin-amd64" +VM_DRIVER="docker" +JOB_NAME="Docker_macOS" +EXTRA_START_ARGS="" +EXPECTED_DEFAULT_DRIVER="hyperkit" + + +mkdir -p cron && gsutil -qm rsync "gs://minikube-builds/${MINIKUBE_LOCATION}/cron" cron || echo "FAILED TO GET CRON FILES" +install cron/cleanup_and_reboot_Darwin.sh $HOME/cleanup_and_reboot.sh || echo "FAILED TO INSTALL CLEANUP" +echo "*/30 * * * * $HOME/cleanup_and_reboot.sh" | crontab +crontab -l + +source common.sh From 30a926f04376d41b2c8a7b81586043366e7e8798 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Tue, 25 Feb 2020 15:16:18 -0800 Subject: [PATCH 18/50] ls volumes --- hack/jenkins/cron/cleanup_and_reboot_Darwin.sh | 2 ++ hack/jenkins/cron/cleanup_and_reboot_Linux.sh | 1 + 2 files changed, 3 insertions(+) diff --git a/hack/jenkins/cron/cleanup_and_reboot_Darwin.sh b/hack/jenkins/cron/cleanup_and_reboot_Darwin.sh index e8f0c73415..75c7dfedb4 100755 --- a/hack/jenkins/cron/cleanup_and_reboot_Darwin.sh +++ b/hack/jenkins/cron/cleanup_and_reboot_Darwin.sh @@ -44,8 +44,10 @@ killall java # clean docker left overs docker rm -f -v $(docker ps -aq) >/dev/null 2>&1 || true docker volume prune -f || true +docker volume ls || true docker system df || true + # macOS specific cleanup sudo rm /var/db/dhcpd_leases || echo "could not clear dhcpd leases" sudo softwareupdate -i -a -R diff --git a/hack/jenkins/cron/cleanup_and_reboot_Linux.sh b/hack/jenkins/cron/cleanup_and_reboot_Linux.sh index c55742970b..e03e6cd06b 100755 --- a/hack/jenkins/cron/cleanup_and_reboot_Linux.sh +++ b/hack/jenkins/cron/cleanup_and_reboot_Linux.sh @@ -39,6 +39,7 @@ killall java # clean docker left overs docker rm -f -v $(docker ps -aq) >/dev/null 2>&1 || true docker volume prune -f || true +docker volume ls || true docker system df || true # Linux-specific cleanup From e2161049a71caf7aa5a39f936bc35194445ba2ed Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Tue, 25 Feb 2020 15:55:35 -0800 Subject: [PATCH 19/50] restart docker on mac --- hack/jenkins/osx_integration_tests_docker.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hack/jenkins/osx_integration_tests_docker.sh b/hack/jenkins/osx_integration_tests_docker.sh index 648c00c636..c2865d83ff 100755 --- a/hack/jenkins/osx_integration_tests_docker.sh +++ b/hack/jenkins/osx_integration_tests_docker.sh @@ -33,6 +33,9 @@ EXTRA_START_ARGS="" EXPECTED_DEFAULT_DRIVER="hyperkit" +# restart docker on mac for a fresh test +osascript -e 'quit app "Docker"'; open -a Docker ; while [ -z "$(docker info 2> /dev/null )" ]; do printf "."; sleep 1; done; echo "" || true + mkdir -p cron && gsutil -qm rsync "gs://minikube-builds/${MINIKUBE_LOCATION}/cron" cron || echo "FAILED TO GET CRON FILES" install cron/cleanup_and_reboot_Darwin.sh $HOME/cleanup_and_reboot.sh || echo "FAILED TO INSTALL CLEANUP" echo "*/30 * * * * $HOME/cleanup_and_reboot.sh" | crontab From e60b034c840f086c8e6affbf99549a886ed98838 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Tue, 25 Feb 2020 17:16:49 -0800 Subject: [PATCH 20/50] Add Daemon Info func to oci --- pkg/drivers/kic/oci/info.go | 239 ++++++++++++++++++++++++++++++++++++ 1 file changed, 239 insertions(+) create mode 100644 pkg/drivers/kic/oci/info.go diff --git a/pkg/drivers/kic/oci/info.go b/pkg/drivers/kic/oci/info.go new file mode 100644 index 0000000000..9eac680458 --- /dev/null +++ b/pkg/drivers/kic/oci/info.go @@ -0,0 +1,239 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package oci + +import ( + "encoding/json" + "os/exec" + "strings" + "time" + + "github.com/pkg/errors" +) + +// SysInfo Info represents common system Information between docker and podman that minikube cares +type SysInfo struct { + CPUs int // CPUs is Number of CPUs + TotalMemory int64 // TotalMemory Total available ram +} + +// dockerSysInfo represents the output of docker system info --format '{{json .}}' +type dockerSysInfo struct { + ID string `json:"ID"` + Containers int `json:"Containers"` + ContainersRunning int `json:"ContainersRunning"` + ContainersPaused int `json:"ContainersPaused"` + ContainersStopped int `json:"ContainersStopped"` + Images int `json:"Images"` + Driver string `json:"Driver"` + DriverStatus [][]string `json:"DriverStatus"` + SystemStatus interface{} `json:"SystemStatus"` + Plugins struct { + Volume []string `json:"Volume"` + Network []string `json:"Network"` + Authorization interface{} `json:"Authorization"` + Log []string `json:"Log"` + } `json:"Plugins"` + MemoryLimit bool `json:"MemoryLimit"` + SwapLimit bool `json:"SwapLimit"` + KernelMemory bool `json:"KernelMemory"` + KernelMemoryTCP bool `json:"KernelMemoryTCP"` + CPUCfsPeriod bool `json:"CpuCfsPeriod"` + CPUCfsQuota bool `json:"CpuCfsQuota"` + CPUShares bool `json:"CPUShares"` + CPUSet bool `json:"CPUSet"` + PidsLimit bool `json:"PidsLimit"` + IPv4Forwarding bool `json:"IPv4Forwarding"` + BridgeNfIptables bool `json:"BridgeNfIptables"` + BridgeNfIP6Tables bool `json:"BridgeNfIp6tables"` + Debug bool `json:"Debug"` + NFd int `json:"NFd"` + OomKillDisable bool `json:"OomKillDisable"` + NGoroutines int `json:"NGoroutines"` + SystemTime time.Time `json:"SystemTime"` + LoggingDriver string `json:"LoggingDriver"` + CgroupDriver string `json:"CgroupDriver"` + NEventsListener int `json:"NEventsListener"` + KernelVersion string `json:"KernelVersion"` + OperatingSystem string `json:"OperatingSystem"` + OSType string `json:"OSType"` + Architecture string `json:"Architecture"` + IndexServerAddress string `json:"IndexServerAddress"` + RegistryConfig struct { + AllowNondistributableArtifactsCIDRs []interface{} `json:"AllowNondistributableArtifactsCIDRs"` + AllowNondistributableArtifactsHostnames []interface{} `json:"AllowNondistributableArtifactsHostnames"` + InsecureRegistryCIDRs []string `json:"InsecureRegistryCIDRs"` + IndexConfigs struct { + DockerIo struct { + Name string `json:"Name"` + Mirrors []interface{} `json:"Mirrors"` + Secure bool `json:"Secure"` + Official bool `json:"Official"` + } `json:"docker.io"` + } `json:"IndexConfigs"` + Mirrors []interface{} `json:"Mirrors"` + } `json:"RegistryConfig"` + NCPU int `json:"NCPU"` + MemTotal int64 `json:"MemTotal"` + GenericResources interface{} `json:"GenericResources"` + DockerRootDir string `json:"DockerRootDir"` + HTTPProxy string `json:"HttpProxy"` + HTTPSProxy string `json:"HttpsProxy"` + NoProxy string `json:"NoProxy"` + Name string `json:"Name"` + Labels []interface{} `json:"Labels"` + ExperimentalBuild bool `json:"ExperimentalBuild"` + ServerVersion string `json:"ServerVersion"` + ClusterStore string `json:"ClusterStore"` + ClusterAdvertise string `json:"ClusterAdvertise"` + Runtimes struct { + Runc struct { + Path string `json:"path"` + } `json:"runc"` + } `json:"Runtimes"` + DefaultRuntime string `json:"DefaultRuntime"` + Swarm struct { + NodeID string `json:"NodeID"` + NodeAddr string `json:"NodeAddr"` + LocalNodeState string `json:"LocalNodeState"` + ControlAvailable bool `json:"ControlAvailable"` + Error string `json:"Error"` + RemoteManagers interface{} `json:"RemoteManagers"` + } `json:"Swarm"` + LiveRestoreEnabled bool `json:"LiveRestoreEnabled"` + Isolation string `json:"Isolation"` + InitBinary string `json:"InitBinary"` + ContainerdCommit struct { + ID string `json:"ID"` + Expected string `json:"Expected"` + } `json:"ContainerdCommit"` + RuncCommit struct { + ID string `json:"ID"` + Expected string `json:"Expected"` + } `json:"RuncCommit"` + InitCommit struct { + ID string `json:"ID"` + Expected string `json:"Expected"` + } `json:"InitCommit"` + SecurityOptions []string `json:"SecurityOptions"` + ProductLicense string `json:"ProductLicense"` + Warnings interface{} `json:"Warnings"` + ClientInfo struct { + Debug bool `json:"Debug"` + Plugins []interface{} `json:"Plugins"` + Warnings interface{} `json:"Warnings"` + } `json:"ClientInfo"` +} + +// podmanSysInfo represents the output of podman system info --format '{{json .}}' +type podmanSysInfo struct { + Host struct { + BuildahVersion string `json:"BuildahVersion"` + CgroupVersion string `json:"CgroupVersion"` + Conmon struct { + Package string `json:"package"` + Path string `json:"path"` + Version string `json:"version"` + } `json:"Conmon"` + Distribution struct { + Distribution string `json:"distribution"` + Version string `json:"version"` + } `json:"Distribution"` + MemFree int `json:"MemFree"` + MemTotal int64 `json:"MemTotal"` + OCIRuntime struct { + Name string `json:"name"` + Package string `json:"package"` + Path string `json:"path"` + Version string `json:"version"` + } `json:"OCIRuntime"` + SwapFree int `json:"SwapFree"` + SwapTotal int `json:"SwapTotal"` + Arch string `json:"arch"` + Cpus int `json:"cpus"` + Eventlogger string `json:"eventlogger"` + Hostname string `json:"hostname"` + Kernel string `json:"kernel"` + Os string `json:"os"` + Rootless bool `json:"rootless"` + Uptime string `json:"uptime"` + } `json:"host"` + Registries struct { + Search []string `json:"search"` + } `json:"registries"` + Store struct { + ConfigFile string `json:"ConfigFile"` + ContainerStore struct { + Number int `json:"number"` + } `json:"ContainerStore"` + GraphDriverName string `json:"GraphDriverName"` + GraphOptions struct { + } `json:"GraphOptions"` + GraphRoot string `json:"GraphRoot"` + GraphStatus struct { + BackingFilesystem string `json:"Backing Filesystem"` + NativeOverlayDiff string `json:"Native Overlay Diff"` + SupportsDType string `json:"Supports d_type"` + UsingMetacopy string `json:"Using metacopy"` + } `json:"GraphStatus"` + ImageStore struct { + Number int `json:"number"` + } `json:"ImageStore"` + RunRoot string `json:"RunRoot"` + VolumePath string `json:"VolumePath"` + } `json:"store"` +} + +// DaemonInfo returns docker system info +func DaemonInfo(ociBin string) (SysInfo, error) { + if ociBin == Podman { + return podmanSystemInfo() + } + return dockerSystemInfo() +} + +func dockerSystemInfo() (SysInfo, error) { + var ds dockerSysInfo + var s SysInfo + cmd := exec.Command(Docker, "system", "info", "--format", "'{{json .}}'") + out, err := cmd.CombinedOutput() + if err != nil { + return s, errors.Wrap(err, "get docker system info") + } + if err := json.Unmarshal([]byte(strings.TrimSpace(string(out))), &ds); err != nil { + return s, errors.Wrapf(err, "unmarshal docker system info") + } + s.CPUs = ds.NCPU + s.TotalMemory = ds.MemTotal + return s, nil +} + +func podmanSystemInfo() (SysInfo, error) { + var ps podmanSysInfo + var s SysInfo + cmd := exec.Command(Podman, "system", "info", "--format", "'{{json .}}'") + out, err := cmd.CombinedOutput() + if err != nil { + return s, errors.Wrap(err, "get podman system info") + } + if err := json.Unmarshal([]byte(strings.TrimSpace(string(out))), &ps); err != nil { + return s, errors.Wrapf(err, "unmarshal podman system info") + } + s.CPUs = ps.Host.Cpus + s.TotalMemory = ps.Host.MemTotal + return s, nil +} From f3c171eaa39f7c2a7452a466e8590399179e5a63 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Tue, 25 Feb 2020 17:34:03 -0800 Subject: [PATCH 21/50] show info for docker damemon --- pkg/drivers/kic/oci/info.go | 2 +- pkg/minikube/machine/info.go | 8 ++++---- pkg/minikube/machine/start.go | 6 +++++- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/pkg/drivers/kic/oci/info.go b/pkg/drivers/kic/oci/info.go index 9eac680458..8d3ef73fc2 100644 --- a/pkg/drivers/kic/oci/info.go +++ b/pkg/drivers/kic/oci/info.go @@ -209,7 +209,7 @@ func DaemonInfo(ociBin string) (SysInfo, error) { func dockerSystemInfo() (SysInfo, error) { var ds dockerSysInfo var s SysInfo - cmd := exec.Command(Docker, "system", "info", "--format", "'{{json .}}'") + cmd := exec.Command(Docker, "system", "info", "--format", "{{json .}}") out, err := cmd.CombinedOutput() if err != nil { return s, errors.Wrap(err, "get docker system info") diff --git a/pkg/minikube/machine/info.go b/pkg/minikube/machine/info.go index 99330d3850..1bae7253e9 100644 --- a/pkg/minikube/machine/info.go +++ b/pkg/minikube/machine/info.go @@ -29,13 +29,13 @@ import ( ) type hostInfo struct { - Memory int + Memory int64 CPUs int - DiskSize int + DiskSize int64 } -func megs(bytes uint64) int { - return int(bytes / 1024 / 1024) +func megs(bytes uint64) int64 { + return int64(bytes / 1024 / 1024) } func getHostInfo() (*hostInfo, error) { diff --git a/pkg/minikube/machine/start.go b/pkg/minikube/machine/start.go index c5cd2fa11f..6549bdbdcc 100644 --- a/pkg/minikube/machine/start.go +++ b/pkg/minikube/machine/start.go @@ -32,6 +32,7 @@ import ( "github.com/juju/mutex" "github.com/pkg/errors" "github.com/spf13/viper" + "k8s.io/minikube/pkg/drivers/kic/oci" "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/constants" @@ -260,8 +261,11 @@ func showHostInfo(cfg config.ClusterConfig) { return } if driver.IsKIC(cfg.Driver) { - info, err := getHostInfo() // TODO medyagh: get docker-machine info for non linux + s, err := oci.DaemonInfo(cfg.Driver) if err == nil { + var info hostInfo + info.CPUs = s.CPUs + info.Memory = s.TotalMemory out.T(out.StartingVM, "Creating Kubernetes in {{.driver_name}} container with (CPUs={{.number_of_cpus}}), Memory={{.memory_size}}MB ({{.host_memory_size}}MB available) ...", out.V{"driver_name": cfg.Driver, "number_of_cpus": cfg.CPUs, "number_of_host_cpus": info.CPUs, "memory_size": cfg.Memory, "host_memory_size": info.Memory}) } return From f64b167c3bb9557ad090e6006c36ebd099aa156c Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Tue, 25 Feb 2020 18:08:53 -0800 Subject: [PATCH 22/50] Display more meaninful system info for docker/podman driver --- pkg/minikube/machine/start.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/minikube/machine/start.go b/pkg/minikube/machine/start.go index 6549bdbdcc..a4a994c1bc 100644 --- a/pkg/minikube/machine/start.go +++ b/pkg/minikube/machine/start.go @@ -260,13 +260,13 @@ func showHostInfo(cfg config.ClusterConfig) { } return } - if driver.IsKIC(cfg.Driver) { + if driver.IsKIC(cfg.Driver) { // TODO:medyagh add free disk space on docker machine s, err := oci.DaemonInfo(cfg.Driver) if err == nil { var info hostInfo info.CPUs = s.CPUs - info.Memory = s.TotalMemory - out.T(out.StartingVM, "Creating Kubernetes in {{.driver_name}} container with (CPUs={{.number_of_cpus}}), Memory={{.memory_size}}MB ({{.host_memory_size}}MB available) ...", out.V{"driver_name": cfg.Driver, "number_of_cpus": cfg.CPUs, "number_of_host_cpus": info.CPUs, "memory_size": cfg.Memory, "host_memory_size": info.Memory}) + info.Memory = megs(uint64(s.TotalMemory)) + out.T(out.StartingVM, "Creating Kubernetes in {{.driver_name}} container with (CPUs={{.number_of_cpus}}) ({{.number_of_host_cpus}} available), Memory={{.memory_size}}MB ({{.host_memory_size}}MB available) ...", out.V{"driver_name": cfg.Driver, "number_of_cpus": cfg.CPUs, "number_of_host_cpus": info.CPUs, "memory_size": cfg.Memory, "host_memory_size": info.Memory}) } return } From 6651e0b5a52d7f16e801b9a82ab2bee40585c698 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Wed, 26 Feb 2020 09:15:25 -0800 Subject: [PATCH 23/50] docker podman sysinfo --- pkg/drivers/kic/oci/info.go | 50 +++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/pkg/drivers/kic/oci/info.go b/pkg/drivers/kic/oci/info.go index 8d3ef73fc2..3245bee2a6 100644 --- a/pkg/drivers/kic/oci/info.go +++ b/pkg/drivers/kic/oci/info.go @@ -31,6 +31,21 @@ type SysInfo struct { TotalMemory int64 // TotalMemory Total available ram } +// DaemonInfo returns common docker/podman daemon system info that minikube cares about +func DaemonInfo(ociBin string) (SysInfo, error) { + var info SysInfo + if ociBin == Podman { + p, err := podmanSystemInfo() + info.CPUs = p.Host.Cpus + info.TotalMemory = p.Host.MemTotal + return info, err + } + d, err := dockerSystemInfo() + info.CPUs = d.NCPU + info.TotalMemory = d.MemTotal + return info, err +} + // dockerSysInfo represents the output of docker system info --format '{{json .}}' type dockerSysInfo struct { ID string `json:"ID"` @@ -198,42 +213,33 @@ type podmanSysInfo struct { } `json:"store"` } -// DaemonInfo returns docker system info -func DaemonInfo(ociBin string) (SysInfo, error) { - if ociBin == Podman { - return podmanSystemInfo() - } - return dockerSystemInfo() -} - -func dockerSystemInfo() (SysInfo, error) { +// dockerSystemInfo returns docker system info --format '{{json .}}' +func dockerSystemInfo() (dockerSysInfo, error) { var ds dockerSysInfo - var s SysInfo + if err := PointToHostDockerDaemon(); err != nil { + return ds, errors.Wrap(err, "point host docker-daemon") + } cmd := exec.Command(Docker, "system", "info", "--format", "{{json .}}") out, err := cmd.CombinedOutput() if err != nil { - return s, errors.Wrap(err, "get docker system info") + return ds, errors.Wrap(err, "get docker system info") } if err := json.Unmarshal([]byte(strings.TrimSpace(string(out))), &ds); err != nil { - return s, errors.Wrapf(err, "unmarshal docker system info") + return ds, errors.Wrapf(err, "unmarshal docker system info") } - s.CPUs = ds.NCPU - s.TotalMemory = ds.MemTotal - return s, nil + return ds, nil } -func podmanSystemInfo() (SysInfo, error) { +// podmanSysInfo returns podman system info --format '{{json .}}' +func podmanSystemInfo() (podmanSysInfo, error) { var ps podmanSysInfo - var s SysInfo cmd := exec.Command(Podman, "system", "info", "--format", "'{{json .}}'") out, err := cmd.CombinedOutput() if err != nil { - return s, errors.Wrap(err, "get podman system info") + return ps, errors.Wrap(err, "get podman system info") } if err := json.Unmarshal([]byte(strings.TrimSpace(string(out))), &ps); err != nil { - return s, errors.Wrapf(err, "unmarshal podman system info") + return ps, errors.Wrapf(err, "unmarshal podman system info") } - s.CPUs = ps.Host.Cpus - s.TotalMemory = ps.Host.MemTotal - return s, nil + return ps, nil } From e8c85543f7a021fbcd9f1babb8b61e9e2d205699 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Wed, 26 Feb 2020 10:57:46 -0800 Subject: [PATCH 24/50] limit time waiting for delete --- pkg/drivers/kic/oci/oci.go | 45 ++++++++++++++++++++++++++++++---- pkg/drivers/kic/oci/volumes.go | 21 +++++++++++++--- 2 files changed, 58 insertions(+), 8 deletions(-) diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go index 630b6db4b9..fa41c26fc2 100644 --- a/pkg/drivers/kic/oci/oci.go +++ b/pkg/drivers/kic/oci/oci.go @@ -17,9 +17,11 @@ limitations under the License. package oci import ( + "context" "os" "path/filepath" "strconv" + "time" "bufio" "bytes" @@ -51,9 +53,18 @@ func DeleteAllContainersByLabel(ociBin string, label string) []error { return nil } for _, c := range cs { - cmd := exec.Command(ociBin, "rm", "-f", "-v", c) - if out, err := cmd.CombinedOutput(); err != nil { - deleteErrs = append(deleteErrs, errors.Wrapf(err, "delete container %s: output %s", c, out)) + _, err := containerStatus(ociBin, c) + // only try to delete if docker/podman inspect returns + // if it doesn't it means docker-daemon is stuck and needs restart + if err != nil { + deleteErrs = append(deleteErrs, errors.Wrapf(err, "delete container %s: -daemon is stuck.", c, ociBin)) + glog.Errorf("%s-daemon seems to be stuck. Please try restarting your %s.", ociBin, ociBin) + } else { + cmd := exec.Command(ociBin, "rm", "-f", "-v", c) + if out, err := cmd.CombinedOutput(); err != nil { + deleteErrs = append(deleteErrs, errors.Wrapf(err, "delete container %s: output %s", c, out)) + } + } } return deleteErrs @@ -429,7 +440,10 @@ func listContainersByLabel(ociBinary string, label string) ([]string, error) { if err := PointToHostDockerDaemon(); err != nil { return nil, errors.Wrap(err, "point host docker-daemon") } - cmd := exec.Command(ociBinary, "ps", "-a", "--filter", fmt.Sprintf("label=%s", label), "--format", "{{.Names}}") + // allow no more than 5 seconds for docker ps + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, ociBinary, "ps", "-a", "--filter", fmt.Sprintf("label=%s", label), "--format", "{{.Names}}") stdout, err := cmd.Output() s := bufio.NewScanner(bytes.NewReader(stdout)) var names []string @@ -439,7 +453,6 @@ func listContainersByLabel(ociBinary string, label string) ([]string, error) { names = append(names, n) } } - return names, err } @@ -461,3 +474,25 @@ func PointToHostDockerDaemon() error { } return nil } + +// containerStatus returns status of a container running,exited,... +func containerStatus(ociBin string, name string) (string, error) { + if ociBin == Docker { + if err := PointToHostDockerDaemon(); err != nil { + return "", errors.Wrap(err, "point host docker-daemon") + } + } + // allow no more than 2 seconds for this. when this takes long this means deadline passed + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, ociBin, "inspect", name, "--format={{.State.Status}}") + out, err := cmd.CombinedOutput() + if ctx.Err() == context.DeadlineExceeded { + glog.Warningf("%s inspect %s took longer than normal. Restarting your docker daemon might fix this issue.") + return strings.TrimSpace(string(out)), fmt.Errorf("inspect %s timeout", name) + } + if err != nil { + return string(out), errors.Wrapf(err, "inspecting container: output %s", out) + } + return strings.TrimSpace(string(out)), nil +} diff --git a/pkg/drivers/kic/oci/volumes.go b/pkg/drivers/kic/oci/volumes.go index a06f471d04..1181aa0652 100644 --- a/pkg/drivers/kic/oci/volumes.go +++ b/pkg/drivers/kic/oci/volumes.go @@ -19,9 +19,11 @@ package oci import ( "bufio" "bytes" + "context" "fmt" "os/exec" "strings" + "time" "github.com/golang/glog" "github.com/pkg/errors" @@ -42,9 +44,15 @@ func DeleteAllVolumesByLabel(ociBin string, label string) []error { if err != nil { return []error{fmt.Errorf("listing volumes by label %q: %v", label, err)} } - for _, v := range vs { - cmd := exec.Command(ociBin, "volume", "rm", "--force", v) + // allow no more than 3 seconds for this. when this takes long this means deadline passed + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, ociBin, "volume", "rm", "--force", v) + if ctx.Err() == context.DeadlineExceeded { + glog.Warningf("removing volume with label %s took longer than normal. Restarting your %s daemon might fix this issue.", label, ociBin) + deleteErrs = append(deleteErrs, fmt.Errorf("delete deadline exceeded for %s", label)) + } if out, err := cmd.CombinedOutput(); err != nil { deleteErrs = append(deleteErrs, fmt.Errorf("deleting volume %s: output: %s", v, string(out))) } @@ -63,9 +71,16 @@ func PruneAllVolumesByLabel(ociBin string, label string) []error { return []error{errors.Wrap(err, "point host docker-daemon")} } } + // allow no more than 5 seconds for this. when this takes long this means deadline passed + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() // try to prune afterwards just in case delete didn't go through - cmd := exec.Command(ociBin, "volume", "prune", "-f", "--filter", "label="+label) + cmd := exec.CommandContext(ctx, ociBin, "volume", "prune", "-f", "--filter", "label="+label) + if ctx.Err() == context.DeadlineExceeded { + glog.Warningf("pruning volume with label %s took longer than normal. Restarting your %s daemon might fix this issue.", label, ociBin) + deleteErrs = append(deleteErrs, fmt.Errorf("prune deadline exceeded for %s", label)) + } if out, err := cmd.CombinedOutput(); err != nil { deleteErrs = append(deleteErrs, errors.Wrapf(err, "prune volume by label %s: %s", label, string(out))) } From 4709af55df13ad06af03d125198905f804d22663 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Wed, 26 Feb 2020 12:44:25 -0800 Subject: [PATCH 25/50] resolve review comments --- cmd/minikube/cmd/delete.go | 5 ++--- pkg/drivers/kic/kic.go | 13 +++++++++-- pkg/drivers/kic/oci/network.go | 4 ++-- pkg/drivers/kic/oci/oci.go | 40 +++++++++++++++++----------------- pkg/drivers/kic/oci/volumes.go | 16 +++++++------- pkg/minikube/machine/delete.go | 33 +++++++++++++++++++++++----- 6 files changed, 71 insertions(+), 40 deletions(-) diff --git a/cmd/minikube/cmd/delete.go b/cmd/minikube/cmd/delete.go index fe90c6d578..f64579112f 100644 --- a/cmd/minikube/cmd/delete.go +++ b/cmd/minikube/cmd/delete.go @@ -116,7 +116,7 @@ func runDelete(cmd *cobra.Command, args []string) { exit.UsageT("usage: minikube delete --all") } delLabel := fmt.Sprintf("%s=%s", oci.CreatedByLabelKey, "true") - errs := oci.DeleteAllContainersByLabel(oci.Docker, delLabel) + errs := oci.DeleteContainersByLabel(oci.Docker, delLabel) if len(errs) > 0 { // it will error if there is no container to delete glog.Infof("error delete containers by label %q (might be okay): %+v", delLabel, err) } @@ -194,7 +194,7 @@ func deleteProfile(profile *pkg_config.Profile) error { viper.Set(pkg_config.MachineProfile, profile.Name) delLabel := fmt.Sprintf("%s=%s", oci.ProfileLabelKey, profile.Name) - errs := oci.DeleteAllContainersByLabel(oci.Docker, delLabel) + errs := oci.DeleteContainersByLabel(oci.Docker, delLabel) if errs != nil { // it will error if there is no container to delete glog.Infof("error deleting containers for %s (might be okay):\n%v", profile.Name, errs) } @@ -207,7 +207,6 @@ func deleteProfile(profile *pkg_config.Profile) error { if len(errs) > 0 { // it will not error if there is nothing to delete glog.Warningf("error pruning volume (might be okay):\n%v", errs) } - api, err := machine.NewAPIClient() if err != nil { delErr := profileDeletionErr(profile.Name, fmt.Sprintf("error getting client %v", err)) diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go index 0ecb6a2ade..a5abcd372d 100644 --- a/pkg/drivers/kic/kic.go +++ b/pkg/drivers/kic/kic.go @@ -17,11 +17,13 @@ limitations under the License. package kic import ( + "context" "fmt" "net" "os/exec" "strconv" "strings" + "time" "github.com/docker/machine/libmachine/drivers" "github.com/docker/machine/libmachine/log" @@ -181,11 +183,18 @@ func (d *Driver) GetURL() (string, error) { // GetState returns the state that the host is in (running, stopped, etc) func (d *Driver) GetState() (state.State, error) { if err := oci.PointToHostDockerDaemon(); err != nil { - return state.Error, errors.Wrap(err, "point host docker-daemon") + return state.Error, errors.Wrap(err, "point host docker daemon") } + // allow no more than 2 seconds for this. when this takes long this means deadline passed + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() - cmd := exec.Command(d.NodeConfig.OCIBinary, "inspect", "-f", "{{.State.Status}}", d.MachineName) + cmd := exec.CommandContext(ctx, d.NodeConfig.OCIBinary, "inspect", "-f", "{{.State.Status}}", d.MachineName) out, err := cmd.CombinedOutput() + if ctx.Err() == context.DeadlineExceeded { + glog.Errorf("GetState for %s took longer than normal. Restarting your %s daemon might fix this issue.", d.MachineName, d.OCIBinary) + return state.Error, fmt.Errorf("inspect %s timeout", d.MachineName) + } o := strings.TrimSpace(string(out)) if err != nil { return state.Error, errors.Wrapf(err, "get container %s status", d.MachineName) diff --git a/pkg/drivers/kic/oci/network.go b/pkg/drivers/kic/oci/network.go index 5e27528b5f..4e28161d84 100644 --- a/pkg/drivers/kic/oci/network.go +++ b/pkg/drivers/kic/oci/network.go @@ -43,7 +43,7 @@ func RoutableHostIPFromInside(ociBin string, containerName string) (net.IP, erro // digDNS will get the IP record for a dns func digDNS(ociBin, containerName, dns string) (net.IP, error) { if err := PointToHostDockerDaemon(); err != nil { - return nil, errors.Wrap(err, "point host docker-daemon") + return nil, errors.Wrap(err, "point host docker daemon") } cmd := exec.Command(ociBin, "exec", "-t", containerName, "dig", "+short", dns) out, err := cmd.CombinedOutput() @@ -59,7 +59,7 @@ func digDNS(ociBin, containerName, dns string) (net.IP, error) { // gets the ip from user's host docker func dockerGatewayIP() (net.IP, error) { if err := PointToHostDockerDaemon(); err != nil { - return nil, errors.Wrap(err, "point host docker-daemon") + return nil, errors.Wrap(err, "point host docker daemon") } cmd := exec.Command(Docker, "network", "ls", "--filter", "name=bridge", "--format", "{{.ID}}") out, err := cmd.CombinedOutput() diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go index fa41c26fc2..784e3250c2 100644 --- a/pkg/drivers/kic/oci/oci.go +++ b/pkg/drivers/kic/oci/oci.go @@ -36,13 +36,13 @@ import ( "strings" ) -// DeleteAllContainersByLabel deletes all containers that have a specific label +// DeleteContainersByLabel deletes all containers that have a specific label // if there no containers found with the given label, it will return nil -func DeleteAllContainersByLabel(ociBin string, label string) []error { +func DeleteContainersByLabel(ociBin string, label string) []error { var deleteErrs []error if ociBin == Docker { if err := PointToHostDockerDaemon(); err != nil { - return []error{errors.Wrap(err, "point host docker-daemon")} + return []error{errors.Wrap(err, "point host docker daemon")} } } cs, err := listContainersByLabel(ociBin, label) @@ -53,12 +53,12 @@ func DeleteAllContainersByLabel(ociBin string, label string) []error { return nil } for _, c := range cs { - _, err := containerStatus(ociBin, c) + _, err := ContainerStatus(ociBin, c) // only try to delete if docker/podman inspect returns - // if it doesn't it means docker-daemon is stuck and needs restart + // if it doesn't it means docker daemon is stuck and needs restart if err != nil { - deleteErrs = append(deleteErrs, errors.Wrapf(err, "delete container %s: -daemon is stuck.", c, ociBin)) - glog.Errorf("%s-daemon seems to be stuck. Please try restarting your %s.", ociBin, ociBin) + deleteErrs = append(deleteErrs, errors.Wrapf(err, "delete container %s: %s daemon is stuck. please try again!", c, ociBin)) + glog.Errorf("%s daemon seems to be stuck. Please try restarting your %s.", ociBin, ociBin) } else { cmd := exec.Command(ociBin, "rm", "-f", "-v", c) if out, err := cmd.CombinedOutput(); err != nil { @@ -73,7 +73,7 @@ func DeleteAllContainersByLabel(ociBin string, label string) []error { // CreateContainerNode creates a new container node func CreateContainerNode(p CreateParams) error { if err := PointToHostDockerDaemon(); err != nil { - return errors.Wrap(err, "point host docker-daemon") + return errors.Wrap(err, "point host docker daemon") } runArgs := []string{ @@ -152,7 +152,7 @@ func CreateContainerNode(p CreateParams) error { // CreateContainer creates a container with "docker/podman run" func createContainer(ociBinary string, image string, opts ...createOpt) ([]string, error) { if err := PointToHostDockerDaemon(); err != nil { - return nil, errors.Wrap(err, "point host docker-daemon") + return nil, errors.Wrap(err, "point host docker daemon") } o := &createOpts{} @@ -196,7 +196,7 @@ func createContainer(ociBinary string, image string, opts ...createOpt) ([]strin // Copy copies a local asset into the container func Copy(ociBinary string, ociID string, targetDir string, fName string) error { if err := PointToHostDockerDaemon(); err != nil { - return errors.Wrap(err, "point host docker-daemon") + return errors.Wrap(err, "point host docker daemon") } if _, err := os.Stat(fName); os.IsNotExist(err) { return errors.Wrapf(err, "error source %s does not exist", fName) @@ -217,7 +217,7 @@ func Copy(ociBinary string, ociID string, targetDir string, fName string) error // only supports TCP ports func HostPortBinding(ociBinary string, ociID string, contPort int) (int, error) { if err := PointToHostDockerDaemon(); err != nil { - return 0, errors.Wrap(err, "point host docker-daemon") + return 0, errors.Wrap(err, "point host docker daemon") } var out []byte var err error @@ -272,7 +272,7 @@ func podmanConttainerIP(name string) (string, string, error) { // dockerContainerIP returns ipv4, ipv6 of container or error func dockerContainerIP(name string) (string, string, error) { if err := PointToHostDockerDaemon(); err != nil { - return "", "", errors.Wrap(err, "point host docker-daemon") + return "", "", errors.Wrap(err, "point host docker daemon") } // retrieve the IP address of the node using docker inspect lines, err := inspect(Docker, name, "{{range .NetworkSettings.Networks}}{{.IPAddress}},{{.GlobalIPv6Address}}{{end}}") @@ -292,7 +292,7 @@ func dockerContainerIP(name string) (string, string, error) { // ContainerID returns id of a container name func ContainerID(ociBinary string, nameOrID string) (string, error) { if err := PointToHostDockerDaemon(); err != nil { - return "", errors.Wrap(err, "point host docker-daemon") + return "", errors.Wrap(err, "point host docker daemon") } cmd := exec.Command(ociBinary, "inspect", "-f", "{{.Id}}", nameOrID) id, err := cmd.CombinedOutput() @@ -310,7 +310,7 @@ func ListOwnedContainers(ociBinary string) ([]string, error) { // inspect return low-level information on containers func inspect(ociBinary string, containerNameOrID, format string) ([]string, error) { if err := PointToHostDockerDaemon(); err != nil { - return nil, errors.Wrap(err, "point host docker-daemon") + return nil, errors.Wrap(err, "point host docker daemon") } cmd := exec.Command(ociBinary, "inspect", "-f", format, @@ -376,7 +376,7 @@ func generateMountBindings(mounts ...Mount) []string { // isUsernsRemapEnabled checks if userns-remap is enabled in docker func isUsernsRemapEnabled(ociBinary string) (bool, error) { if err := PointToHostDockerDaemon(); err != nil { - return false, errors.Wrap(err, "point host docker-daemon") + return false, errors.Wrap(err, "point host docker daemon") } cmd := exec.Command(ociBinary, "info", "--format", "'{{json .SecurityOptions}}'") var buff bytes.Buffer @@ -438,7 +438,7 @@ func withPortMappings(portMappings []PortMapping) createOpt { // listContainersByLabel returns all the container names with a specified label func listContainersByLabel(ociBinary string, label string) ([]string, error) { if err := PointToHostDockerDaemon(); err != nil { - return nil, errors.Wrap(err, "point host docker-daemon") + return nil, errors.Wrap(err, "point host docker daemon") } // allow no more than 5 seconds for docker ps ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) @@ -475,11 +475,11 @@ func PointToHostDockerDaemon() error { return nil } -// containerStatus returns status of a container running,exited,... -func containerStatus(ociBin string, name string) (string, error) { +// ContainerStatus returns status of a container running,exited,... +func ContainerStatus(ociBin string, name string) (string, error) { if ociBin == Docker { if err := PointToHostDockerDaemon(); err != nil { - return "", errors.Wrap(err, "point host docker-daemon") + return "", errors.Wrap(err, "point host docker daemon") } } // allow no more than 2 seconds for this. when this takes long this means deadline passed @@ -488,7 +488,7 @@ func containerStatus(ociBin string, name string) (string, error) { cmd := exec.CommandContext(ctx, ociBin, "inspect", name, "--format={{.State.Status}}") out, err := cmd.CombinedOutput() if ctx.Err() == context.DeadlineExceeded { - glog.Warningf("%s inspect %s took longer than normal. Restarting your docker daemon might fix this issue.") + glog.Warningf("%s inspect %s took longer than normal. Restarting your %s daemon might fix this issue.", ociBin, name, ociBin) return strings.TrimSpace(string(out)), fmt.Errorf("inspect %s timeout", name) } if err != nil { diff --git a/pkg/drivers/kic/oci/volumes.go b/pkg/drivers/kic/oci/volumes.go index 1181aa0652..d24bd7bb96 100644 --- a/pkg/drivers/kic/oci/volumes.go +++ b/pkg/drivers/kic/oci/volumes.go @@ -36,7 +36,7 @@ func DeleteAllVolumesByLabel(ociBin string, label string) []error { glog.Infof("trying to delete all %s volumes with label %s", ociBin, label) if ociBin == Docker { if err := PointToHostDockerDaemon(); err != nil { - return []error{errors.Wrap(err, "point host docker-daemon")} + return []error{errors.Wrap(err, "point host docker daemon")} } } @@ -68,22 +68,22 @@ func PruneAllVolumesByLabel(ociBin string, label string) []error { glog.Infof("trying to prune all %s volumes with label %s", ociBin, label) if ociBin == Docker { if err := PointToHostDockerDaemon(); err != nil { - return []error{errors.Wrap(err, "point host docker-daemon")} + return []error{errors.Wrap(err, "point host docker daemon")} } } - // allow no more than 5 seconds for this. when this takes long this means deadline passed - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + // allow no more than 3 seconds for this. when this takes long this means deadline passed + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() // try to prune afterwards just in case delete didn't go through cmd := exec.CommandContext(ctx, ociBin, "volume", "prune", "-f", "--filter", "label="+label) + if out, err := cmd.CombinedOutput(); err != nil { + deleteErrs = append(deleteErrs, errors.Wrapf(err, "prune volume by label %s: %s", label, string(out))) + } if ctx.Err() == context.DeadlineExceeded { glog.Warningf("pruning volume with label %s took longer than normal. Restarting your %s daemon might fix this issue.", label, ociBin) deleteErrs = append(deleteErrs, fmt.Errorf("prune deadline exceeded for %s", label)) } - if out, err := cmd.CombinedOutput(); err != nil { - deleteErrs = append(deleteErrs, errors.Wrapf(err, "prune volume by label %s: %s", label, string(out))) - } return deleteErrs } @@ -108,7 +108,7 @@ func allVolumesByLabel(ociBin string, label string) ([]string, error) { // TODO: this should be fixed as a part of https://github.com/kubernetes/minikube/issues/6530 func createDockerVolume(name string) error { if err := PointToHostDockerDaemon(); err != nil { - return errors.Wrap(err, "point host docker-daemon") + return errors.Wrap(err, "point host docker daemon") } cmd := exec.Command(Docker, "volume", "create", name, "--label", fmt.Sprintf("%s=%s", ProfileLabelKey, name), "--label", fmt.Sprintf("%s=%s", CreatedByLabelKey, "true")) if out, err := cmd.CombinedOutput(); err != nil { diff --git a/pkg/minikube/machine/delete.go b/pkg/minikube/machine/delete.go index 9a7fa00a81..c18e4c9183 100644 --- a/pkg/minikube/machine/delete.go +++ b/pkg/minikube/machine/delete.go @@ -17,7 +17,9 @@ limitations under the License. package machine import ( + "context" "os/exec" + "time" "github.com/docker/machine/libmachine" "github.com/docker/machine/libmachine/mcnerror" @@ -29,10 +31,30 @@ import ( "k8s.io/minikube/pkg/minikube/out" ) -// deleteOrphanedKIC attempts to delete an orphaned docker instance -func deleteOrphanedKIC(name string) { - cmd := exec.Command(oci.Docker, "rm", "-f", "-v", name) - err := cmd.Run() +// deleteOrphanedKIC attempts to delete an orphaned docker instance for machines without a config file +// used as last effort clean up not returning errors, wont warn user. +func deleteOrphanedKIC(ociBin string, name string) { + if !(ociBin == oci.Podman || ociBin == oci.Docker) { + return + } + if ociBin == oci.Docker { + if err := oci.PointToHostDockerDaemon(); err != nil { + glog.Infof("Failed to point to host docker-damon: %v", err) + return + } + } + + _, err := oci.ContainerStatus(ociBin, name) + if err != nil { + glog.Infof("couldn't inspect container %q before deleting, %s-daemon might needs a restart!: %v", name, ociBin, err) + return + } + // allow no more than 5 seconds for delting the container + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + cmd := exec.CommandContext(ctx, ociBin, "rm", "-f", "-v", name) + err = cmd.Run() if err == nil { glog.Infof("Found stale kic container and successfully cleaned it up!") } @@ -42,7 +64,8 @@ func deleteOrphanedKIC(name string) { func DeleteHost(api libmachine.API, machineName string) error { host, err := api.Load(machineName) if err != nil && host == nil { - deleteOrphanedKIC(machineName) + deleteOrphanedKIC(oci.Docker, machineName) + deleteOrphanedKIC(oci.Podman, machineName) // Keep going even if minikube does not know about the host } From a57a282027d0e56747a3a08c0cc14bd2cb825d9c Mon Sep 17 00:00:00 2001 From: Jose Donizetti Date: Wed, 26 Feb 2020 18:27:24 -0300 Subject: [PATCH 26/50] kic: fix known_hosts issue --- pkg/minikube/tunnel/kic/ssh_conn.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/minikube/tunnel/kic/ssh_conn.go b/pkg/minikube/tunnel/kic/ssh_conn.go index 51b06f8ce0..53c7304a3c 100644 --- a/pkg/minikube/tunnel/kic/ssh_conn.go +++ b/pkg/minikube/tunnel/kic/ssh_conn.go @@ -33,6 +33,7 @@ func createSSHConn(name, sshPort, sshKey string, svc v1.Service) *sshConn { // extract sshArgs sshArgs := []string{ // TODO: document the options here + "-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking no", "-N", "docker@127.0.0.1", From 8d0c41392b81ae595f16a61067ca2c1ff7cf9fcc Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Wed, 26 Feb 2020 13:50:42 -0800 Subject: [PATCH 27/50] continue instead of nest --- pkg/drivers/kic/oci/oci.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go index 784e3250c2..2d510aa0e3 100644 --- a/pkg/drivers/kic/oci/oci.go +++ b/pkg/drivers/kic/oci/oci.go @@ -59,13 +59,13 @@ func DeleteContainersByLabel(ociBin string, label string) []error { if err != nil { deleteErrs = append(deleteErrs, errors.Wrapf(err, "delete container %s: %s daemon is stuck. please try again!", c, ociBin)) glog.Errorf("%s daemon seems to be stuck. Please try restarting your %s.", ociBin, ociBin) - } else { - cmd := exec.Command(ociBin, "rm", "-f", "-v", c) - if out, err := cmd.CombinedOutput(); err != nil { - deleteErrs = append(deleteErrs, errors.Wrapf(err, "delete container %s: output %s", c, out)) - } - + continue } + cmd := exec.Command(ociBin, "rm", "-f", "-v", c) + if out, err := cmd.CombinedOutput(); err != nil { + deleteErrs = append(deleteErrs, errors.Wrapf(err, "delete container %s: output %s", c, out)) + } + } return deleteErrs } From f390d4bf3053e79b5edf5268de2c81ec0a8bd1ee Mon Sep 17 00:00:00 2001 From: Jose Donizetti Date: Wed, 26 Feb 2020 19:29:34 -0300 Subject: [PATCH 28/50] kic on mac: add service cmd support --- cmd/minikube/cmd/service.go | 98 +++++++++++++++++------ pkg/minikube/tunnel/kic/service_tunnel.go | 78 ++++++++++++++++++ pkg/minikube/tunnel/kic/ssh_conn.go | 2 +- pkg/minikube/tunnel/kic/ssh_tunnel.go | 32 +++++++- 4 files changed, 185 insertions(+), 25 deletions(-) create mode 100644 pkg/minikube/tunnel/kic/service_tunnel.go diff --git a/cmd/minikube/cmd/service.go b/cmd/minikube/cmd/service.go index 65e1f69297..b7ec05a6c3 100644 --- a/cmd/minikube/cmd/service.go +++ b/cmd/minikube/cmd/service.go @@ -20,8 +20,13 @@ import ( "fmt" "net/url" "os" + "os/signal" + "path/filepath" "runtime" + "strconv" + "strings" "text/template" + "time" "github.com/golang/glog" "github.com/pkg/browser" @@ -32,9 +37,11 @@ import ( "k8s.io/minikube/pkg/minikube/config" pkg_config "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/exit" + "k8s.io/minikube/pkg/minikube/localpath" "k8s.io/minikube/pkg/minikube/machine" "k8s.io/minikube/pkg/minikube/out" "k8s.io/minikube/pkg/minikube/service" + "k8s.io/minikube/pkg/minikube/tunnel/kic" ) const defaultServiceFormatTemplate = "http://{{.IP}}:{{.Port}}" @@ -85,34 +92,17 @@ var serviceCmd = &cobra.Command{ exit.WithError("Error getting config", err) } + if runtime.GOOS == "darwin" && cfg.Driver == oci.Docker { + startKicServiceTunnel(svc, cfg.Name) + return + } + urls, err := service.WaitForService(api, namespace, svc, serviceURLTemplate, serviceURLMode, https, wait, interval) if err != nil { exit.WithError("Error opening service", err) } - if runtime.GOOS == "darwin" && cfg.Driver == oci.Docker { - out.FailureT("Opening service in browser is not implemented yet for docker driver on Mac.\nThe following issue is tracking the in progress work:\nhttps://github.com/kubernetes/minikube/issues/6778") - exit.WithCodeT(exit.Unavailable, "Not yet implemented for docker driver on MacOS.") - } - - for _, u := range urls { - _, err := url.Parse(u) - if err != nil { - glog.Warningf("failed to parse url %q: %v (will not open)", u, err) - out.String(fmt.Sprintf("%s\n", u)) - continue - } - - if serviceURLMode { - out.String(fmt.Sprintf("%s\n", u)) - continue - } - - out.T(out.Celebrate, "Opening service {{.namespace_name}}/{{.service_name}} in default browser...", out.V{"namespace_name": namespace, "service_name": svc}) - if err := browser.OpenURL(u); err != nil { - exit.WithError(fmt.Sprintf("open url failed: %s", u), err) - } - } + openURLs(svc, urls) }, } @@ -126,3 +116,65 @@ func init() { serviceCmd.PersistentFlags().StringVar(&serviceURLFormat, "format", defaultServiceFormatTemplate, "Format to output service URL in. This format will be applied to each url individually and they will be printed one at a time.") } + +func startKicServiceTunnel(svc, configName string) { + ctrlC := make(chan os.Signal, 1) + signal.Notify(ctrlC, os.Interrupt) + + clientset, err := service.K8s.GetClientset(1 * time.Second) + if err != nil { + exit.WithError("error creating clientset", err) + } + + port, err := oci.HostPortBinding(oci.Docker, configName, 22) + if err != nil { + exit.WithError("error getting ssh port", err) + } + sshPort := strconv.Itoa(port) + sshKey := filepath.Join(localpath.MiniPath(), "machines", configName, "id_rsa") + + serviceTunnel := kic.NewServiceTunnel(sshPort, sshKey, clientset.CoreV1()) + urls, err := serviceTunnel.Start(svc, namespace) + if err != nil { + exit.WithError("error starting tunnel", err) + } + + // wait for tunnel to come up + time.Sleep(1 * time.Second) + + data := [][]string{{namespace, svc, "", strings.Join(urls, "\n")}} + service.PrintServiceList(os.Stdout, data) + + openURLs(svc, urls) + + <-ctrlC + + err = serviceTunnel.Stop() + if err != nil { + exit.WithError("error stopping tunnel", err) + } + + return +} + +func openURLs(svc string, urls []string) { + for _, u := range urls { + _, err := url.Parse(u) + if err != nil { + glog.Warningf("failed to parse url %q: %v (will not open)", u, err) + out.String(fmt.Sprintf("%s\n", u)) + continue + } + + if serviceURLMode { + out.String(fmt.Sprintf("%s\n", u)) + continue + } + + out.T(out.Celebrate, "Opening service {{.namespace_name}}/{{.service_name}} in default browser...", out.V{"namespace_name": namespace, "service_name": svc}) + if err := browser.OpenURL(u); err != nil { + exit.WithError(fmt.Sprintf("open url failed: %s", u), err) + } + } + +} diff --git a/pkg/minikube/tunnel/kic/service_tunnel.go b/pkg/minikube/tunnel/kic/service_tunnel.go new file mode 100644 index 0000000000..5cc6bff7ac --- /dev/null +++ b/pkg/minikube/tunnel/kic/service_tunnel.go @@ -0,0 +1,78 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kic + +import ( + "fmt" + + "github.com/golang/glog" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + typed_core "k8s.io/client-go/kubernetes/typed/core/v1" +) + +// ServiceTunnel ... +type ServiceTunnel struct { + sshPort string + sshKey string + v1Core typed_core.CoreV1Interface + sshConn *sshConn +} + +// NewServiceTunnel ... +func NewServiceTunnel(sshPort, sshKey string, v1Core typed_core.CoreV1Interface) *ServiceTunnel { + return &ServiceTunnel{ + sshPort: sshPort, + sshKey: sshKey, + v1Core: v1Core, + } +} + +// Start ... +func (t *ServiceTunnel) Start(svcName, namespace string) ([]string, error) { + svc, err := t.v1Core.Services(namespace).Get(svcName, metav1.GetOptions{}) + if err != nil { + glog.Errorf("error listing services: %v", err) + return nil, err + } + + t.sshConn = createSSHConn(svcName, t.sshPort, t.sshKey, svc) + + go func() { + err = t.sshConn.startAndWait() + if err != nil { + glog.Errorf("error starting ssh tunnel: %v", err) + } + }() + + urls := make([]string, 0, len(svc.Spec.Ports)) + for _, port := range svc.Spec.Ports { + urls = append(urls, fmt.Sprintf("http://127.0.0.1:%d", port.Port)) + } + + return urls, nil +} + +// Stop ... +func (t *ServiceTunnel) Stop() error { + err := t.sshConn.stop() + if err != nil { + glog.Errorf("error stopping ssh tunnel: %v", err) + return err + } + + return nil +} diff --git a/pkg/minikube/tunnel/kic/ssh_conn.go b/pkg/minikube/tunnel/kic/ssh_conn.go index 53c7304a3c..4c98f0836f 100644 --- a/pkg/minikube/tunnel/kic/ssh_conn.go +++ b/pkg/minikube/tunnel/kic/ssh_conn.go @@ -29,7 +29,7 @@ type sshConn struct { cmd *exec.Cmd } -func createSSHConn(name, sshPort, sshKey string, svc v1.Service) *sshConn { +func createSSHConn(name, sshPort, sshKey string, svc *v1.Service) *sshConn { // extract sshArgs sshArgs := []string{ // TODO: document the options here diff --git a/pkg/minikube/tunnel/kic/ssh_tunnel.go b/pkg/minikube/tunnel/kic/ssh_tunnel.go index 3dca0a1d34..65acaf325b 100644 --- a/pkg/minikube/tunnel/kic/ssh_tunnel.go +++ b/pkg/minikube/tunnel/kic/ssh_tunnel.go @@ -104,7 +104,7 @@ func (t *SSHTunnel) startConnection(svc v1.Service) { } // create new ssh conn - newSSHConn := createSSHConn(uniqName, t.sshPort, t.sshKey, svc) + newSSHConn := createSSHConn(uniqName, t.sshPort, t.sshKey, &svc) t.conns[newSSHConn.name] = newSSHConn go func() { @@ -147,3 +147,33 @@ func sshConnUniqName(service v1.Service) string { return strings.Join(n, "") } + +// StartServiceTunnel ... +func (t *SSHTunnel) StartServiceTunnel(svcName string) error { + svc, err := t.v1Core.Services("default").Get(svcName, metav1.GetOptions{}) + if err != nil { + glog.Errorf("error listing services: %v", err) + } + + newSSHConn := createSSHConn(svcName, t.sshPort, t.sshKey, svc) + + go func() { + err := newSSHConn.startAndWait() + if err != nil { + glog.Errorf("error starting ssh tunnel: %v", err) + } + }() + + //err = t.LoadBalancerEmulator.PatchServiceIP(t.v1Core.RESTClient(), svc, "127.0.0.1") + //if err != nil { + // glog.Errorf("error patching service: %v", err) + //} + + <-t.ctx.Done() + _, err = t.LoadBalancerEmulator.Cleanup() + if err != nil { + glog.Errorf("error cleaning up: %v", err) + } + + return err +} From 5f73506ff6a09eb21e36de91af8f4d620f77ddf6 Mon Sep 17 00:00:00 2001 From: Jose Donizetti Date: Wed, 26 Feb 2020 20:51:36 -0300 Subject: [PATCH 29/50] Remove old code --- pkg/minikube/tunnel/kic/ssh_tunnel.go | 30 --------------------------- 1 file changed, 30 deletions(-) diff --git a/pkg/minikube/tunnel/kic/ssh_tunnel.go b/pkg/minikube/tunnel/kic/ssh_tunnel.go index 65acaf325b..bc5fd53a4f 100644 --- a/pkg/minikube/tunnel/kic/ssh_tunnel.go +++ b/pkg/minikube/tunnel/kic/ssh_tunnel.go @@ -147,33 +147,3 @@ func sshConnUniqName(service v1.Service) string { return strings.Join(n, "") } - -// StartServiceTunnel ... -func (t *SSHTunnel) StartServiceTunnel(svcName string) error { - svc, err := t.v1Core.Services("default").Get(svcName, metav1.GetOptions{}) - if err != nil { - glog.Errorf("error listing services: %v", err) - } - - newSSHConn := createSSHConn(svcName, t.sshPort, t.sshKey, svc) - - go func() { - err := newSSHConn.startAndWait() - if err != nil { - glog.Errorf("error starting ssh tunnel: %v", err) - } - }() - - //err = t.LoadBalancerEmulator.PatchServiceIP(t.v1Core.RESTClient(), svc, "127.0.0.1") - //if err != nil { - // glog.Errorf("error patching service: %v", err) - //} - - <-t.ctx.Done() - _, err = t.LoadBalancerEmulator.Cleanup() - if err != nil { - glog.Errorf("error cleaning up: %v", err) - } - - return err -} From 1752d7c004f225ac9c4e0167c7a452e1e9a07006 Mon Sep 17 00:00:00 2001 From: Jose Donizetti Date: Wed, 26 Feb 2020 22:09:48 -0300 Subject: [PATCH 30/50] kic on mac: add random port support to service cmd --- pkg/minikube/tunnel/kic/service_tunnel.go | 10 ++++-- pkg/minikube/tunnel/kic/ssh_conn.go | 44 +++++++++++++++++++++++ 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/pkg/minikube/tunnel/kic/service_tunnel.go b/pkg/minikube/tunnel/kic/service_tunnel.go index 5cc6bff7ac..6f4222f1b0 100644 --- a/pkg/minikube/tunnel/kic/service_tunnel.go +++ b/pkg/minikube/tunnel/kic/service_tunnel.go @@ -49,7 +49,11 @@ func (t *ServiceTunnel) Start(svcName, namespace string) ([]string, error) { return nil, err } - t.sshConn = createSSHConn(svcName, t.sshPort, t.sshKey, svc) + t.sshConn, err = createSSHConnWithRandomPorts(svcName, t.sshPort, t.sshKey, svc) + if err != nil { + glog.Errorf("error creating ssh conn: %v", err) + return nil, err + } go func() { err = t.sshConn.startAndWait() @@ -59,8 +63,8 @@ func (t *ServiceTunnel) Start(svcName, namespace string) ([]string, error) { }() urls := make([]string, 0, len(svc.Spec.Ports)) - for _, port := range svc.Spec.Ports { - urls = append(urls, fmt.Sprintf("http://127.0.0.1:%d", port.Port)) + for _, port := range t.sshConn.ports { + urls = append(urls, fmt.Sprintf("http://127.0.0.1:%d", port)) } return urls, nil diff --git a/pkg/minikube/tunnel/kic/ssh_conn.go b/pkg/minikube/tunnel/kic/ssh_conn.go index 4c98f0836f..d619a44bc5 100644 --- a/pkg/minikube/tunnel/kic/ssh_conn.go +++ b/pkg/minikube/tunnel/kic/ssh_conn.go @@ -20,6 +20,8 @@ import ( "fmt" "os/exec" + "github.com/phayes/freeport" + v1 "k8s.io/api/core/v1" ) @@ -27,6 +29,7 @@ type sshConn struct { name string service string cmd *exec.Cmd + ports []int } func createSSHConn(name, sshPort, sshKey string, svc *v1.Service) *sshConn { @@ -61,6 +64,47 @@ func createSSHConn(name, sshPort, sshKey string, svc *v1.Service) *sshConn { } } +func createSSHConnWithRandomPorts(name, sshPort, sshKey string, svc *v1.Service) (*sshConn, error) { + // extract sshArgs + sshArgs := []string{ + // TODO: document the options here + "-o", "UserKnownHostsFile=/dev/null", + "-o", "StrictHostKeyChecking no", + "-N", + "docker@127.0.0.1", + "-p", sshPort, + "-i", sshKey, + } + + usedPorts := make([]int, 0, len(svc.Spec.Ports)) + + for _, port := range svc.Spec.Ports { + freeport, err := freeport.GetFreePort() + if err != nil { + return nil, err + } + + arg := fmt.Sprintf( + "-L %d:%s:%d", + freeport, + svc.Spec.ClusterIP, + port.Port, + ) + + sshArgs = append(sshArgs, arg) + usedPorts = append(usedPorts, freeport) + } + + cmd := exec.Command("ssh", sshArgs...) + + return &sshConn{ + name: name, + service: svc.Name, + cmd: cmd, + ports: usedPorts, + }, nil +} + func (c *sshConn) startAndWait() error { fmt.Printf("starting tunnel for %s\n", c.service) err := c.cmd.Start() From 5d8f2541f6093b87a501e81fe43093917a165934 Mon Sep 17 00:00:00 2001 From: Jose Donizetti Date: Wed, 26 Feb 2020 22:16:50 -0300 Subject: [PATCH 31/50] kic on mac: add service warn --- cmd/minikube/cmd/service.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/minikube/cmd/service.go b/cmd/minikube/cmd/service.go index b7ec05a6c3..357da16827 100644 --- a/cmd/minikube/cmd/service.go +++ b/cmd/minikube/cmd/service.go @@ -175,6 +175,8 @@ func openURLs(svc string, urls []string) { if err := browser.OpenURL(u); err != nil { exit.WithError(fmt.Sprintf("open url failed: %s", u), err) } + + out.T(out.WarningType, "Because you are using docker driver on Mac, the terminal needs to be open to run it.") } } From 7a6ff5838469b5deebd48db3e43f924073dcefad Mon Sep 17 00:00:00 2001 From: Jose Donizetti Date: Wed, 26 Feb 2020 22:21:44 -0300 Subject: [PATCH 32/50] kic on mac: improve error reporting service tunnel --- pkg/minikube/tunnel/kic/service_tunnel.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pkg/minikube/tunnel/kic/service_tunnel.go b/pkg/minikube/tunnel/kic/service_tunnel.go index 6f4222f1b0..e1be04d97d 100644 --- a/pkg/minikube/tunnel/kic/service_tunnel.go +++ b/pkg/minikube/tunnel/kic/service_tunnel.go @@ -20,6 +20,8 @@ import ( "fmt" "github.com/golang/glog" + "github.com/pkg/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" typed_core "k8s.io/client-go/kubernetes/typed/core/v1" ) @@ -45,14 +47,12 @@ func NewServiceTunnel(sshPort, sshKey string, v1Core typed_core.CoreV1Interface) func (t *ServiceTunnel) Start(svcName, namespace string) ([]string, error) { svc, err := t.v1Core.Services(namespace).Get(svcName, metav1.GetOptions{}) if err != nil { - glog.Errorf("error listing services: %v", err) - return nil, err + return nil, errors.Wrap(err, "getting service") } t.sshConn, err = createSSHConnWithRandomPorts(svcName, t.sshPort, t.sshKey, svc) if err != nil { - glog.Errorf("error creating ssh conn: %v", err) - return nil, err + return nil, errors.Wrap(err, "creating ssh conn") } go func() { @@ -74,8 +74,7 @@ func (t *ServiceTunnel) Start(svcName, namespace string) ([]string, error) { func (t *ServiceTunnel) Stop() error { err := t.sshConn.stop() if err != nil { - glog.Errorf("error stopping ssh tunnel: %v", err) - return err + return errors.Wrap(err, "stopping ssh tunnel") } return nil From 5e30d2f8c93de2c9d54ab79c736e72d4ac616eb1 Mon Sep 17 00:00:00 2001 From: Jose Donizetti Date: Wed, 26 Feb 2020 22:29:28 -0300 Subject: [PATCH 33/50] Fix linter --- cmd/minikube/cmd/service.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/cmd/minikube/cmd/service.go b/cmd/minikube/cmd/service.go index 357da16827..de0205fbcc 100644 --- a/cmd/minikube/cmd/service.go +++ b/cmd/minikube/cmd/service.go @@ -153,8 +153,6 @@ func startKicServiceTunnel(svc, configName string) { if err != nil { exit.WithError("error stopping tunnel", err) } - - return } func openURLs(svc string, urls []string) { From 551c5701890a4387eff17b8dbd2e563562d44ff2 Mon Sep 17 00:00:00 2001 From: Black-Hole <158blackhole@gmail.com> Date: Thu, 27 Feb 2020 10:41:38 +0800 Subject: [PATCH 34/50] refactor(i18n): optimizing Chinese translation --- translations/zh-CN.json | 112 ++++++++++++++++++++-------------------- 1 file changed, 56 insertions(+), 56 deletions(-) diff --git a/translations/zh-CN.json b/translations/zh-CN.json index 4b6ce53343..6f01a1242f 100644 --- a/translations/zh-CN.json +++ b/translations/zh-CN.json @@ -281,19 +281,19 @@ "Log into or run a command on a machine with SSH; similar to 'docker-machine ssh'": "", "Log into or run a command on a machine with SSH; similar to 'docker-machine ssh'.": "", "Message Size: {{.size}}": "", - "Minikube is a CLI tool that provisions and manages single-node Kubernetes clusters optimized for development workflows.": "", + "Minikube is a CLI tool that provisions and manages single-node Kubernetes clusters optimized for development workflows.": "Minikube 是一个命令行工具,它提供和管理针对开发工作流程优化的单节点 Kubernetes 集群。", "Minikube is a tool for managing local Kubernetes clusters.": "", - "Modify minikube config": "", - "Modify minikube's kubernetes addons": "", + "Modify minikube config": "修改 minikube 配置", + "Modify minikube's kubernetes addons": "修改 minikube 的 kubernetes 插件", "Mount type: {{.name}}": "", "Mounting host path {{.sourcePath}} into VM as {{.destinationPath}} ...": "", - "Mounts the specified directory into minikube": "", - "Mounts the specified directory into minikube.": "", + "Mounts the specified directory into minikube": "将指定的目录挂载到 minikube", + "Mounts the specified directory into minikube.": "将指定的目录挂载到 minikube。", "Multiple errors deleting profiles": "删除配置文件时出现多个错误", "Multiple minikube profiles were found -": "发现了多个 minikube 配置文件 -", "NIC Type used for host only network. One of Am79C970A, Am79C973, 82540EM, 82543GC, 82545EM, or virtio (virtualbox driver only)": "网卡类型仅用于主机网络。Am79C970A, Am79C973, 82540EM, 82543GC, 82545EM 之一,或 virtio(仅限 VirtualBox 驱动程序)", "NOTE: This process must stay alive for the mount to be accessible ...": "", - "Networking and Connectivity Commands:": "", + "Networking and Connectivity Commands:": "网络和连接命令:", "No minikube profile was found. You can create one using `minikube start`.": "", "Node may be unable to resolve external DNS records": "", "Node operations": "", @@ -314,11 +314,11 @@ "Options: {{.options}}": "", "Outputs minikube shell completion for the given shell (bash or zsh)": "", "Outputs minikube shell completion for the given shell (bash or zsh)\n\n\tThis depends on the bash-completion binary. Example installation instructions:\n\tOS X:\n\t\t$ brew install bash-completion\n\t\t$ source $(brew --prefix)/etc/bash_completion\n\t\t$ minikube completion bash \u003e ~/.minikube-completion # for bash users\n\t\t$ minikube completion zsh \u003e ~/.minikube-completion # for zsh users\n\t\t$ source ~/.minikube-completion\n\tUbuntu:\n\t\t$ apt-get install bash-completion\n\t\t$ source /etc/bash-completion\n\t\t$ source \u003c(minikube completion bash) # for bash users\n\t\t$ source \u003c(minikube completion zsh) # for zsh users\n\n\tAdditionally, you may want to output the completion to a file and source in your .bashrc\n\n\tNote for zsh users: [1] zsh completions are only supported in versions of zsh \u003e= 5.2": "", - "Pause": "", - "Paused kubelet and {{.count}} containers": "", - "Paused kubelet and {{.count}} containers in: {{.namespaces}}": "", - "Permissions: {{.octalMode}} ({{.writtenMode}})": "", - "Please enter a value:": "", + "Pause": "暂停", + "Paused kubelet and {{.count}} containers": "已暂停 kubelet 和 {{.count}} 个容器", + "Paused kubelet and {{.count}} containers in: {{.namespaces}}": "已暂停 {{.namespaces}} 中的 kubelet 和 {{.count}} 个容器", + "Permissions: {{.octalMode}} ({{.writtenMode}})": "权限: {{.octalMode}} ({{.writtenMode}})", + "Please enter a value:": "请输入一个值:", "Please install the minikube hyperkit VM driver, or select an alternative --vm-driver": "", "Please install the minikube kvm2 VM driver, or select an alternative --vm-driver": "", "Please make sure the service you are looking for is deployed or is in the correct namespace.": "", @@ -327,23 +327,23 @@ "Populates the specified folder with documentation in markdown about minikube": "", "Powering off \"{{.profile_name}}\" via SSH ...": "正在通过 SSH 关闭“{{.profile_name}}”…", "Preparing Kubernetes {{.k8sVersion}} on {{.runtime}} {{.runtimeVersion}} ...": "正在 {{.runtime}} {{.runtimeVersion}} 中准备 Kubernetes {{.k8sVersion}}…", - "Print current and latest version number": "", - "Print the version of minikube": "", - "Print the version of minikube.": "", - "Problems detected in {{.entry}}:": "", - "Problems detected in {{.name}}:": "", - "Profile gets or sets the current minikube profile": "", + "Print current and latest version number": "打印当前和最新版本版本", + "Print the version of minikube": "打印 minikube 版本", + "Print the version of minikube.": "打印 minikube 版本。", + "Problems detected in {{.entry}}:": "在 {{.entry}} 中 检测到问题:", + "Problems detected in {{.name}}:": "在 {{.name}} 中 检测到问题:", + "Profile gets or sets the current minikube profile": "获取或设置当前的 minikube 配置文件", "Profile name \"{{.profilename}}\" is minikube keyword. To delete profile use command minikube delete -p \u003cprofile name\u003e": "配置文件名称 \"{{.profilename}}\" 是 minikube 的一个关键字。使用 minikube delete -p \u003cprofile name\u003e 命令 删除配置文件", "Provide VM UUID to restore MAC address (hyperkit driver only)": "提供虚拟机 UUID 以恢复 MAC 地址(仅限 hyperkit 驱动程序)", "Pulling images ...": "拉取镜像 ...", "Reboot to complete VirtualBox installation, verify that VirtualBox is not blocked by your system, and/or use another hypervisor": "重启以完成 VirtualBox 安装,检查 VirtualBox 未被您的操作系统禁用,或者使用其他的管理程序。", "Rebuild libvirt with virt-network support": "", - "Received {{.name}} signal": "", - "Reconfiguring existing host ...": "", + "Received {{.name}} signal": "收到 {{.name}} 信号", + "Reconfiguring existing host ...": "重新配置现有主机", "Registry mirrors to pass to the Docker daemon": "传递给 Docker 守护进程的注册表镜像", "Reinstall VirtualBox and reboot. Alternatively, try the kvm2 driver: https://minikube.sigs.k8s.io/docs/reference/drivers/kvm2/": "", "Reinstall VirtualBox and verify that it is not blocked: System Preferences -\u003e Security \u0026 Privacy -\u003e General -\u003e Some system software was blocked from loading": "", - "Related issues:": "", + "Related issues:": "相关问题:", "Relaunching Kubernetes using {{.bootstrapper}} ...": "正在使用 {{.bootstrapper}} 重新启动 Kubernetes…", "Removed all traces of the \"{{.name}}\" cluster.": "", "Removing {{.directory}} ...": "正在移除 {{.directory}}…", @@ -351,15 +351,15 @@ "Requested disk size {{.requested_size}} is less than minimum of {{.minimum_size}}": "请求的磁盘大小 {{.requested_size}} 小于最小值 {{.minimum_size}}", "Requested memory allocation ({{.memory}}MB) is less than the default memory allocation of {{.default_memorysize}}MB. Beware that minikube might not work correctly or crash unexpectedly.": "请求的内存分配 ({{.memory}}MB) 小于默认内存分配 {{.default_memorysize}}MB。请注意 minikube 可能无法正常运行或可能会意外崩溃。", "Requested memory allocation {{.requested_size}} is less than the minimum allowed of {{.minimum_size}}": "请求的内存分配 {{.requested_size}} 小于允许的 {{.minimum_size}} 最小值", - "Retrieve the ssh identity key path of the specified cluster": "", - "Retrieve the ssh identity key path of the specified cluster.": "", - "Retrieves the IP address of the running cluster": "", + "Retrieve the ssh identity key path of the specified cluster": "检索指定集群的 ssh 密钥路径", + "Retrieve the ssh identity key path of the specified cluster.": "检索指定集群的 ssh 密钥路径。", + "Retrieves the IP address of the running cluster": "检索正在运行的群集的 IP 地址", "Retrieves the IP address of the running cluster, and writes it to STDOUT.": "", "Retrieves the IP address of the running cluster, checks it\n\t\t\twith IP in kubeconfig, and corrects kubeconfig if incorrect.": "", "Returns the value of PROPERTY_NAME from the minikube config file. Can be overwritten at runtime by flags or environmental variables.": "", "Run 'kubectl describe pod coredns -n kube-system' and check for a firewall or DNS conflict": "", "Run 'minikube delete' to delete the stale VM, or and ensure that minikube is running as the same user you are issuing this command with": "执行 'minikube delete' 以删除过时的虚拟机,或者确保 minikube 以与您发出此命令的用户相同的用户身份运行", - "Run kubectl": "", + "Run kubectl": "运行 kubectl", "Run minikube from the C: drive.": "", "Run the kubernetes client, download it if necessary. Remember -- after kubectl!\n\nExamples:\nminikube kubectl -- --help\nminikube kubectl -- get pods --namespace kube-system": "", "Run the minikube command as an Administrator": "", @@ -372,11 +372,11 @@ "Set this flag to delete the '.minikube' folder from your user directory.": "设置这个标志来删除您用户目录下的 '.minikube' 文件夹。", "Sets an individual value in a minikube config file": "", "Sets the PROPERTY_NAME config value to PROPERTY_VALUE\n\tThese values can be overwritten by flags or environment variables at runtime.": "", - "Sets up docker env variables; similar to '$(docker-machine env)'": "", - "Sets up docker env variables; similar to '$(docker-machine env)'.": "", - "Sets up podman env variables; similar to '$(podman-machine env)'": "", - "Sets up podman env variables; similar to '$(podman-machine env)'.": "", - "Setting profile failed": "", + "Sets up docker env variables; similar to '$(docker-machine env)'": "设置 docker env 变量;类似于 '$(docker-machine env)'", + "Sets up docker env variables; similar to '$(docker-machine env)'.": "设置 docker env 变量;类似于 '$(docker-machine env)'。", + "Sets up podman env variables; similar to '$(podman-machine env)'": "设置 podman env 变量;类似于 '$(podman-machine env)'", + "Sets up podman env variables; similar to '$(podman-machine env)'.": "设置 podman env 变量;类似于 '$(podman-machine env)'。", + "Setting profile failed": "设置配置文件失败", "Show a list of global command-line options (applies to all commands).": "显示全局命令行选项列表 (应用于所有命令)。", "Show only log entries which point to known problems": "", "Show only the most recent journal entries, and continuously print new entries as they are appended to the journal.": "", @@ -400,7 +400,7 @@ "Stopping \"{{.profile_name}}\" in {{.driver_name}} ...": "", "Stops a local kubernetes cluster running in Virtualbox. This command stops the VM\nitself, leaving all files intact. The cluster can be started again with the \"start\" command.": "", "Stops a node in a cluster.": "", - "Stops a running local kubernetes cluster": "", + "Stops a running local kubernetes cluster": "停止正在运行的本地 kubernetes 集群", "Successfully deleted all profiles": "成功删除所有配置文件", "Successfully deleted profile \\\"{{.name}}\\\"": "成功删除配置文件 \\\"{{.name}}\\\"", "Successfully mounted {{.sourcePath}} to {{.destinationPath}}": "", @@ -418,8 +418,8 @@ "The 'none' driver does not respect the --memory flag": "'none' 驱动程序不遵循 --memory 标志", "The 'none' driver does not support multiple profiles: https://minikube.sigs.k8s.io/docs/reference/drivers/none/": "", "The 'none' driver provides limited isolation and may reduce system security and reliability.": "“none”驱动程序提供有限的隔离功能,并且可能会降低系统安全性和可靠性。", - "The '{{.addonName}}' addon is enabled": "", - "The '{{.driver}}' driver requires elevated permissions. The following commands will be executed:\\n\\n{{ .example }}\\n": "", + "The '{{.addonName}}' addon is enabled": "启动 '{{.addonName}}' 插件", + "The '{{.driver}}' driver requires elevated permissions. The following commands will be executed:\\n\\n{{ .example }}\\n": "'{{.driver}}' 驱动程序需要提升权限,将执行以下命令:\\n\\n{{ .example }}\\n", "The CIDR to be used for service cluster IPs.": "需要用于服务集群 IP 的 CIDR。", "The CIDR to be used for the minikube VM (virtualbox driver only)": "需要用于 minikube 虚拟机的 CIDR(仅限 virtualbox 驱动程序)", "The KVM QEMU connection URI. (kvm2 driver only)": "KVM QEMU 连接 URI。(仅限 kvm2 驱动程序)", @@ -479,7 +479,7 @@ "To see addons list for other profiles use: `minikube addons -p name list`": "", "To start minikube with HyperV Powershell must be in your PATH`": "", "To use kubectl or minikube commands as your own user, you may need to relocate them. For example, to overwrite your own settings, run:": "如需以您自己的用户身份使用 kubectl 或 minikube 命令,您可能需要重新定位该命令。例如,如需覆盖您的自定义设置,请运行:", - "Troubleshooting Commands:": "", + "Troubleshooting Commands:": "故障排除命令ƒ", "Trying to delete invalid profile {{.profile}}": "尝试删除无效的配置文件 {{.profile}}", "Unable to bind flags": "无法绑定标志", "Unable to determine a default driver to use. Try specifying --vm-driver, or see https://minikube.sigs.k8s.io/docs/start/": "无法确定要使用的默认驱动。尝试通过 --vm-dirver 指定,或者查阅 https://minikube.sigs.k8s.io/docs/start/", @@ -515,33 +515,33 @@ "Update server returned an empty list": "", "Upgrade to QEMU v3.1.0+, run 'virt-host-validate', or ensure that you are not running in a nested VM environment.": "", "Upgrading from Kubernetes {{.old}} to {{.new}}": "正在从 Kubernetes {{.old}} 升级到 {{.new}}", - "Usage": "", - "Usage: minikube completion SHELL": "", - "Usage: minikube delete": "", + "Usage": "使用方法", + "Usage: minikube completion SHELL": "使用方法:minikube completion SHELL", + "Usage: minikube delete": "使用方法:minikube delete", "Usage: minikube delete --all --purge": "使用方法:minikube delete --all --purge", - "Usage: minikube node [add|start|stop|delete]": "", - "Use \"{{.CommandPath}} [command] --help\" for more information about a command.": "", + "Usage: minikube node [add|start|stop|delete]": "使用方法:minikube node [add|start|stop|delete]", + "Use \"{{.CommandPath}} [command] --help\" for more information about a command.": "使用 \"{{.CommandPath}} [command] --help\" 可以获取有关命令的更多信息", "Use 'kubect get po -A' to find the correct and namespace name": "使用 'kubect get po -A' 来查询正确的命名空间名称", - "Use -A to specify all namespaces": "", - "Use VirtualBox to remove the conflicting VM and/or network interfaces": "", + "Use -A to specify all namespaces": "使用 -A 指定所有 namespaces", + "Use VirtualBox to remove the conflicting VM and/or network interfaces": "使用 VirtualBox 删除有冲突的 虚拟机 和/或 网络接口", "Use native Golang SSH client (default true). Set to 'false' to use the command line 'ssh' command when accessing the docker machine. Useful for the machine drivers when they will not start with 'Waiting for SSH'.": "", - "User ID: {{.userID}}": "", + "User ID: {{.userID}}": "用户 ID: {{.userID}}", "Userspace file server is shutdown": "", "Userspace file server:": "", "Using image repository {{.name}}": "正在使用镜像存储库 {{.name}}", "Using the '{{.runtime}}' runtime with the 'none' driver is an untested configuration!": "同时使用 'none' 驱动以及 '{{.runtime}}' 运行时是未经测试过的配置!", - "Using the running {{.driver_name}} \"{{.profile_name}}\" VM ...": "", - "Using the {{.driver}} driver based on existing profile": "", - "Using the {{.driver}} driver based on user configuration": "", + "Using the running {{.driver_name}} \"{{.profile_name}}\" VM ...": "使用正在运行的 {{.driver_name}} \"{{.profile_name}}\" 虚拟机", + "Using the {{.driver}} driver based on existing profile": "根据现有的配置文件使用 {{.driver}} 驱动程序", + "Using the {{.driver}} driver based on user configuration": "根据用户配置使用 {{.driver}} 驱动程序", "VM driver is one of: %v": "虚拟机驱动程序是以下项之一:%v", "VM is unable to access {{.repository}}, you may need to configure a proxy or set --image-repository": "虚拟机无权访问 {{.repository}},或许您需要配置代理或者设置 --image-repository", "VM may be unable to resolve external DNS records": "虚拟机可能无法解析外部 DNS 记录", - "Verify that your HTTP_PROXY and HTTPS_PROXY environment variables are set correctly.": "", - "Verify the IP address of the running cluster in kubeconfig.": "", - "Verifying dashboard health ...": "", - "Verifying proxy health ...": "", + "Verify that your HTTP_PROXY and HTTPS_PROXY environment variables are set correctly.": "验证是否正确设置了 HTTP_PROXY 和 HTTPS_PROXY 环境变量。", + "Verify the IP address of the running cluster in kubeconfig.": "在 kubeconfig 中验证正在运行的集群 IP 地址。", + "Verifying dashboard health ...": "正在验证 dashboard 运行情况 ...", + "Verifying proxy health ...": "正在验证 proxy 运行状况 ...", "Verifying:": "正在验证:", - "Version: {{.version}}": "", + "Version: {{.version}}": "版本: {{.version}}", "VirtualBox and Hyper-V are having a conflict. Use '--vm-driver=hyperv' or disable Hyper-V using: 'bcdedit /set hypervisorlaunchtype off'": "", "VirtualBox cannot create a network, probably because it conflicts with an existing network that minikube no longer knows about. Try running 'minikube delete'": "", "VirtualBox is broken. Disable real-time anti-virus software, reboot, and reinstall VirtualBox if the problem continues.": "", @@ -577,10 +577,10 @@ "config view failed": "", "creating api client": "", "dashboard service is not running: {{.error}}": "", - "disable failed": "", + "disable failed": "禁用失败", "dry-run mode. Validates configuration, but does not mutate system state": "", "dry-run validation complete!": "", - "enable failed": "", + "enable failed": "开启失败", "error creating clientset": "", "error creating machine client": "", "error parsing the input ip address for mount": "", @@ -594,7 +594,7 @@ "loading config": "", "logdir set failed": "", "machine '{{.name}}' does not exist. Proceeding ahead with recreating VM.": "", - "max time to wait per Kubernetes core services to be healthy.": "", + "max time to wait per Kubernetes core services to be healthy.": "每个 Kubernetes 核心服务保持健康所需的最长时间。", "minikube addons list --output OUTPUT. json, list": "", "minikube is exiting due to an error. If the above message is not useful, open an issue:": "由于出错 minikube 正在退出。如果以上信息没有帮助,请提交问题反馈:", "minikube is unable to access the Google Container Registry. You may need to configure it to use a HTTP proxy.": "", @@ -610,7 +610,7 @@ "namespaces to pause": "", "namespaces to unpause": "", "not enough arguments ({{.ArgCount}}).\\nusage: minikube config set PROPERTY_NAME PROPERTY_VALUE": "", - "pause containers": "", + "pause containers": "暂停容器", "profile sets the current minikube profile, or gets the current profile if no arguments are provided. This is used to run and manage multiple minikube instance. You can return to the default minikube profile by running `minikube profile default`": "", "reload cached images.": "重新加载缓存的镜像", "reloads images previously added using the 'cache add' subcommand": "重新加载之前通过子命令 'cache add' 添加的镜像", @@ -621,16 +621,16 @@ "status text failure": "", "toom any arguments ({{.ArgCount}}).\\nusage: minikube config set PROPERTY_NAME PROPERTY_VALUE": "", "tunnel creates a route to services deployed with type LoadBalancer and sets their Ingress to their ClusterIP. for a detailed example see https://minikube.sigs.k8s.io/docs/tasks/loadbalancer": "", - "tunnel makes services of type LoadBalancer accessible on localhost": "", + "tunnel makes services of type LoadBalancer accessible on localhost": "隧道使本地主机上可以访问 LoadBalancer 类型的服务", "unable to bind flags": "", "unable to delete minikube config folder": "无法删除 minikube 配置目录", "unable to set logtostderr": "", - "unpause Kubernetes": "", + "unpause Kubernetes": "恢复 Kubernetes", "unset failed": "", "unsets PROPERTY_NAME from the minikube config file. Can be overwritten by flags or environmental variables": "", "unsets an individual value in a minikube config file": "", "unsupported or missing driver: {{.name}}": "不支持或者缺失驱动:{{.name}}", - "update config": "", + "update config": "更新配置", "usage: minikube addons configure ADDON_NAME": "", "usage: minikube addons disable ADDON_NAME": "", "usage: minikube addons enable ADDON_NAME": "", From 884c30a5f64ee21590f29130c9dbdb634f801e0d Mon Sep 17 00:00:00 2001 From: Song Shukun Date: Thu, 27 Feb 2020 19:21:08 +0900 Subject: [PATCH 35/50] Fix: disable a disabled addon cause error --- pkg/addons/addons.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/addons/addons.go b/pkg/addons/addons.go index e1ca836c37..cc0f5d5d83 100644 --- a/pkg/addons/addons.go +++ b/pkg/addons/addons.go @@ -118,6 +118,9 @@ func enableOrDisableAddon(name, val, profile string) error { if alreadySet { glog.Warningf("addon %s should already be in state %v", name, val) + if !enable { + return nil + } } if name == "istio" && enable { From c412a8ae4441c3d4867954907d6f398e6aafa233 Mon Sep 17 00:00:00 2001 From: Jose Donizetti Date: Fri, 28 Feb 2020 12:53:51 -0300 Subject: [PATCH 36/50] Fix start for existing profile --- cmd/minikube/cmd/start.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index f56ad75c1e..cd15bbeea9 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -436,12 +436,6 @@ func selectDriver(existing *config.ClusterConfig) registry.DriverState { // Technically unrelated, but important to perform before detection driver.SetLibvirtURI(viper.GetString(kvmQemuURI)) - if viper.GetString("vm-driver") != "" { - ds := driver.Status(viper.GetString("vm-driver")) - out.T(out.Sparkle, `Using the {{.driver}} driver based on user configuration`, out.V{"driver": ds.String()}) - return ds - } - // By default, the driver is whatever we used last time if existing != nil && existing.Driver != "" { ds := driver.Status(existing.Driver) @@ -449,6 +443,12 @@ func selectDriver(existing *config.ClusterConfig) registry.DriverState { return ds } + if viper.GetString("vm-driver") != "" { + ds := driver.Status(viper.GetString("vm-driver")) + out.T(out.Sparkle, `Using the {{.driver}} driver based on user configuration`, out.V{"driver": ds.String()}) + return ds + } + pick, alts := driver.Suggest(driver.Choices()) if pick.Name == "" { exit.WithCodeT(exit.Config, "Unable to determine a default driver to use. Try specifying --vm-driver, or see https://minikube.sigs.k8s.io/docs/start/") From 954f5a6b7448c82fd307e0bd4da1e119a3468049 Mon Sep 17 00:00:00 2001 From: Jose Donizetti Date: Fri, 28 Feb 2020 14:57:53 -0300 Subject: [PATCH 37/50] kic: fix service list for docker on osx --- cmd/minikube/cmd/service_list.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/cmd/minikube/cmd/service_list.go b/cmd/minikube/cmd/service_list.go index d2bc661c66..9d86d5a8b8 100644 --- a/cmd/minikube/cmd/service_list.go +++ b/cmd/minikube/cmd/service_list.go @@ -66,17 +66,18 @@ var serviceListCmd = &cobra.Command{ if len(serviceURL.URLs) == 0 { data = append(data, []string{serviceURL.Namespace, serviceURL.Name, "No node port"}) } else { - data = append(data, []string{serviceURL.Namespace, serviceURL.Name, "", strings.Join(serviceURL.URLs, "\n")}) + serviceURLs := strings.Join(serviceURL.URLs, "\n") + // if we are running Docker on OSX we empty the internal service URLs + if runtime.GOOS == "darwin" && cfg.Driver == oci.Docker { + serviceURLs = "" + } + + data = append(data, []string{serviceURL.Namespace, serviceURL.Name, "", serviceURLs}) } - } + service.PrintServiceList(os.Stdout, data) - if runtime.GOOS == "darwin" && cfg.Driver == oci.Docker { - out.FailureT("Accessing service is not implemented yet for docker driver on Mac.\nThe following issue is tracking the in progress work::\nhttps://github.com/kubernetes/minikube/issues/6778") - exit.WithCodeT(exit.Failure, "Not yet implemented for docker driver on MacOS.") - } - }, } From 77fbafb620c64d2fe8443a247f37dab87c6ebcf0 Mon Sep 17 00:00:00 2001 From: Jose Donizetti Date: Fri, 28 Feb 2020 17:05:44 -0300 Subject: [PATCH 38/50] kic: fix unprivileged port bind tunnel docker for mac --- pkg/minikube/tunnel/kic/ssh_conn.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/minikube/tunnel/kic/ssh_conn.go b/pkg/minikube/tunnel/kic/ssh_conn.go index d619a44bc5..6674db4af6 100644 --- a/pkg/minikube/tunnel/kic/ssh_conn.go +++ b/pkg/minikube/tunnel/kic/ssh_conn.go @@ -36,6 +36,7 @@ func createSSHConn(name, sshPort, sshKey string, svc *v1.Service) *sshConn { // extract sshArgs sshArgs := []string{ // TODO: document the options here + "ssh", "-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking no", "-N", @@ -55,7 +56,7 @@ func createSSHConn(name, sshPort, sshKey string, svc *v1.Service) *sshConn { sshArgs = append(sshArgs, arg) } - cmd := exec.Command("ssh", sshArgs...) + cmd := exec.Command("sudo", sshArgs...) return &sshConn{ name: name, From 71a83acabb5557ad8489de3308e1bd5361bfac73 Mon Sep 17 00:00:00 2001 From: Keith Schaab Date: Fri, 28 Feb 2020 12:39:21 -0800 Subject: [PATCH 39/50] Update loadbalancer.md --- site/content/en/docs/Tasks/loadbalancer.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/site/content/en/docs/Tasks/loadbalancer.md b/site/content/en/docs/Tasks/loadbalancer.md index 3807363165..55fd5fbfbd 100644 --- a/site/content/en/docs/Tasks/loadbalancer.md +++ b/site/content/en/docs/Tasks/loadbalancer.md @@ -14,7 +14,7 @@ A LoadBalancer service is the standard way to expose a service to the internet. ## Using `minikube tunnel` -Services of type `LoadBalancer` can be exposed via the `minikube tunnel` command. It will run in a separate terminal until Ctrl-C is hit. +Services of type `LoadBalancer` can be exposed via the `minikube tunnel` command. It will run in the terminal until Ctrl-C is hit. ## Example @@ -25,7 +25,7 @@ it will ask for password. minikube tunnel ``` -`minikube tunnel` runs as a separate daemon, creating a network route on the host to the service CIDR of the cluster using the cluster's IP address as a gateway. The tunnel command exposes the external IP directly to any program running on the host operating system. +`minikube tunnel` runs as a process, creating a network route on the host to the service CIDR of the cluster using the cluster's IP address as a gateway. The tunnel command exposes the external IP directly to any program running on the host operating system.
From 11c0612b5cb4a1b016a849e0e14d856be1bd04ba Mon Sep 17 00:00:00 2001 From: Jose Donizetti Date: Fri, 28 Feb 2020 18:34:10 -0300 Subject: [PATCH 40/50] kic: only ask sudo for privileged ports tunnel docker for mac --- pkg/minikube/tunnel/kic/ssh_conn.go | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/pkg/minikube/tunnel/kic/ssh_conn.go b/pkg/minikube/tunnel/kic/ssh_conn.go index 6674db4af6..7f8728bcdc 100644 --- a/pkg/minikube/tunnel/kic/ssh_conn.go +++ b/pkg/minikube/tunnel/kic/ssh_conn.go @@ -36,7 +36,6 @@ func createSSHConn(name, sshPort, sshKey string, svc *v1.Service) *sshConn { // extract sshArgs sshArgs := []string{ // TODO: document the options here - "ssh", "-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking no", "-N", @@ -45,6 +44,8 @@ func createSSHConn(name, sshPort, sshKey string, svc *v1.Service) *sshConn { "-i", sshKey, } + askForSudo := false + var privilegedPorts []int32 for _, port := range svc.Spec.Ports { arg := fmt.Sprintf( "-L %d:%s:%d", @@ -53,10 +54,27 @@ func createSSHConn(name, sshPort, sshKey string, svc *v1.Service) *sshConn { port.Port, ) + // check if any port is privileged + if port.Port < 1024 { + privilegedPorts = append(privilegedPorts, port.Port) + askForSudo = true + } + sshArgs = append(sshArgs, arg) } - cmd := exec.Command("sudo", sshArgs...) + command := "ssh" + + if askForSudo { + // TODO: use out package + fmt.Printf("The service %s requires priviledged ports to be exposed: %v\n", svc.Name, privilegedPorts) + fmt.Printf("sudo permission will be asked for it.\n") + + command = "sudo" + sshArgs = append([]string{"ssh"}, sshArgs...) + } + + cmd := exec.Command(command, sshArgs...) return &sshConn{ name: name, From f02a0a53b74a037a49a321f545984be015917de8 Mon Sep 17 00:00:00 2001 From: Keith Schaab Date: Fri, 28 Feb 2020 14:00:49 -0800 Subject: [PATCH 41/50] Update loadbalancer.md --- site/content/en/docs/Tasks/loadbalancer.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/content/en/docs/Tasks/loadbalancer.md b/site/content/en/docs/Tasks/loadbalancer.md index 55fd5fbfbd..7aa1f9c7bd 100644 --- a/site/content/en/docs/Tasks/loadbalancer.md +++ b/site/content/en/docs/Tasks/loadbalancer.md @@ -14,7 +14,7 @@ A LoadBalancer service is the standard way to expose a service to the internet. ## Using `minikube tunnel` -Services of type `LoadBalancer` can be exposed via the `minikube tunnel` command. It will run in the terminal until Ctrl-C is hit. +Services of type `LoadBalancer` can be exposed via the `minikube tunnel` command. It must be continuously run in a terminal to keep the `LoadBalancer` running. Ctrl-C in the terminal can be used to terminate the process at which time the network routes will be cleaned up. ## Example From 44467579a6d5174e2a8fee84cb61eb8548aa140f Mon Sep 17 00:00:00 2001 From: Keith Schaab Date: Fri, 28 Feb 2020 14:02:16 -0800 Subject: [PATCH 42/50] Update loadbalancer.md --- site/content/en/docs/Tasks/loadbalancer.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/content/en/docs/Tasks/loadbalancer.md b/site/content/en/docs/Tasks/loadbalancer.md index 7aa1f9c7bd..4ec2f0f06d 100644 --- a/site/content/en/docs/Tasks/loadbalancer.md +++ b/site/content/en/docs/Tasks/loadbalancer.md @@ -14,7 +14,7 @@ A LoadBalancer service is the standard way to expose a service to the internet. ## Using `minikube tunnel` -Services of type `LoadBalancer` can be exposed via the `minikube tunnel` command. It must be continuously run in a terminal to keep the `LoadBalancer` running. Ctrl-C in the terminal can be used to terminate the process at which time the network routes will be cleaned up. +Services of type `LoadBalancer` can be exposed via the `minikube tunnel` command. It must be run in a separate terminal window to keep the `LoadBalancer` running. Ctrl-C in the terminal can be used to terminate the process at which time the network routes will be cleaned up. ## Example From df70b7900d72b4d707294800392db976c39fa178 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sat, 29 Feb 2020 13:59:55 +0100 Subject: [PATCH 43/50] Negotiate Docker API Version before image query Avoids error working with older Docker daemons Error response from daemon: client version 1.41 is too new. Maximum supported API version is 1.39 --- pkg/minikube/image/image.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/minikube/image/image.go b/pkg/minikube/image/image.go index d2a45f37fa..d7face059f 100644 --- a/pkg/minikube/image/image.go +++ b/pkg/minikube/image/image.go @@ -44,6 +44,7 @@ var defaultPlatform = v1.Platform{ func DigestByDockerLib(imgClient *client.Client, imgName string) string { ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) defer cancel() + imgClient.NegotiateAPIVersion(ctx) img, _, err := imgClient.ImageInspectWithRaw(ctx, imgName) if err != nil && !client.IsErrNotFound(err) { glog.Infof("couldn't find image digest %s from local daemon: %v ", imgName, err) From da1ee8bcae868b001b71cd42ed94c9a40c90eb3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sat, 29 Feb 2020 18:28:20 +0100 Subject: [PATCH 44/50] Add fallback command when nslookup is missing The "dnsutils" package with nslookup is not installed in all installations, but the node can still resolve DNS lookups... Also restore the correct order of the type parameter, that was lost in a previous commit (when changing from querytype). --- pkg/minikube/node/machine.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/minikube/node/machine.go b/pkg/minikube/node/machine.go index d0c4021222..1263c17d15 100644 --- a/pkg/minikube/node/machine.go +++ b/pkg/minikube/node/machine.go @@ -154,12 +154,15 @@ func trySSH(h *host.Host, ip string) { func tryLookup(r command.Runner) { // DNS check - if rr, err := r.RunCmd(exec.Command("nslookup", "kubernetes.io", "-type=ns")); err != nil { + if rr, err := r.RunCmd(exec.Command("nslookup", "-type=ns", "kubernetes.io")); err != nil { glog.Infof("%s failed: %v which might be okay will retry nslookup without query type", rr.Args, err) // will try with without query type for ISOs with different busybox versions. if _, err = r.RunCmd(exec.Command("nslookup", "kubernetes.io")); err != nil { glog.Warningf("nslookup failed: %v", err) - out.WarningT("Node may be unable to resolve external DNS records") + // try with the older "host" command, instead of the newer "nslookup" + if _, err = r.RunCmd(exec.Command("host", "kubernetes.io")); err != nil { + out.WarningT("Node may be unable to resolve external DNS records") + } } } } From d6e414326c512098b8eef200e38c3e2ce3afaa00 Mon Sep 17 00:00:00 2001 From: Jose Donizetti Date: Sat, 29 Feb 2020 20:26:20 -0300 Subject: [PATCH 45/50] Change service wait and interval defaults --- pkg/minikube/service/service.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/minikube/service/service.go b/pkg/minikube/service/service.go index 8a695f7072..3f3c159cd4 100644 --- a/pkg/minikube/service/service.go +++ b/pkg/minikube/service/service.go @@ -48,9 +48,9 @@ import ( const ( defaultK8sClientTimeout = 60 * time.Second // DefaultWait is the default wait time, in seconds - DefaultWait = 20 + DefaultWait = 2 // DefaultInterval is the default interval, in seconds - DefaultInterval = 6 + DefaultInterval = 1 ) // K8sClient represents a kubernetes client From 960d402e1e73f3d9bb757ad6d57db9a345d6f158 Mon Sep 17 00:00:00 2001 From: Jose Donizetti Date: Sat, 29 Feb 2020 20:50:07 -0300 Subject: [PATCH 46/50] refactor kic tunnel to use out pkg --- pkg/minikube/tunnel/kic/ssh_conn.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/pkg/minikube/tunnel/kic/ssh_conn.go b/pkg/minikube/tunnel/kic/ssh_conn.go index 7f8728bcdc..bedb2cd473 100644 --- a/pkg/minikube/tunnel/kic/ssh_conn.go +++ b/pkg/minikube/tunnel/kic/ssh_conn.go @@ -21,8 +21,9 @@ import ( "os/exec" "github.com/phayes/freeport" - v1 "k8s.io/api/core/v1" + + "k8s.io/minikube/pkg/minikube/out" ) type sshConn struct { @@ -66,9 +67,13 @@ func createSSHConn(name, sshPort, sshKey string, svc *v1.Service) *sshConn { command := "ssh" if askForSudo { - // TODO: use out package - fmt.Printf("The service %s requires priviledged ports to be exposed: %v\n", svc.Name, privilegedPorts) - fmt.Printf("sudo permission will be asked for it.\n") + out.T( + out.WarningType, + "The service {{.service}} requires privileged ports to be exposed: {{.ports}}", + out.V{"service": svc.Name, "ports": fmt.Sprintf("%v", privilegedPorts)}, + ) + + out.T(out.Permissions, "sudo permission will be asked for it.") command = "sudo" sshArgs = append([]string{"ssh"}, sshArgs...) @@ -125,7 +130,8 @@ func createSSHConnWithRandomPorts(name, sshPort, sshKey string, svc *v1.Service) } func (c *sshConn) startAndWait() error { - fmt.Printf("starting tunnel for %s\n", c.service) + out.T(out.Running, "Starting tunnel for service {{.service}}.", out.V{"service": c.service}) + err := c.cmd.Start() if err != nil { return err @@ -138,6 +144,7 @@ func (c *sshConn) startAndWait() error { } func (c *sshConn) stop() error { - fmt.Printf("stopping tunnel for %s\n", c.service) + out.T(out.Stopping, "Stopping tunnel for service {{.service}}.", out.V{"service": c.service}) + return c.cmd.Process.Kill() } From 6d1612f8990e795698fc590c340a89341d1614e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sun, 1 Mar 2020 12:23:23 +0100 Subject: [PATCH 47/50] Allow running lint (golangci-lint) using docker --- Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile b/Makefile index b7fee83129..7f49be2b11 100755 --- a/Makefile +++ b/Makefile @@ -365,7 +365,11 @@ out/linters/golangci-lint-$(GOLINT_VERSION): # this one is meant for local use .PHONY: lint lint: pkg/minikube/assets/assets.go pkg/minikube/translate/translations.go out/linters/golangci-lint-$(GOLINT_VERSION) ## Run lint +ifeq ($(MINIKUBE_BUILD_IN_DOCKER),y) + docker run --rm -v $(pwd):/app -w /app golangci/golangci-lint:$(GOLINT_VERSION) golangci-lint run ${GOLINT_OPTIONS} ./... +else ./out/linters/golangci-lint-$(GOLINT_VERSION) run ${GOLINT_OPTIONS} ./... +endif # lint-ci is slower version of lint and is meant to be used in ci (travis) to avoid out of memory leaks. .PHONY: lint-ci From 21e5e1873d91d2cd38729fcde0f652c8eaf100f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sun, 1 Mar 2020 12:23:48 +0100 Subject: [PATCH 48/50] Skip native drivers when running lint in docker They require libvirt and hyperkit host packages installed --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7f49be2b11..e9d700129c 100755 --- a/Makefile +++ b/Makefile @@ -366,7 +366,8 @@ out/linters/golangci-lint-$(GOLINT_VERSION): .PHONY: lint lint: pkg/minikube/assets/assets.go pkg/minikube/translate/translations.go out/linters/golangci-lint-$(GOLINT_VERSION) ## Run lint ifeq ($(MINIKUBE_BUILD_IN_DOCKER),y) - docker run --rm -v $(pwd):/app -w /app golangci/golangci-lint:$(GOLINT_VERSION) golangci-lint run ${GOLINT_OPTIONS} ./... + docker run --rm -v $(pwd):/app -w /app golangci/golangci-lint:$(GOLINT_VERSION) \ + golangci-lint run ${GOLINT_OPTIONS} --skip-dirs "cmd/drivers/kvm|cmd/drivers/hyperkit|pkg/drivers/kvm|pkg/drivers/hyperkit" ./... else ./out/linters/golangci-lint-$(GOLINT_VERSION) run ${GOLINT_OPTIONS} ./... endif From d8e203e32518ec868d5cf523fa240803879fbf6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sun, 1 Mar 2020 12:21:07 +0100 Subject: [PATCH 49/50] Don't download the golangci-lint binary for docker It will instead download the golangci/golangci-lint docker image --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e9d700129c..71f547e32d 100755 --- a/Makefile +++ b/Makefile @@ -364,11 +364,12 @@ out/linters/golangci-lint-$(GOLINT_VERSION): # this one is meant for local use .PHONY: lint -lint: pkg/minikube/assets/assets.go pkg/minikube/translate/translations.go out/linters/golangci-lint-$(GOLINT_VERSION) ## Run lint ifeq ($(MINIKUBE_BUILD_IN_DOCKER),y) +lint: pkg/minikube/assets/assets.go pkg/minikube/translate/translations.go docker run --rm -v $(pwd):/app -w /app golangci/golangci-lint:$(GOLINT_VERSION) \ golangci-lint run ${GOLINT_OPTIONS} --skip-dirs "cmd/drivers/kvm|cmd/drivers/hyperkit|pkg/drivers/kvm|pkg/drivers/hyperkit" ./... else +lint: pkg/minikube/assets/assets.go pkg/minikube/translate/translations.go out/linters/golangci-lint-$(GOLINT_VERSION) ## Run lint ./out/linters/golangci-lint-$(GOLINT_VERSION) run ${GOLINT_OPTIONS} ./... endif From 07d1402588ffe1ec1114b200458dfdb29ebae449 Mon Sep 17 00:00:00 2001 From: Prasad Katti Date: Sun, 1 Mar 2020 13:11:49 -0800 Subject: [PATCH 50/50] minor fix in roadmap doc --- site/content/en/docs/Contributing/roadmap.en.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/content/en/docs/Contributing/roadmap.en.md b/site/content/en/docs/Contributing/roadmap.en.md index 02d21ce9f5..37b135a78f 100644 --- a/site/content/en/docs/Contributing/roadmap.en.md +++ b/site/content/en/docs/Contributing/roadmap.en.md @@ -6,7 +6,7 @@ description: > Development Roadmap --- -This roadmap is a living document outlining the major technical improvements which we would like to see in minikube during 2019, divided by how they apply to our [guiding principles](principles.md) +This roadmap is a living document outlining the major technical improvements which we would like to see in minikube during 2020, divided by how they apply to our [guiding principles](principles.md) Please send a PR to suggest any improvements to it.