Merge branch 'master' of https://github.com/kubernetes/minikube into systemd

cmd/minikube/cmd/config/configure.go

@@ -18,8 +18,10 @@ package config
 
 import (
 	"io/ioutil"
+	"net"
 
 	"github.com/spf13/cobra"
+	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/exit"
 	"k8s.io/minikube/pkg/minikube/out"
 	"k8s.io/minikube/pkg/minikube/service"
@@ -184,6 +186,30 @@ var addonsConfigureCmd = &cobra.Command{
 				out.WarningT("ERROR creating `registry-creds-acr` secret")
 			}
 
+		case "metallb":
+			profile := ClusterFlagValue()
+			cfg, err := config.Load(profile)
+			if err != nil {
+				out.ErrT(out.FatalType, "Failed to load config {{.profile}}", out.V{"profile": profile})
+			}
+
+			validator := func(s string) bool {
+				return net.ParseIP(s) != nil
+			}
+
+			if cfg.KubernetesConfig.LoadBalancerStartIP == "" {
+				cfg.KubernetesConfig.LoadBalancerStartIP = AskForStaticValidatedValue("-- Enter Load Balancer Start IP: ", validator)
+			}
+
+			if cfg.KubernetesConfig.LoadBalancerEndIP == "" {
+				cfg.KubernetesConfig.LoadBalancerEndIP = AskForStaticValidatedValue("-- Enter Load Balancer End IP: ", validator)
+			}
+
+			err = config.SaveProfile(profile, cfg)
+			if err != nil {
+				out.ErrT(out.FatalType, "Failed to save config {{.profile}}", out.V{"profile": profile})
+			}
+
 		default:
 			out.FailureT("{{.name}} has no available configuration options", out.V{"name": addon})
 			return

cmd/minikube/cmd/config/prompt.go

@@ -153,3 +153,23 @@ func posString(slice []string, element string) int {
 func containsString(slice []string, element string) bool {
 	return posString(slice, element) != -1
 }
+
+// AskForStaticValidatedValue asks for a single value to enter and check for valid input
+func AskForStaticValidatedValue(s string, validator func(s string) bool) string {
+	reader := bufio.NewReader(os.Stdin)
+
+	for {
+		response := getStaticValue(reader, s)
+
+		// Can't have zero length
+		if len(response) == 0 {
+			out.Err("--Error, please enter a value:")
+			continue
+		}
+		if !validator(response) {
+			out.Err("--Invalid input, please enter a value:")
+			continue
+		}
+		return response
+	}
+}

cmd/minikube/cmd/service.go

@@ -80,11 +80,6 @@ var serviceCmd = &cobra.Command{
 		cname := ClusterFlagValue()
 		co := mustload.Healthy(cname)
 
-		if driver.NeedsPortForward(co.Config.Driver) {
-			startKicServiceTunnel(svc, cname)
-			return
-		}
-
 		urls, err := service.WaitForService(co.API, co.Config.Name, namespace, svc, serviceURLTemplate, serviceURLMode, https, wait, interval)
 		if err != nil {
 			var s *service.SVCNotFoundError
@@ -95,6 +90,11 @@ You may select another namespace by using 'minikube service {{.service}} -n <nam
 			exit.WithError("Error opening service", err)
 		}
 
+		if driver.NeedsPortForward(co.Config.Driver) {
+			startKicServiceTunnel(svc, cname)
+			return
+		}
+
 		openURLs(svc, urls)
 	},
 }

deploy/addons/metallb/metallb-config.yaml.tmpl

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  namespace: metallb-system
+  name: config
+data:
+  config: |
+    address-pools:
+    - name: default
+      protocol: layer2
+      addresses:
+      - {{ .LoadBalancerStartIP }}-{{ .LoadBalancerEndIP }}

deploy/addons/metallb/metallb.yaml

@@ -0,0 +1,293 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  labels:
+    app: metallb
+  name: metallb-system
+---
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  labels:
+    app: metallb
+  name: speaker
+  namespace: metallb-system
+spec:
+  allowPrivilegeEscalation: false
+  allowedCapabilities:
+  - NET_ADMIN
+  - NET_RAW
+  - SYS_ADMIN
+  fsGroup:
+    rule: RunAsAny
+  hostNetwork: true
+  hostPorts:
+  - max: 7472
+    min: 7472
+  privileged: true
+  runAsUser:
+    rule: RunAsAny
+  seLinux:
+    rule: RunAsAny
+  supplementalGroups:
+    rule: RunAsAny
+  volumes:
+  - '*'
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    app: metallb
+  name: controller
+  namespace: metallb-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    app: metallb
+  name: speaker
+  namespace: metallb-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app: metallb
+  name: metallb-system:controller
+rules:
+- apiGroups:
+  - ''
+  resources:
+  - services
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+- apiGroups:
+  - ''
+  resources:
+  - services/status
+  verbs:
+  - update
+- apiGroups:
+  - ''
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app: metallb
+  name: metallb-system:speaker
+rules:
+- apiGroups:
+  - ''
+  resources:
+  - services
+  - endpoints
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ''
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+- apiGroups:
+  - extensions
+  resourceNames:
+  - speaker
+  resources:
+  - podsecuritypolicies
+  verbs:
+  - use
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    app: metallb
+  name: config-watcher
+  namespace: metallb-system
+rules:
+- apiGroups:
+  - ''
+  resources:
+  - configmaps
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app: metallb
+  name: metallb-system:controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: metallb-system:controller
+subjects:
+- kind: ServiceAccount
+  name: controller
+  namespace: metallb-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app: metallb
+  name: metallb-system:speaker
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: metallb-system:speaker
+subjects:
+- kind: ServiceAccount
+  name: speaker
+  namespace: metallb-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    app: metallb
+  name: config-watcher
+  namespace: metallb-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: config-watcher
+subjects:
+- kind: ServiceAccount
+  name: controller
+- kind: ServiceAccount
+  name: speaker
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    app: metallb
+    component: speaker
+  name: speaker
+  namespace: metallb-system
+spec:
+  selector:
+    matchLabels:
+      app: metallb
+      component: speaker
+  template:
+    metadata:
+      annotations:
+        prometheus.io/port: '7472'
+        prometheus.io/scrape: 'true'
+      labels:
+        app: metallb
+        component: speaker
+    spec:
+      containers:
+      - args:
+        - --port=7472
+        - --config=config
+        env:
+        - name: METALLB_NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        - name: METALLB_HOST
+          valueFrom:
+            fieldRef:
+              fieldPath: status.hostIP
+        image: metallb/speaker:v0.8.2
+        imagePullPolicy: IfNotPresent
+        name: speaker
+        ports:
+        - containerPort: 7472
+          name: monitoring
+        resources:
+          limits:
+            cpu: 100m
+            memory: 100Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            add:
+            - NET_ADMIN
+            - NET_RAW
+            - SYS_ADMIN
+            drop:
+            - ALL
+          readOnlyRootFilesystem: true
+      hostNetwork: true
+      nodeSelector:
+        beta.kubernetes.io/os: linux
+      serviceAccountName: speaker
+      terminationGracePeriodSeconds: 0
+      tolerations:
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/master
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: metallb
+    component: controller
+  name: controller
+  namespace: metallb-system
+spec:
+  revisionHistoryLimit: 3
+  selector:
+    matchLabels:
+      app: metallb
+      component: controller
+  template:
+    metadata:
+      annotations:
+        prometheus.io/port: '7472'
+        prometheus.io/scrape: 'true'
+      labels:
+        app: metallb
+        component: controller
+    spec:
+      containers:
+      - args:
+        - --port=7472
+        - --config=config
+        image: metallb/controller:v0.8.2
+        imagePullPolicy: IfNotPresent
+        name: controller
+        ports:
+        - containerPort: 7472
+          name: monitoring
+        resources:
+          limits:
+            cpu: 100m
+            memory: 100Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - all
+          readOnlyRootFilesystem: true
+      nodeSelector:
+        beta.kubernetes.io/os: linux
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65534
+      serviceAccountName: controller
+      terminationGracePeriodSeconds: 0

pkg/addons/config.go

@@ -129,4 +129,9 @@ var Addons = []*Addon{
 		set:       SetBool,
 		callbacks: []setFn{enableOrDisableStorageClasses},
 	},
+	{
+		name:      "metallb",
+		set:       SetBool,
+		callbacks: []setFn{enableOrDisableAddon},
+	},
 }

pkg/drivers/kic/kic.go

@@ -99,7 +99,7 @@ func (d *Driver) Create() error {
 		},
 	)
 
-	exists, err := oci.ContainerExists(d.OCIBinary, params.Name)
+	exists, err := oci.ContainerExists(d.OCIBinary, params.Name, true)
 	if err != nil {
 		glog.Warningf("failed to check if container already exists: %v", err)
 	}
@@ -234,26 +234,7 @@ func (d *Driver) GetURL() (string, error) {
 
 // GetState returns the state that the host is in (running, stopped, etc)
 func (d *Driver) GetState() (state.State, error) {
-	out, err := oci.WarnIfSlow(d.NodeConfig.OCIBinary, "inspect", "-f", "{{.State.Status}}", d.MachineName)
-	if err != nil {
-		return state.Error, err
-	}
-
-	o := strings.TrimSpace(string(out))
-	switch o {
-	case "running":
-		return state.Running, nil
-	case "exited":
-		return state.Stopped, nil
-	case "paused":
-		return state.Paused, nil
-	case "restarting":
-		return state.Starting, nil
-	case "dead":
-		return state.Error, nil
-	default:
-		return state.None, fmt.Errorf("unknown state")
-	}
+	return oci.ContainerStatus(d.OCIBinary, d.MachineName, true)
 }
 
 // Kill stops a host forcefully, including any containers that we are managing.

pkg/drivers/kic/oci/cli_runner.go

@@ -0,0 +1,137 @@
+/*
+Copyright 2019 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package oci
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"os/exec"
+	"strings"
+	"time"
+
+	"github.com/golang/glog"
+	"k8s.io/minikube/pkg/minikube/out"
+)
+
+// RunResult holds the results of a Runner
+type RunResult struct {
+	Stdout   bytes.Buffer
+	Stderr   bytes.Buffer
+	ExitCode int
+	Args     []string // the args that was passed to Runner
+}
+
+// Command returns a human readable command string that does not induce eye fatigue
+func (rr RunResult) Command() string {
+	var sb strings.Builder
+	sb.WriteString(rr.Args[0])
+	for _, a := range rr.Args[1:] {
+		if strings.Contains(a, " ") {
+			sb.WriteString(fmt.Sprintf(` "%s"`, a))
+			continue
+		}
+		sb.WriteString(fmt.Sprintf(" %s", a))
+	}
+	return sb.String()
+}
+
+// Output returns human-readable output for an execution result
+func (rr RunResult) Output() string {
+	var sb strings.Builder
+	if rr.Stdout.Len() > 0 {
+		sb.WriteString(fmt.Sprintf("-- stdout --\n%s\n-- /stdout --", rr.Stdout.Bytes()))
+	}
+	if rr.Stderr.Len() > 0 {
+		sb.WriteString(fmt.Sprintf("\n** stderr ** \n%s\n** /stderr **", rr.Stderr.Bytes()))
+	}
+	return sb.String()
+}
+
+// runCmd runs a command exec.Command against docker daemon or podman
+func runCmd(cmd *exec.Cmd, warnSlow ...bool) (*RunResult, error) {
+	warn := false
+	if len(warnSlow) > 0 {
+		warn = warnSlow[0]
+	}
+
+	killTime := 19 * time.Second // this will be applied only if warnSlow is true
+	warnTime := 2 * time.Second
+	ctx, cancel := context.WithTimeout(context.Background(), killTime)
+	defer cancel()
+
+	if cmd.Args[1] == "volume" || cmd.Args[1] == "ps" { // volume and ps requires more time than inspect
+		killTime = 30 * time.Second
+		warnTime = 3 * time.Second
+	}
+
+	if warn { // convert exec.Command to with context
+		cmdWithCtx := exec.CommandContext(ctx, cmd.Args[0], cmd.Args[1:]...)
+		cmdWithCtx.Stdout = cmd.Stdout //copying the original command
+		cmdWithCtx.Stderr = cmd.Stderr
+		cmd = cmdWithCtx
+	}
+
+	rr := &RunResult{Args: cmd.Args}
+	glog.Infof("Run: %v", rr.Command())
+
+	var outb, errb io.Writer
+	if cmd.Stdout == nil {
+		var so bytes.Buffer
+		outb = io.MultiWriter(&so, &rr.Stdout)
+	} else {
+		outb = io.MultiWriter(cmd.Stdout, &rr.Stdout)
+	}
+
+	if cmd.Stderr == nil {
+		var se bytes.Buffer
+		errb = io.MultiWriter(&se, &rr.Stderr)
+	} else {
+		errb = io.MultiWriter(cmd.Stderr, &rr.Stderr)
+	}
+
+	cmd.Stdout = outb
+	cmd.Stderr = errb
+
+	start := time.Now()
+	err := cmd.Run()
+	elapsed := time.Since(start)
+	if warn {
+		if elapsed > warnTime {
+			out.WarningT(`Executing "{{.command}}" took an unusually long time: {{.duration}}`, out.V{"command": rr.Command(), "duration": elapsed})
+			out.ErrT(out.Tip, `Restarting the {{.name}} service may improve performance.`, out.V{"name": cmd.Args[0]})
+		}
+
+		if ctx.Err() == context.DeadlineExceeded {
+			return rr, fmt.Errorf("%q timed out after %s", rr.Command(), killTime)
+		}
+	}
+
+	if exitError, ok := err.(*exec.ExitError); ok {
+		rr.ExitCode = exitError.ExitCode()
+	}
+	// Decrease log spam
+	if elapsed > (1 * time.Second) {
+		glog.Infof("Completed: %s: (%s)", rr.Command(), elapsed)
+	}
+	if err == nil {
+		return rr, nil
+	}
+
+	return rr, fmt.Errorf("%s: %v\nstdout:\n%s\nstderr:\n%s", rr.Command(), err, rr.Stdout.String(), rr.Stderr.String())
+}

pkg/drivers/kic/oci/info.go

@@ -216,15 +216,12 @@ type podmanSysInfo struct {
 // dockerSystemInfo returns docker system info --format '{{json .}}'
 func dockerSystemInfo() (dockerSysInfo, error) {
 	var ds dockerSysInfo
-
-	cmd := exec.Command(Docker, "system", "info", "--format", "{{json .}}")
-	out, err := cmd.CombinedOutput()
-
+	rr, err := runCmd(exec.Command(Docker, "system", "info", "--format", "{{json .}}"))
 	if err != nil {
 		return ds, errors.Wrap(err, "get docker system info")
 	}
 
-	if err := json.Unmarshal([]byte(strings.TrimSpace(string(out))), &ds); err != nil {
+	if err := json.Unmarshal([]byte(strings.TrimSpace(rr.Stdout.String())), &ds); err != nil {
 		return ds, errors.Wrapf(err, "unmarshal docker system info")
 	}
 
@@ -234,12 +231,12 @@ func dockerSystemInfo() (dockerSysInfo, error) {
 // podmanSysInfo returns podman system info --format '{{json .}}'
 func podmanSystemInfo() (podmanSysInfo, error) {
 	var ps podmanSysInfo
-	cmd := exec.Command(Podman, "system", "info", "--format", "'{{json .}}'")
-	out, err := cmd.CombinedOutput()
+	rr, err := runCmd(exec.Command(Podman, "system", "info", "--format", "'{{json .}}'"))
 	if err != nil {
 		return ps, errors.Wrap(err, "get podman system info")
 	}
-	if err := json.Unmarshal([]byte(strings.TrimSpace(string(out))), &ps); err != nil {
+
+	if err := json.Unmarshal([]byte(strings.TrimSpace(rr.Stdout.String())), &ps); err != nil {
 		return ps, errors.Wrapf(err, "unmarshal podman system info")
 	}
 	return ps, nil

pkg/drivers/kic/oci/network.go

@@ -43,12 +43,10 @@ func RoutableHostIPFromInside(ociBin string, containerName string) (net.IP, erro
 
 // digDNS will get the IP record for a dns
 func digDNS(ociBin, containerName, dns string) (net.IP, error) {
-	cmd := exec.Command(ociBin, "exec", "-t", containerName, "dig", "+short", dns)
-	out, err := cmd.CombinedOutput()
-	ip := net.ParseIP(strings.TrimSpace(string(out)))
-
+	rr, err := runCmd(exec.Command(ociBin, "exec", "-t", containerName, "dig", "+short", dns))
+	ip := net.ParseIP(strings.TrimSpace(rr.Stdout.String()))
 	if err != nil {
-		return ip, errors.Wrapf(err, "resolve dns to ip: %s", string(out))
+		return ip, errors.Wrapf(err, "resolve dns to ip")
 	}
 
 	glog.Infof("got host ip for mount in container by digging dns: %s", ip.String())
@@ -58,23 +56,19 @@ func digDNS(ociBin, containerName, dns string) (net.IP, error) {
 // dockerGatewayIP gets the default gateway ip for the docker bridge on the user's host machine
 // gets the ip from user's host docker
 func dockerGatewayIP() (net.IP, error) {
-	cmd := exec.Command(Docker, "network", "ls", "--filter", "name=bridge", "--format", "{{.ID}}")
-	out, err := cmd.CombinedOutput()
-
+	rr, err := runCmd(exec.Command(Docker, "network", "ls", "--filter", "name=bridge", "--format", "{{.ID}}"))
 	if err != nil {
-		return nil, errors.Wrapf(err, "get network bridge. output: %s", string(out))
+		return nil, errors.Wrapf(err, "get network bridge")
 	}
 
-	bridgeID := strings.TrimSpace(string(out))
-	cmd = exec.Command(Docker, "inspect",
-		"--format", "{{(index .IPAM.Config 0).Gateway}}", bridgeID)
-	out, err = cmd.CombinedOutput()
-
+	bridgeID := strings.TrimSpace(rr.Stdout.String())
+	rr, err = runCmd(exec.Command(Docker, "inspect",
+		"--format", "{{(index .IPAM.Config 0).Gateway}}", bridgeID))
 	if err != nil {
-		return nil, errors.Wrapf(err, "inspect IP gatway for bridge network: %q. output: %s", string(out), bridgeID)
+		return nil, errors.Wrapf(err, "inspect IP bridge network %q.", bridgeID)
 	}
 
-	ip := net.ParseIP(strings.TrimSpace(string(out)))
+	ip := net.ParseIP(strings.TrimSpace(rr.Stdout.String()))
 	glog.Infof("got host ip for mount in container by inspect docker network: %s", ip.String())
 	return ip, nil
 }
@@ -84,26 +78,24 @@ func dockerGatewayIP() (net.IP, error) {
 // will return the docker assigned port:
 // 32769, nil
 // only supports TCP ports
-func ForwardedPort(ociBinary string, ociID string, contPort int) (int, error) {
-	var out []byte
+func ForwardedPort(ociBin string, ociID string, contPort int) (int, error) {
+	var rr *RunResult
 	var err error
 
-	if ociBinary == Podman {
+	if ociBin == Podman {
 		//podman inspect -f "{{range .NetworkSettings.Ports}}{{if eq .ContainerPort "80"}}{{.HostPort}}{{end}}{{end}}"
-		cmd := exec.Command(ociBinary, "inspect", "-f", fmt.Sprintf("{{range .NetworkSettings.Ports}}{{if eq .ContainerPort %s}}{{.HostPort}}{{end}}{{end}}", fmt.Sprint(contPort)), ociID)
-		out, err = cmd.CombinedOutput()
+		rr, err = runCmd(exec.Command(ociBin, "inspect", "-f", fmt.Sprintf("{{range .NetworkSettings.Ports}}{{if eq .ContainerPort %s}}{{.HostPort}}{{end}}{{end}}", fmt.Sprint(contPort)), ociID))
 		if err != nil {
-			return 0, errors.Wrapf(err, "get host-bind port %d for %q, output %s", contPort, ociID, out)
+			return 0, errors.Wrapf(err, "get port %d for %q", contPort, ociID)
 		}
 	} else {
-		cmd := exec.Command(ociBinary, "inspect", "-f", fmt.Sprintf("'{{(index (index .NetworkSettings.Ports \"%d/tcp\") 0).HostPort}}'", contPort), ociID)
-		out, err = cmd.CombinedOutput()
+		rr, err = runCmd(exec.Command(ociBin, "inspect", "-f", fmt.Sprintf("'{{(index (index .NetworkSettings.Ports \"%d/tcp\") 0).HostPort}}'", contPort), ociID))
 		if err != nil {
-			return 0, errors.Wrapf(err, "get host-bind port %d for %q, output %s", contPort, ociID, out)
+			return 0, errors.Wrapf(err, "get port %d for %q", contPort, ociID)
 		}
 	}
 
-	o := strings.TrimSpace(string(out))
+	o := strings.TrimSpace(rr.Stdout.String())
 	o = strings.Trim(o, "'")
 	p, err := strconv.Atoi(o)
 
@@ -115,8 +107,8 @@ func ForwardedPort(ociBinary string, ociID string, contPort int) (int, error) {
 }
 
 // ContainerIPs returns ipv4,ipv6, error of a container by their name
-func ContainerIPs(ociBinary string, name string) (string, string, error) {
-	if ociBinary == Podman {
+func ContainerIPs(ociBin string, name string) (string, string, error) {
+	if ociBin == Podman {
 		return podmanConttainerIP(name)
 	}
 	return dockerContainerIP(name)
@@ -124,14 +116,13 @@ func ContainerIPs(ociBinary string, name string) (string, string, error) {
 
 // podmanConttainerIP returns ipv4, ipv6 of container or error
 func podmanConttainerIP(name string) (string, string, error) {
-	cmd := exec.Command(Podman, "inspect",
+	rr, err := runCmd(exec.Command(Podman, "inspect",
 		"-f", "{{.NetworkSettings.IPAddress}}",
-		name)
-	out, err := cmd.CombinedOutput()
+		name))
 	if err != nil {
 		return "", "", errors.Wrapf(err, "podman inspect ip %s", name)
 	}
-	output := strings.TrimSpace(string(out))
+	output := strings.TrimSpace(rr.Stdout.String())
 	if err == nil && output == "" { // podman returns empty for 127.0.0.1
 		return DefaultBindIPV4, "", nil
 	}

pkg/drivers/kic/oci/oci.go

@@ -17,7 +17,6 @@ limitations under the License.
 package oci
 
 import (
-	"context"
 	"os"
 	"path/filepath"
 	"time"
@@ -30,7 +29,6 @@ import (
 	"github.com/pkg/errors"
 	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/localpath"
-	"k8s.io/minikube/pkg/minikube/out"
 	"k8s.io/minikube/pkg/util/retry"
 
 	"fmt"
@@ -64,9 +62,9 @@ func DeleteContainersByLabel(ociBin string, label string) []error {
 		if err := ShutDown(ociBin, c); err != nil {
 			glog.Infof("couldn't shut down %s (might be okay): %v ", c, err)
 		}
-		cmd := exec.Command(ociBin, "rm", "-f", "-v", c)
-		if out, err := cmd.CombinedOutput(); err != nil {
-			deleteErrs = append(deleteErrs, errors.Wrapf(err, "delete container %s: output %s", c, out))
+
+		if _, err := runCmd(exec.Command(ociBin, "rm", "-f", "-v", c)); err != nil {
+			deleteErrs = append(deleteErrs, errors.Wrapf(err, "delete container %s: output %s", c, err))
 		}
 
 	}
@@ -84,9 +82,9 @@ func DeleteContainer(ociBin string, name string) error {
 	if err := ShutDown(ociBin, name); err != nil {
 		glog.Infof("couldn't shut down %s (might be okay): %v ", name, err)
 	}
-	cmd := exec.Command(ociBin, "rm", "-f", "-v", name)
-	if out, err := cmd.CombinedOutput(); err != nil {
-		return errors.Wrapf(err, "delete container %s: output %s", name, out)
+
+	if _, err := runCmd(exec.Command(ociBin, "rm", "-f", "-v", name)); err != nil {
+		return errors.Wrapf(err, "delete %s", name)
 	}
 	return nil
 }
@@ -188,7 +186,7 @@ func CreateContainerNode(p CreateParams) error {
 }
 
 // CreateContainer creates a container with "docker/podman run"
-func createContainer(ociBinary string, image string, opts ...createOpt) error {
+func createContainer(ociBin string, image string, opts ...createOpt) error {
 	o := &createOpts{}
 	for _, opt := range opts {
 		o = opt(o)
@@ -205,7 +203,7 @@ func createContainer(ociBinary string, image string, opts ...createOpt) error {
 	args := []string{"run"}
 
 	// to run nested container from privileged container in podman https://bugzilla.redhat.com/show_bug.cgi?id=1687713
-	if ociBinary == Podman {
+	if ociBin == Podman {
 		args = append(args, "--cgroup-manager", "cgroupfs")
 	}
 
@@ -213,87 +211,33 @@ func createContainer(ociBinary string, image string, opts ...createOpt) error {
 	args = append(args, image)
 	args = append(args, o.ContainerArgs...)
 
-	out, err := exec.Command(ociBinary, args...).CombinedOutput()
-	if err != nil {
-		return errors.Wrapf(err, "failed args: %v output: %s", args, out)
-	}
-
-	return nil
-}
-
-// Copy copies a local asset into the container
-func Copy(ociBinary string, ociID string, targetDir string, fName string) error {
-	if _, err := os.Stat(fName); os.IsNotExist(err) {
-		return errors.Wrapf(err, "error source %s does not exist", fName)
-	}
-
-	destination := fmt.Sprintf("%s:%s", ociID, targetDir)
-	cmd := exec.Command(ociBinary, "cp", fName, destination)
-	if err := cmd.Run(); err != nil {
-		return errors.Wrapf(err, "error copying %s into node", fName)
+	if _, err := runCmd(exec.Command(ociBin, args...)); err != nil {
+		return err
 	}
 
 	return nil
 }
 
 // ContainerID returns id of a container name
-func ContainerID(ociBinary string, nameOrID string) (string, error) {
-	cmd := exec.Command(ociBinary, "inspect", "-f", "{{.Id}}", nameOrID)
-	out, err := cmd.CombinedOutput()
-
+func ContainerID(ociBin string, nameOrID string) (string, error) {
+	rr, err := runCmd(exec.Command(ociBin, "inspect", "-f", "{{.Id}}", nameOrID))
 	if err != nil { // don't return error if not found, only return empty string
-		if strings.Contains(string(out), "Error: No such object:") || strings.Contains(string(out), "unable to find") {
+		if strings.Contains(rr.Stdout.String(), "Error: No such object:") || strings.Contains(rr.Stdout.String(), "unable to find") {
 			err = nil
 		}
-		out = []byte{}
+		return "", err
 	}
-
-	return string(out), err
-}
-
-// WarnIfSlow runs an oci command, warning about performance issues
-func WarnIfSlow(args ...string) ([]byte, error) {
-	killTime := 19 * time.Second
-	warnTime := 2 * time.Second
-
-	if args[1] == "volume" || args[1] == "ps" { // volume and ps requires more time than inspect
-		killTime = 30 * time.Second
-		warnTime = 3 * time.Second
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), killTime)
-	defer cancel()
-
-	start := time.Now()
-	glog.Infof("executing with %s timeout: %v", args, killTime)
-	cmd := exec.CommandContext(ctx, args[0], args[1:]...)
-	stdout, err := cmd.Output()
-	d := time.Since(start)
-	if d > warnTime {
-		out.WarningT(`Executing "{{.command}}" took an unusually long time: {{.duration}}`, out.V{"command": strings.Join(cmd.Args, " "), "duration": d})
-		out.ErrT(out.Tip, `Restarting the {{.name}} service may improve performance.`, out.V{"name": args[0]})
-	}
-
-	if ctx.Err() == context.DeadlineExceeded {
-		return stdout, fmt.Errorf("%q timed out after %s", strings.Join(cmd.Args, " "), killTime)
-	}
-	if err != nil {
-		if exitErr, ok := err.(*exec.ExitError); ok {
-			return stdout, fmt.Errorf("%q failed: %v: %s", strings.Join(cmd.Args, " "), exitErr, exitErr.Stderr)
-		}
-		return stdout, fmt.Errorf("%q failed: %v", strings.Join(cmd.Args, " "), err)
-	}
-	return stdout, nil
+	return rr.Stdout.String(), nil
 }
 
 // ContainerExists checks if container name exists (either running or exited)
-func ContainerExists(ociBin string, name string) (bool, error) {
-	out, err := WarnIfSlow(ociBin, "ps", "-a", "--format", "{{.Names}}")
+func ContainerExists(ociBin string, name string, warnSlow ...bool) (bool, error) {
+	rr, err := runCmd(exec.Command(ociBin, "ps", "-a", "--format", "{{.Names}}"), warnSlow...)
 	if err != nil {
-		return false, errors.Wrapf(err, string(out))
+		return false, err
 	}
 
-	containers := strings.Split(string(out), "\n")
+	containers := strings.Split(rr.Stdout.String(), "\n")
 	for _, c := range containers {
 		if strings.TrimSpace(c) == name {
 			return true, nil
@@ -305,15 +249,13 @@ func ContainerExists(ociBin string, name string) (bool, error) {
 
 // IsCreatedByMinikube returns true if the container was created by minikube
 // with default assumption that it is not created by minikube when we don't know for sure
-func IsCreatedByMinikube(ociBinary string, nameOrID string) bool {
-	cmd := exec.Command(ociBinary, "inspect", nameOrID, "--format", "{{.Config.Labels}}")
-	out, err := cmd.CombinedOutput()
-
+func IsCreatedByMinikube(ociBin string, nameOrID string) bool {
+	rr, err := runCmd(exec.Command(ociBin, "inspect", nameOrID, "--format", "{{.Config.Labels}}"))
 	if err != nil {
 		return false
 	}
 
-	if strings.Contains(string(out), fmt.Sprintf("%s:true", CreatedByLabelKey)) {
+	if strings.Contains(rr.Stdout.String(), fmt.Sprintf("%s:true", CreatedByLabelKey)) {
 		return true
 	}
 
@@ -321,20 +263,19 @@ func IsCreatedByMinikube(ociBinary string, nameOrID string) bool {
 }
 
 // ListOwnedContainers lists all the containres that kic driver created on user's machine using a label
-func ListOwnedContainers(ociBinary string) ([]string, error) {
-	return ListContainersByLabel(ociBinary, ProfileLabelKey)
+func ListOwnedContainers(ociBin string) ([]string, error) {
+	return ListContainersByLabel(ociBin, ProfileLabelKey)
 }
 
 // inspect return low-level information on containers
-func inspect(ociBinary string, containerNameOrID, format string) ([]string, error) {
-
-	cmd := exec.Command(ociBinary, "inspect",
+func inspect(ociBin string, containerNameOrID, format string) ([]string, error) {
+	cmd := exec.Command(ociBin, "inspect",
 		"-f", format,
 		containerNameOrID) // ... against the "node" container
 	var buff bytes.Buffer
 	cmd.Stdout = &buff
 	cmd.Stderr = &buff
-	err := cmd.Run()
+	_, err := runCmd(cmd)
 	scanner := bufio.NewScanner(&buff)
 	var lines []string
 	for scanner.Scan() {
@@ -390,13 +331,13 @@ func generateMountBindings(mounts ...Mount) []string {
 }
 
 // isUsernsRemapEnabled checks if userns-remap is enabled in docker
-func isUsernsRemapEnabled(ociBinary string) bool {
-	cmd := exec.Command(ociBinary, "info", "--format", "'{{json .SecurityOptions}}'")
+func isUsernsRemapEnabled(ociBin string) bool {
+	cmd := exec.Command(ociBin, "info", "--format", "'{{json .SecurityOptions}}'")
 	var buff bytes.Buffer
 	cmd.Stdout = &buff
 	cmd.Stderr = &buff
-	err := cmd.Run()
-	if err != nil {
+
+	if _, err := runCmd(cmd); err != nil {
 		return false
 	}
 
@@ -453,12 +394,12 @@ func withPortMappings(portMappings []PortMapping) createOpt {
 }
 
 // ListContainersByLabel returns all the container names with a specified label
-func ListContainersByLabel(ociBinary string, label string) ([]string, error) {
-	stdout, err := WarnIfSlow(ociBinary, "ps", "-a", "--filter", fmt.Sprintf("label=%s", label), "--format", "{{.Names}}")
+func ListContainersByLabel(ociBin string, label string, warnSlow ...bool) ([]string, error) {
+	rr, err := runCmd(exec.Command(ociBin, "ps", "-a", "--filter", fmt.Sprintf("label=%s", label), "--format", "{{.Names}}"), warnSlow...)
 	if err != nil {
 		return nil, err
 	}
-	s := bufio.NewScanner(bytes.NewReader(stdout))
+	s := bufio.NewScanner(bytes.NewReader(rr.Stdout.Bytes()))
 	var names []string
 	for s.Scan() {
 		n := strings.TrimSpace(s.Text())
@@ -489,9 +430,10 @@ func PointToHostDockerDaemon() error {
 }
 
 // ContainerStatus returns status of a container running,exited,...
-func ContainerStatus(ociBin string, name string) (state.State, error) {
-	out, err := WarnIfSlow(ociBin, "inspect", name, "--format={{.State.Status}}")
-	o := strings.TrimSpace(string(out))
+func ContainerStatus(ociBin string, name string, warnSlow ...bool) (state.State, error) {
+	cmd := exec.Command(ociBin, "inspect", name, "--format={{.State.Status}}")
+	rr, err := runCmd(cmd, warnSlow...)
+	o := strings.TrimSpace(rr.Stdout.String())
 	switch o {
 	case "running":
 		return state.Running, nil
@@ -508,13 +450,12 @@ func ContainerStatus(ociBin string, name string) (state.State, error) {
 	}
 }
 
-// Shutdown will run command to shut down the container
+// ShutDown will run command to shut down the container
 // to ensure the containers process and networking bindings are all closed
 // to avoid containers getting stuck before delete https://github.com/kubernetes/minikube/issues/7657
 func ShutDown(ociBin string, name string) error {
-	cmd := exec.Command(ociBin, "exec", "--privileged", "-t", name, "/bin/bash", "-c", "sudo init 0")
-	if out, err := cmd.CombinedOutput(); err != nil {
-		glog.Infof("error shutdown %s output %q : %v", name, out, err)
+	if _, err := runCmd(exec.Command(ociBin, "exec", "--privileged", "-t", name, "/bin/bash", "-c", "sudo init 0")); err != nil {
+		glog.Infof("error shutdown %s: %v", name, err)
 	}
 	// helps with allowing docker realize the container is exited and report its status correctly.
 	time.Sleep(time.Second * 1)

pkg/drivers/kic/oci/volumes.go

@@ -29,7 +29,7 @@ import (
 
 // DeleteAllVolumesByLabel deletes all volumes that have a specific label
 // if there is no volume to delete it will return nil
-func DeleteAllVolumesByLabel(ociBin string, label string) []error {
+func DeleteAllVolumesByLabel(ociBin string, label string, warnSlow ...bool) []error {
 	var deleteErrs []error
 	glog.Infof("trying to delete all %s volumes with label %s", ociBin, label)
 
@@ -40,7 +40,7 @@ func DeleteAllVolumesByLabel(ociBin string, label string) []error {
 	}
 
 	for _, v := range vs {
-		if _, err := WarnIfSlow(ociBin, "volume", "rm", "--force", v); err != nil {
+		if _, err := runCmd(exec.Command(ociBin, "volume", "rm", "--force", v), warnSlow...); err != nil {
 			deleteErrs = append(deleteErrs, fmt.Errorf("deleting %q", v))
 		}
 	}
@@ -51,11 +51,11 @@ func DeleteAllVolumesByLabel(ociBin string, label string) []error {
 // PruneAllVolumesByLabel deletes all volumes that have a specific label
 // if there is no volume to delete it will return nil
 // example: docker volume prune -f --filter label=name.minikube.sigs.k8s.io=minikube
-func PruneAllVolumesByLabel(ociBin string, label string) []error {
+func PruneAllVolumesByLabel(ociBin string, label string, warnSlow ...bool) []error {
 	var deleteErrs []error
 	glog.Infof("trying to prune all %s volumes with label %s", ociBin, label)
-
-	if _, err := WarnIfSlow(ociBin, "volume", "prune", "-f", "--filter", "label="+label); err != nil {
+	cmd := exec.Command(ociBin, "volume", "prune", "-f", "--filter", "label="+label)
+	if _, err := runCmd(cmd, warnSlow...); err != nil {
 		deleteErrs = append(deleteErrs, errors.Wrapf(err, "prune volume by label %s", label))
 	}
 
@@ -65,9 +65,8 @@ func PruneAllVolumesByLabel(ociBin string, label string) []error {
 // allVolumesByLabel returns name of all docker volumes by a specific label
 // will not return error if there is no volume found.
 func allVolumesByLabel(ociBin string, label string) ([]string, error) {
-	cmd := exec.Command(ociBin, "volume", "ls", "--filter", "label="+label, "--format", "{{.Name}}")
-	stdout, err := cmd.Output()
-	s := bufio.NewScanner(bytes.NewReader(stdout))
+	rr, err := runCmd(exec.Command(ociBin, "volume", "ls", "--filter", "label="+label, "--format", "{{.Name}}"))
+	s := bufio.NewScanner(bytes.NewReader(rr.Stdout.Bytes()))
 	var vols []string
 	for s.Scan() {
 		v := strings.TrimSpace(s.Text())
@@ -82,9 +81,8 @@ func allVolumesByLabel(ociBin string, label string) ([]string, error) {
 // to the volume named volumeName
 func ExtractTarballToVolume(tarballPath, volumeName, imageName string) error {
 	cmd := exec.Command(Docker, "run", "--rm", "--entrypoint", "/usr/bin/tar", "-v", fmt.Sprintf("%s:/preloaded.tar:ro", tarballPath), "-v", fmt.Sprintf("%s:/extractDir", volumeName), imageName, "-I", "lz4", "-xvf", "/preloaded.tar", "-C", "/extractDir")
-	glog.Infof("executing: %s", cmd.Args)
-	if out, err := cmd.CombinedOutput(); err != nil {
-		return errors.Wrapf(err, "output %s", string(out))
+	if _, err := runCmd(cmd); err != nil {
+		return err
 	}
 	return nil
 }
@@ -93,10 +91,8 @@ func ExtractTarballToVolume(tarballPath, volumeName, imageName string) error {
 // Caution ! if volume already exists does NOT return an error and will not apply the minikube labels on it.
 // TODO: this should be fixed as a part of https://github.com/kubernetes/minikube/issues/6530
 func createDockerVolume(profile string, nodeName string) error {
-	cmd := exec.Command(Docker, "volume", "create", nodeName, "--label", fmt.Sprintf("%s=%s", ProfileLabelKey, profile), "--label", fmt.Sprintf("%s=%s", CreatedByLabelKey, "true"))
-	glog.Infof("executing: %s", cmd.Args)
-	if out, err := cmd.CombinedOutput(); err != nil {
-		return errors.Wrapf(err, "output %s", string(out))
+	if _, err := runCmd(exec.Command(Docker, "volume", "create", nodeName, "--label", fmt.Sprintf("%s=%s", ProfileLabelKey, profile), "--label", fmt.Sprintf("%s=%s", CreatedByLabelKey, "true"))); err != nil {
+		return err
 	}
 	return nil
 }

pkg/minikube/assets/addons.go

@@ -355,6 +355,20 @@ var Addons = map[string]*Addon{
 			"0640",
 			false),
 	}, false, "ingress-dns"),
+	"metallb": NewAddon([]*BinAsset{
+		MustBinAsset(
+			"deploy/addons/metallb/metallb.yaml",
+			vmpath.GuestAddonsDir,
+			"metallb.yaml",
+			"0640",
+			false),
+		MustBinAsset(
+			"deploy/addons/metallb/metallb-config.yaml.tmpl",
+			vmpath.GuestAddonsDir,
+			"metallb-config.yaml",
+			"0640",
+			true),
+	}, false, "metallb"),
 }
 
 // GenerateTemplateData generates template data for template assets
@@ -368,13 +382,17 @@ func GenerateTemplateData(cfg config.KubernetesConfig) interface{} {
 		ea = "-" + runtime.GOARCH
 	}
 	opts := struct {
-		Arch            string
-		ExoticArch      string
-		ImageRepository string
+		Arch                string
+		ExoticArch          string
+		ImageRepository     string
+		LoadBalancerStartIP string
+		LoadBalancerEndIP   string
 	}{
-		Arch:            a,
-		ExoticArch:      ea,
-		ImageRepository: cfg.ImageRepository,
+		Arch:                a,
+		ExoticArch:          ea,
+		ImageRepository:     cfg.ImageRepository,
+		LoadBalancerStartIP: cfg.LoadBalancerStartIP,
+		LoadBalancerEndIP:   cfg.LoadBalancerEndIP,
 	}
 
 	return opts

pkg/minikube/config/types.go

@@ -70,19 +70,21 @@ type ClusterConfig struct {
 
 // KubernetesConfig contains the parameters used to configure the VM Kubernetes.
 type KubernetesConfig struct {
-	KubernetesVersion string
-	ClusterName       string
-	APIServerName     string
-	APIServerNames    []string
-	APIServerIPs      []net.IP
-	DNSDomain         string
-	ContainerRuntime  string
-	CRISocket         string
-	NetworkPlugin     string
-	FeatureGates      string // https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/
-	ServiceCIDR       string // the subnet which kubernetes services will be deployed to
-	ImageRepository   string
-	ExtraOptions      ExtraOptionSlice
+	KubernetesVersion   string
+	ClusterName         string
+	APIServerName       string
+	APIServerNames      []string
+	APIServerIPs        []net.IP
+	DNSDomain           string
+	ContainerRuntime    string
+	CRISocket           string
+	NetworkPlugin       string
+	FeatureGates        string // https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/
+	ServiceCIDR         string // the subnet which kubernetes services will be deployed to
+	ImageRepository     string
+	LoadBalancerStartIP string // currently only used by MetalLB addon
+	LoadBalancerEndIP   string // currently only used by MetalLB addon
+	ExtraOptions        ExtraOptionSlice
 
 	ShouldLoadCachedImages bool
 	EnableDefaultCNI       bool

pkg/minikube/machine/delete.go

@@ -42,7 +42,7 @@ func deleteOrphanedKIC(ociBin string, name string) {
 
 	_, err := oci.ContainerStatus(ociBin, name)
 	if err != nil {
-		glog.Infof("couldn't inspect container %q before deleting, %s-daemon might needs a restart!: %v", name, ociBin, err)
+		glog.Infof("couldn't inspect container %q before deleting: %v", name, err)
 		return
 	}
 	// allow no more than 5 seconds for delting the container

pkg/minikube/registry/drvs/hyperv/hyperv.go

@@ -91,17 +91,19 @@ func status() registry.State {
 
 	cmd := exec.CommandContext(ctx, path, "@(Get-Wmiobject Win32_ComputerSystem).HypervisorPresent")
 	out, err := cmd.CombinedOutput()
-	if string(out) != "True\r\n" {
-		errorMessage := fmt.Errorf("%s failed:\n%s", strings.Join(cmd.Args, " "), out)
-		fixMessage := "Start PowerShell as Administrator, and run: 'Enable-WindowsOptionalFeature -Online -FeatureName Microsoft-Hyper-V -All'"
 
-		// If timed out, prompt different error and suggestion messages
-		// See https://github.com/kubernetes/minikube/issues/6579
-		if ctx.Err() != nil {
-			errorMessage = fmt.Errorf("%s exited unexpectedly:\n%s", strings.Join(cmd.Args, " "), ctx.Err())
-			fixMessage = "If you have Hyper-V configured correctly, please try start again with `--force` specified"
-		}
+	if err != nil {
+		errorMessage := fmt.Errorf("%s failed:\n%s", strings.Join(cmd.Args, " "), out)
+		fixMessage := "Start PowerShell as an Administrator"
 		return registry.State{Installed: false, Error: errorMessage, Fix: fixMessage, Doc: docURL}
 	}
+
+	// Get-Wmiobject does not return an error code for false
+	if strings.TrimSpace(string(out)) != "True" {
+		errorMessage := fmt.Errorf("%s returned %q", strings.Join(cmd.Args, " "), out)
+		fixMessage := "Enable Hyper-V: Start PowerShell as Administrator, and run: 'Enable-WindowsOptionalFeature -Online -FeatureName Microsoft-Hyper-V -All'"
+		return registry.State{Installed: false, Error: errorMessage, Fix: fixMessage, Doc: docURL}
+	}
+
 	return registry.State{Installed: true, Healthy: true}
 }