Merge branch 'master' of https://github.com/kubernetes/minikube into metrics-code

Makefile

@@ -23,7 +23,7 @@ KUBERNETES_VERSION ?= $(shell egrep "DefaultKubernetesVersion =" pkg/minikube/co
 KIC_VERSION ?= $(shell egrep "Version =" pkg/drivers/kic/types.go | cut -d \" -f2)
 
 # Default to .0 for higher cache hit rates, as build increments typically don't require new ISO versions
-ISO_VERSION ?= v1.15.0
+ISO_VERSION ?= v1.15.2-snapshot2
 # Dashes are valid in semver, but not Linux packaging. Use ~ to delimit alpha/beta
 DEB_VERSION ?= $(subst -,~,$(RAW_VERSION))
 RPM_VERSION ?= $(DEB_VERSION)
@@ -582,8 +582,6 @@ storage-provisioner-image-%: out/storage-provisioner-%
 
 .PHONY: kic-base-image
 kic-base-image: ## builds the kic base image and tags local/kicbase:latest and local/kicbase:$(KIC_VERSION)-$(COMMIT_SHORT)
-	docker rmi -f local/kicbase:latest || true
-	docker rmi -f local/kicbase:$(KIC_VERSION) || true
 	docker build -f ./deploy/kicbase/Dockerfile -t local/kicbase:$(KIC_VERSION)  --build-arg COMMIT_SHA=${VERSION}-$(COMMIT) --cache-from $(KIC_BASE_IMAGE_GCR) ./deploy/kicbase
 	docker tag local/kicbase:$(KIC_VERSION) local/kicbase:latest
 	docker tag local/kicbase:$(KIC_VERSION) local/kicbase:$(KIC_VERSION)-$(COMMIT_SHORT)

cmd/minikube/cmd/config/profile_list.go

@@ -130,7 +130,7 @@ func profileStatus(p *config.Profile, api libmachine.API) string {
 
 func renderProfilesTable(ps [][]string) {
 	table := tablewriter.NewWriter(os.Stdout)
-	table.SetHeader([]string{"Profile", "VM Driver", "Runtime", "IP", "Port", "Version", "Status"})
+	table.SetHeader([]string{"Profile", "VM Driver", "Runtime", "IP", "Port", "Version", "Status", "Nodes"})
 	table.SetAutoFormatHeaders(false)
 	table.SetBorders(tablewriter.Border{Left: true, Top: true, Right: true, Bottom: true})
 	table.SetCenterSeparator("|")
@@ -146,7 +146,7 @@ func profilesToTableData(profiles []*config.Profile) [][]string {
 			exit.Error(reason.GuestCpConfig, "error getting primary control plane", err)
 		}
 
-		data = append(data, []string{p.Name, p.Config.Driver, p.Config.KubernetesConfig.ContainerRuntime, cp.IP, strconv.Itoa(cp.Port), p.Config.KubernetesConfig.KubernetesVersion, p.Status})
+		data = append(data, []string{p.Name, p.Config.Driver, p.Config.KubernetesConfig.ContainerRuntime, cp.IP, strconv.Itoa(cp.Port), p.Config.KubernetesConfig.KubernetesVersion, p.Status, strconv.Itoa(len(p.Config.Nodes))})
 	}
 	return data
 }

cmd/minikube/cmd/docker-env.go

@@ -27,6 +27,7 @@ import (
 	"os/exec"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/spf13/cobra"
 	"k8s.io/klog/v2"
@@ -151,15 +152,35 @@ func (EnvNoProxyGetter) GetNoProxyVar() (string, string) {
 	return noProxyVar, noProxyValue
 }
 
+// ensureDockerd ensures dockerd inside minikube is running before a docker-env  command
+func ensureDockerd(name string, r command.Runner) {
+	if ok := isDockerActive(r); ok {
+		return
+	}
+	mustRestartDockerd(name, r)
+}
+
 // isDockerActive checks if Docker is active
 func isDockerActive(r command.Runner) bool {
 	return sysinit.New(r).Active("docker")
 }
 
-func mustRestartDocker(name string, runner command.Runner) {
+// mustRestartDockerd will attempt to reload dockerd if fails, will try restart and exit if fails again
+func mustRestartDockerd(name string, runner command.Runner) {
+	// Docker Docs: https://docs.docker.com/config/containers/live-restore
+	// On Linux, you can avoid a restart (and avoid any downtime for your containers) by reloading the Docker daemon.
+	klog.Warningf("dockerd is not active will try to reload it...")
+	if err := sysinit.New(runner).Reload("docker"); err != nil {
+		klog.Warningf("will try to restart dockerd because reload failed: %v", err)
 		if err := sysinit.New(runner).Restart("docker"); err != nil {
 			exit.Message(reason.RuntimeRestart, `The Docker service within '{{.name}}' is not active`, out.V{"name": name})
 		}
+		// if we get to the point that we have to restart docker (instead of reload)
+		// will need to wait for apisever container to come up, this usually takes 5 seconds
+		// verifying apisever using kverify would add code complexity for a rare case.
+		klog.Warningf("waiting 5 seconds to ensure apisever container is up...")
+		time.Sleep(time.Second * 5)
+	}
 }
 
 // dockerEnvCmd represents the docker-env command
@@ -205,10 +226,7 @@ var dockerEnvCmd = &cobra.Command{
 				out.V{"runtime": co.Config.KubernetesConfig.ContainerRuntime})
 		}
 
-		if ok := isDockerActive(co.CP.Runner); !ok {
+		ensureDockerd(cname, co.CP.Runner)
-			klog.Warningf("dockerd is not active will try to restart it...")
-			mustRestartDocker(cname, co.CP.Runner)
-		}
 
 		port := constants.DockerDaemonPort
 		if driver.NeedsPortForward(driverName) {
@@ -238,8 +256,9 @@ var dockerEnvCmd = &cobra.Command{
 			out, err := tryDockerConnectivity("docker", ec)
 			if err != nil { // docker might be up but been loaded with wrong certs/config
 				// to fix issues like this #8185
-				klog.Warningf("couldn't connect to docker inside minikube. will try to restart dockerd service... output: %s error: %v", string(out), err)
+				// even though docker maybe running just fine it could be holding on to old certs and needs a refresh
-				mustRestartDocker(cname, co.CP.Runner)
+				klog.Warningf("couldn't connect to docker inside minikube.  output: %s error: %v", string(out), err)
+				mustRestartDockerd(cname, co.CP.Runner)
 			}
 		}
 

cmd/minikube/cmd/start.go

@@ -60,6 +60,7 @@ import (
 	"k8s.io/minikube/pkg/minikube/out/register"
 	"k8s.io/minikube/pkg/minikube/reason"
 	"k8s.io/minikube/pkg/minikube/style"
+	pkgtrace "k8s.io/minikube/pkg/trace"
 
 	"k8s.io/minikube/pkg/minikube/registry"
 	"k8s.io/minikube/pkg/minikube/translate"
@@ -129,6 +130,10 @@ func runStart(cmd *cobra.Command, args []string) {
 	register.SetEventLogPath(localpath.EventLog(ClusterFlagValue()))
 
 	out.SetJSON(outputFormat == "json")
+	if err := pkgtrace.Initialize(viper.GetString(trace)); err != nil {
+		exit.Message(reason.Usage, "error initializing tracing: {{.Error}}", out.V{"Error": err.Error()})
+	}
+	defer pkgtrace.Cleanup()
 	displayVersion(version.GetVersion())
 
 	// No need to do the update check if no one is going to see it
@@ -243,6 +248,17 @@ func runStart(cmd *cobra.Command, args []string) {
 				})
 			}
 		}
+
+		if existing.KubernetesConfig.ContainerRuntime == "crio" {
+			// Stop and start again if it's crio because it's broken above v1.17.3
+			out.WarningT("Due to issues with CRI-O post v1.17.3, we need to restart your cluster.")
+			out.WarningT("See details at https://github.com/kubernetes/minikube/issues/8861")
+			stopProfile(existing.Name)
+			starter, err = provisionWithDriver(cmd, ds, existing)
+			if err != nil {
+				exitGuestProvision(err)
+			}
+		}
 	}
 
 	kubeconfig, err := startWithDriver(cmd, starter, existing)

cmd/minikube/cmd/start_flags.go

@@ -108,6 +108,7 @@ const (
 	kicBaseImage            = "base-image"
 	ports                   = "ports"
 	startNamespace          = "namespace"
+	trace                   = "trace"
 )
 
 var (
@@ -152,6 +153,7 @@ func initMinikubeFlags() {
 	startCmd.Flags().Bool(deleteOnFailure, false, "If set, delete the current cluster if start fails and try again. Defaults to false.")
 	startCmd.Flags().Bool(forceSystemd, false, "If set, force the container runtime to use sytemd as cgroup manager. Currently available for docker and crio. Defaults to false.")
 	startCmd.Flags().StringVarP(&outputFormat, "output", "o", "text", "Format to print stdout in. Options include: [text,json]")
+	startCmd.Flags().StringP(trace, "", "", "Send trace events. Options include: [gcp]")
 }
 
 // initKubernetesFlags inits the commandline flags for Kubernetes related options

cmd/minikube/cmd/stop.go

@@ -46,6 +46,7 @@ var (
 	stopAll               bool
 	keepActive            bool
 	scheduledStopDuration time.Duration
+	cancelScheduledStop   bool
 )
 
 // stopCmd represents the stop command
@@ -60,6 +61,8 @@ func init() {
 	stopCmd.Flags().BoolVar(&stopAll, "all", false, "Set flag to stop all profiles (clusters)")
 	stopCmd.Flags().BoolVar(&keepActive, "keep-context-active", false, "keep the kube-context active after cluster is stopped. Defaults to false.")
 	stopCmd.Flags().DurationVar(&scheduledStopDuration, "schedule", 0*time.Second, "Set flag to stop cluster after a set amount of time (e.g. --schedule=5m)")
+	stopCmd.Flags().BoolVar(&cancelScheduledStop, "cancel-scheduled", false, "cancel any existing scheduled stop requests")
+
 	if err := stopCmd.Flags().MarkHidden("schedule"); err != nil {
 		klog.Info("unable to mark --schedule flag as hidden")
 	}
@@ -97,6 +100,11 @@ func runStop(cmd *cobra.Command, args []string) {
 
 	// Kill any existing scheduled stops
 	schedule.KillExisting(profilesToStop)
+	if cancelScheduledStop {
+		register.Reg.SetStep(register.Done)
+		out.Step(style.Stopped, `All existing scheduled stops cancelled`)
+		return
+	}
 
 	if scheduledStopDuration != 0 {
 		if err := schedule.Daemonize(profilesToStop, scheduledStopDuration); err != nil {

deploy/iso/minikube-iso/package/Config.in

@@ -13,6 +13,7 @@ menu "System tools"
     source "$BR2_EXTERNAL_MINIKUBE_PATH/package/gluster/Config.in"
     source "$BR2_EXTERNAL_MINIKUBE_PATH/package/vbox-guest/Config.in"
     source "$BR2_EXTERNAL_MINIKUBE_PATH/package/containerd-bin/Config.in"
+    source "$BR2_EXTERNAL_MINIKUBE_PATH/package/buildkit-bin/Config.in"
     source "$BR2_EXTERNAL_MINIKUBE_PATH/package/falco-module/Config.in"
     source "$BR2_EXTERNAL_MINIKUBE_PATH/package/scheduled-stop/Config.in"
 endmenu

deploy/iso/minikube-iso/package/buildkit-bin/Config.in

@@ -0,0 +1,4 @@
+config BR2_PACKAGE_BUILDKIT_BIN
+	bool "buildkit-bin"
+	default y
+	depends on BR2_x86_64

deploy/iso/minikube-iso/package/buildkit-bin/buildkit-bin.hash

@@ -0,0 +1 @@
+sha256 33bcaa49b31bc3a277ac75d32fce3f5442d39f53a1799b8624e985279b579f74  buildkit-v0.7.2.linux-amd64.tar.gz

deploy/iso/minikube-iso/package/buildkit-bin/buildkit-bin.mk

@@ -0,0 +1,20 @@
+################################################################################
+#
+# buildkit-bin
+#
+################################################################################
+
+BUILDKIT_BIN_VERSION = v0.7.2
+BUILDKIT_BIN_SITE = https://github.com/moby/buildkit/releases/download/$(BUILDKIT_BIN_VERSION)
+BUILDKIT_BIN_SOURCE = buildkit-$(BUILDKIT_BIN_VERSION).linux-amd64.tar.gz
+
+define BUILDKIT_BIN_INSTALL_TARGET_CMDS
+	$(INSTALL) -D -m 0755 \
+		$(@D)/buildctl \
+		$(TARGET_DIR)/usr/bin/buildctl
+	$(INSTALL) -D -m 0755 \
+		$(@D)/buildkitd \
+		$(TARGET_DIR)/usr/sbin/buildkitd
+endef
+
+$(eval $(generic-package))

deploy/kicbase/Dockerfile

@@ -21,6 +21,8 @@
 # for a kubernetes node image, it doesn't contain much we don't need
 FROM ubuntu:focal-20200925
 
+ARG BUILDKIT_VERSION="v0.7.2"
+
 # copy in static files (configs, scripts)
 COPY 10-network-security.conf /etc/sysctl.d/10-network-security.conf
 COPY 11-tcp-mtu-probing.conf /etc/sysctl.d/11-tcp-mtu-probing.conf
@@ -106,6 +108,17 @@ RUN sh -c "echo 'deb https://download.docker.com/linux/ubuntu focal stable' > /e
     apt-key add - < docker.key && \
     clean-install docker-ce docker-ce-cli containerd.io
 
+# install buildkit
+RUN export ARCH=$(dpkg --print-architecture | sed 's/ppc64el/ppc64le/' | sed 's/armhf/arm-v7/') \
+ && echo "Installing buildkit ..." \
+    && export BUILDKIT_BASE_URL="https://github.com/moby/buildkit/releases/download/${BUILDKIT_VERSION}" \
+    && curl -sSL --retry 5 --output /tmp/buildkit.tgz "${BUILDKIT_BASE_URL}/buildkit-${BUILDKIT_VERSION}.linux-${ARCH}.tar.gz" \
+    && tar -C /usr/local -xzvf /tmp/buildkit.tgz \
+    && rm -rf /tmp/buildkit.tgz \
+    && chmod 755 /usr/local/bin/buildctl \
+    && chmod 755 /usr/local/bin/buildkit-runc \
+    && chmod 755 /usr/local/bin/buildkitd
+
 # Install cri-o/podman dependencies:
 RUN sh -c "echo 'deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_20.04/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list" && \
     curl -LO https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_20.04/Release.key && \

go.mod

@@ -3,8 +3,9 @@ module k8s.io/minikube
 go 1.15
 
 require (
-	cloud.google.com/go/storage v1.8.0
+	cloud.google.com/go/storage v1.10.0
 	github.com/Azure/azure-sdk-for-go v42.3.0+incompatible
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v0.13.0
 	github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5 // indirect
 	github.com/Parallels/docker-machine-parallels/v2 v2.0.1
 	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
@@ -74,21 +75,25 @@ require (
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f
 	github.com/zchee/go-vmnet v0.0.0-20161021174912-97ebf9174097
+	go.opencensus.io v0.22.4
+	go.opentelemetry.io/otel v0.13.0
+	go.opentelemetry.io/otel/sdk v0.13.0
 	golang.org/x/build v0.0.0-20190927031335-2835ba2e683f
 	golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a
 	golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6
 	golang.org/x/mod v0.3.0
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
-	golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a
+	golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208
 	golang.org/x/sys v0.0.0-20200523222454-059865788121
-	golang.org/x/text v0.3.2
+	golang.org/x/text v0.3.3
-	google.golang.org/api v0.25.0
+	google.golang.org/api v0.29.0
 	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
 	gopkg.in/yaml.v2 v2.3.0
 	gotest.tools/v3 v3.0.2 // indirect
 	k8s.io/api v0.17.4
 	k8s.io/apimachinery v0.17.4
 	k8s.io/client-go v0.17.4
+	k8s.io/klog v1.0.0
 	k8s.io/klog/v2 v2.4.0
 	k8s.io/kubectl v0.0.0
 	k8s.io/kubernetes v1.18.5

go.sum

@@ -17,12 +17,15 @@ cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bP
 cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
 cloud.google.com/go v0.57.0 h1:EpMNVUorLiZIELdMZbCYX/ByTFCdoYopYAGxaGVz9ms=
 cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
+cloud.google.com/go v0.61.0 h1:NLQf5e1OMspfNT1RAHOB3ublr1TW3YTXO8OiWwVjK2U=
+cloud.google.com/go v0.61.0/go.mod h1:XukKJg4Y7QsUu0Hxg3qQKUWR4VuWivmyMK2+rUyxAqw=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
 cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
 cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
 cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
 cloud.google.com/go/bigquery v1.7.0 h1:a/O/bK/vWrYGOTFtH8di4rBxMZnmkjy+Y5LxpDwo+dA=
 cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
+cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
 cloud.google.com/go/datastore v1.1.0 h1:/May9ojXjRkPBNVrq+oWLqmWCkr4OU5uRY29bu0mRyQ=
 cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
@@ -37,6 +40,8 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo
 cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
 cloud.google.com/go/storage v1.8.0 h1:86K1Gel7BQ9/WmNWn7dTKMvTLFzwtBe5FNqYbi9X35g=
 cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
+cloud.google.com/go/storage v1.10.0 h1:STgFzyU5/8miMl0//zKh2aQeTyeaUH3WN9bSUiJ09bA=
+cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 code.gitea.io/sdk/gitea v0.12.0/go.mod h1:z3uwDV/b9Ls47NGukYM9XhnHtqPh/J+t40lsUrR6JDY=
 contrib.go.opencensus.io/exporter/aws v0.0.0-20181029163544-2befc13012d0/go.mod h1:uu1P0UCM/6RbsMrgPa98ll8ZcHM858i/AD06a9aLRCA=
 contrib.go.opencensus.io/exporter/ocagent v0.5.0/go.mod h1:ImxhfLRpxoYiSq891pBrLVhN+qmP8BTVvdH2YLs7Gl0=
@@ -84,10 +89,14 @@ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbt
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/DataDog/sketches-go v0.0.1/go.mod h1:Q5DbzQ+3AkgGwymQO7aZFNP7ns2lZKGtvRBzRXfdi60=
 github.com/Djarvur/go-err113 v0.0.0-20200410182137-af658d038157/go.mod h1:4UJr5HIiMZrwgkSPdsjy2uOQExX/WEILpIrO9UPGuXs=
 github.com/Djarvur/go-err113 v0.1.0/go.mod h1:4UJr5HIiMZrwgkSPdsjy2uOQExX/WEILpIrO9UPGuXs=
 github.com/GoogleCloudPlatform/cloudsql-proxy v0.0.0-20191009163259-e802c2cb94ae/go.mod h1:mjwGPas4yKduTyubHvD1Atl9r1rUq8DfVy+gkVvZ+oo=
 github.com/GoogleCloudPlatform/k8s-cloud-provider v0.0.0-20190822182118-27a4ced34534/go.mod h1:iroGtC8B3tQiqtds1l+mgk/BBOrxbqjH+eUfFQYRc14=
+github.com/GoogleCloudPlatform/opentelemetry-operations-go v0.13.0 h1:Gx9IxOjR9ug5Ad8YbafxJ6N2g6oDj/Q419tYHdd1od4=
+github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v0.13.0 h1:fjKUtfldCPIF4nIzAAj3LzP8Lrd3DuRIMiFdOsj4fLc=
+github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v0.13.0/go.mod h1:q/paYxLXKVhwfC3lzLfhtL54fAx14wzMN9DundQOBMc=
 github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab/go.mod h1:3VYc5hodBMJ5+l/7J4xAyMeuM2PNuepvHlGs8yilUCA=
 github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd h1:sjQovDkwrZp8u+gxLtPgKGjk5hCxuy2hrRejBTA9xFU=
 github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E=
@@ -171,6 +180,7 @@ github.com/bazelbuild/bazel-gazelle v0.19.1-0.20191105222053-70208cbdc798/go.mod
 github.com/bazelbuild/buildtools v0.0.0-20190731111112-f720930ceb60/go.mod h1:5JP0TXzWDHXv8qvxRC4InIazwdyDseBDbzESUMKk1yU=
 github.com/bazelbuild/buildtools v0.0.0-20190917191645-69366ca98f89/go.mod h1:5JP0TXzWDHXv8qvxRC4InIazwdyDseBDbzESUMKk1yU=
 github.com/bazelbuild/rules_go v0.0.0-20190719190356-6dae44dc5cab/go.mod h1:MC23Dc/wkXEyk3Wpq6lCqz0ZAYOZDw2DR5y3N1q2i7M=
+github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
@@ -527,6 +537,7 @@ github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.4.1 h1:/exdXoGamhu5ONeUJH0deniYLWYvQwW66yvlfiiKTu0=
 github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
@@ -547,6 +558,7 @@ github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPg
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible h1:xmapqc1AyLoB+ddYT6r04bD9lIjlOqGaREovi0SzFaE=
 github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f h1:Jnx61latede7zDD3DiiP4gmNz33uK0U5HDUaF0a/HVQ=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
@@ -554,6 +566,7 @@ github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hf
 github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/rpmpack v0.0.0-20191226140753-aa36bfddb3a0/go.mod h1:RaTPr0KUf2K7fnZYLNDrr8rxAamWs3iNywJLtQ2AzBg=
 github.com/google/slowjam v0.0.0-20200530021616-df27e642fe7b h1:x3aElhKtGmXLo6RI2FJSBaPBT0acmn2LFfKVP1CqH8o=
@@ -575,6 +588,7 @@ github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTV
 github.com/googleapis/gnostic v0.2.2/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
 github.com/googleapis/gnostic v0.3.0 h1:CcQijm0XKekKjP/YCz28LXVSpgguuB+nCxaSjCe09y0=
 github.com/googleapis/gnostic v0.3.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
+github.com/googleinterns/cloud-operations-api-mock v0.0.0-20200709193332-a1e58c29bdd3/go.mod h1:h/KNeRx7oYU4SpA4SoY7W2/NxDKEEVuwA6j9A27L4OI=
 github.com/gookit/color v1.2.4/go.mod h1:AhIE+pS6D4Ql0SQWbBeXPHw7gY0/sjHoA4s/n1KB7xg=
 github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
@@ -1094,6 +1108,7 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 h1:b6uOv7YOFK0TYG7HtkIgExQo+2RdLuwRft63jn2HWj8=
@@ -1158,6 +1173,7 @@ github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSf
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/zchee/go-vmnet v0.0.0-20161021174912-97ebf9174097 h1:Ucx5I1l1+TWXvqFmBigYu4Ub4MLvUuUU/whjoUvV95I=
 github.com/zchee/go-vmnet v0.0.0-20161021174912-97ebf9174097/go.mod h1:lFZSWRIpCfE/pt91hHBBpV6+x87YlCjsp+aIR2qCPPU=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
@@ -1175,6 +1191,13 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.4 h1:LYy1Hy3MJdrCdMwwzxA/dRok4ejH+RwNGbuoD9fCjto=
+go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opentelemetry.io v0.1.0 h1:EANZoRCOP+A3faIlw/iN6YEWoYb1vleZRKm1EvH8T48=
+go.opentelemetry.io/otel v0.13.0 h1:2isEnyzjjJZq6r2EKMsFj4TxiQiexsM04AVhwbR/oBA=
+go.opentelemetry.io/otel v0.13.0/go.mod h1:dlSNewoRYikTkotEnxdmuBHgzT+k/idJSfDv/FxEnOY=
+go.opentelemetry.io/otel/sdk v0.13.0 h1:4VCfpKamZ8GtnepXxMRurSpHpMKkcxhtO33z1S4rGDQ=
+go.opentelemetry.io/otel/sdk v0.13.0/go.mod h1:dKvLH8Uu8LcEPlSAUsfW7kMGaJBhk/1NYvpPZ6wIMbU=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
@@ -1211,6 +1234,7 @@ golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37 h1:cg5LA/zNPRzIXIWSCxQW10Rvpy94aQh3LT/ShoCpkHw=
 golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM=
 golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -1300,8 +1324,13 @@ golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2 h1:eDrdRpKgkcCqKZQwyZRyeFZgfqt37SL7Kv3tok06cKE=
 golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1323,6 +1352,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03iWnKLEWinaScsxF2Vm2o=
 golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20171026204733-164713f0dfce/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -1378,6 +1409,7 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200523222454-059865788121 h1:rITEj+UZHYC927n8GT97eC3zrpzXdb/voyeOuVKS46o=
 golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1387,6 +1419,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -1467,8 +1501,12 @@ golang.org/x/tools v0.0.0-20200426102838-f3a5411a4c3b/go.mod h1:EkVYQZoAsY45+roY
 golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200502202811-ed308ab3e770/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200527183253-8e7acdbce89d h1:SR+e35rACZFBohNb4Om1ibX6N3iO0FtdbwqGSuD9dBU=
 golang.org/x/tools v0.0.0-20200527183253-8e7acdbce89d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200701151220-7cb253f4c4f8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200713011307-fd294ab11aed/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -1499,6 +1537,9 @@ google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/
 google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
 google.golang.org/api v0.25.0 h1:LodzhlzZEUfhXzNUMIfVlf9Gr6Ua5MMtoFWh7+f47qA=
 google.golang.org/api v0.25.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.29.0 h1:BaiDisFir8O4IJxvAabCGGkQ6yCJegNQqSVoYUNAnbk=
+google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -1536,9 +1577,15 @@ google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfG
 google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
 google.golang.org/genproto v0.0.0-20200527145253-8367513e4ece h1:1YM0uhfumvoDu9sx8+RyWwTI63zoCQvI23IYFRlvte0=
 google.golang.org/genproto v0.0.0-20200527145253-8367513e4ece/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
+google.golang.org/genproto v0.0.0-20200605102947-12044bf5ea91/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
+google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
+google.golang.org/genproto v0.0.0-20200711021454-869866162049/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200715011427-11fb19a81f2c h1:6DWnZZ6EY/59QRRQttZKiktVL23UuQYs7uy75MhhLRM=
+google.golang.org/genproto v0.0.0-20200715011427-11fb19a81f2c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -1555,6 +1602,9 @@ google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8
 google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
 google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4=
 google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
+google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.32.0 h1:zWTV+LMdc3kaiJMSTOFz2UgSBgx8RNQoTGiZu3fR9S0=
+google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -1565,6 +1615,8 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA=
 google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
+google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 gopkg.in/airbrake/gobrake.v2 v2.0.9 h1:7z2uVWwn7oVeeugY1DtlPAy5H+KYgB1KeKTnqjNatLo=
 gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
@@ -1612,6 +1664,7 @@ gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools v2.1.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=

hack/conformance_tests.sh

@@ -27,28 +27,30 @@ set -ex -o pipefail
 readonly PROFILE_NAME="k8sconformance"
 readonly MINIKUBE=${1:-./out/minikube}
 shift || true
+readonly START_ARGS=${@:-}
 
 # Requires a fully running Kubernetes cluster.
 "${MINIKUBE}" delete -p "${PROFILE_NAME}" || true
-"${MINIKUBE}" start -p "${PROFILE_NAME}" --wait=all
+"${MINIKUBE}" start -p "${PROFILE_NAME}" ${START_ARGS} --wait=all --nodes=2
 kubectl --context "${PROFILE_NAME}" get pods --all-namespaces
 "${MINIKUBE}" status -p "${PROFILE_NAME}"
 
-go get -u -v github.com/vmware-tanzu/sonobuoy
+curl -LO https://github.com/vmware-tanzu/sonobuoy/releases/download/v0.19.0/sonobuoy_0.19.0_linux_amd64.tar.gz || true
+tar -xzf sonobuoy_0.19.0_linux_amd64.tar.gz
 
 
-sonobuoy run --wait
+./sonobuoy run --mode=certified-conformance --wait --alsologtostderr
 outdir="$(mktemp -d)"
-sonobuoy retrieve "${outdir}"
+./sonobuoy retrieve "${outdir}"
 
 cwd=$(pwd)
 
 cd "${outdir}"
 mkdir ./results; tar xzf *.tar.gz -C ./results
 
-version=$(${MINIKUBE} version  | cut -d" " -f3)
+version=$(${cwd}/${MINIKUBE} version  | cut -d" " -f3)
 
-mkdir "minikube-${version}"
+mkdir -p "minikube-${version}"
 cd "minikube-${version}"
 
 cat <<EOF >PRODUCT.yaml
@@ -67,6 +69,6 @@ cat <<EOF >README.md
 ./hack/conformance_tests.sh $MINIKUBE $START_ARGS
 EOF
 
-cp ../results/plugins/e2e/results/* .
+cp -r ../results/plugins/e2e/results/global/* .
 cd ..
 cp -r "minikube-${version}" "${cwd}"

hack/jenkins/build/minikube_build_upload_latest.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# Copyright 2020 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script builds the minikube binary for all 3 platforms and uploads them.
+# The binaries are built on master and uploaded to a latest bucket.
+
+set -eux -o pipefail
+
+readonly bucket="minikube/latest"
+
+# Make sure the right golang version is installed based on Makefile
+WANT_GOLANG_VERSION=$(grep '^GO_VERSION' Makefile | awk '{ print $3 }')
+./hack/jenkins/installers/check_install_golang.sh $WANT_GOLANG_VERSION /usr/local
+
+
+declare -rx GOPATH=/var/lib/jenkins/go
+
+make cross && failed=$? || failed=$?
+if [[ "${failed}" -ne 0 ]]; then
+  echo "build failed"
+  exit "${failed}"
+fi
+gsutil cp out/minikube-* "gs://${bucket}"

pkg/drivers/kic/types.go

@@ -24,9 +24,9 @@ import (
 
 const (
 	// Version is the current version of kic
-	Version = "v0.0.14"
+	Version = "v0.0.15-snapshot2"
 	// SHA of the kic base image
-	baseImageSHA = "2bd97b482faf5b6a403ac39dd5e7c6fe2006425c6663a12f94f64f5f81a7787e"
+	baseImageSHA = "0973e4bcdfef0dc8c5a581ecfcca5e36fa6a1cc8773e832ecfd31de3d2b6bf46"
 )
 
 var (

pkg/minikube/bootstrapper/images/images.go

@@ -29,7 +29,7 @@ import (
 // Pause returns the image name to pull for a given Kubernetes version
 func Pause(v semver.Version, mirror string) string {
 	// Should match `PauseVersion` in:
-	// https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/constants/constants.go
+	// https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/constants/constants_unix.go
 	pv := "3.2"
 	if semver.MustParseRange("<1.18.0-alpha.0")(v) {
 		pv = "3.1"
@@ -37,7 +37,7 @@ func Pause(v semver.Version, mirror string) string {
 	return path.Join(kubernetesRepo(mirror), "pause"+archTag(false)+pv)
 }
 
-// essentials returns images needed too bootstrap a kubenretes
+// essentials returns images needed too bootstrap a Kubernetes
 func essentials(mirror string, v semver.Version) []string {
 	imgs := []string{
 		componentImage("kube-proxy", v, mirror),

pkg/minikube/out/register/register.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 
 	"k8s.io/klog/v2"
+	"k8s.io/minikube/pkg/trace"
 )
 
 const (
@@ -117,6 +118,7 @@ func (r *Register) currentStep() string {
 
 // SetStep sets the current step
 func (r *Register) SetStep(s RegStep) {
+	defer trace.StartSpan(string(s))
 	if r.first == RegStep("") {
 		_, ok := r.steps[s]
 		if ok {
@@ -124,6 +126,8 @@ func (r *Register) SetStep(s RegStep) {
 		} else {
 			klog.Errorf("unexpected first step: %q", r.first)
 		}
+	} else {
+		trace.EndSpan(string(r.current))
 	}
 
 	r.current = s

pkg/minikube/sysinit/openrc.go

@@ -20,6 +20,7 @@ package sysinit
 import (
 	"bytes"
 	"context"
+	"fmt"
 	"html/template"
 	"os/exec"
 	"path"
@@ -128,6 +129,12 @@ func (s *OpenRC) Restart(svc string) error {
 	return err
 }
 
+// Reload reloads a service
+// currently only used by our docker-env that doesn't need openrc implementation
+func (s *OpenRC) Reload(svc string) error {
+	return fmt.Errorf("reload is not implemented for OpenRC yet ! Please implement if needed")
+}
+
 // Stop stops a service
 func (s *OpenRC) Stop(svc string) error {
 	rr, err := s.r.RunCmd(exec.Command("sudo", "service", svc, "stop"))

pkg/minikube/sysinit/sysinit.go

@@ -50,6 +50,9 @@ type Manager interface {
 	// Restart restarts a service
 	Restart(string) error
 
+	// Reload restarts a service
+	Reload(string) error
+
 	// Stop stops a service
 	Stop(string) error
 

pkg/minikube/sysinit/systemd.go

@@ -34,8 +34,8 @@ func (s *Systemd) Name() string {
 	return "systemd"
 }
 
-// reload reloads systemd configuration
+// daemonReload reloads systemd configuration
-func (s *Systemd) reload() error {
+func (s *Systemd) daemonReload() error {
 	_, err := s.r.RunCmd(exec.Command("sudo", "systemctl", "daemon-reload"))
 	return err
 }
@@ -63,7 +63,7 @@ func (s *Systemd) Enable(svc string) error {
 
 // Start starts a service
 func (s *Systemd) Start(svc string) error {
-	if err := s.reload(); err != nil {
+	if err := s.daemonReload(); err != nil {
 		return err
 	}
 	_, err := s.r.RunCmd(exec.Command("sudo", "systemctl", "start", svc))
@@ -72,13 +72,22 @@ func (s *Systemd) Start(svc string) error {
 
 // Restart restarts a service
 func (s *Systemd) Restart(svc string) error {
-	if err := s.reload(); err != nil {
+	if err := s.daemonReload(); err != nil {
 		return err
 	}
 	_, err := s.r.RunCmd(exec.Command("sudo", "systemctl", "restart", svc))
 	return err
 }
 
+// Reload reloads a service
+func (s *Systemd) Reload(svc string) error {
+	if err := s.daemonReload(); err != nil {
+		return err
+	}
+	_, err := s.r.RunCmd(exec.Command("sudo", "systemctl", "reload", svc))
+	return err
+}
+
 // Stop stops a service
 func (s *Systemd) Stop(svc string) error {
 	_, err := s.r.RunCmd(exec.Command("sudo", "systemctl", "stop", svc))

pkg/provision/buildroot.go

@@ -79,6 +79,9 @@ Requires= minikube-automount.service docker.socket
 
 [Service]
 Type=notify
+Restart=on-failure
+StartLimitBurst=3
+StartLimitIntervalSec=60
 `
 	if noPivot {
 		klog.Warning("Using fundamentally insecure --no-pivot option")

pkg/provision/ubuntu.go

@@ -82,6 +82,9 @@ Requires=docker.socket
 
 [Service]
 Type=notify
+Restart=on-failure
+StartLimitBurst=3
+StartLimitIntervalSec=60
 `
 	if noPivot {
 		klog.Warning("Using fundamentally insecure --no-pivot option")

pkg/trace/gcp.go

@@ -0,0 +1,104 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package trace
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"go.opentelemetry.io/otel/api/trace"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	"k8s.io/klog"
+
+	texporter "github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace"
+	"github.com/pkg/errors"
+	"go.opentelemetry.io/otel/api/global"
+)
+
+const (
+	projectEnvVar = "MINIKUBE_GCP_PROJECT_ID"
+	// this is the name of the parent span to help identify it
+	// in the Cloud Trace UI.
+	parentSpanName = "minikube start"
+)
+
+type gcpTracer struct {
+	projectID string
+	parentCtx context.Context
+	trace.Tracer
+	spans   map[string]trace.Span
+	cleanup func()
+}
+
+// StartSpan starts a span for the next step of
+// `minikube start` via the GCP tracer
+func (t *gcpTracer) StartSpan(name string) {
+	_, span := t.Tracer.Start(t.parentCtx, name)
+	t.spans[name] = span
+}
+
+// EndSpan ends the most recent span, indicating
+// that one step of `minikube start` has completed
+func (t *gcpTracer) EndSpan(name string) {
+	span, ok := t.spans[name]
+	if !ok {
+		klog.Warningf("cannot end span %s as it was never started", name)
+		return
+	}
+	span.End()
+}
+
+func (t *gcpTracer) Cleanup() {
+	span, ok := t.spans[parentSpanName]
+	if ok {
+		span.End()
+	}
+	t.cleanup()
+}
+
+func initGCPTracer() (*gcpTracer, error) {
+	projectID := os.Getenv(projectEnvVar)
+	if projectID == "" {
+		return nil, fmt.Errorf("GCP tracer requires a valid GCP project id set via the %s env variable", projectEnvVar)
+	}
+
+	_, flush, err := texporter.InstallNewPipeline(
+		[]texporter.Option{
+			texporter.WithProjectID(projectID),
+		},
+		sdktrace.WithConfig(sdktrace.Config{
+			DefaultSampler: sdktrace.AlwaysSample(),
+		}),
+	)
+	if err != nil {
+		return nil, errors.Wrap(err, "installing pipeline")
+	}
+
+	t := global.Tracer(parentSpanName)
+
+	ctx, span := t.Start(context.Background(), parentSpanName)
+	return &gcpTracer{
+		projectID: projectID,
+		parentCtx: ctx,
+		cleanup:   flush,
+		Tracer:    t,
+		spans: map[string]trace.Span{
+			parentSpanName: span,
+		},
+	}, nil
+}

pkg/trace/trace.go

@@ -0,0 +1,78 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package trace
+
+import (
+	"fmt"
+
+	"github.com/pkg/errors"
+)
+
+var (
+	tracer minikubeTracer
+)
+
+type minikubeTracer interface {
+	StartSpan(string)
+	EndSpan(string)
+	Cleanup()
+}
+
+// Initialize intializes the global tracer variable
+func Initialize(t string) error {
+	tr, err := getTracer(t)
+	if err != nil {
+		return errors.Wrap(err, "getting tracer")
+	}
+	tracer = tr
+	return nil
+}
+
+func getTracer(t string) (minikubeTracer, error) {
+	switch t {
+	case "gcp":
+		return initGCPTracer()
+	case "":
+		return nil, nil
+	}
+	return nil, fmt.Errorf("%s is not a valid tracer, valid tracers include: [gcp]", t)
+}
+
+// StartSpan starts a span with the given name
+func StartSpan(name string) {
+	if tracer == nil {
+		return
+	}
+	tracer.StartSpan(name)
+}
+
+// EndSpan ends a span with the given name
+func EndSpan(name string) {
+	if tracer == nil {
+		return
+	}
+	tracer.EndSpan(name)
+}
+
+// Cleanup is responsible for trace related cleanup,
+// such as flushing all data
+func Cleanup() {
+	if tracer == nil {
+		return
+	}
+	tracer.Cleanup()
+}

site/content/en/docs/commands/start.md

@@ -26,7 +26,7 @@ minikube start [flags]
       --apiserver-names strings           A set of apiserver names which are used in the generated certificate for kubernetes.  This can be used if you want to make the apiserver available from outside the machine
       --apiserver-port int                The apiserver listening port (default 8443)
       --auto-update-drivers               If set, automatically updates drivers to the latest version. Defaults to true. (default true)
-      --base-image string                 The base image to use for docker/podman drivers. Intended for local development. (default "gcr.io/k8s-minikube/kicbase:v0.0.14@sha256:2bd97b482faf5b6a403ac39dd5e7c6fe2006425c6663a12f94f64f5f81a7787e")
+      --base-image string                 The base image to use for docker/podman drivers. Intended for local development. (default "gcr.io/k8s-minikube/kicbase:v0.0.15-snapshot2@sha256:0973e4bcdfef0dc8c5a581ecfcca5e36fa6a1cc8773e832ecfd31de3d2b6bf46")
       --cache-images                      If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --driver=none. (default true)
       --cni string                        CNI plug-in to use. Valid options: auto, bridge, calico, cilium, flannel, kindnet, or path to a CNI manifest (default: auto)
       --container-runtime string          The container runtime to be used (docker, cri-o, containerd). (default "docker")
@@ -64,7 +64,7 @@ minikube start [flags]
       --insecure-registry strings         Insecure Docker registries to pass to the Docker daemon.  The default service CIDR range will automatically be added.
       --install-addons                    If set, install addons. Defaults to true. (default true)
       --interactive                       Allow user prompts for more information (default true)
-      --iso-url strings                   Locations to fetch the minikube ISO from. (default [https://storage.googleapis.com/minikube/iso/minikube-v1.15.0.iso,https://github.com/kubernetes/minikube/releases/download/v1.15.0/minikube-v1.15.0.iso,https://kubernetes.oss-cn-hangzhou.aliyuncs.com/minikube/iso/minikube-v1.15.0.iso])
+      --iso-url strings                   Locations to fetch the minikube ISO from. (default [https://storage.googleapis.com/minikube/iso/minikube-v1.15.2-snapshot2.iso,https://github.com/kubernetes/minikube/releases/download/v1.15.2-snapshot2/minikube-v1.15.2-snapshot2.iso,https://kubernetes.oss-cn-hangzhou.aliyuncs.com/minikube/iso/minikube-v1.15.2-snapshot2.iso])
       --keep-context                      This will keep the existing kubectl context and will create a minikube context.
       --kubernetes-version string         The Kubernetes version that the minikube VM will use (ex: v1.2.3, 'stable' for v1.19.4, 'latest' for v1.20.0-beta.1). Defaults to 'stable'.
       --kvm-gpu                           Enable experimental NVIDIA GPU support in minikube

site/content/en/docs/commands/stop.md

@@ -21,6 +21,7 @@ minikube stop [flags]
 
 ```
       --all                   Set flag to stop all profiles (clusters)
+      --cancel-scheduled      cancel any existing scheduled stop requests
       --keep-context-active   keep the kube-context active after cluster is stopped. Defaults to false.
   -o, --output string         Format to print stdout in. Options include: [text,json] (default "text")
 ```

site/content/en/docs/handbook/kubectl.md

@@ -10,8 +10,7 @@ aliases:
 By default, [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) gets configured to access the kubernetes cluster control plane
 inside minikube when the `minikube start` command is executed. 
 
-However if `kubectl` is not installed locally, kubectl can be used inside the minikube
+However if `kubectl` is not installed locally, minikube already includes kubectl which can be used like this:
-as well.
 
 `minikube kubectl -- <kubectl commands>`
 
@@ -36,3 +35,8 @@ Exposing the deployment with a NodePort service
 For more help
 
 `minikube kubectl -- --help`
+
+### Shell autocompletion
+
+After applying the alias or the symbolic link you can follow https://kubernetes.io/docs/tasks/tools/install-kubectl/#enabling-shell-autocompletion to enable shell-autocompletion. 
+When using zsh and the alias approach you also have to execute `setopt complete_aliases`.

site/content/en/docs/tutorials/telemetry.md

@@ -0,0 +1,24 @@
+---
+title: "Telemetry"
+linkTitle: "telemetry"
+weight: 1
+date: 2020-11-24
+---
+
+## Overview
+
+minikube provides telemetry suppport via [OpenTelemetry tracing](https://opentelemetry.io/about/) to collect trace data for `minikube start`.
+
+Currently, minikube supports the following exporters for tracing data:
+- [Stackdriver](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/master/exporter/stackdriverexporter)
+
+To collect trace data with minikube and the Stackdriver exporter, run:
+
+```
+MINIKUBE_GCP_PROJECT_ID=<project ID> minikube start --output json --trace gcp
+```
+
+## Contributing
+There are many exporters available via [OpenTelemetry community contributions](https://github.com/open-telemetry/opentelemetry-collector-contrib).
+
+If you would like to see additional exporters, please create an [issue](https://github.com/kubernetes/minikube/issues) or refer to our [contribution][https://minikube.sigs.k8s.io/docs/contrib/] guidelines and submit a pull request. Thank you!

test/integration/scheduled_stop_test.go

@@ -50,7 +50,7 @@ func TestScheduledStopWindows(t *testing.T) {
 	startMinikube(ctx, t, profile)
 
 	// schedule a stop for 5m from now
-	scheduledStopMinikube(ctx, t, profile, "5m")
+	stopMinikube(ctx, t, profile, []string{"--schedule", "5m"})
 
 	// make sure the systemd service is running
 	rr, err := Run(t, exec.CommandContext(ctx, Target(), []string{"ssh", "-p", profile, "--", "sudo", "systemctl", "show", constants.ScheduledStopSystemdService, "--no-page"}...))
@@ -62,12 +62,12 @@ func TestScheduledStopWindows(t *testing.T) {
 	}
 
 	// reschedule stop for 5 seconds from now
-	scheduledStopMinikube(ctx, t, profile, "5s")
+	stopMinikube(ctx, t, profile, []string{"--schedule", "5s"})
 
 	// sleep for 5 seconds
 	time.Sleep(5 * time.Second)
 	// make sure minikube status is "Stopped"
-	ensureMinikubeStatusStopped(ctx, t, profile)
+	ensureMinikubeStatus(ctx, t, profile, state.Stopped.String())
 }
 
 func TestScheduledStopUnix(t *testing.T) {
@@ -83,20 +83,31 @@ func TestScheduledStopUnix(t *testing.T) {
 	startMinikube(ctx, t, profile)
 
 	// schedule a stop for 5 min from now and make sure PID is created
-	scheduledStopMinikube(ctx, t, profile, "5m")
+	stopMinikube(ctx, t, profile, []string{"--schedule", "5m"})
 	pid := checkPID(t, profile)
 	if !processRunning(t, pid) {
 		t.Fatalf("process %v is not running", pid)
 	}
 
-	// redo scheduled stop to be in 10s
+	// schedule a second stop which should cancel the first scheduled stop
-	scheduledStopMinikube(ctx, t, profile, "10s")
+	stopMinikube(ctx, t, profile, []string{"--schedule", "8s"})
 	if processRunning(t, pid) {
 		t.Fatalf("process %v running but should have been killed on reschedule of stop", pid)
 	}
 	checkPID(t, profile)
-	// make sure minikube status is "Stopped"
+
-	ensureMinikubeStatusStopped(ctx, t, profile)
+	// cancel the shutdown and make sure minikube is still running after 8 seconds
+	// sleep 12 just to be safe
+	stopMinikube(ctx, t, profile, []string{"--cancel-scheduled"})
+	time.Sleep(12 * time.Second)
+	ensureMinikubeStatus(ctx, t, profile, state.Running.String())
+
+	// schedule another stop, make sure minikube status is "Stopped"
+	stopMinikube(ctx, t, profile, []string{"--schedule", "5s"})
+	if processRunning(t, pid) {
+		t.Fatalf("process %v running but should have been killed on reschedule of stop", pid)
+	}
+	ensureMinikubeStatus(ctx, t, profile, state.Stopped.String())
 }
 
 func startMinikube(ctx context.Context, t *testing.T, profile string) {
@@ -107,8 +118,9 @@ func startMinikube(ctx context.Context, t *testing.T, profile string) {
 	}
 }
 
-func scheduledStopMinikube(ctx context.Context, t *testing.T, profile string, stop string) {
+func stopMinikube(ctx context.Context, t *testing.T, profile string, additionalArgs []string) {
-	args := []string{"stop", "-p", profile, "--schedule", stop}
+	args := []string{"stop", "-p", profile}
+	args = append(args, additionalArgs...)
 	rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
 	if err != nil {
 		t.Fatalf("starting minikube: %v\n%s", err, rr.Output())
@@ -144,14 +156,13 @@ func processRunning(t *testing.T, pid string) bool {
 	t.Log("signal error was: ", err)
 	return err == nil
 }
-
+func ensureMinikubeStatus(ctx context.Context, t *testing.T, profile, wantStatus string) {
-func ensureMinikubeStatusStopped(ctx context.Context, t *testing.T, profile string) {
 	// wait allotted time to make sure minikube status is "Stopped"
 	checkStatus := func() error {
 		ctx, cancel := context.WithDeadline(ctx, time.Now().Add(10*time.Second))
 		defer cancel()
 		got := Status(ctx, t, Target(), profile, "Host", profile)
-		if got != state.Stopped.String() {
+		if got != wantStatus {
 			return fmt.Errorf("expected post-stop host status to be -%q- but got *%q*", state.Stopped, got)
 		}
 		return nil

test/integration/start_stop_delete_test.go

@@ -110,7 +110,7 @@ func TestStartStop(t *testing.T) {
 						{"UserAppExistsAfterStop", validateAppExistsAfterStop},
 						{"AddonExistsAfterStop", validateAddonAfterStop},
 						{"VerifyKubernetesImages", validateKubernetesImages},
-						{"Pause", validatePauseAfterSart},
+						{"Pause", validatePauseAfterStart},
 					}
 					for _, stc := range serialTests {
 						if ctx.Err() == context.DeadlineExceeded {
@@ -233,7 +233,7 @@ func validateKubernetesImages(ctx context.Context, t *testing.T, profile string,
 	}
 }
 
-func validatePauseAfterSart(ctx context.Context, t *testing.T, profile string, tcName string, tcVersion string, startArgs []string) {
+func validatePauseAfterStart(ctx context.Context, t *testing.T, profile string, tcName string, tcVersion string, startArgs []string) {
 	defer PostMortemLogs(t, profile)
 	testPause(ctx, t, profile)
 }
@@ -317,8 +317,16 @@ func testPulledImages(ctx context.Context, t *testing.T, profile string, version
 	}
 	sort.Strings(want)
 	sort.Strings(gotImages)
-	if diff := cmp.Diff(want, gotImages); diff != "" {
+	// check if we got all the images we want, ignoring any extraneous ones in cache (eg, may be created by other tests)
-		t.Errorf("%s images mismatch (-want +got):\n%s", version, diff)
+	missing := false
+	for _, img := range want {
+		if sort.SearchStrings(gotImages, img) == len(gotImages) {
+			missing = true
+			break
+		}
+	}
+	if missing {
+		t.Errorf("%s images missing (-want +got):\n%s", version, cmp.Diff(want, gotImages))
 	}
 }