From 3672753100f9f7b68f1cb67d51b851c4043a836d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Tue, 10 Nov 2020 16:54:39 +0100 Subject: [PATCH 01/57] Change back to the version-pinned cri-o channel --- deploy/kicbase/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/kicbase/Dockerfile b/deploy/kicbase/Dockerfile index 66e6fd7bbe..4fef0263af 100644 --- a/deploy/kicbase/Dockerfile +++ b/deploy/kicbase/Dockerfile @@ -113,8 +113,8 @@ RUN sh -c "echo 'deb https://download.opensuse.org/repositories/devel:/kubic:/li clean-install containers-common catatonit conmon containernetworking-plugins cri-tools podman-plugins varlink # install cri-o based on https://github.com/cri-o/cri-o/blob/release-1.18/README.md#installing-cri-o -RUN sh -c "echo 'deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/1.18/xUbuntu_20.04/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:1.18.list" && \ - curl -LO https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/1.18/xUbuntu_20.04/Release.key && \ +RUN sh -c "echo 'deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/1.18:/1.18.4/xUbuntu_20.04/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:1.18.list" && \ + curl -LO https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/1.18:/1.18.4/xUbuntu_20.04/Release.key && \ apt-key add - < Release.key && \ clean-install cri-o cri-o-runc From 64d38c813ad16f72dd6b2cb9679e597d134c3915 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Fri, 23 Oct 2020 20:16:41 +0200 Subject: [PATCH 02/57] Upgrade podman to version 2.1.1 and remove varlink Add the podman group, to remove "sudo varlink bridge" Use the podman socket /run/podman/podman.sock instead --- .../minikube-iso/board/coreos/minikube/users | 2 +- deploy/iso/minikube-iso/package/Config.in | 1 - .../minikube-iso/package/podman/override.conf | 4 ++ .../minikube-iso/package/podman/podman.conf | 1 + .../minikube-iso/package/podman/podman.hash | 1 + .../iso/minikube-iso/package/podman/podman.mk | 45 ++++++++++++++++--- .../minikube-iso/package/varlink/Config.in | 3 -- .../minikube-iso/package/varlink/varlink.hash | 4 -- .../minikube-iso/package/varlink/varlink.mk | 9 ---- 9 files changed, 46 insertions(+), 24 deletions(-) create mode 100644 deploy/iso/minikube-iso/package/podman/override.conf create mode 100644 deploy/iso/minikube-iso/package/podman/podman.conf delete mode 100644 deploy/iso/minikube-iso/package/varlink/Config.in delete mode 100644 deploy/iso/minikube-iso/package/varlink/varlink.hash delete mode 100644 deploy/iso/minikube-iso/package/varlink/varlink.mk diff --git a/deploy/iso/minikube-iso/board/coreos/minikube/users b/deploy/iso/minikube-iso/board/coreos/minikube/users index 8e967e11f1..cdff9ff1f9 100644 --- a/deploy/iso/minikube-iso/board/coreos/minikube/users +++ b/deploy/iso/minikube-iso/board/coreos/minikube/users @@ -1 +1 @@ -docker 1000 docker 1000 =tcuser /home/docker /bin/bash wheel,vboxsf - +docker 1000 docker 1000 =tcuser /home/docker /bin/bash wheel,vboxsf,podman - diff --git a/deploy/iso/minikube-iso/package/Config.in b/deploy/iso/minikube-iso/package/Config.in index dc7a8c85c2..e1ee83243a 100644 --- a/deploy/iso/minikube-iso/package/Config.in +++ b/deploy/iso/minikube-iso/package/Config.in @@ -1,7 +1,6 @@ menu "System tools" source "$BR2_EXTERNAL_MINIKUBE_PATH/package/runc-master/Config.in" source "$BR2_EXTERNAL_MINIKUBE_PATH/package/podman/Config.in" - source "$BR2_EXTERNAL_MINIKUBE_PATH/package/varlink/Config.in" source "$BR2_EXTERNAL_MINIKUBE_PATH/package/conmon/Config.in" source "$BR2_EXTERNAL_MINIKUBE_PATH/package/crio-bin/Config.in" source "$BR2_EXTERNAL_MINIKUBE_PATH/package/crictl-bin/Config.in" diff --git a/deploy/iso/minikube-iso/package/podman/override.conf b/deploy/iso/minikube-iso/package/podman/override.conf new file mode 100644 index 0000000000..b762370a4a --- /dev/null +++ b/deploy/iso/minikube-iso/package/podman/override.conf @@ -0,0 +1,4 @@ +[Socket] +SocketMode=0660 +SocketUser=root +SocketGroup=podman diff --git a/deploy/iso/minikube-iso/package/podman/podman.conf b/deploy/iso/minikube-iso/package/podman/podman.conf new file mode 100644 index 0000000000..8e31190ab8 --- /dev/null +++ b/deploy/iso/minikube-iso/package/podman/podman.conf @@ -0,0 +1 @@ +d /run/podman 0770 root podman diff --git a/deploy/iso/minikube-iso/package/podman/podman.hash b/deploy/iso/minikube-iso/package/podman/podman.hash index 29f2e1a338..3d5a50be25 100644 --- a/deploy/iso/minikube-iso/package/podman/podman.hash +++ b/deploy/iso/minikube-iso/package/podman/podman.hash @@ -1 +1,2 @@ sha256 a16846fe076aaf2c9ea2e854c3baba9fb838d916be7fb4b5be332e6c92d907d4 v1.9.3.tar.gz +sha256 5ebaa6e0dbd7fd1863f70d2bc71dc8a94e195c3339c17e3cac4560c9ec5747f8 v2.1.1.tar.gz diff --git a/deploy/iso/minikube-iso/package/podman/podman.mk b/deploy/iso/minikube-iso/package/podman/podman.mk index a5c297b7f2..7c76bcfbcb 100644 --- a/deploy/iso/minikube-iso/package/podman/podman.mk +++ b/deploy/iso/minikube-iso/package/podman/podman.mk @@ -1,12 +1,15 @@ -PODMAN_DUMMY = DUMMY -PODMAN_VERSION = v1.9.3 -PODMAN_COMMIT = 5d44534fff6877b1cb15b760242279ae6293154c +PODMAN_VERSION = v2.1.1 +PODMAN_COMMIT = 9f6d6ba0b314d86521b66183c9ce48eaa2da1de2 PODMAN_SITE = https://github.com/containers/podman/archive PODMAN_SOURCE = $(PODMAN_VERSION).tar.gz PODMAN_LICENSE = Apache-2.0 PODMAN_LICENSE_FILES = LICENSE PODMAN_DEPENDENCIES = host-go +ifeq ($(BR2_INIT_SYSTEMD),y) +# need libsystemd for journal +PODMAN_DEPENDENCIES += systemd +endif PODMAN_GOPATH = $(@D)/_output PODMAN_BIN_ENV = \ @@ -17,11 +20,24 @@ PODMAN_BIN_ENV = \ PATH=$(PODMAN_GOPATH)/bin:$(BR_PATH) +define PODMAN_USERS + - -1 podman -1 - - - - - +endef + +define PODMAN_MOD_VENDOR_MAKEFILE + # "build flag -mod=vendor only valid when using modules" + sed -e 's|-mod=vendor ||' -i $(@D)/Makefile +endef + +PODMAN_POST_EXTRACT_HOOKS += PODMAN_MOD_VENDOR_MAKEFILE + define PODMAN_CONFIGURE_CMDS + mkdir -p $(PODMAN_GOPATH) && mv $(@D)/vendor $(PODMAN_GOPATH)/src + mkdir -p $(PODMAN_GOPATH)/src/github.com/containers - ln -sf $(@D) $(PODMAN_GOPATH)/src/github.com/containers/libpod - mkdir -p $(PODMAN_GOPATH)/src/github.com/varlink - ln -sf $(@D)/vendor/github.com/varlink/go $(PODMAN_GOPATH)/src/github.com/varlink/go + ln -sf $(@D) $(PODMAN_GOPATH)/src/github.com/containers/podman + + ln -sf $(@D) $(PODMAN_GOPATH)/src/github.com/containers/podman/v2 endef define PODMAN_BUILD_CMDS @@ -35,4 +51,21 @@ define PODMAN_INSTALL_TARGET_CMDS $(INSTALL) -m 644 $(@D)/cni/87-podman-bridge.conflist $(TARGET_DIR)/etc/cni/net.d/87-podman-bridge.conflist endef +define PODMAN_INSTALL_INIT_SYSTEMD + $(INSTALL) -D -m 644 \ + $(@D)/contrib/systemd/system/podman.service \ + $(TARGET_DIR)/usr/lib/systemd/system/podman.service + $(INSTALL) -D -m 644 \ + $(@D)/contrib/systemd/system/podman.socket \ + $(TARGET_DIR)/usr/lib/systemd/system/podman.socket + + # Allow running podman-remote as a user in the group "podman" + $(INSTALL) -D -m 644 \ + $(PODMAN_PKGDIR)/override.conf \ + $(TARGET_DIR)/usr/lib/systemd/system/podman.socket.d/override.conf + $(INSTALL) -D -m 644 \ + $(PODMAN_PKGDIR)/podman.conf \ + $(TARGET_DIR)/usr/lib/tmpfiles.d/podman.conf +endef + $(eval $(generic-package)) diff --git a/deploy/iso/minikube-iso/package/varlink/Config.in b/deploy/iso/minikube-iso/package/varlink/Config.in deleted file mode 100644 index ef047e1560..0000000000 --- a/deploy/iso/minikube-iso/package/varlink/Config.in +++ /dev/null @@ -1,3 +0,0 @@ -config BR2_PACKAGE_VARLINK - bool "varlink" - default y diff --git a/deploy/iso/minikube-iso/package/varlink/varlink.hash b/deploy/iso/minikube-iso/package/varlink/varlink.hash deleted file mode 100644 index 64b07416e6..0000000000 --- a/deploy/iso/minikube-iso/package/varlink/varlink.hash +++ /dev/null @@ -1,4 +0,0 @@ -sha256 3857f109574750403b233b5fdf73f1852d8decc33dac8f73bd49f2003b69ad22 16.tar.gz -sha256 0dcb451f32033154c56710c216e67f245923fe2b011321271f6670e5a2285ce6 17.tar.gz -sha256 7a32543643116ad105da4ddb2f8030de7dcad1cdb3feb1a214ae5e7b65a6a198 18.tar.gz -sha256 0e316138ef6abc34363b05d0caf6df2e389a93b832e8d971e3ae64b48ba96133 19.tar.gz diff --git a/deploy/iso/minikube-iso/package/varlink/varlink.mk b/deploy/iso/minikube-iso/package/varlink/varlink.mk deleted file mode 100644 index 06a26e788f..0000000000 --- a/deploy/iso/minikube-iso/package/varlink/varlink.mk +++ /dev/null @@ -1,9 +0,0 @@ -VARLINK_VERSION = 19 -VARLINK_SITE = https://github.com/varlink/libvarlink/archive -VARLINK_SOURCE = $(VARLINK_VERSION).tar.gz -VARLINK_LICENSE = Apache-2.0 -VARLINK_LICENSE_FILES = LICENSE - -VARLINK_NEEDS_HOST_PYTHON = python3 - -$(eval $(meson-package)) From a8754ad470923367b39ca3a36c301786ef106957 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sat, 24 Oct 2020 10:43:05 +0200 Subject: [PATCH 03/57] Upgrade podman to version 2.1.1 and remove varlink Add the podman group, to remove "sudo varlink bridge" Use the podman socket /run/podman/podman.sock instead --- deploy/kicbase/Dockerfile | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/deploy/kicbase/Dockerfile b/deploy/kicbase/Dockerfile index 66e6fd7bbe..4970738ab4 100644 --- a/deploy/kicbase/Dockerfile +++ b/deploy/kicbase/Dockerfile @@ -110,7 +110,7 @@ RUN sh -c "echo 'deb https://download.docker.com/linux/ubuntu focal stable' > /e RUN sh -c "echo 'deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_20.04/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list" && \ curl -LO https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_20.04/Release.key && \ apt-key add - < Release.key && \ - clean-install containers-common catatonit conmon containernetworking-plugins cri-tools podman-plugins varlink + clean-install containers-common catatonit conmon containernetworking-plugins cri-tools podman-plugins # install cri-o based on https://github.com/cri-o/cri-o/blob/release-1.18/README.md#installing-cri-o RUN sh -c "echo 'deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/1.18/xUbuntu_20.04/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:1.18.list" && \ @@ -119,10 +119,17 @@ RUN sh -c "echo 'deb https://download.opensuse.org/repositories/devel:/kubic:/li clean-install cri-o cri-o-runc # install podman -RUN sh -c "echo 'deb https://dl.bintray.com/afbjorklund/podman focal main' > /etc/apt/sources.list.d/podman.list" && \ - curl -L https://bintray.com/user/downloadSubjectPublicKey?username=afbjorklund -o afbjorklund-public.key.asc && \ - apt-key add - < afbjorklund-public.key.asc && \ - clean-install podman=1.9.3~1 +RUN sh -c "echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_20.04/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list" && \ + curl -LO https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_20.04/Release.key && \ + apt-key add - < Release.key && \ + clean-install podman && \ + addgroup --system podman && \ + mkdir -p /etc/systemd/system/podman.socket.d && \ + printf "[Socket]\nSocketMode=0660\nSocketUser=root\nSocketGroup=podman\n" \ + > /etc/systemd/system/podman.socket.d/override.conf && \ + mkdir -p /etc/tmpfiles.d && \ + echo "d /run/podman 0770 root podman" > /etc/tmpfiles.d/podman.conf && \ + systemd-tmpfiles --create # automount service COPY automount/minikube-automount /usr/sbin/minikube-automount @@ -133,7 +140,7 @@ RUN ln -fs /usr/lib/systemd/system/minikube-automount.service \ # disable non-docker runtimes by default RUN systemctl disable containerd && systemctl disable crio && rm /etc/crictl.yaml # enable docker which is default -RUN systemctl enable docker +RUN systemctl enable docker.service && systemctl enable podman.socket # making SSH work for docker container # based on https://github.com/rastasheep/ubuntu-sshd/blob/master/18.04/Dockerfile RUN mkdir /var/run/sshd @@ -148,6 +155,7 @@ EXPOSE 22 # create docker user for minikube ssh. to match VM using "docker" as username RUN adduser --ingroup docker --disabled-password --gecos '' docker RUN adduser docker sudo +RUN adduser docker podman RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers USER docker RUN mkdir /home/docker/.ssh From b5e1d7ef17dd36a533d17f1f1ec53f1916221a99 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Fri, 20 Nov 2020 14:27:55 -0800 Subject: [PATCH 04/57] metrics --- hack/metrics/README.md | 4 ++ hack/metrics/metrics.go | 138 +++++++++++++++++++++++++++++++++++++++ hack/metrics/minikube.go | 26 ++++++++ 3 files changed, 168 insertions(+) create mode 100644 hack/metrics/README.md create mode 100644 hack/metrics/metrics.go create mode 100644 hack/metrics/minikube.go diff --git a/hack/metrics/README.md b/hack/metrics/README.md new file mode 100644 index 0000000000..ab82671fc6 --- /dev/null +++ b/hack/metrics/README.md @@ -0,0 +1,4 @@ +This script runs `minikube start` in a loop times how long it takes. +It exports this data to Stackdriver via the OpenTelemetry API. + +This script is used to track minikube performance and prevent regressions. diff --git a/hack/metrics/metrics.go b/hack/metrics/metrics.go new file mode 100644 index 0000000000..8a7582cada --- /dev/null +++ b/hack/metrics/metrics.go @@ -0,0 +1,138 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "fmt" + "log" + "os" + "os/exec" + "path/filepath" + "runtime" + "time" + + _ "cloud.google.com/go/storage" + "contrib.go.opencensus.io/exporter/stackdriver" + "github.com/pkg/errors" + "go.opencensus.io/stats" + "go.opencensus.io/stats/view" + "go.opencensus.io/tag" + "go.opencensus.io/trace" +) + +const ( + projectEnvVar = "MINIKUBE_GCP_PROJECT_ID" + customMetricName = "custom.googleapis.com/minikube/start_time" + profile = "cloud-monitoring" +) + +var ( + // The task latency in milliseconds. + latencyS = stats.Float64("repl/start_time", "start time in seconds", "s") +) + +func main() { + if err := execute(); err != nil { + fmt.Println(err) + os.Exit(1) + } +} + +func execute() error { + projectID := os.Getenv(projectEnvVar) + if projectID == "" { + return nil, fmt.Errorf("GCP tracer requires a valid GCP project id set via the %s env variable", projectEnvVar) + } + + osMethod, err := tag.NewKey("os") + if err != nil { + return errors.Wrap(err, "new tag key") + } + + ctx, err := tag.New(context.Background(), tag.Insert(osMethod, runtime.GOOS), tag.Insert(driverMethod, "docker")) + if err != nil { + return errors.Wrap(err, "new tag") + } + // Register the view. It is imperative that this step exists, + // otherwise recorded metrics will be dropped and never exported. + v := &view.View{ + Name: customMetricName, + Measure: latencyS, + Description: "minikube start time", + Aggregation: view.LastValue(), + } + if err := view.Register(v); err != nil { + log.Fatalf("Failed to register the view: %v", err) + } + + sd, err := stackdriver.NewExporter(stackdriver.Options{ + ProjectID: projectID, + // MetricPrefix helps uniquely identify your metrics. + MetricPrefix: "minikube_start", + // ReportingInterval sets the frequency of reporting metrics + // to stackdriver backend. + ReportingInterval: 1 * time.Second, + }) + if err != nil { + return errors.Wrap(err, "getting stackdriver exporter") + } + // It is imperative to invoke flush before your main function exits + defer sd.Flush() + + // Register it as a trace exporter + trace.RegisterExporter(sd) + + if err := sd.StartMetricsExporter(); err != nil { + log.Fatalf("Error starting metric exporter: %v", err) + } + defer sd.StopMetricsExporter() + + // Record 2p0 start time values + for { + st := minikubeStartTime(ctx) + fmt.Printf("Latency: %f\n", st) + stats.Record(ctx, latencyS.M(st)) + time.Sleep(2 * time.Second) + } +} + +func minikubeStartTime(ctx context.Context) (float64, error) { + defer deleteMinikube(ctx, profile) + + minikube := filepath.Join(os.Getenv("HOME"), "minikube/out/minikube") + + cmd := exec.CommandContext(ctx, minikube, "start", "--driver=docker", "-p", profile) + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + + t := time.Now() + log.Print("Running minikube start....") + if err := cmd.Run(); err != nil { + log.Fatal(err) + } + return time.Since(t).Seconds() +} + +func deleteMinikube(ctx context.Context) { + cmd := exec.CommandContext(ctx, minikube, "delete", "-p", profile) + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + log.Printf("error deleting: %v", err) + } +} diff --git a/hack/metrics/minikube.go b/hack/metrics/minikube.go new file mode 100644 index 0000000000..712a8dd516 --- /dev/null +++ b/hack/metrics/minikube.go @@ -0,0 +1,26 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +const ( + repo = "https://github.com/kubernetes/minikube" +) + +// buildMinikubeAtHEAD copies minikube to a temp dir +func buildMinikubeAtHEAD() error { + +} From fee0f5310a1febf984799ed59af1c86883fb75a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Mon, 16 Nov 2020 17:40:57 +0100 Subject: [PATCH 05/57] Add private network implementation for podman Most of it the same as docker, except for the options. i.e. libnetwork "bridge" plugin vs. cni "bridge" plugin --- cmd/minikube/cmd/delete.go | 2 +- pkg/drivers/kic/kic.go | 2 +- pkg/drivers/kic/oci/errors.go | 5 +- pkg/drivers/kic/oci/network.go | 21 +++-- pkg/drivers/kic/oci/network_create.go | 117 +++++++++++++++++--------- pkg/drivers/kic/oci/oci.go | 11 +-- 6 files changed, 102 insertions(+), 56 deletions(-) diff --git a/cmd/minikube/cmd/delete.go b/cmd/minikube/cmd/delete.go index 9b45c58240..8ce08f8d85 100644 --- a/cmd/minikube/cmd/delete.go +++ b/cmd/minikube/cmd/delete.go @@ -260,7 +260,7 @@ func deletePossibleKicLeftOver(cname string, driverName string) { klog.Warningf("error deleting volumes (might be okay).\nTo see the list of volumes run: 'docker volume ls'\n:%v", errs) } - errs = oci.DeleteKICNetworks() + errs = oci.DeleteKICNetworks(bin) if errs != nil { klog.Warningf("error deleting leftover networks (might be okay).\nTo see the list of networks: 'docker network ls'\n:%v", errs) } diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go index 4c47a478cf..156079675d 100644 --- a/pkg/drivers/kic/kic.go +++ b/pkg/drivers/kic/kic.go @@ -320,7 +320,7 @@ func (d *Driver) Remove() error { return fmt.Errorf("expected no container ID be found for %q after delete. but got %q", d.MachineName, id) } - if err := oci.RemoveNetwork(d.NodeConfig.ClusterName); err != nil { + if err := oci.RemoveNetwork(d.OCIBinary, d.NodeConfig.ClusterName); err != nil { klog.Warningf("failed to remove network (which might be okay) %s: %v", d.NodeConfig.ClusterName, err) } return nil diff --git a/pkg/drivers/kic/oci/errors.go b/pkg/drivers/kic/oci/errors.go index 6089d88cb7..ce4aa00be4 100644 --- a/pkg/drivers/kic/oci/errors.go +++ b/pkg/drivers/kic/oci/errors.go @@ -85,14 +85,15 @@ func LogContainerDebug(ociBin string, name string) string { } else { klog.Infof("postmortem docker info: %+v", di) } - logDockerNetworkInspect(name) + logDockerNetworkInspect(ociBin, name) } else { pi, err := podmanSystemInfo() if err != nil { - klog.Warningf("couldn't get postmortem info, failed to to run podman info: %v", err) + klog.Warningf("couldn't get postmortem podman info: %v", err) } else { klog.Infof("postmortem podman info: %+v", pi) } + logDockerNetworkInspect(ociBin, name) } if rr.Stdout.Len() == 0 { diff --git a/pkg/drivers/kic/oci/network.go b/pkg/drivers/kic/oci/network.go index a3c7edb94f..de2b742adf 100644 --- a/pkg/drivers/kic/oci/network.go +++ b/pkg/drivers/kic/oci/network.go @@ -35,7 +35,7 @@ import ( func RoutableHostIPFromInside(ociBin string, clusterName string, containerName string) (net.IP, error) { if ociBin == Docker { if runtime.GOOS == "linux" { - info, err := dockerNetworkInspect(clusterName) + info, err := containerNetworkInspect(ociBin, clusterName) if err != nil { if errors.Is(err, ErrNetworkNotFound) { klog.Infof("The container %s is not attached to a network, this could be because the cluster was created by minikube 0 { args = append(args, "-o") - args = append(args, fmt.Sprintf("com.docker.network.driver.mtu=%d", mtu)) - } + args = append(args, "--ip-masq") + args = append(args, "-o") + args = append(args, "--icc") - rr, err := runCmd(exec.Command(Docker, args...)) + // adding MTU option because #9528 + if mtu > 0 { + args = append(args, "-o") + args = append(args, fmt.Sprintf("com.docker.network.driver.mtu=%d", mtu)) + } + + args = append(args, fmt.Sprintf("--label=%s=%s", CreatedByLabelKey, "true")) + } + args = append(args, name) + + rr, err := runCmd(exec.Command(ociBin, args...)) if err != nil { // Pool overlaps with other one on this address space if strings.Contains(rr.Output(), "Pool overlaps") { @@ -135,6 +137,16 @@ type netInfo struct { mtu int } +func containerNetworkInspect(ociBin string, name string) (netInfo, error) { + if ociBin == Docker { + return dockerNetworkInspect(name) + } + if ociBin == Podman { + return podmanNetworkInspect(name) + } + return netInfo{}, fmt.Errorf("%s unknown", ociBin) +} + // networkInspect is only used to unmarshal the docker network inspect output and translate it to netInfo type networkInspect struct { Name string @@ -153,7 +165,7 @@ func dockerNetworkInspect(name string) (netInfo, error) { cmd := exec.Command(Docker, "network", "inspect", name, "--format", `{"Name": "{{.Name}}","Driver": "{{.Driver}}","Subnet": "{{range .IPAM.Config}}{{.Subnet}}{{end}}","Gateway": "{{range .IPAM.Config}}{{.Gateway}}{{end}}","MTU": {{(index .Options "com.docker.network.driver.mtu")}},{{$first := true}} "ContainerIPs": [{{range $k,$v := .Containers }}{{if $first}}{{$first = false}}{{else}}, {{end}}"{{$v.IPv4Address}}"{{end}}]}`) rr, err := runCmd(cmd) if err != nil { - logDockerNetworkInspect(name) + logDockerNetworkInspect(Docker, name) if strings.Contains(rr.Output(), "No such network") { return info, ErrNetworkNotFound @@ -177,8 +189,39 @@ func dockerNetworkInspect(name string) (netInfo, error) { return info, nil } -func logDockerNetworkInspect(name string) { - cmd := exec.Command(Docker, "network", "inspect", name) +func podmanNetworkInspect(name string) (netInfo, error) { + var info = netInfo{name: name} + cmd := exec.Command(Podman, "network", "inspect", name, "--format", `{{(index .IPAM.Config 0).Subnet}},{{(index .IPAM.Config 0).Gateway}}`) + rr, err := runCmd(cmd) + if err != nil { + logDockerNetworkInspect(Podman, name) + if strings.Contains(rr.Output(), "No such network") { + + return info, ErrNetworkNotFound + } + return info, err + } + + // results looks like 172.17.0.0/16,172.17.0.1,1500 + vals := strings.Split(strings.TrimSpace(rr.Stdout.String()), ",") + if len(vals) == 0 { + return info, fmt.Errorf("empty list network inspect: %q", rr.Output()) + } + + if len(vals) > 0 { + info.gateway = net.ParseIP(vals[1]) + } + + _, info.subnet, err = net.ParseCIDR(vals[0]) + if err != nil { + return info, errors.Wrapf(err, "parse subnet for %s", name) + } + + return info, nil +} + +func logDockerNetworkInspect(ociBin string, name string) { + cmd := exec.Command(ociBin, "network", "inspect", name) klog.Infof("running %v to gather additional debugging logs...", cmd.Args) rr, err := runCmd(cmd) if err != nil { @@ -188,11 +231,11 @@ func logDockerNetworkInspect(name string) { } // RemoveNetwork removes a network -func RemoveNetwork(name string) error { - if !networkExists(name) { +func RemoveNetwork(ociBin string, name string) error { + if !networkExists(ociBin, name) { return nil } - rr, err := runCmd(exec.Command(Docker, "network", "remove", name)) + rr, err := runCmd(exec.Command(ociBin, "network", "rm", name)) if err != nil { if strings.Contains(rr.Output(), "No such network") { return ErrNetworkNotFound @@ -206,8 +249,8 @@ func RemoveNetwork(name string) error { return err } -func networkExists(name string) bool { - _, err := dockerNetworkInspect(name) +func networkExists(ociBin string, name string) bool { + _, err := containerNetworkInspect(ociBin, name) if err != nil && !errors.Is(err, ErrNetworkNotFound) { // log unexpected error klog.Warningf("Error inspecting docker network %s: %v", name, err) } @@ -216,12 +259,8 @@ func networkExists(name string) bool { // networkNamesByLabel returns all network names created by a label func networkNamesByLabel(ociBin string, label string) ([]string, error) { - if ociBin != Docker { - return nil, fmt.Errorf("%s not supported", ociBin) - } - // docker network ls --filter='label=created_by.minikube.sigs.k8s.io=true' --format '{{.Name}}' - rr, err := runCmd(exec.Command(Docker, "network", "ls", fmt.Sprintf("--filter=label=%s", label), "--format", "{{.Name}}")) + rr, err := runCmd(exec.Command(ociBin, "network", "ls", fmt.Sprintf("--filter=label=%s", label), "--format", "{{.Name}}")) if err != nil { return nil, err } @@ -235,14 +274,14 @@ func networkNamesByLabel(ociBin string, label string) ([]string, error) { } // DeleteKICNetworks deletes all networks created by kic -func DeleteKICNetworks() []error { +func DeleteKICNetworks(ociBin string) []error { var errs []error - ns, err := networkNamesByLabel(Docker, CreatedByLabelKey+"=true") + ns, err := networkNamesByLabel(ociBin, CreatedByLabelKey) if err != nil { return []error{errors.Wrap(err, "list all volume")} } for _, n := range ns { - err := RemoveNetwork(n) + err := RemoveNetwork(ociBin, n) if err != nil { errs = append(errs, err) } diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go index 087d6224f4..63c45fc283 100644 --- a/pkg/drivers/kic/oci/oci.go +++ b/pkg/drivers/kic/oci/oci.go @@ -146,6 +146,12 @@ func CreateContainerNode(p CreateParams) error { // label th enode wuth the node ID "--label", p.NodeLabel, } + // to provide a static IP + if p.Network != "" && p.IP != "" { + runArgs = append(runArgs, "--network", p.Network) + runArgs = append(runArgs, "--ip", p.IP) + } + memcgSwap := true if runtime.GOOS == "linux" { if _, err := os.Stat("/sys/fs/cgroup/memory/memsw.limit_in_bytes"); os.IsNotExist(err) { @@ -170,11 +176,6 @@ func CreateContainerNode(p CreateParams) error { virtualization = "podman" // VIRTUALIZATION_PODMAN } if p.OCIBinary == Docker { - // to provide a static IP for docker - if p.Network != "" && p.IP != "" { - runArgs = append(runArgs, "--network", p.Network) - runArgs = append(runArgs, "--ip", p.IP) - } runArgs = append(runArgs, "--volume", fmt.Sprintf("%s:/var", p.Name)) // ignore apparmore github actions docker: https://github.com/kubernetes/minikube/issues/7624 runArgs = append(runArgs, "--security-opt", "apparmor=unconfined") From d370a3e164485efcf295fcde723e2fe2ab941f31 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Mon, 23 Nov 2020 12:56:48 -0800 Subject: [PATCH 06/57] fix up script --- hack/metrics/metrics.go | 11 +++++------ site/content/en/docs/commands/start.md | 1 + 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/hack/metrics/metrics.go b/hack/metrics/metrics.go index 8a7582cada..ad8602beb2 100644 --- a/hack/metrics/metrics.go +++ b/hack/metrics/metrics.go @@ -42,7 +42,7 @@ const ( ) var ( - // The task latency in milliseconds. + // The task latency in seconds latencyS = stats.Float64("repl/start_time", "start time in seconds", "s") ) @@ -56,7 +56,7 @@ func main() { func execute() error { projectID := os.Getenv(projectEnvVar) if projectID == "" { - return nil, fmt.Errorf("GCP tracer requires a valid GCP project id set via the %s env variable", projectEnvVar) + return fmt.Errorf("metrics collector requires a valid GCP project id set via the %s env variable", projectEnvVar) } osMethod, err := tag.NewKey("os") @@ -64,7 +64,7 @@ func execute() error { return errors.Wrap(err, "new tag key") } - ctx, err := tag.New(context.Background(), tag.Insert(osMethod, runtime.GOOS), tag.Insert(driverMethod, "docker")) + ctx, err := tag.New(context.Background(), tag.Insert(osMethod, runtime.GOOS)) if err != nil { return errors.Wrap(err, "new tag") } @@ -102,17 +102,16 @@ func execute() error { } defer sd.StopMetricsExporter() - // Record 2p0 start time values for { st := minikubeStartTime(ctx) fmt.Printf("Latency: %f\n", st) stats.Record(ctx, latencyS.M(st)) - time.Sleep(2 * time.Second) + time.Sleep(30 * time.Second) } } func minikubeStartTime(ctx context.Context) (float64, error) { - defer deleteMinikube(ctx, profile) + defer deleteMinikube(ctx) minikube := filepath.Join(os.Getenv("HOME"), "minikube/out/minikube") diff --git a/site/content/en/docs/commands/start.md b/site/content/en/docs/commands/start.md index 122f0b407e..2cc6dd0d8b 100644 --- a/site/content/en/docs/commands/start.md +++ b/site/content/en/docs/commands/start.md @@ -87,6 +87,7 @@ minikube start [flags] --preload If set, download tarball of preloaded images if available to improve start time. Defaults to true. (default true) --registry-mirror strings Registry mirrors to pass to the Docker daemon --service-cluster-ip-range string The CIDR to be used for service cluster IPs. (default "10.96.0.0/12") + --trace string Send trace events. Options include: [gcp] --uuid string Provide VM UUID to restore MAC address (hyperkit driver only) --vm Filter to use only VM Drivers --vm-driver driver DEPRECATED, use driver instead. From 46f82272e01a33eb029d7274dcc51a34558d2642 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Mon, 23 Nov 2020 16:14:06 -0800 Subject: [PATCH 07/57] Add metrics script to collect data for cloud monitoring --- go.sum | 3 ++ hack/metrics/metrics.go | 28 +++++++++++-------- hack/metrics/minikube.go | 59 +++++++++++++++++++++++++++++++++++++--- 3 files changed, 75 insertions(+), 15 deletions(-) diff --git a/go.sum b/go.sum index 22abe6c76e..d1f38e9872 100644 --- a/go.sum +++ b/go.sum @@ -40,6 +40,7 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX code.gitea.io/sdk/gitea v0.12.0/go.mod h1:z3uwDV/b9Ls47NGukYM9XhnHtqPh/J+t40lsUrR6JDY= contrib.go.opencensus.io/exporter/aws v0.0.0-20181029163544-2befc13012d0/go.mod h1:uu1P0UCM/6RbsMrgPa98ll8ZcHM858i/AD06a9aLRCA= contrib.go.opencensus.io/exporter/ocagent v0.5.0/go.mod h1:ImxhfLRpxoYiSq891pBrLVhN+qmP8BTVvdH2YLs7Gl0= +contrib.go.opencensus.io/exporter/stackdriver v0.12.1 h1:Dll2uFfOVI3fa8UzsHyP6z0M6fEc9ZTAMo+Y3z282Xg= contrib.go.opencensus.io/exporter/stackdriver v0.12.1/go.mod h1:iwB6wGarfphGGe/e5CWqyUk/cLzKnWsOKPVW3no6OTw= contrib.go.opencensus.io/integrations/ocsql v0.1.4/go.mod h1:8DsSdjz3F+APR+0z0WkU1aRorQCFfRxvqjUUPMbF3fE= contrib.go.opencensus.io/resource v0.1.1/go.mod h1:F361eGI91LCmW1I/Saf+rX0+OFcigGlFvXwEGEnkRLA= @@ -137,6 +138,7 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/apex/log v1.1.4/go.mod h1:AlpoD9aScyQfJDVHmLMEcx4oU6LqzkWp4Mg9GdAcEvQ= +github.com/apex/log v1.3.0 h1:1fyfbPvUwD10nMoh3hY6MXzvZShJQn9/ck7ATgAt5pA= github.com/apex/log v1.3.0/go.mod h1:jd8Vpsr46WAe3EZSQ/IUMs2qQD/GOycT5rPWCO1yGcs= github.com/apex/logs v0.0.4/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo= github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE= @@ -202,6 +204,7 @@ github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QH github.com/cenkalti/backoff/v4 v4.1.0 h1:c8LkOFQTzuO0WBM/ae5HdGQuZPfPxp7lqBRwQRm4fSc= github.com/cenkalti/backoff/v4 v4.1.0/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/prettybench v0.0.0-20150116022406-03b8cfe5406c/go.mod h1:Xe6ZsFhtM8HrDku0pxJ3/Lr51rwykrzgFwpmTzleatY= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= diff --git a/hack/metrics/metrics.go b/hack/metrics/metrics.go index ad8602beb2..04b26fbed5 100644 --- a/hack/metrics/metrics.go +++ b/hack/metrics/metrics.go @@ -22,7 +22,6 @@ import ( "log" "os" "os/exec" - "path/filepath" "runtime" "time" @@ -98,12 +97,16 @@ func execute() error { trace.RegisterExporter(sd) if err := sd.StartMetricsExporter(); err != nil { - log.Fatalf("Error starting metric exporter: %v", err) + return errors.Wrap(err, "starting metric exporter") } defer sd.StopMetricsExporter() for { - st := minikubeStartTime(ctx) + st, err := minikubeStartTime(ctx) + if err != nil { + log.Printf("error collecting start time: %v", err) + continue + } fmt.Printf("Latency: %f\n", st) stats.Record(ctx, latencyS.M(st)) time.Sleep(30 * time.Second) @@ -111,24 +114,27 @@ func execute() error { } func minikubeStartTime(ctx context.Context) (float64, error) { - defer deleteMinikube(ctx) + minikubePath, err := downloadMinikube() + if err != nil { + return 0, errors.Wrap(err, "downloading minikube") + } + defer os.Remove(minikubePath) + defer deleteMinikube(ctx, minikubePath) - minikube := filepath.Join(os.Getenv("HOME"), "minikube/out/minikube") - - cmd := exec.CommandContext(ctx, minikube, "start", "--driver=docker", "-p", profile) + cmd := exec.CommandContext(ctx, minikubePath, "start", "--driver=docker", "-p", profile) cmd.Stdout = os.Stderr cmd.Stderr = os.Stderr t := time.Now() log.Print("Running minikube start....") if err := cmd.Run(); err != nil { - log.Fatal(err) + return 0, errors.Wrapf(err, "running %v", cmd.Args) } - return time.Since(t).Seconds() + return time.Since(t).Seconds(), nil } -func deleteMinikube(ctx context.Context) { - cmd := exec.CommandContext(ctx, minikube, "delete", "-p", profile) +func deleteMinikube(ctx context.Context, minikubePath string) { + cmd := exec.CommandContext(ctx, minikubePath, "delete", "-p", profile) cmd.Stdout = os.Stderr cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { diff --git a/hack/metrics/minikube.go b/hack/metrics/minikube.go index 712a8dd516..d6b7adf86c 100644 --- a/hack/metrics/minikube.go +++ b/hack/metrics/minikube.go @@ -16,11 +16,62 @@ limitations under the License. package main -const ( - repo = "https://github.com/kubernetes/minikube" +import ( + "context" + "fmt" + "io/ioutil" + "os" + "runtime" + + "log" + + "cloud.google.com/go/storage" + "github.com/pkg/errors" ) -// buildMinikubeAtHEAD copies minikube to a temp dir -func buildMinikubeAtHEAD() error { +const ( + bucketName = "priya-test-bucket/latest" +) +// download minikube latest to a tmp file +func downloadMinikube() (string, error) { + b := binary() + tmp, err := ioutil.TempFile("", b) + if err != nil { + return "", errors.Wrap(err, "creating tmp file") + } + if err := tmp.Close(); err != nil { + return "", errors.Wrap(err, "closing tmp file") + } + client, err := storage.NewClient(context.Background()) + if err != nil { + return "", errors.Wrap(err, "creating client") + } + ctx := context.Background() + rc, err := client.Bucket(bucketName).Object(b).NewReader(ctx) + if err != nil { + return "", errors.Wrap(err, "gcs new reader") + } + defer rc.Close() + + data, err := ioutil.ReadAll(rc) + if err != nil { + return "", errors.Wrap(err, "ioutil read all") + } + log.Printf("downloading gs://%s/%s to %v", bucketName, b, tmp.Name()) + if err := ioutil.WriteFile(tmp.Name(), data, 0777); err != nil { + return "", errors.Wrap(err, "writing file") + } + if err := os.Chmod(tmp.Name(), 0700); err != nil { + return "", errors.Wrap(err, "chmod") + } + return tmp.Name(), nil +} + +func binary() string { + b := fmt.Sprintf("minikube-%s-amd64", runtime.GOOS) + if runtime.GOOS == "windows" { + b += ".exe" + } + return b } From b01bad0ebf63c820c1b5e78874ce7056a009c849 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Mon, 23 Nov 2020 16:18:46 -0800 Subject: [PATCH 08/57] add --memory flag to constrain that variable --- hack/metrics/metrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/metrics/metrics.go b/hack/metrics/metrics.go index 04b26fbed5..0c0b7153b5 100644 --- a/hack/metrics/metrics.go +++ b/hack/metrics/metrics.go @@ -121,7 +121,7 @@ func minikubeStartTime(ctx context.Context) (float64, error) { defer os.Remove(minikubePath) defer deleteMinikube(ctx, minikubePath) - cmd := exec.CommandContext(ctx, minikubePath, "start", "--driver=docker", "-p", profile) + cmd := exec.CommandContext(ctx, minikubePath, "start", "--driver=docker", "-p", profile, "--memory=2000") cmd.Stdout = os.Stderr cmd.Stderr = os.Stderr From 5b2a552091a905225d84c8f0eaf0bf298a70738d Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Mon, 23 Nov 2020 16:21:13 -0800 Subject: [PATCH 09/57] cleanup --- hack/metrics/README.md | 2 +- hack/metrics/metrics.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hack/metrics/README.md b/hack/metrics/README.md index ab82671fc6..68df5ac838 100644 --- a/hack/metrics/README.md +++ b/hack/metrics/README.md @@ -1,4 +1,4 @@ -This script runs `minikube start` in a loop times how long it takes. +This script runs `minikube start` in a loop and measures how long it takes. It exports this data to Stackdriver via the OpenTelemetry API. This script is used to track minikube performance and prevent regressions. diff --git a/hack/metrics/metrics.go b/hack/metrics/metrics.go index 0c0b7153b5..bf74e3ba39 100644 --- a/hack/metrics/metrics.go +++ b/hack/metrics/metrics.go @@ -76,7 +76,7 @@ func execute() error { Aggregation: view.LastValue(), } if err := view.Register(v); err != nil { - log.Fatalf("Failed to register the view: %v", err) + return errors.Wrap(err, "registering view") } sd, err := stackdriver.NewExporter(stackdriver.Options{ From 2a0a01a18cd93046ae2c0b70045783d5754eae9c Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Tue, 24 Nov 2020 10:14:09 -0800 Subject: [PATCH 10/57] UPDate --- hack/metrics/metrics.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/hack/metrics/metrics.go b/hack/metrics/metrics.go index bf74e3ba39..142d3e2013 100644 --- a/hack/metrics/metrics.go +++ b/hack/metrics/metrics.go @@ -22,7 +22,6 @@ import ( "log" "os" "os/exec" - "runtime" "time" _ "cloud.google.com/go/storage" @@ -63,10 +62,6 @@ func execute() error { return errors.Wrap(err, "new tag key") } - ctx, err := tag.New(context.Background(), tag.Insert(osMethod, runtime.GOOS)) - if err != nil { - return errors.Wrap(err, "new tag") - } // Register the view. It is imperative that this step exists, // otherwise recorded metrics will be dropped and never exported. v := &view.View{ @@ -74,6 +69,7 @@ func execute() error { Measure: latencyS, Description: "minikube start time", Aggregation: view.LastValue(), + TagKeys: []tag.Key{osMethod}, } if err := view.Register(v); err != nil { return errors.Wrap(err, "registering view") @@ -100,7 +96,7 @@ func execute() error { return errors.Wrap(err, "starting metric exporter") } defer sd.StopMetricsExporter() - + ctx := context.Background() for { st, err := minikubeStartTime(ctx) if err != nil { From ad3e4a98d0e513eb8a9a6d9319e30125c41bdb10 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Tue, 24 Nov 2020 11:36:34 -0800 Subject: [PATCH 11/57] Add --cancel-scheduled-stop flag and add to integration test to make sure it works --- cmd/minikube/cmd/stop.go | 8 ++++++ test/integration/scheduled_stop_test.go | 37 ++++++++++++++++--------- 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/cmd/minikube/cmd/stop.go b/cmd/minikube/cmd/stop.go index 4cce5d09fa..e8d537e9c5 100644 --- a/cmd/minikube/cmd/stop.go +++ b/cmd/minikube/cmd/stop.go @@ -46,6 +46,7 @@ var ( stopAll bool keepActive bool scheduledStopDuration time.Duration + cancelScheduledStop bool ) // stopCmd represents the stop command @@ -60,6 +61,8 @@ func init() { stopCmd.Flags().BoolVar(&stopAll, "all", false, "Set flag to stop all profiles (clusters)") stopCmd.Flags().BoolVar(&keepActive, "keep-context-active", false, "keep the kube-context active after cluster is stopped. Defaults to false.") stopCmd.Flags().DurationVar(&scheduledStopDuration, "schedule", 0*time.Second, "Set flag to stop cluster after a set amount of time (e.g. --schedule=5m)") + stopCmd.Flags().BoolVar(&cancelScheduledStop, "cancel-scheduled-stop", false, "cancel any existing scheduled stop requests") + if err := stopCmd.Flags().MarkHidden("schedule"); err != nil { klog.Info("unable to mark --schedule flag as hidden") } @@ -97,6 +100,11 @@ func runStop(cmd *cobra.Command, args []string) { // Kill any existing scheduled stops schedule.KillExisting(profilesToStop) + if cancelScheduledStop { + register.Reg.SetStep(register.Done) + out.Step(style.Stopped, `All existing scheduled stops cancelled`) + return + } if scheduledStopDuration != 0 { if err := schedule.Daemonize(profilesToStop, scheduledStopDuration); err != nil { diff --git a/test/integration/scheduled_stop_test.go b/test/integration/scheduled_stop_test.go index a9d833a22b..597f9a2200 100644 --- a/test/integration/scheduled_stop_test.go +++ b/test/integration/scheduled_stop_test.go @@ -50,7 +50,7 @@ func TestScheduledStopWindows(t *testing.T) { startMinikube(ctx, t, profile) // schedule a stop for 5m from now - scheduledStopMinikube(ctx, t, profile, "5m") + stopMinikube(ctx, t, profile, []string{"--schedule", "5m"}) // make sure the systemd service is running rr, err := Run(t, exec.CommandContext(ctx, Target(), []string{"ssh", "-p", profile, "--", "sudo", "systemctl", "show", constants.ScheduledStopSystemdService, "--no-page"}...)) @@ -62,12 +62,12 @@ func TestScheduledStopWindows(t *testing.T) { } // reschedule stop for 5 seconds from now - scheduledStopMinikube(ctx, t, profile, "5s") + stopMinikube(ctx, t, profile, []string{"--schedule", "5s"}) // sleep for 5 seconds time.Sleep(5 * time.Second) // make sure minikube status is "Stopped" - ensureMinikubeStatusStopped(ctx, t, profile) + ensureMinikubeStatus(ctx, t, profile, state.Stopped.String()) } func TestScheduledStopUnix(t *testing.T) { @@ -83,20 +83,31 @@ func TestScheduledStopUnix(t *testing.T) { startMinikube(ctx, t, profile) // schedule a stop for 5 min from now and make sure PID is created - scheduledStopMinikube(ctx, t, profile, "5m") + stopMinikube(ctx, t, profile, []string{"--schedule", "5m"}) pid := checkPID(t, profile) if !processRunning(t, pid) { t.Fatalf("process %v is not running", pid) } - // redo scheduled stop to be in 10s - scheduledStopMinikube(ctx, t, profile, "10s") + // schedule a second stop which should cancel the first scheduled stop + stopMinikube(ctx, t, profile, []string{"--schedule", "8s"}) if processRunning(t, pid) { t.Fatalf("process %v running but should have been killed on reschedule of stop", pid) } checkPID(t, profile) - // make sure minikube status is "Stopped" - ensureMinikubeStatusStopped(ctx, t, profile) + + // cancel the shutdown and make sure minikube is still running after 8 seconds + // sleep 12 just to be safe + stopMinikube(ctx, t, profile, []string{"--cancel-scheduled-stop"}) + time.Sleep(12 * time.Second) + ensureMinikubeStatus(ctx, t, profile, state.Running.String()) + + // schedule another stop, make sure minikube status is "Stopped" + stopMinikube(ctx, t, profile, []string{"--schedule", "5s"}) + if processRunning(t, pid) { + t.Fatalf("process %v running but should have been killed on reschedule of stop", pid) + } + ensureMinikubeStatus(ctx, t, profile, state.Stopped.String()) } func startMinikube(ctx context.Context, t *testing.T, profile string) { @@ -107,8 +118,9 @@ func startMinikube(ctx context.Context, t *testing.T, profile string) { } } -func scheduledStopMinikube(ctx context.Context, t *testing.T, profile string, stop string) { - args := []string{"stop", "-p", profile, "--schedule", stop} +func stopMinikube(ctx context.Context, t *testing.T, profile string, additionalArgs []string) { + args := []string{"stop", "-p", profile} + args = append(args, additionalArgs...) rr, err := Run(t, exec.CommandContext(ctx, Target(), args...)) if err != nil { t.Fatalf("starting minikube: %v\n%s", err, rr.Output()) @@ -144,14 +156,13 @@ func processRunning(t *testing.T, pid string) bool { t.Log("signal error was: ", err) return err == nil } - -func ensureMinikubeStatusStopped(ctx context.Context, t *testing.T, profile string) { +func ensureMinikubeStatus(ctx context.Context, t *testing.T, profile, wantStatus string) { // wait allotted time to make sure minikube status is "Stopped" checkStatus := func() error { ctx, cancel := context.WithDeadline(ctx, time.Now().Add(10*time.Second)) defer cancel() got := Status(ctx, t, Target(), profile, "Host", profile) - if got != state.Stopped.String() { + if got != wantStatus { return fmt.Errorf("expected post-stop host status to be -%q- but got *%q*", state.Stopped, got) } return nil From b7eaaf8c7512c66b593c475f76a64d068b3e04f1 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Tue, 24 Nov 2020 11:36:41 -0800 Subject: [PATCH 12/57] update docs --- site/content/en/docs/commands/stop.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/site/content/en/docs/commands/stop.md b/site/content/en/docs/commands/stop.md index 9f54204ab1..d5a2c19563 100644 --- a/site/content/en/docs/commands/stop.md +++ b/site/content/en/docs/commands/stop.md @@ -20,9 +20,10 @@ minikube stop [flags] ### Options ``` - --all Set flag to stop all profiles (clusters) - --keep-context-active keep the kube-context active after cluster is stopped. Defaults to false. - -o, --output string Format to print stdout in. Options include: [text,json] (default "text") + --all Set flag to stop all profiles (clusters) + --cancel-scheduled-stop cancel any existing scheduled stop requests + --keep-context-active keep the kube-context active after cluster is stopped. Defaults to false. + -o, --output string Format to print stdout in. Options include: [text,json] (default "text") ``` ### Options inherited from parent commands From 70cb886a393d1ebd55efa5cf470fbce944e5d0a2 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Tue, 24 Nov 2020 17:29:46 -0800 Subject: [PATCH 13/57] shorten flag name to --cancel-scheduled --- cmd/minikube/cmd/stop.go | 2 +- site/content/en/docs/commands/stop.md | 8 ++++---- test/integration/scheduled_stop_test.go | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/minikube/cmd/stop.go b/cmd/minikube/cmd/stop.go index e8d537e9c5..4fd0fadabc 100644 --- a/cmd/minikube/cmd/stop.go +++ b/cmd/minikube/cmd/stop.go @@ -61,7 +61,7 @@ func init() { stopCmd.Flags().BoolVar(&stopAll, "all", false, "Set flag to stop all profiles (clusters)") stopCmd.Flags().BoolVar(&keepActive, "keep-context-active", false, "keep the kube-context active after cluster is stopped. Defaults to false.") stopCmd.Flags().DurationVar(&scheduledStopDuration, "schedule", 0*time.Second, "Set flag to stop cluster after a set amount of time (e.g. --schedule=5m)") - stopCmd.Flags().BoolVar(&cancelScheduledStop, "cancel-scheduled-stop", false, "cancel any existing scheduled stop requests") + stopCmd.Flags().BoolVar(&cancelScheduledStop, "cancel-scheduled", false, "cancel any existing scheduled stop requests") if err := stopCmd.Flags().MarkHidden("schedule"); err != nil { klog.Info("unable to mark --schedule flag as hidden") diff --git a/site/content/en/docs/commands/stop.md b/site/content/en/docs/commands/stop.md index d5a2c19563..8b96e3de89 100644 --- a/site/content/en/docs/commands/stop.md +++ b/site/content/en/docs/commands/stop.md @@ -20,10 +20,10 @@ minikube stop [flags] ### Options ``` - --all Set flag to stop all profiles (clusters) - --cancel-scheduled-stop cancel any existing scheduled stop requests - --keep-context-active keep the kube-context active after cluster is stopped. Defaults to false. - -o, --output string Format to print stdout in. Options include: [text,json] (default "text") + --all Set flag to stop all profiles (clusters) + --cancel-scheduled cancel any existing scheduled stop requests + --keep-context-active keep the kube-context active after cluster is stopped. Defaults to false. + -o, --output string Format to print stdout in. Options include: [text,json] (default "text") ``` ### Options inherited from parent commands diff --git a/test/integration/scheduled_stop_test.go b/test/integration/scheduled_stop_test.go index 597f9a2200..508a385999 100644 --- a/test/integration/scheduled_stop_test.go +++ b/test/integration/scheduled_stop_test.go @@ -98,7 +98,7 @@ func TestScheduledStopUnix(t *testing.T) { // cancel the shutdown and make sure minikube is still running after 8 seconds // sleep 12 just to be safe - stopMinikube(ctx, t, profile, []string{"--cancel-scheduled-stop"}) + stopMinikube(ctx, t, profile, []string{"--cancel-scheduled"}) time.Sleep(12 * time.Second) ensureMinikubeStatus(ctx, t, profile, state.Running.String()) From fc8fd7bf273b27fdc197c1bffd65a1fb72d54a90 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Wed, 25 Nov 2020 15:17:54 -0800 Subject: [PATCH 14/57] Update conformance tests for multinode testing: --- hack/conformance_tests.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/hack/conformance_tests.sh b/hack/conformance_tests.sh index b4509e8e9c..cec198a741 100755 --- a/hack/conformance_tests.sh +++ b/hack/conformance_tests.sh @@ -27,19 +27,21 @@ set -ex -o pipefail readonly PROFILE_NAME="k8sconformance" readonly MINIKUBE=${1:-./out/minikube} shift || true +readonly START_ARGS=${@:-} # Requires a fully running Kubernetes cluster. "${MINIKUBE}" delete -p "${PROFILE_NAME}" || true -"${MINIKUBE}" start -p "${PROFILE_NAME}" --wait=all +"${MINIKUBE}" start -p "${PROFILE_NAME}" ${START_ARGS} --wait=all --nodes=2 kubectl --context "${PROFILE_NAME}" get pods --all-namespaces "${MINIKUBE}" status -p "${PROFILE_NAME}" -go get -u -v github.com/vmware-tanzu/sonobuoy +curl -LO https://github.com/vmware-tanzu/sonobuoy/releases/download/v0.19.0/sonobuoy_0.19.0_linux_amd64.tar.gz || true +tar -xzf sonobuoy_0.19.0_linux_amd64.tar.gz -sonobuoy run --wait +./sonobuoy run --mode=certified-conformance --wait --alsologtostderr outdir="$(mktemp -d)" -sonobuoy retrieve "${outdir}" +./sonobuoy retrieve "${outdir}" cwd=$(pwd) From 22a99c55cd73cb26cada21dd2e272b24a1b8ac0e Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Wed, 25 Nov 2020 17:48:37 -0800 Subject: [PATCH 15/57] small fixes --- hack/conformance_tests.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hack/conformance_tests.sh b/hack/conformance_tests.sh index cec198a741..439649449f 100755 --- a/hack/conformance_tests.sh +++ b/hack/conformance_tests.sh @@ -48,9 +48,9 @@ cwd=$(pwd) cd "${outdir}" mkdir ./results; tar xzf *.tar.gz -C ./results -version=$(${MINIKUBE} version | cut -d" " -f3) +version=$(${cwd}/${MINIKUBE} version | cut -d" " -f3) -mkdir "minikube-${version}" +mkdir -p "minikube-${version}" cd "minikube-${version}" cat <PRODUCT.yaml @@ -69,6 +69,6 @@ cat <README.md ./hack/conformance_tests.sh $MINIKUBE $START_ARGS EOF -cp ../results/plugins/e2e/results/* . +cp -r ../results/plugins/e2e/results/global/* . cd .. cp -r "minikube-${version}" "${cwd}" From 5b8670c4e952f7cf0e1f46dafc8be80e05c0c062 Mon Sep 17 00:00:00 2001 From: Tharun Date: Fri, 27 Nov 2020 01:22:02 +0530 Subject: [PATCH 16/57] fix unable to set memory in config Signed-off-by: Tharun --- cmd/minikube/cmd/config/config.go | 4 ++-- cmd/minikube/cmd/config/set_test.go | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/cmd/minikube/cmd/config/config.go b/cmd/minikube/cmd/config/config.go index 08f4608f29..5869c1d175 100644 --- a/cmd/minikube/cmd/config/config.go +++ b/cmd/minikube/cmd/config/config.go @@ -92,8 +92,8 @@ var settings = []Setting{ }, { name: "memory", - set: SetInt, - validations: []setFn{IsPositive}, + set: SetString, + validations: []setFn{IsValidDiskSize}, callbacks: []setFn{RequiresRestartMsg}, }, { diff --git a/cmd/minikube/cmd/config/set_test.go b/cmd/minikube/cmd/config/set_test.go index 5d73523fff..7469b3798a 100644 --- a/cmd/minikube/cmd/config/set_test.go +++ b/cmd/minikube/cmd/config/set_test.go @@ -38,6 +38,10 @@ func TestSetNotAllowed(t *testing.T) { if err == nil || err.Error() != "run validations for \"driver\" with value of \"123456\": [driver \"123456\" is not supported]" { t.Fatalf("Set did not return error for unallowed value: %+v", err) } + err = Set("memory", "10a") + if err == nil || err.Error() != "run validations for \"memory\" with value of \"10a\": [invalid disk size: invalid size: '10a']" { + t.Fatalf("Set did not return error for unallowed value: %+v", err) + } } func TestSetOK(t *testing.T) { From 6b4dc381f98f97ef99489abd2ee7c3e59c4eb714 Mon Sep 17 00:00:00 2001 From: Brian Li Date: Fri, 27 Nov 2020 10:14:01 +0800 Subject: [PATCH 17/57] fix base image when using with custom image repository --- pkg/minikube/node/cache.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/minikube/node/cache.go b/pkg/minikube/node/cache.go index 3017d31472..e775ed3ca1 100644 --- a/pkg/minikube/node/cache.go +++ b/pkg/minikube/node/cache.go @@ -124,6 +124,7 @@ func beginDownloadKicBaseImage(g *errgroup.Group, cc *config.ClusterConfig, down baseImg := cc.KicBaseImage if baseImg == kic.BaseImage && len(cc.KubernetesConfig.ImageRepository) != 0 { baseImg = strings.Replace(baseImg, "gcr.io/k8s-minikube", cc.KubernetesConfig.ImageRepository, 1) + cc.KicBaseImage = baseImg } var finalImg string // If we end up using a fallback image, notify the user From dae980a3556846a259a2ac81c1db63f97ce7f39a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sun, 29 Nov 2020 22:56:06 +0100 Subject: [PATCH 18/57] Add support for restoring existing podman env If user had a podman environment configured before podman-env, restore it on unset (similar to existing code for docker-env) --- cmd/minikube/cmd/podman-env.go | 35 +++++++++++++++++++++++++++-- pkg/drivers/kic/oci/env.go | 10 +++++++++ pkg/minikube/constants/constants.go | 3 +++ 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/cmd/minikube/cmd/podman-env.go b/cmd/minikube/cmd/podman-env.go index a0290d78e9..361186e56a 100644 --- a/cmd/minikube/cmd/podman-env.go +++ b/cmd/minikube/cmd/podman-env.go @@ -29,6 +29,7 @@ import ( "github.com/docker/machine/libmachine/drivers" "github.com/docker/machine/libmachine/ssh" "github.com/spf13/cobra" + "k8s.io/minikube/pkg/drivers/kic/oci" "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/driver" @@ -39,9 +40,27 @@ import ( "k8s.io/minikube/pkg/minikube/shell" ) -var podmanEnv1Tmpl = fmt.Sprintf("{{ .Prefix }}%s{{ .Delimiter }}{{ .VarlinkBridge }}{{ .Suffix }}{{ .Prefix }}%s{{ .Delimiter }}{{ .MinikubePodmanProfile }}{{ .Suffix }}{{ .UsageHint }}", constants.PodmanVarlinkBridgeEnv, constants.MinikubeActivePodmanEnv) +var podmanEnv1Tmpl = fmt.Sprintf( + "{{ .Prefix }}%s{{ .Delimiter }}{{ .VarlinkBridge }}{{ .Suffix }}"+ + "{{ .Prefix }}%s{{ .Delimiter }}{{ .MinikubePodmanProfile }}{{ .Suffix }}"+ + "{{ .UsageHint }}", + constants.PodmanVarlinkBridgeEnv, + constants.MinikubeActivePodmanEnv) -var podmanEnv2Tmpl = fmt.Sprintf("{{ .Prefix }}%s{{ .Delimiter }}{{ .ContainerHost }}{{ .Suffix }}{{ if .ContainerSSHKey }}{{ .Prefix }}%s{{ .Delimiter }}{{ .ContainerSSHKey}}{{ .Suffix }}{{ end }}{{ .Prefix }}%s{{ .Delimiter }}{{ .MinikubePodmanProfile }}{{ .Suffix }}{{ .UsageHint }}", constants.PodmanContainerHostEnv, constants.PodmanContainerSSHKeyEnv, constants.MinikubeActivePodmanEnv) +var podmanEnv2Tmpl = fmt.Sprintf( + "{{ .Prefix }}%s{{ .Delimiter }}{{ .ContainerHost }}{{ .Suffix }}"+ + "{{ if .ContainerSSHKey }}"+ + "{{ .Prefix }}%s{{ .Delimiter }}{{ .ContainerSSHKey}}{{ .Suffix }}"+ + "{{ end }}"+ + "{{ if .ExistingContainerHost }}"+ + "{{ .Prefix }}%s{{ .Delimiter }}{{ .ExistingContainerHost }}{{ .Suffix }}"+ + "{{ end }}"+ + "{{ .Prefix }}%s{{ .Delimiter }}{{ .MinikubePodmanProfile }}{{ .Suffix }}"+ + "{{ .UsageHint }}", + constants.PodmanContainerHostEnv, + constants.PodmanContainerSSHKeyEnv, + constants.ExistingContainerHostEnv, + constants.MinikubeActivePodmanEnv) // PodmanShellConfig represents the shell config for Podman type PodmanShellConfig struct { @@ -50,6 +69,8 @@ type PodmanShellConfig struct { ContainerHost string ContainerSSHKey string MinikubePodmanProfile string + + ExistingContainerHost string } var podmanUnset bool @@ -65,6 +86,9 @@ func podmanShellCfgSet(ec PodmanEnvConfig, envMap map[string]string) *PodmanShel s.VarlinkBridge = envMap[constants.PodmanVarlinkBridgeEnv] s.ContainerHost = envMap[constants.PodmanContainerHostEnv] s.ContainerSSHKey = envMap[constants.PodmanContainerSSHKeyEnv] + + s.ExistingContainerHost = envMap[constants.ExistingContainerHostEnv] + s.MinikubePodmanProfile = envMap[constants.MinikubeActivePodmanEnv] return s @@ -259,6 +283,13 @@ func podmanEnvVars(ec PodmanEnvConfig) map[string]string { for k, v := range env0 { env[k] = v } + if os.Getenv(constants.MinikubeActivePodmanEnv) == "" { + e := constants.PodmanContainerHostEnv + if v := oci.InitialEnv(e); v != "" { + key := constants.ExistingContainerHostEnv + env[key] = v + } + } return env } diff --git a/pkg/drivers/kic/oci/env.go b/pkg/drivers/kic/oci/env.go index 8bedcf4d0c..0cc3ab4e12 100644 --- a/pkg/drivers/kic/oci/env.go +++ b/pkg/drivers/kic/oci/env.go @@ -25,6 +25,7 @@ import ( var initialEnvs = make(map[string]string) func init() { + // docker for _, env := range constants.DockerDaemonEnvs { if v, set := os.LookupEnv(env); set { initialEnvs[env] = v @@ -34,6 +35,15 @@ func init() { initialEnvs[exEnv] = v } } + // podman + env := constants.PodmanContainerHostEnv + if v, set := os.LookupEnv(env); set { + initialEnvs[env] = v + } + exEnv := constants.ExistingContainerHostEnv + if v, set := os.LookupEnv(exEnv); set { + initialEnvs[exEnv] = v + } } // InitialEnv returns the value of the environment variable env before any environment changes made by minikube diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index 1b3e794bed..5853a5893a 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -93,6 +93,9 @@ const ( ExistingDockerCertPathEnv = MinikubeExistingPrefix + "DOCKER_CERT_PATH" // ExistingDockerTLSVerifyEnv is used to save original docker environment ExistingDockerTLSVerifyEnv = MinikubeExistingPrefix + "DOCKER_TLS_VERIFY" + + // ExistingContainerHostEnv is used to save original podman environment + ExistingContainerHostEnv = MinikubeExistingPrefix + "CONTAINER_HOST" ) var ( From a2bc2fe64cafcae02a6a24f87ae0bbc6d36afd18 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Mon, 30 Nov 2020 10:24:15 -0800 Subject: [PATCH 19/57] Enable tracing when collecting metrics --- hack/metrics/README.md | 8 ++++++++ hack/metrics/metrics.go | 13 +++++++------ hack/metrics/minikube.go | 2 +- pkg/trace/gcp.go | 7 ++++--- 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/hack/metrics/README.md b/hack/metrics/README.md index 68df5ac838..03dd6187bd 100644 --- a/hack/metrics/README.md +++ b/hack/metrics/README.md @@ -1,4 +1,12 @@ This script runs `minikube start` in a loop and measures how long it takes. It exports this data to Stackdriver via the OpenTelemetry API. +To run this script, run: + +``` +MINIKUBE_GCP_PROJECT_ID= go run hack/metrics/*.go +``` + This script is used to track minikube performance and prevent regressions. + +_Note: this script will export data to both Cloud Monitoring and Cloud Trace in the provided GCP project_ diff --git a/hack/metrics/metrics.go b/hack/metrics/metrics.go index 142d3e2013..363511b6a8 100644 --- a/hack/metrics/metrics.go +++ b/hack/metrics/metrics.go @@ -31,10 +31,10 @@ import ( "go.opencensus.io/stats/view" "go.opencensus.io/tag" "go.opencensus.io/trace" + pkgtrace "k8s.io/minikube/pkg/trace" ) const ( - projectEnvVar = "MINIKUBE_GCP_PROJECT_ID" customMetricName = "custom.googleapis.com/minikube/start_time" profile = "cloud-monitoring" ) @@ -52,9 +52,9 @@ func main() { } func execute() error { - projectID := os.Getenv(projectEnvVar) + projectID := os.Getenv(pkgtrace.ProjectEnvVar) if projectID == "" { - return fmt.Errorf("metrics collector requires a valid GCP project id set via the %s env variable", projectEnvVar) + return fmt.Errorf("metrics collector requires a valid GCP project id set via the %s env variable", pkgtrace.ProjectEnvVar) } osMethod, err := tag.NewKey("os") @@ -98,7 +98,7 @@ func execute() error { defer sd.StopMetricsExporter() ctx := context.Background() for { - st, err := minikubeStartTime(ctx) + st, err := minikubeStartTime(ctx, projectID) if err != nil { log.Printf("error collecting start time: %v", err) continue @@ -109,7 +109,7 @@ func execute() error { } } -func minikubeStartTime(ctx context.Context) (float64, error) { +func minikubeStartTime(ctx context.Context, projectID string) (float64, error) { minikubePath, err := downloadMinikube() if err != nil { return 0, errors.Wrap(err, "downloading minikube") @@ -117,7 +117,8 @@ func minikubeStartTime(ctx context.Context) (float64, error) { defer os.Remove(minikubePath) defer deleteMinikube(ctx, minikubePath) - cmd := exec.CommandContext(ctx, minikubePath, "start", "--driver=docker", "-p", profile, "--memory=2000") + cmd := exec.CommandContext(ctx, minikubePath, "start", "--driver=docker", "-p", profile, "--memory=2000", "--trace=gcp") + cmd.Env = append(os.Environ(), fmt.Sprintf("%s=%s", pkgtrace.ProjectEnvVar, projectID)) cmd.Stdout = os.Stderr cmd.Stderr = os.Stderr diff --git a/hack/metrics/minikube.go b/hack/metrics/minikube.go index d6b7adf86c..16086873e5 100644 --- a/hack/metrics/minikube.go +++ b/hack/metrics/minikube.go @@ -30,7 +30,7 @@ import ( ) const ( - bucketName = "priya-test-bucket/latest" + bucketName = "minikube/latest" ) // download minikube latest to a tmp file diff --git a/pkg/trace/gcp.go b/pkg/trace/gcp.go index 8a4bf57b0d..212d198fb5 100644 --- a/pkg/trace/gcp.go +++ b/pkg/trace/gcp.go @@ -31,7 +31,8 @@ import ( ) const ( - projectEnvVar = "MINIKUBE_GCP_PROJECT_ID" + // ProjectEnvVar is the name of the env variable that the user must pass in their GCP project ID through + ProjectEnvVar = "MINIKUBE_GCP_PROJECT_ID" // this is the name of the parent span to help identify it // in the Cloud Trace UI. parentSpanName = "minikube start" @@ -72,9 +73,9 @@ func (t *gcpTracer) Cleanup() { } func initGCPTracer() (*gcpTracer, error) { - projectID := os.Getenv(projectEnvVar) + projectID := os.Getenv(ProjectEnvVar) if projectID == "" { - return nil, fmt.Errorf("GCP tracer requires a valid GCP project id set via the %s env variable", projectEnvVar) + return nil, fmt.Errorf("GCP tracer requires a valid GCP project id set via the %s env variable", ProjectEnvVar) } _, flush, err := texporter.InstallNewPipeline( From 2b43ff99899ce1fdbf47c7280276b5d15d1db929 Mon Sep 17 00:00:00 2001 From: Steven Powell Date: Mon, 30 Nov 2020 15:46:05 -0700 Subject: [PATCH 20/57] Auto detect user OS and default tabs to it --- site/content/en/docs/drivers/vmware.md | 12 ++++---- site/content/en/docs/handbook/pushing.md | 12 ++++---- site/content/en/docs/start/_index.md | 12 ++++---- site/layouts/shortcodes/linuxtab.html | 3 ++ site/layouts/shortcodes/mactab.html | 3 ++ site/layouts/shortcodes/windowstab.html | 3 ++ site/static/js/tabs.js | 36 ++++++++++++++++++------ 7 files changed, 54 insertions(+), 27 deletions(-) create mode 100644 site/layouts/shortcodes/linuxtab.html create mode 100644 site/layouts/shortcodes/mactab.html create mode 100644 site/layouts/shortcodes/windowstab.html diff --git a/site/content/en/docs/drivers/vmware.md b/site/content/en/docs/drivers/vmware.md index 7351904e4e..81b2ba3010 100644 --- a/site/content/en/docs/drivers/vmware.md +++ b/site/content/en/docs/drivers/vmware.md @@ -10,15 +10,15 @@ aliases: The vmware driver supports virtualization across all VMware based hypervisors. {{% tabs %}} -{{% tab "macOS" %}} +{{% mactab %}} {{% readfile file="/docs/drivers/includes/vmware_macos_usage.inc" %}} -{{% /tab %}} -{{% tab "Linux" %}} +{{% /mactab %}} +{{% linuxtab %}} No documentation is available yet. -{{% /tab %}} -{{% tab "Windows" %}} +{{% /linuxtab %}} +{{% windowstab %}} No documentation is available yet. -{{% /tab %}} +{{% /windowstab %}} {{% /tabs %}} ## Issues diff --git a/site/content/en/docs/handbook/pushing.md b/site/content/en/docs/handbook/pushing.md index a7419a58b2..5a43517bdc 100644 --- a/site/content/en/docs/handbook/pushing.md +++ b/site/content/en/docs/handbook/pushing.md @@ -136,7 +136,7 @@ eval $(minikube podman-env) You should now be able to use podman client on the command line on your host machine talking to the podman service inside the minikube VM: {{% tabs %}} -{{% tab "Linux" %}} +{{% linuxtab %}} ```shell podman-remote help @@ -146,8 +146,8 @@ podman-remote help Note: On Linux the remote client is called "podman-remote", while the local program is called "podman". {{% /pageinfo %}} -{{% /tab %}} -{{% tab "macOS" %}} +{{% /linuxtab %}} +{{% mactab %}} ```shell podman help @@ -157,8 +157,8 @@ podman help Note: On macOS the remote client is called "podman", since there is no local "podman" program available. {{% /pageinfo %}} -{{% /tab %}} -{{% tab "Windows" %}} +{{% /mactab %}} +{{% windowstab %}} ```shell podman help @@ -168,7 +168,7 @@ podman help Note: On Windows the remote client is called "podman", since there is no local "podman" program available. {{% /pageinfo %}} -{{% /tab %}} +{{% /windowstab %}} {{% /tabs %}} diff --git a/site/content/en/docs/start/_index.md b/site/content/en/docs/start/_index.md index d3ede45804..b15b010c7b 100644 --- a/site/content/en/docs/start/_index.md +++ b/site/content/en/docs/start/_index.md @@ -22,7 +22,7 @@ All you need is Docker (or similarly compatible) container or a Virtual Machine

1Installation

{{% tabs %}} -{{% tab "Linux" %}} +{{% linuxtab %}} For Linux users, we provide 3 easy download options: @@ -47,8 +47,8 @@ curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-latest sudo rpm -ivh minikube-latest.x86_64.rpm ``` -{{% /tab %}} -{{% tab "macOS" %}} +{{% /linuxtab %}} +{{% mactab %}} If the [Brew Package Manager](https://brew.sh/) installed: @@ -70,8 +70,8 @@ curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-darwin sudo install minikube-darwin-amd64 /usr/local/bin/minikube ``` -{{% /tab %}} -{{% tab "Windows" %}} +{{% /mactab %}} +{{% windowstab %}} If the [Chocolatey Package Manager](https://chocolatey.org/) is installed, use it to install minikube: @@ -81,7 +81,7 @@ choco install minikube Otherwise, download and run the [Windows installer](https://storage.googleapis.com/minikube/releases/latest/minikube-installer.exe) -{{% /tab %}} +{{% /windowstab %}} {{% /tabs %}}

2Start your cluster

diff --git a/site/layouts/shortcodes/linuxtab.html b/site/layouts/shortcodes/linuxtab.html new file mode 100644 index 0000000000..2e229e4949 --- /dev/null +++ b/site/layouts/shortcodes/linuxtab.html @@ -0,0 +1,3 @@ +
+ {{ .Inner }} +
diff --git a/site/layouts/shortcodes/mactab.html b/site/layouts/shortcodes/mactab.html new file mode 100644 index 0000000000..7e8d3c75eb --- /dev/null +++ b/site/layouts/shortcodes/mactab.html @@ -0,0 +1,3 @@ +
+ {{ .Inner }} +
diff --git a/site/layouts/shortcodes/windowstab.html b/site/layouts/shortcodes/windowstab.html new file mode 100644 index 0000000000..bb9e9a1ada --- /dev/null +++ b/site/layouts/shortcodes/windowstab.html @@ -0,0 +1,3 @@ +
+ {{ .Inner }} +
diff --git a/site/static/js/tabs.js b/site/static/js/tabs.js index 06ae6a5562..88800d84d0 100644 --- a/site/static/js/tabs.js +++ b/site/static/js/tabs.js @@ -2,18 +2,21 @@ function initTabs() { $('.tab-content').find('.tab-pane').each(function(idx, item) { var navTabs = $(this).closest('.code-tabs').find('.nav-tabs'), - title = $(this).attr('title'); - navTabs.append('
  • '+title+'
    • '); + title = $(this).attr('title'), + os = $(this).attr('os'); + navTabs.append('
  • '+title+'
    • '); }); - + $('.code-tabs ul.nav-tabs').each(function() { - $(this).find("li:first").addClass('active'); - }) - - $('.code-tabs .tab-content').each(function() { - $(this).find("div:first").addClass('active'); + let tabSelector = getTabSelector(this); + $(this).find('li'+tabSelector).addClass('active'); }); - + + $('.code-tabs .tab-content').each(function() { + let tabSelector = getTabSelector(this); + $(this).find('div'+tabSelector).addClass('active'); + }) + $('.nav-tabs a').click(function(e){ e.preventDefault(); var tab = $(this).parent(), @@ -25,3 +28,18 @@ function initTabs() { tabPane.addClass('active'); }); } + +const getTabSelector = currElement => { + let osSelector = '.'+getUserOS(); + let hasMatchingOSTab = $(currElement).find(osSelector).length; + return hasMatchingOSTab ? osSelector : ':first'; +} + +const getUserOS = () => { + let os = ['Linux', 'Mac', 'Windows']; + let userAgent = navigator.userAgent; + for (let currentOS of os) { + if (userAgent.indexOf(currentOS) !== -1) return currentOS; + } + return 'Linux'; +} From e1aa07feb2e878bdf6719b1740724219b29b3f55 Mon Sep 17 00:00:00 2001 From: msedzins Date: Mon, 30 Nov 2020 22:53:27 +0000 Subject: [PATCH 21/57] Ability to use a custom TLS certificate with the Ingress Tutorial added --- cmd/minikube/cmd/config/configure.go | 17 +++++++ deploy/addons/ingress/ingress-dp.yaml.tmpl | 3 ++ pkg/minikube/assets/addons.go | 2 + pkg/minikube/config/types.go | 1 + .../en/docs/tutorials/custom_cert_ingress.md | 44 +++++++++++++++++++ 5 files changed, 67 insertions(+) create mode 100644 site/content/en/docs/tutorials/custom_cert_ingress.md diff --git a/cmd/minikube/cmd/config/configure.go b/cmd/minikube/cmd/config/configure.go index 5b4d5301b6..b5800cf2fb 100644 --- a/cmd/minikube/cmd/config/configure.go +++ b/cmd/minikube/cmd/config/configure.go @@ -19,6 +19,7 @@ package config import ( "io/ioutil" "net" + "regexp" "github.com/spf13/cobra" "k8s.io/minikube/pkg/minikube/config" @@ -204,6 +205,22 @@ var addonsConfigureCmd = &cobra.Command{ cfg.KubernetesConfig.LoadBalancerEndIP = AskForStaticValidatedValue("-- Enter Load Balancer End IP: ", validator) } + if err := config.SaveProfile(profile, cfg); err != nil { + out.ErrT(style.Fatal, "Failed to save config {{.profile}}", out.V{"profile": profile}) + } + case "ingress": + profile := ClusterFlagValue() + _, cfg := mustload.Partial(profile) + + validator := func(s string) bool { + format := regexp.MustCompile("^.+/.+$") + return format.MatchString(s) + } + + if cfg.KubernetesConfig.CustomIngressCert == "" { + cfg.KubernetesConfig.CustomIngressCert = AskForStaticValidatedValue("-- Enter custom cert(format is \"namespace/secret\"): ", validator) + } + if err := config.SaveProfile(profile, cfg); err != nil { out.ErrT(style.Fatal, "Failed to save config {{.profile}}", out.V{"profile": profile}) } diff --git a/deploy/addons/ingress/ingress-dp.yaml.tmpl b/deploy/addons/ingress/ingress-dp.yaml.tmpl index 56f9dc3443..fda40ff2ba 100644 --- a/deploy/addons/ingress/ingress-dp.yaml.tmpl +++ b/deploy/addons/ingress/ingress-dp.yaml.tmpl @@ -65,6 +65,9 @@ spec: - --validating-webhook=:8443 - --validating-webhook-certificate=/usr/local/certificates/cert - --validating-webhook-key=/usr/local/certificates/key + {{if .CustomIngressCert}} + - --default-ssl-certificate={{ .CustomIngressCert }} + {{end}} securityContext: capabilities: drop: diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go index 609b103818..85aa09d225 100644 --- a/pkg/minikube/assets/addons.go +++ b/pkg/minikube/assets/addons.go @@ -489,6 +489,7 @@ func GenerateTemplateData(cfg config.KubernetesConfig) interface{} { ImageRepository string LoadBalancerStartIP string LoadBalancerEndIP string + CustomIngressCert string StorageProvisionerVersion string }{ Arch: a, @@ -496,6 +497,7 @@ func GenerateTemplateData(cfg config.KubernetesConfig) interface{} { ImageRepository: cfg.ImageRepository, LoadBalancerStartIP: cfg.LoadBalancerStartIP, LoadBalancerEndIP: cfg.LoadBalancerEndIP, + CustomIngressCert: cfg.CustomIngressCert, StorageProvisionerVersion: version.GetStorageProvisionerVersion(), } diff --git a/pkg/minikube/config/types.go b/pkg/minikube/config/types.go index 1d19604bc4..d95e9f7c85 100644 --- a/pkg/minikube/config/types.go +++ b/pkg/minikube/config/types.go @@ -92,6 +92,7 @@ type KubernetesConfig struct { ImageRepository string LoadBalancerStartIP string // currently only used by MetalLB addon LoadBalancerEndIP string // currently only used by MetalLB addon + CustomIngressCert string // used by Ingress addon ExtraOptions ExtraOptionSlice ShouldLoadCachedImages bool diff --git a/site/content/en/docs/tutorials/custom_cert_ingress.md b/site/content/en/docs/tutorials/custom_cert_ingress.md new file mode 100644 index 0000000000..1f419e6207 --- /dev/null +++ b/site/content/en/docs/tutorials/custom_cert_ingress.md @@ -0,0 +1,44 @@ +--- +title: "How to use custom TLS certificate with ingress addon" +linkTitle: "Using custom TLS certificate with ingress addon" +weight: 1 +date: 2020-11-30 +--- + +## Overview + +- This tutorial will show you how to configure custom TLS certificatate for ingress addon. + +## Tutorial + +- Start minikube +``` +$ minikube start +``` + +- Create TLS secret which contains custom certificate and private key +``` +$ kubectl -n kube-system create secret tls mkcert --key key.pem --cert cert.pem +``` + +- Configure ingress addon +``` +$ minikube addons configure ingress +-- Enter custom cert(format is "namespace/secret"): kube-system/mkcert +✅ ingress was successfully configured +``` + +- Enable ingress addon (disable first when already enabled) +``` +$ minikube addons disable ingress +🌑 "The 'ingress' addon is disabled + +$ minikube addons enable ingress +🔎 Verifying ingress addon... +🌟 The 'ingress' addon is enabled +``` +- Verify if custom certificate was enabled +``` +$ kubectl -n kube-system get deployment ingress-nginx-controller -o yaml | grep "kube-system" +- --default-ssl-certificate=kube-system/mkcert +``` \ No newline at end of file From 2a11844320af6d827b0573c40d56af76b5373bec Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Mon, 30 Nov 2020 17:01:15 -0800 Subject: [PATCH 22/57] Only rebuild latest if commit has changed Also store commit in the metadata of binaries in GCS so it's easy to figure out what commit they were built at. --- hack/jenkins/build/minikube_build_upload_latest.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/hack/jenkins/build/minikube_build_upload_latest.sh b/hack/jenkins/build/minikube_build_upload_latest.sh index b311e0e8ad..b6b50cc86c 100755 --- a/hack/jenkins/build/minikube_build_upload_latest.sh +++ b/hack/jenkins/build/minikube_build_upload_latest.sh @@ -28,9 +28,18 @@ WANT_GOLANG_VERSION=$(grep '^GO_VERSION' Makefile | awk '{ print $3 }') declare -rx GOPATH=/var/lib/jenkins/go +GIT_COMMIT_AT_HEAD=$(git rev-parse HEAD | xargs) +MINIKUBE_LATEST_COMMIT=$(gsutil stat gs://minikube/latest/minikube-linux-amd64 | grep commit | awk '{ print $2 }' | xargs) + +if [ "$GIT_COMMIT_AT_HEAD" = "$MINIKUBE_LATEST_COMMIT" ]; then + echo "The current uploaded binary is already latest, skipping build" + exit 0 +fi + make cross && failed=$? || failed=$? if [[ "${failed}" -ne 0 ]]; then echo "build failed" exit "${failed}" fi gsutil cp out/minikube-* "gs://${bucket}" +gsutil setmeta -r -h "x-goog-meta-commit:$GIT_COMMIT_AT_HEAD" "gs://${bucket}" From 8314c631f547497459dbde129ad3e2b916488f32 Mon Sep 17 00:00:00 2001 From: edtrist Date: Tue, 1 Dec 2020 02:02:08 +0000 Subject: [PATCH 23/57] Update Docsy theme to latest version --- site/themes/docsy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/themes/docsy b/site/themes/docsy index dd303fd19c..2536303cad 160000 --- a/site/themes/docsy +++ b/site/themes/docsy @@ -1 +1 @@ -Subproject commit dd303fd19cc13ffc01bcbe86ff54f21e423d04de +Subproject commit 2536303cad19991c673037f4f16332075141364a From ae4fa9a2ee25a11e1c125987dcd5b1d9c839fee2 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Mon, 30 Nov 2020 19:03:30 -0800 Subject: [PATCH 24/57] Address code review comments` --- hack/metrics/metrics.go | 26 +++++++++++----- hack/metrics/minikube.go | 67 +++++++++++++++++++++++++++------------- 2 files changed, 65 insertions(+), 28 deletions(-) diff --git a/hack/metrics/metrics.go b/hack/metrics/metrics.go index 363511b6a8..cd0c2ded93 100644 --- a/hack/metrics/metrics.go +++ b/hack/metrics/metrics.go @@ -19,6 +19,7 @@ package main import ( "context" "fmt" + "io/ioutil" "log" "os" "os/exec" @@ -97,8 +98,20 @@ func execute() error { } defer sd.StopMetricsExporter() ctx := context.Background() + + // create a tmp file to download minikube to + tmp, err := ioutil.TempFile("", binary()) + if err != nil { + return errors.Wrap(err, "creating tmp file") + } + if err := tmp.Close(); err != nil { + return errors.Wrap(err, "closing file") + } + defer os.Remove(tmp.Name()) + + // start loop where we will track minikube start time for { - st, err := minikubeStartTime(ctx, projectID) + st, err := minikubeStartTime(ctx, projectID, tmp.Name()) if err != nil { log.Printf("error collecting start time: %v", err) continue @@ -109,12 +122,10 @@ func execute() error { } } -func minikubeStartTime(ctx context.Context, projectID string) (float64, error) { - minikubePath, err := downloadMinikube() - if err != nil { +func minikubeStartTime(ctx context.Context, projectID, minikubePath string) (float64, error) { + if err := downloadMinikube(ctx, minikubePath); err != nil { return 0, errors.Wrap(err, "downloading minikube") } - defer os.Remove(minikubePath) defer deleteMinikube(ctx, minikubePath) cmd := exec.CommandContext(ctx, minikubePath, "start", "--driver=docker", "-p", profile, "--memory=2000", "--trace=gcp") @@ -123,11 +134,12 @@ func minikubeStartTime(ctx context.Context, projectID string) (float64, error) { cmd.Stderr = os.Stderr t := time.Now() - log.Print("Running minikube start....") + log.Printf("Running [%v]....", cmd.Args) if err := cmd.Run(); err != nil { return 0, errors.Wrapf(err, "running %v", cmd.Args) } - return time.Since(t).Seconds(), nil + totalTime := time.Since(t).Seconds() + return totalTime, nil } func deleteMinikube(ctx context.Context, minikubePath string) { diff --git a/hack/metrics/minikube.go b/hack/metrics/minikube.go index 16086873e5..d8c6149ea7 100644 --- a/hack/metrics/minikube.go +++ b/hack/metrics/minikube.go @@ -21,7 +21,9 @@ import ( "fmt" "io/ioutil" "os" + "os/exec" "runtime" + "strings" "log" @@ -33,39 +35,62 @@ const ( bucketName = "minikube/latest" ) -// download minikube latest to a tmp file -func downloadMinikube() (string, error) { - b := binary() - tmp, err := ioutil.TempFile("", b) +// download minikube latest to file +func downloadMinikube(ctx context.Context, minikubePath string) error { + client, err := storage.NewClient(ctx) if err != nil { - return "", errors.Wrap(err, "creating tmp file") + return errors.Wrap(err, "creating client") } - if err := tmp.Close(); err != nil { - return "", errors.Wrap(err, "closing tmp file") + + if localMinikubeIsLatest(ctx, minikubePath, client) { + log.Print("local minikube is latest, skipping download...") + return nil } - client, err := storage.NewClient(context.Background()) + + os.Remove(minikubePath) + // download minikube binary from GCS + obj := client.Bucket("minikube").Object(fmt.Sprintf("latest/%s", binary())) + rc, err := obj.NewReader(ctx) if err != nil { - return "", errors.Wrap(err, "creating client") - } - ctx := context.Background() - rc, err := client.Bucket(bucketName).Object(b).NewReader(ctx) - if err != nil { - return "", errors.Wrap(err, "gcs new reader") + return errors.Wrap(err, "gcs new reader") } defer rc.Close() data, err := ioutil.ReadAll(rc) if err != nil { - return "", errors.Wrap(err, "ioutil read all") + return errors.Wrap(err, "ioutil read all") } - log.Printf("downloading gs://%s/%s to %v", bucketName, b, tmp.Name()) - if err := ioutil.WriteFile(tmp.Name(), data, 0777); err != nil { - return "", errors.Wrap(err, "writing file") + log.Printf("downloading gs://%s/%s to %v", bucketName, binary(), minikubePath) + if err := ioutil.WriteFile(minikubePath, data, 0777); err != nil { + return errors.Wrap(err, "writing minikubePath") } - if err := os.Chmod(tmp.Name(), 0700); err != nil { - return "", errors.Wrap(err, "chmod") + if err := os.Chmod(minikubePath, 0700); err != nil { + return errors.Wrap(err, "chmod") } - return tmp.Name(), nil + return nil +} + +// localMinikubeIsLatest returns true if the local version of minikube +// matches the latest version in GCS +func localMinikubeIsLatest(ctx context.Context, minikubePath string, client *storage.Client) bool { + log.Print("checking if local minikube is latest...") + obj := client.Bucket("minikube").Object(fmt.Sprintf("latest/%s", binary())) + attrs, err := obj.Attrs(ctx) + if err != nil { + log.Printf("error getting %s object attrs: %v", obj.ObjectName(), err) + return false + } + gcsMinikubeVersion, ok := attrs.Metadata["commit"] + if !ok { + log.Printf("there is no commit: %v", attrs.Metadata) + return false + } + currentMinikubeVersion, err := exec.Command(minikubePath, "version", "--output=json").Output() + if err != nil { + log.Printf("error running [%s version]: %v", minikubePath, err) + return false + } + return strings.Contains(string(currentMinikubeVersion), gcsMinikubeVersion) } func binary() string { From 8590560f838617b8ce6bfb6601bdd16ec0632f52 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Mon, 30 Nov 2020 19:21:36 -0800 Subject: [PATCH 25/57] use same storage.ObjectHandle for downloading file and getting commmit --- hack/metrics/minikube.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/hack/metrics/minikube.go b/hack/metrics/minikube.go index d8c6149ea7..5b9d649205 100644 --- a/hack/metrics/minikube.go +++ b/hack/metrics/minikube.go @@ -32,7 +32,7 @@ import ( ) const ( - bucketName = "minikube/latest" + bucketName = "minikube" ) // download minikube latest to file @@ -41,15 +41,15 @@ func downloadMinikube(ctx context.Context, minikubePath string) error { if err != nil { return errors.Wrap(err, "creating client") } + obj := client.Bucket("minikube").Object(fmt.Sprintf("latest/%s", binary())) - if localMinikubeIsLatest(ctx, minikubePath, client) { + if localMinikubeIsLatest(ctx, minikubePath, obj) { log.Print("local minikube is latest, skipping download...") return nil } os.Remove(minikubePath) // download minikube binary from GCS - obj := client.Bucket("minikube").Object(fmt.Sprintf("latest/%s", binary())) rc, err := obj.NewReader(ctx) if err != nil { return errors.Wrap(err, "gcs new reader") @@ -72,9 +72,8 @@ func downloadMinikube(ctx context.Context, minikubePath string) error { // localMinikubeIsLatest returns true if the local version of minikube // matches the latest version in GCS -func localMinikubeIsLatest(ctx context.Context, minikubePath string, client *storage.Client) bool { +func localMinikubeIsLatest(ctx context.Context, minikubePath string, obj *storage.ObjectHandle) bool { log.Print("checking if local minikube is latest...") - obj := client.Bucket("minikube").Object(fmt.Sprintf("latest/%s", binary())) attrs, err := obj.Attrs(ctx) if err != nil { log.Printf("error getting %s object attrs: %v", obj.ObjectName(), err) From 71de27fb756f67453109dbab5d580b00a4b91f72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Tue, 1 Dec 2020 17:11:08 +0100 Subject: [PATCH 26/57] Only recommend virtualbox when running on amd64 Remove the "recommendation" from the arm64 deb --- Makefile | 5 +++++ installers/linux/deb/minikube_deb_template/DEBIAN/control | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 836d7f78b8..03d081c1cd 100644 --- a/Makefile +++ b/Makefile @@ -462,6 +462,11 @@ out/minikube_$(DEB_VERSION)-0_%.deb: out/minikube-linux-% chmod 0755 out/minikube_$(DEB_VERSION)/DEBIAN sed -E -i 's/--VERSION--/'$(DEB_VERSION)'/g' out/minikube_$(DEB_VERSION)/DEBIAN/control sed -E -i 's/--ARCH--/'$*'/g' out/minikube_$(DEB_VERSION)/DEBIAN/control + if [ "$*" = "amd64" ]; then \ + sed -E -i 's/--RECOMMENDS--/virtualbox/' out/minikube_$(DEB_VERSION)/DEBIAN/control; \ + else \ + sed -E -i '/Recommends: --RECOMMENDS--/d' out/minikube_$(DEB_VERSION)/DEBIAN/control; \ + fi mkdir -p out/minikube_$(DEB_VERSION)/usr/bin cp $< out/minikube_$(DEB_VERSION)/usr/bin/minikube fakeroot dpkg-deb --build out/minikube_$(DEB_VERSION) $@ diff --git a/installers/linux/deb/minikube_deb_template/DEBIAN/control b/installers/linux/deb/minikube_deb_template/DEBIAN/control index ad9d368ba0..3b29fe8094 100644 --- a/installers/linux/deb/minikube_deb_template/DEBIAN/control +++ b/installers/linux/deb/minikube_deb_template/DEBIAN/control @@ -3,7 +3,7 @@ Version: --VERSION-- Section: base Priority: optional Architecture: --ARCH-- -Recommends: virtualbox +Recommends: --RECOMMENDS-- Maintainer: Thomas Strömberg Description: Minikube minikube is a tool that makes it easy to run Kubernetes locally. From 01ff1deb7465493e5c016b7f0b875836c8b20422 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Tue, 1 Dec 2020 17:27:48 +0100 Subject: [PATCH 27/57] Upgrade podman to 2.2.0 --- deploy/iso/minikube-iso/package/podman/podman.hash | 1 + deploy/iso/minikube-iso/package/podman/podman.mk | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/deploy/iso/minikube-iso/package/podman/podman.hash b/deploy/iso/minikube-iso/package/podman/podman.hash index 3d5a50be25..04fbbf474d 100644 --- a/deploy/iso/minikube-iso/package/podman/podman.hash +++ b/deploy/iso/minikube-iso/package/podman/podman.hash @@ -1,2 +1,3 @@ sha256 a16846fe076aaf2c9ea2e854c3baba9fb838d916be7fb4b5be332e6c92d907d4 v1.9.3.tar.gz sha256 5ebaa6e0dbd7fd1863f70d2bc71dc8a94e195c3339c17e3cac4560c9ec5747f8 v2.1.1.tar.gz +sha256 ec5473e51fa28f29af323473fc484f742dc7df23d06d8ba9f217f13382893a71 v2.2.0.tar.gz diff --git a/deploy/iso/minikube-iso/package/podman/podman.mk b/deploy/iso/minikube-iso/package/podman/podman.mk index 7c76bcfbcb..f95aeb8929 100644 --- a/deploy/iso/minikube-iso/package/podman/podman.mk +++ b/deploy/iso/minikube-iso/package/podman/podman.mk @@ -1,5 +1,5 @@ -PODMAN_VERSION = v2.1.1 -PODMAN_COMMIT = 9f6d6ba0b314d86521b66183c9ce48eaa2da1de2 +PODMAN_VERSION = v2.2.0 +PODMAN_COMMIT = db1d2ff111ee9b012779ff3a5279a982520ccda4 PODMAN_SITE = https://github.com/containers/podman/archive PODMAN_SOURCE = $(PODMAN_VERSION).tar.gz PODMAN_LICENSE = Apache-2.0 From ee1da89f11cbe42b295c1081c52436368eaebf5d Mon Sep 17 00:00:00 2001 From: edtrist Date: Tue, 1 Dec 2020 18:25:41 +0000 Subject: [PATCH 28/57] Enable Prism code highlighting / copy buttons --- site/config.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/site/config.toml b/site/config.toml index 7a9c6cf087..3e6c1da9db 100644 --- a/site/config.toml +++ b/site/config.toml @@ -92,6 +92,9 @@ github_subdir = "site" # enabling local search https://www.docsy.dev/docs/adding-content/navigation/#configure-local-search-with-lunr offlineSearch = true +# Enable syntax highlighting and copy buttons on code blocks with Prism +prism_syntax_highlighting = true + # User interface configuration [params.ui] # Enable to show the side bar menu in its compact state. From aa64b8bf7fec5daa816ba368225cd8c4ef0a945f Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Tue, 1 Dec 2020 11:47:23 -0800 Subject: [PATCH 29/57] bump gopogh --- .github/workflows/iso.yml | 2 +- .github/workflows/kic_image.yml | 2 +- .github/workflows/master.yml | 22 +++++++++++----------- .github/workflows/pr.yml | 22 +++++++++++----------- hack/jenkins/common.sh | 4 ++-- 5 files changed, 26 insertions(+), 26 deletions(-) diff --git a/.github/workflows/iso.yml b/.github/workflows/iso.yml index 07e03d7197..d06a930eee 100644 --- a/.github/workflows/iso.yml +++ b/.github/workflows/iso.yml @@ -82,7 +82,7 @@ jobs: - name: Install gopogh shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-linux-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64 sudo install gopogh-linux-amd64 /usr/local/bin/gopogh sudo apt-get install -y jq - name: Run Integration Test diff --git a/.github/workflows/kic_image.yml b/.github/workflows/kic_image.yml index b3ddee81b2..cadf209cab 100644 --- a/.github/workflows/kic_image.yml +++ b/.github/workflows/kic_image.yml @@ -60,7 +60,7 @@ jobs: - name: Install gopogh shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-linux-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64 sudo install gopogh-linux-amd64 /usr/local/bin/gopogh sudo apt-get install -y jq rm -f gopogh-linux-amd64 || true diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 73d297e07e..d89d6156bf 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -120,7 +120,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-linux-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64 sudo install gopogh-linux-amd64 /usr/local/bin/gopogh - name: Download Binaries uses: actions/download-artifact@v1 @@ -205,7 +205,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-darwin-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-darwin-amd64 sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh - name: Install docker shell: bash @@ -350,7 +350,7 @@ jobs: continue-on-error: true shell: powershell run: | - (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe") + (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe") choco install -y kubernetes-cli choco install -y jq choco install -y caffeine @@ -487,7 +487,7 @@ jobs: shell: powershell run: | $ErrorActionPreference = "SilentlyContinue" - (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe") + (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe") choco install -y kubernetes-cli choco install -y jq choco install -y caffeine @@ -592,7 +592,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-linux-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64 sudo install gopogh-linux-amd64 /usr/local/bin/gopogh - name: Download Binaries uses: actions/download-artifact@v1 @@ -689,7 +689,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-linux-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64 sudo install gopogh-linux-amd64 /usr/local/bin/gopogh - name: Download Binaries uses: actions/download-artifact@v1 @@ -771,7 +771,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-darwin-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-darwin-amd64 sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh - name: Install docker shell: bash @@ -883,7 +883,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-linux-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64 sudo install gopogh-linux-amd64 /usr/local/bin/gopogh - name: Download Binaries uses: actions/download-artifact@v1 @@ -967,7 +967,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-darwin-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-darwin-amd64 sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh - name: Download Binaries uses: actions/download-artifact@v1 @@ -1074,7 +1074,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-linux-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64 sudo install gopogh-linux-amd64 /usr/local/bin/gopogh - name: Download Binaries uses: actions/download-artifact@v1 @@ -1156,7 +1156,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-darwin-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-darwin-amd64 sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh - name: Download Binaries uses: actions/download-artifact@v1 diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 8e0021e663..c84945b0df 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -118,7 +118,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-linux-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64 sudo install gopogh-linux-amd64 /usr/local/bin/gopogh - name: Download Binaries uses: actions/download-artifact@v1 @@ -203,7 +203,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-darwin-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-darwin-amd64 sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh - name: Install docker shell: bash @@ -348,7 +348,7 @@ jobs: continue-on-error: true shell: powershell run: | - (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe") + (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe") choco install -y kubernetes-cli choco install -y jq choco install -y caffeine @@ -485,7 +485,7 @@ jobs: shell: powershell run: | $ErrorActionPreference = "SilentlyContinue" - (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe") + (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe") choco install -y kubernetes-cli choco install -y jq choco install -y caffeine @@ -590,7 +590,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-linux-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64 sudo install gopogh-linux-amd64 /usr/local/bin/gopogh - name: Download Binaries uses: actions/download-artifact@v1 @@ -687,7 +687,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-linux-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64 sudo install gopogh-linux-amd64 /usr/local/bin/gopogh - name: Download Binaries uses: actions/download-artifact@v1 @@ -769,7 +769,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-darwin-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-darwin-amd64 sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh - name: Install docker shell: bash @@ -881,7 +881,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-linux-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64 sudo install gopogh-linux-amd64 /usr/local/bin/gopogh - name: Download Binaries uses: actions/download-artifact@v1 @@ -965,7 +965,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-darwin-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-darwin-amd64 sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh - name: Download Binaries uses: actions/download-artifact@v1 @@ -1072,7 +1072,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-linux-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64 sudo install gopogh-linux-amd64 /usr/local/bin/gopogh - name: Download Binaries uses: actions/download-artifact@v1 @@ -1154,7 +1154,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-darwin-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-darwin-amd64 sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh - name: Download Binaries uses: actions/download-artifact@v1 diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh index 0f8cec2a20..aebdb5cea0 100755 --- a/hack/jenkins/common.sh +++ b/hack/jenkins/common.sh @@ -347,9 +347,9 @@ fi echo ">> Installing gopogh" if [ "$(uname)" != "Darwin" ]; then - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-linux-amd64 && sudo install gopogh-linux-amd64 /usr/local/bin/gopogh + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64 && sudo install gopogh-linux-amd64 /usr/local/bin/gopogh else - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-darwin-amd64 && sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-darwin-amd64 && sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh fi echo ">> Running gopogh" From d1410363820008be447956431d6fa91f53cf4030 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Tue, 1 Dec 2020 14:29:09 -0800 Subject: [PATCH 30/57] bump storage provsioner to multi arch --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 836d7f78b8..3d522f8a68 100644 --- a/Makefile +++ b/Makefile @@ -92,7 +92,7 @@ SHA512SUM=$(shell command -v sha512sum || echo "shasum -a 512") GVISOR_TAG ?= latest # storage provisioner tag to push changes to -STORAGE_PROVISIONER_TAG ?= v3 +STORAGE_PROVISIONER_TAG ?= v4 STORAGE_PROVISIONER_MANIFEST ?= $(REGISTRY)/storage-provisioner:$(STORAGE_PROVISIONER_TAG) STORAGE_PROVISIONER_IMAGE ?= $(REGISTRY)/storage-provisioner-$(GOARCH):$(STORAGE_PROVISIONER_TAG) From 2c6dc34f04535dfb19022f46f780b526db3dfed3 Mon Sep 17 00:00:00 2001 From: edtrist Date: Tue, 1 Dec 2020 19:30:51 +0000 Subject: [PATCH 31/57] Styling updates after Docsy update --- site/assets/scss/_variables_project.scss | 16 +++++++++++++++- site/layouts/partials/hooks/head-end.html | 2 +- site/layouts/partials/sidebar-tree.html | 2 +- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/site/assets/scss/_variables_project.scss b/site/assets/scss/_variables_project.scss index 7bc0efdb22..536b3b4a01 100644 --- a/site/assets/scss/_variables_project.scss +++ b/site/assets/scss/_variables_project.scss @@ -180,7 +180,7 @@ section.td-box--height-auto { // Allow code tags to span most of a window length (default is 80%) pre { max-width: 99% !important; - font-family: 'Inconsolata', monospace !important; + font-family: 'SFMono-Regular', Menlo, Monaco, Consolas, 'liberation mono', 'courier new', monospace !important; font-size: 13px !important; } @@ -197,6 +197,17 @@ div.td-content { padding: 0.5em !important; margin-top: 1.25em; margin-bottom: 1.25em; + background-color: $gray-100; + max-width: 99% !important; + + pre { + background-color: inherit !important; + padding: 0 !important; + + code { + font-family: inherit !important; + } + } } } @@ -214,3 +225,6 @@ div.td-content { max-width: 460px; } +div.code-toolbar > .toolbar { + top: -.3em !important; +} diff --git a/site/layouts/partials/hooks/head-end.html b/site/layouts/partials/hooks/head-end.html index ed11ad10a1..bf082de3bd 100644 --- a/site/layouts/partials/hooks/head-end.html +++ b/site/layouts/partials/hooks/head-end.html @@ -1,5 +1,5 @@ - + \ No newline at end of file diff --git a/site/layouts/partials/sidebar-tree.html b/site/layouts/partials/sidebar-tree.html index 029abae4c5..b1527f944f 100644 --- a/site/layouts/partials/sidebar-tree.html +++ b/site/layouts/partials/sidebar-tree.html @@ -12,7 +12,7 @@ {{ end }} -