diff --git a/.travis.yml b/.travis.yml index cd5ca58f96..d2edd1570c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ env: - GOPROXY=https://proxy.golang.org matrix: include: - - go: 1.12.7 + - go: 1.12.9 - language: python before_install: pip install flake8 script: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics diff --git a/Makefile b/Makefile index 1d2e641e1b..1bff47bda8 100755 --- a/Makefile +++ b/Makefile @@ -22,6 +22,9 @@ ISO_VERSION ?= v$(VERSION_MAJOR).$(VERSION_MINOR).0 VERSION ?= v$(VERSION_MAJOR).$(VERSION_MINOR).$(VERSION_BUILD) DEB_VERSION ?= $(VERSION_MAJOR).$(VERSION_MINOR).$(VERSION_BUILD) RPM_VERSION ?= $(VERSION_MAJOR).$(VERSION_MINOR).$(VERSION_BUILD) +# used by hack/jenkins/release_build_and_upload.sh and KVM_BUILD_IMAGE, see also BUILD_IMAGE below +GO_VERSION ?= 1.12.9 + INSTALL_SIZE ?= $(shell du out/minikube-windows-amd64.exe | cut -f1) BUILDROOT_BRANCH ?= 2018.05.3 REGISTRY?=gcr.io/k8s-minikube @@ -31,8 +34,8 @@ COMMIT_NO := $(shell git rev-parse HEAD 2> /dev/null || true) COMMIT ?= $(if $(shell git status --porcelain --untracked-files=no),"${COMMIT_NO}-dirty","${COMMIT_NO}") HYPERKIT_BUILD_IMAGE ?= karalabe/xgo-1.12.x -# NOTE: "latest" as of 2019-07-12. kube-cross images aren't updated as often as Kubernetes -BUILD_IMAGE ?= k8s.gcr.io/kube-cross:v1.12.7-1 +# NOTE: "latest" as of 2019-08-15. kube-cross images aren't updated as often as Kubernetes +BUILD_IMAGE ?= k8s.gcr.io/kube-cross:v$(GO_VERSION)-1 ISO_BUILD_IMAGE ?= $(REGISTRY)/buildroot-image KVM_BUILD_IMAGE ?= $(REGISTRY)/kvm-build-image:$(GO_VERSION) @@ -44,10 +47,7 @@ MINIKUBE_UPLOAD_LOCATION := gs://${MINIKUBE_BUCKET} MINIKUBE_RELEASES_URL=https://github.com/kubernetes/minikube/releases/download KERNEL_VERSION ?= 4.15 - -# Currently *only* used for the KVM_BUILD_IMAGE, see also BUILD_IMAGE above -GO_VERSION ?= 1.12.7 - +# latest from https://github.com/golangci/golangci-lint/releases GOLINT_VERSION ?= v1.17.1 # Limit number of default jobs, to avoid the CI builds running out of memory GOLINT_JOBS ?= 4 @@ -101,7 +101,7 @@ SOURCE_DIRS = $(CMD_SOURCE_DIRS) test SOURCE_PACKAGES = ./cmd/... ./pkg/... ./test/... # kvm2 ldflags -KVM2_LDFLAGS := -X k8s.io/minikube/pkg/drivers/kvm.version=$(VERSION) -X k8s.io/minikube/pkg/drivers/kvm.gitCommitID=$(COMMIT) +KVM2_LDFLAGS := -X k8s.io/minikube/pkg/drivers/kvm.version=$(VERSION) -X k8s.io/minikube/pkg/drivers/kvm.gitCommitID=$(COMMIT) # hyperkit ldflags HYPERKIT_LDFLAGS := -X k8s.io/minikube/pkg/drivers/hyperkit.version=$(VERSION) -X k8s.io/minikube/pkg/drivers/hyperkit.gitCommitID=$(COMMIT) diff --git a/README.md b/README.md index 57fc71e234..3057bfa60d 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ minikube is a Kubernetes [#sig-cluster-lifecycle](https://github.com/kubernetes/ * [**#minikube on Kubernetes Slack**](https://kubernetes.slack.com) - Live chat with minikube developers! * [minikube-users mailing list](https://groups.google.com/forum/#!forum/minikube-users) * [minikube-dev mailing list](https://groups.google.com/forum/#!forum/minikube-dev) -* [Bi-weekly office hours, Mondays @ 10am PST](https://tinyurl.com/minikube-oh) +* [Bi-weekly office hours, Mondays @ 11am PST](https://tinyurl.com/minikube-oh) * [Contributing](https://minikube.sigs.k8s.io/docs/contributing/) * [Development Roadmap](https://minikube.sigs.k8s.io/docs/contributing/roadmap/) diff --git a/cmd/minikube/cmd/cache.go b/cmd/minikube/cmd/cache.go index 8192a9e0d3..b2795e4851 100644 --- a/cmd/minikube/cmd/cache.go +++ b/cmd/minikube/cmd/cache.go @@ -67,7 +67,7 @@ var deleteCacheCmd = &cobra.Command{ } func imagesInConfigFile() ([]string, error) { - configFile, err := config.ReadConfig() + configFile, err := config.ReadConfig(constants.ConfigFile) if err != nil { return nil, err } diff --git a/cmd/minikube/cmd/config/config.go b/cmd/minikube/cmd/config/config.go index 9e3d9d637b..66779daec9 100644 --- a/cmd/minikube/cmd/config/config.go +++ b/cmd/minikube/cmd/config/config.go @@ -290,7 +290,7 @@ func configurableFields() string { // ListConfigMap list entries from config file func ListConfigMap(name string) ([]string, error) { - configFile, err := config.ReadConfig() + configFile, err := config.ReadConfig(constants.ConfigFile) if err != nil { return nil, err } @@ -310,7 +310,7 @@ func AddToConfigMap(name string, images []string) error { return err } // Set the values - cfg, err := config.ReadConfig() + cfg, err := config.ReadConfig(constants.ConfigFile) if err != nil { return err } @@ -337,7 +337,7 @@ func DeleteFromConfigMap(name string, images []string) error { return err } // Set the values - cfg, err := config.ReadConfig() + cfg, err := config.ReadConfig(constants.ConfigFile) if err != nil { return err } diff --git a/cmd/minikube/cmd/config/set.go b/cmd/minikube/cmd/config/set.go index 3019c793e2..c1db6372a4 100644 --- a/cmd/minikube/cmd/config/set.go +++ b/cmd/minikube/cmd/config/set.go @@ -56,7 +56,7 @@ func Set(name string, value string) error { } // Set the value - config, err := pkgConfig.ReadConfig() + config, err := pkgConfig.ReadConfig(constants.ConfigFile) if err != nil { return err } diff --git a/cmd/minikube/cmd/config/unset.go b/cmd/minikube/cmd/config/unset.go index fc91390ed8..ee9cff74f4 100644 --- a/cmd/minikube/cmd/config/unset.go +++ b/cmd/minikube/cmd/config/unset.go @@ -44,7 +44,7 @@ func init() { // Unset unsets a property func Unset(name string) error { - m, err := pkgConfig.ReadConfig() + m, err := pkgConfig.ReadConfig(constants.ConfigFile) if err != nil { return err } diff --git a/cmd/minikube/cmd/config/view.go b/cmd/minikube/cmd/config/view.go index 3ea025e922..33f79e12cd 100644 --- a/cmd/minikube/cmd/config/view.go +++ b/cmd/minikube/cmd/config/view.go @@ -55,7 +55,7 @@ For the list of accessible variables for the template, see the struct values her // View displays the current config func View() error { - cfg, err := config.ReadConfig() + cfg, err := config.ReadConfig(constants.ConfigFile) if err != nil { return err } diff --git a/cmd/minikube/cmd/delete.go b/cmd/minikube/cmd/delete.go index 3921c5dfb8..afaec01344 100644 --- a/cmd/minikube/cmd/delete.go +++ b/cmd/minikube/cmd/delete.go @@ -63,7 +63,7 @@ func runDelete(cmd *cobra.Command, args []string) { cc, err := pkg_config.Load() if err != nil && !os.IsNotExist(err) { - out.ErrT(out.Sad, "Error loading profile config: {{.error}}", out.V{"name": profile}) + out.ErrT(out.Sad, "Error loading profile {{.name}}: {{.error}}", out.V{"name": profile, "error": err}) } // In the case of "none", we want to uninstall Kubernetes as there is no VM to delete @@ -71,27 +71,38 @@ func runDelete(cmd *cobra.Command, args []string) { uninstallKubernetes(api, cc.KubernetesConfig, viper.GetString(cmdcfg.Bootstrapper)) } + if err := killMountProcess(); err != nil { + out.T(out.FailureType, "Failed to kill mount process: {{.error}}", out.V{"error": err}) + } + if err = cluster.DeleteHost(api); err != nil { - switch err := errors.Cause(err).(type) { + switch errors.Cause(err).(type) { case mcnerror.ErrHostDoesNotExist: out.T(out.Meh, `"{{.name}}" cluster does not exist`, out.V{"name": profile}) default: - exit.WithError("Failed to delete cluster", err) + out.T(out.FailureType, "Failed to delete cluster: {{.error}}", out.V{"error": err}) + out.T(out.Notice, `You may need to manually remove the "{{.name}}" VM from your hypervisor`, out.V{"name": profile}) } } - if err := killMountProcess(); err != nil { - out.FatalT("Failed to kill mount process: {{.error}}", out.V{"error": err}) + // In case DeleteHost didn't complete the job. + machineDir := filepath.Join(constants.GetMinipath(), "machines", profile) + if _, err := os.Stat(machineDir); err == nil { + out.T(out.DeletingHost, `Removing {{.directory}} ...`, out.V{"directory": machineDir}) + err := os.RemoveAll(machineDir) + if err != nil { + exit.WithError("Unable to remove machine directory: %v", err) + } } - if err := pkg_config.DeleteProfile(viper.GetString(pkg_config.MachineProfile)); err != nil { + if err := pkg_config.DeleteProfile(profile); err != nil { if os.IsNotExist(err) { - out.T(out.Meh, `"{{.profile_name}}" profile does not exist`, out.V{"profile_name": profile}) + out.T(out.Meh, `"{{.name}}" profile does not exist`, out.V{"name": profile}) os.Exit(0) } exit.WithError("Failed to remove profile", err) } - out.T(out.Crushed, `The "{{.cluster_name}}" cluster has been deleted.`, out.V{"cluster_name": profile}) + out.T(out.Crushed, `The "{{.name}}" cluster has been deleted.`, out.V{"name": profile}) machineName := pkg_config.GetMachineName() if err := kubeconfig.DeleteContext(constants.KubeconfigPath, machineName); err != nil { diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index ffddbccdeb..c429d1f301 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -102,6 +102,7 @@ const ( dnsProxy = "dns-proxy" hostDNSResolver = "host-dns-resolver" waitUntilHealthy = "wait" + waitTimeout = "wait-timeout" ) var ( @@ -111,7 +112,7 @@ var ( insecureRegistry []string apiServerNames []string apiServerIPs []net.IP - extraOptions pkgutil.ExtraOptionSlice + extraOptions cfg.ExtraOptionSlice ) func init() { @@ -148,6 +149,7 @@ func initMinikubeFlags() { startCmd.Flags().String(networkPlugin, "", "The name of the network plugin.") startCmd.Flags().Bool(enableDefaultCNI, false, "Enable the default CNI plugin (/etc/cni/net.d/k8s.conf). Used in conjunction with \"--network-plugin=cni\".") startCmd.Flags().Bool(waitUntilHealthy, true, "Wait until Kubernetes core services are healthy before exiting.") + startCmd.Flags().Duration(waitTimeout, 3*time.Minute, "max time to wait per Kubernetes core services to be healthy.") } // initKubernetesFlags inits the commandline flags for kubernetes related options @@ -321,7 +323,7 @@ func runStart(cmd *cobra.Command, args []string) { // special ops for none driver, like change minikube directory. prepareNone(viper.GetString(vmDriver)) if viper.GetBool(waitUntilHealthy) { - if err := bs.WaitCluster(config.KubernetesConfig); err != nil { + if err := bs.WaitCluster(config.KubernetesConfig, viper.GetDuration(waitTimeout)); err != nil { exit.WithError("Wait failed", err) } } @@ -538,8 +540,8 @@ func validateConfig() { // check that kubeadm extra args contain only whitelisted parameters for param := range extraOptions.AsMap().Get(kubeadm.Kubeadm) { - if !pkgutil.ContainsString(kubeadm.KubeadmExtraArgsWhitelist[kubeadm.KubeadmCmdParam], param) && - !pkgutil.ContainsString(kubeadm.KubeadmExtraArgsWhitelist[kubeadm.KubeadmConfigParam], param) { + if !cfg.ContainsParam(kubeadm.KubeadmExtraArgsWhitelist[kubeadm.KubeadmCmdParam], param) && + !cfg.ContainsParam(kubeadm.KubeadmExtraArgsWhitelist[kubeadm.KubeadmConfigParam], param) { exit.UsageT("Sorry, the kubeadm.{{.parameter_name}} parameter is currently not supported by --extra-config", out.V{"parameter_name": param}) } } diff --git a/deploy/addons/storage-provisioner-gluster/README.md b/deploy/addons/storage-provisioner-gluster/README.md index 9ae656db47..72a350ca9c 100644 --- a/deploy/addons/storage-provisioner-gluster/README.md +++ b/deploy/addons/storage-provisioner-gluster/README.md @@ -1,5 +1,5 @@ ## storage-provisioner-gluster addon -[Gluster](https://gluster.org/), a scalable network filesystem that provides dynamic provisioning of PersistenVolumeClaims. +[Gluster](https://gluster.org/), a scalable network filesystem that provides dynamic provisioning of PersistentVolumeClaims. ### Starting Minikube This addon works within Minikube, without any additional configuration. diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh index 8e0c7c0b2a..2edffaafa6 100755 --- a/hack/jenkins/common.sh +++ b/hack/jenkins/common.sh @@ -71,11 +71,12 @@ echo ">> Downloading test inputs from ${MINIKUBE_LOCATION} ..." gsutil -qm cp \ "gs://minikube-builds/${MINIKUBE_LOCATION}/minikube-${OS_ARCH}" \ "gs://minikube-builds/${MINIKUBE_LOCATION}/docker-machine-driver"-* \ - "gs://minikube-builds/${MINIKUBE_LOCATION}/e2e-${OS_ARCH}" \ - "gs://minikube-builds/${MINIKUBE_LOCATION}/gvisor-addon" out + "gs://minikube-builds/${MINIKUBE_LOCATION}/e2e-${OS_ARCH}" out gsutil -qm cp "gs://minikube-builds/${MINIKUBE_LOCATION}/testdata"/* testdata/ +gsutil -qm cp "gs://minikube-builds/${MINIKUBE_LOCATION}/gvisor-addon" testdata/ + # Set the executable bit on the e2e binary and out binary export MINIKUBE_BIN="out/minikube-${OS_ARCH}" @@ -125,6 +126,18 @@ for stale_dir in ${TEST_ROOT}/*; do rmdir "${stale_dir}" || true done + +# sometimes tests left over zombie procs that won't exit +# for example: +# jenkins 20041 0.0 0.0 0 0 ? Z Aug19 0:00 [minikube-linux-] +zombie_defuncts=$(ps -A -ostat,ppid | awk '/[zZ]/ && !a[$2]++ {print $2}') +if [[ "${zombie_defuncts}" != "" ]]; then + echo "Found zombie defunct procs to kill..." + ps -f -p ${zombie_defuncts} || true + sudo -E kill ${zombie_defuncts} || true +fi + + if type -P virsh; then virsh -c qemu:///system list --all virsh -c qemu:///system list --all \ @@ -165,6 +178,7 @@ if type -P vboxmanage; then vboxmanage list vms || true fi + if type -P hdiutil; then hdiutil info | grep -E "/dev/disk[1-9][^s]" || true hdiutil info \ @@ -198,6 +212,14 @@ if [[ "${VM_DRIVER}" == "hyperkit" ]]; then fi fi +vboxprocs=$(pgrep VBox || true) +if [[ "${vboxprocs}" != "" ]]; then + echo "error: killing left over virtualbox processes ..." + ps -f -p ${vboxprocs} || true + sudo -E kill ${vboxprocs} || true +fi + + kprocs=$(pgrep kubectl || true) if [[ "${kprocs}" != "" ]]; then echo "error: killing hung kubectl processes ..." @@ -241,6 +263,16 @@ export MINIKUBE_HOME="${TEST_HOME}/.minikube" export MINIKUBE_WANTREPORTERRORPROMPT=False export KUBECONFIG="${TEST_HOME}/kubeconfig" +# Build the gvisor image. This will be copied into minikube and loaded by ctr. +# Used by TestContainerd for Gvisor Test. +# TODO: move this to integration test setup. +chmod +x ./testdata/gvisor-addon +# skipping gvisor mac because ofg https://github.com/kubernetes/minikube/issues/5137 +if [ "$(uname)" != "Darwin" ]; then + docker build -t gcr.io/k8s-minikube/gvisor-addon:latest -f testdata/gvisor-addon-Dockerfile ./testdata +fi + + # Display the default image URL echo "" echo ">> ISO URL" @@ -274,9 +306,6 @@ ${SUDO_PREFIX} rm -f "${KUBECONFIG}" || true rmdir "${TEST_HOME}" echo ">> ${TEST_HOME} completed at $(date)" -# Build the gvisor image. This will be copied into minikube and loaded by ctr. -docker build -t gcr.io/k8s-minikube/gvisor-addon:latest -f testdata/gvisor-addon-Dockerfile out - if [[ "${MINIKUBE_LOCATION}" != "master" ]]; then readonly target_url="https://storage.googleapis.com/minikube-builds/logs/${MINIKUBE_LOCATION}/${JOB_NAME}.txt" curl -s "https://api.github.com/repos/kubernetes/minikube/statuses/${COMMIT}?access_token=$access_token" \ diff --git a/hack/jenkins/installers/check_install_golang.sh b/hack/jenkins/installers/check_install_golang.sh new file mode 100755 index 0000000000..72abf97250 --- /dev/null +++ b/hack/jenkins/installers/check_install_golang.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eux -o pipefail + +if (($# < 2)); then + echo "ERROR: given ! ($#) number of parameters but expect 2." + echo "USAGE: ./check_and_install_golang.sh VERSION_TO_INSTALL INSTALL_PATH" + exit 1 +fi + +VERSION_TO_INSTALL=${1} +INSTALL_PATH=${2} + +# installs or updates golang if right version doesn't exists +function check_and_install_golang() { + if ! go version &>/dev/null; then + echo "WARNING: No golang installation found in your enviroment." + install_golang "$VERSION_TO_INSTALL" "$INSTALL_PATH" + return + fi + + # golang has been installed and check its version + if [[ $(go version) =~ (([0-9]+)\.([0-9]+).([0-9]+).([\.0-9]*)) ]]; then + HOST_VERSION=${BASH_REMATCH[1]} + if [ $HOST_VERSION = $VERSION_TO_INSTALL ]; then + echo "go version on the host looks good : $HOST_VERSION" + else + echo "WARNING: expected go version to be $VERSION_TO_INSTALL but got $HOST_VERSION" + install_golang "$VERSION_TO_INSTALL" "$INSTALL_PATH" + fi + else + echo "ERROR: Failed to parse golang version: $HOST_VERSION" + return + fi +} + +# install_golang takes two parameters version and path to install. +function install_golang() { + echo "Installing golang version: $1 on $2" + pushd /tmp >/dev/null + # using sudo because previously installed versions might have been installed by a different user. + # as it was the case on jenkins VM. + sudo curl -qL -O "https://storage.googleapis.com/golang/go${1}.linux-amd64.tar.gz" && + sudo tar xfa go${1}.linux-amd64.tar.gz && + sudo rm -rf "${2}/go" && + sudo mv go "${2}/" && sudo chown -R $(whoami): ${2}/go + popd >/dev/null +} + +check_and_install_golang diff --git a/hack/jenkins/minikube_cross_build_and_upload.sh b/hack/jenkins/minikube_cross_build_and_upload.sh index adbb0189da..2e95fea227 100755 --- a/hack/jenkins/minikube_cross_build_and_upload.sh +++ b/hack/jenkins/minikube_cross_build_and_upload.sh @@ -25,6 +25,11 @@ set -eux -o pipefail readonly bucket="minikube-builds" +# Make sure the right golang version is installed based on Makefile +WANT_GOLANG_VERSION=$(grep '^GO_VERSION' Makefile | awk '{ print $3 }') +./hack/jenkins/installers/check_install_golang.sh $WANT_GOLANG_VERSION /usr/local + + declare -rx BUILD_IN_DOCKER=y declare -rx GOPATH=/var/lib/jenkins/go declare -rx ISO_BUCKET="${bucket}/${ghprbPullId}" diff --git a/hack/jenkins/release_build_and_upload.sh b/hack/jenkins/release_build_and_upload.sh index 3bb3f3d97c..4f9259843e 100755 --- a/hack/jenkins/release_build_and_upload.sh +++ b/hack/jenkins/release_build_and_upload.sh @@ -31,6 +31,11 @@ export DEB_VERSION=${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_BUILD} export RPM_VERSION=${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_BUILD} export GOPATH=~/go +# Make sure the right golang version is installed based on Makefile +WANT_GOLANG_VERSION=$(grep '^GO_VERSION' Makefile | awk '{ print $3 }') +./hack/jenkins/installers/check_install_golang.sh $WANT_GOLANG_VERSION /usr/local + + # Make sure the tag matches the Makefile cat Makefile | grep "VERSION_MAJOR ?=" | grep $VERSION_MAJOR cat Makefile | grep "VERSION_MINOR ?=" | grep $VERSION_MINOR diff --git a/pkg/util/kubernetes.go b/pkg/kapi/kapi.go similarity index 61% rename from pkg/util/kubernetes.go rename to pkg/kapi/kapi.go index 259149b5ba..c4564e18d2 100644 --- a/pkg/util/kubernetes.go +++ b/pkg/kapi/kapi.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package util +package kapi import ( "context" @@ -29,15 +29,13 @@ import ( meta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/clientcmd" watchtools "k8s.io/client-go/tools/watch" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" + kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/minikube/pkg/minikube/proxy" ) @@ -48,30 +46,8 @@ var ( ReasonableStartTime = time.Minute * 5 ) -// PodStore stores pods -type PodStore struct { - cache.Store - stopCh chan struct{} - Reflector *cache.Reflector -} - -// List lists the pods -func (s *PodStore) List() []*core.Pod { - objects := s.Store.List() - pods := make([]*core.Pod, 0) - for _, o := range objects { - pods = append(pods, o.(*core.Pod)) - } - return pods -} - -// Stop stops the pods -func (s *PodStore) Stop() { - close(s.stopCh) -} - -// GetClient gets the client from config -func GetClient(kubectlContext ...string) (kubernetes.Interface, error) { +// Client gets the kubernetes client from default kubeconfig +func Client(kubectlContext ...string) (kubernetes.Interface, error) { loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() configOverrides := &clientcmd.ConfigOverrides{} if kubectlContext != nil { @@ -92,50 +68,16 @@ func GetClient(kubectlContext ...string) (kubernetes.Interface, error) { return client, nil } -// NewPodStore creates a new PodStore -func NewPodStore(c kubernetes.Interface, namespace string, label fmt.Stringer, field fmt.Stringer) *PodStore { - lw := &cache.ListWatch{ - ListFunc: func(options meta.ListOptions) (runtime.Object, error) { - options.LabelSelector = label.String() - options.FieldSelector = field.String() - obj, err := c.CoreV1().Pods(namespace).List(options) - return runtime.Object(obj), err - }, - WatchFunc: func(options meta.ListOptions) (watch.Interface, error) { - options.LabelSelector = label.String() - options.FieldSelector = field.String() - return c.CoreV1().Pods(namespace).Watch(options) - }, - } - store := cache.NewStore(cache.MetaNamespaceKeyFunc) - stopCh := make(chan struct{}) - reflector := cache.NewReflector(lw, &core.Pod{}, store, 0) - go reflector.Run(stopCh) - return &PodStore{Store: store, stopCh: stopCh, Reflector: reflector} -} - -// StartPods starts all pods -func StartPods(c kubernetes.Interface, namespace string, pod core.Pod, waitForRunning bool) error { - pod.ObjectMeta.Labels["name"] = pod.Name - if waitForRunning { - label := labels.SelectorFromSet(labels.Set(map[string]string{"name": pod.Name})) - err := WaitForPodsWithLabelRunning(c, namespace, label) - if err != nil { - return fmt.Errorf("error waiting for pod %s to be running: %v", pod.Name, err) - } - } - return nil -} - // WaitForPodsWithLabelRunning waits for all matching pods to become Running and at least one matching pod exists. -func WaitForPodsWithLabelRunning(c kubernetes.Interface, ns string, label labels.Selector) error { +func WaitForPodsWithLabelRunning(c kubernetes.Interface, ns string, label labels.Selector, timeOut ...time.Duration) error { + start := time.Now() glog.Infof("Waiting for pod with label %q in ns %q ...", ns, label) lastKnownPodNumber := -1 - return wait.PollImmediate(constants.APICallRetryInterval, ReasonableStartTime, func() (bool, error) { + f := func() (bool, error) { listOpts := meta.ListOptions{LabelSelector: label.String()} pods, err := c.CoreV1().Pods(ns).List(listOpts) if err != nil { - glog.Infof("error getting Pods with label selector %q [%v]\n", label.String(), err) + glog.Infof("temporary error: getting Pods with label selector %q : [%v]\n", label.String(), err) return false, nil } @@ -150,45 +92,23 @@ func WaitForPodsWithLabelRunning(c kubernetes.Interface, ns string, label labels for _, pod := range pods.Items { if pod.Status.Phase != core.PodRunning { + glog.Infof("waiting for pod %q, current state: %s: [%v]\n", label.String(), pod.Status.Phase, err) return false, nil } } return true, nil - }) + } + t := ReasonableStartTime + if timeOut != nil { + t = timeOut[0] + } + err := wait.PollImmediate(kconst.APICallRetryInterval, t, f) + glog.Infof("duration metric: took %s to wait for %s ...", time.Since(start), label) + return err } -// WaitForPodDelete waits for a pod to be deleted -func WaitForPodDelete(c kubernetes.Interface, ns string, label fmt.Stringer) error { - return wait.PollImmediate(constants.APICallRetryInterval, ReasonableMutateTime, func() (bool, error) { - listOpts := meta.ListOptions{LabelSelector: label.String()} - pods, err := c.CoreV1().Pods(ns).List(listOpts) - if err != nil { - glog.Infof("error getting Pods with label selector %q [%v]\n", label.String(), err) - return false, nil - } - return len(pods.Items) == 0, nil - }) -} - -// WaitForEvent waits for the given event to appear -func WaitForEvent(c kubernetes.Interface, ns string, reason string) error { - return wait.PollImmediate(constants.APICallRetryInterval, ReasonableMutateTime, func() (bool, error) { - events, err := c.EventsV1beta1().Events("default").List(meta.ListOptions{}) - if err != nil { - glog.Infof("error getting events: %v", err) - return false, nil - } - for _, e := range events.Items { - if e.Reason == reason { - return true, nil - } - } - return false, nil - }) -} - -// WaitForRCToStabilize waits till the RC has a matching generation/replica count between spec and status. +// WaitForRCToStabilize waits till the RC has a matching generation/replica count between spec and status. used by integration tests func WaitForRCToStabilize(c kubernetes.Interface, ns, name string, timeout time.Duration) error { options := meta.ListOptions{FieldSelector: fields.Set{ "metadata.name": name, @@ -222,7 +142,7 @@ func WaitForRCToStabilize(c kubernetes.Interface, ns, name string, timeout time. return err } -// WaitForDeploymentToStabilize waits till the Deployment has a matching generation/replica count between spec and status. +// WaitForDeploymentToStabilize waits till the Deployment has a matching generation/replica count between spec and status. used by integrationt tests func WaitForDeploymentToStabilize(c kubernetes.Interface, ns, name string, timeout time.Duration) error { options := meta.ListOptions{FieldSelector: fields.Set{ "metadata.name": name, @@ -281,32 +201,6 @@ func WaitForService(c kubernetes.Interface, namespace, name string, exist bool, return nil } -// WaitForServiceEndpointsNum waits until the amount of endpoints that implement service to expectNum. -func WaitForServiceEndpointsNum(c kubernetes.Interface, namespace, serviceName string, expectNum int, interval, timeout time.Duration) error { - return wait.Poll(interval, timeout, func() (bool, error) { - glog.Infof("Waiting for amount of service:%s endpoints to be %d", serviceName, expectNum) - list, err := c.CoreV1().Endpoints(namespace).List(meta.ListOptions{}) - if err != nil { - return false, err - } - - for _, e := range list.Items { - if e.Name == serviceName && countEndpointsNum(&e) == expectNum { - return true, nil - } - } - return false, nil - }) -} - -func countEndpointsNum(e *core.Endpoints) int { - num := 0 - for _, sub := range e.Subsets { - num += len(sub.Addresses) - } - return num -} - // IsRetryableAPIError returns if this error is retryable or not func IsRetryableAPIError(err error) bool { return apierr.IsTimeout(err) || apierr.IsServerTimeout(err) || apierr.IsTooManyRequests(err) || apierr.IsInternalError(err) diff --git a/pkg/minikube/bootstrapper/bootstrapper.go b/pkg/minikube/bootstrapper/bootstrapper.go index 25a0b3a396..d564c6c1a0 100644 --- a/pkg/minikube/bootstrapper/bootstrapper.go +++ b/pkg/minikube/bootstrapper/bootstrapper.go @@ -18,6 +18,7 @@ package bootstrapper import ( "net" + "time" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/constants" @@ -39,7 +40,7 @@ type Bootstrapper interface { UpdateCluster(config.KubernetesConfig) error RestartCluster(config.KubernetesConfig) error DeleteCluster(config.KubernetesConfig) error - WaitCluster(config.KubernetesConfig) error + WaitCluster(config.KubernetesConfig, time.Duration) error // LogCommands returns a map of log type to a command which will display that log. LogCommands(LogOptions) map[string]string SetupCerts(cfg config.KubernetesConfig) error diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 87e0d72ea3..1c8e60c12b 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -35,6 +35,8 @@ import ( "golang.org/x/sync/errgroup" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/wait" + kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/minikube/pkg/kapi" "k8s.io/minikube/pkg/minikube/assets" "k8s.io/minikube/pkg/minikube/bootstrapper" "k8s.io/minikube/pkg/minikube/command" @@ -182,13 +184,13 @@ func (k *Bootstrapper) LogCommands(o bootstrapper.LogOptions) map[string]string } // createFlagsFromExtraArgs converts kubeadm extra args into flags to be supplied from the commad linne -func createFlagsFromExtraArgs(extraOptions util.ExtraOptionSlice) string { +func createFlagsFromExtraArgs(extraOptions config.ExtraOptionSlice) string { kubeadmExtraOpts := extraOptions.AsMap().Get(Kubeadm) // kubeadm allows only a small set of parameters to be supplied from the command line when the --config param // is specified, here we remove those that are not allowed for opt := range kubeadmExtraOpts { - if !util.ContainsString(KubeadmExtraArgsWhitelist[KubeadmCmdParam], opt) { + if !config.ContainsParam(KubeadmExtraArgsWhitelist[KubeadmCmdParam], opt) { // kubeadmExtraOpts is a copy so safe to delete delete(kubeadmExtraOpts, opt) } @@ -198,7 +200,7 @@ func createFlagsFromExtraArgs(extraOptions util.ExtraOptionSlice) string { // StartCluster starts the cluster func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error { - version, err := ParseKubernetesVersion(k8s.KubernetesVersion) + version, err := parseKubernetesVersion(k8s.KubernetesVersion) if err != nil { return errors.Wrap(err, "parsing kubernetes version") } @@ -307,13 +309,13 @@ func addAddons(files *[]assets.CopyableFile, data interface{}) error { } // WaitCluster blocks until Kubernetes appears to be healthy. -func (k *Bootstrapper) WaitCluster(k8s config.KubernetesConfig) error { +func (k *Bootstrapper) WaitCluster(k8s config.KubernetesConfig, timeout time.Duration) error { // Do not wait for "k8s-app" pods in the case of CNI, as they are managed // by a CNI plugin which is usually started after minikube has been brought // up. Otherwise, minikube won't start, as "k8s-app" pods are not ready. componentsOnly := k8s.NetworkPlugin == "cni" out.T(out.WaitingPods, "Waiting for:") - client, err := util.GetClient() + client, err := kapi.Client() if err != nil { return errors.Wrap(err, "k8s client") } @@ -326,13 +328,12 @@ func (k *Bootstrapper) WaitCluster(k8s config.KubernetesConfig) error { } for _, p := range PodsByLayer { - if componentsOnly && p.key != "component" { + if componentsOnly && p.key != "component" { // skip component check if network plugin is cni continue } - out.String(" %s", p.name) selector := labels.SelectorFromSet(labels.Set(map[string]string{p.key: p.value})) - if err := util.WaitForPodsWithLabelRunning(client, "kube-system", selector); err != nil { + if err := kapi.WaitForPodsWithLabelRunning(client, "kube-system", selector, timeout); err != nil { return errors.Wrap(err, fmt.Sprintf("waiting for %s=%s", p.key, p.value)) } } @@ -342,7 +343,7 @@ func (k *Bootstrapper) WaitCluster(k8s config.KubernetesConfig) error { // RestartCluster restarts the Kubernetes cluster configured by kubeadm func (k *Bootstrapper) RestartCluster(k8s config.KubernetesConfig) error { - version, err := ParseKubernetesVersion(k8s.KubernetesVersion) + version, err := parseKubernetesVersion(k8s.KubernetesVersion) if err != nil { return errors.Wrap(err, "parsing kubernetes version") } @@ -398,8 +399,9 @@ func (k *Bootstrapper) waitForAPIServer(k8s config.KubernetesConfig) error { return fmt.Errorf("apiserver process never appeared") } - glog.Infof("Waiting for apiserver status ...") - return wait.PollImmediate(time.Millisecond*300, time.Minute*1, func() (bool, error) { + start := time.Now() + glog.Infof("Waiting for apiserver ...") + f := func() (bool, error) { status, err := k.GetAPIServerStatus(net.ParseIP(k8s.NodeIP), k8s.NodePort) glog.Infof("apiserver status: %s, err: %v", status, err) if err != nil { @@ -409,7 +411,10 @@ func (k *Bootstrapper) waitForAPIServer(k8s config.KubernetesConfig) error { return false, nil } return true, nil - }) + } + err = wait.PollImmediate(kconst.APICallRetryInterval, kconst.DefaultControlPlaneTimeout, f) + glog.Infof("duration metric: took %s to wait for apiserver status ...", time.Since(start)) + return err } // DeleteCluster removes the components that were started earlier @@ -425,7 +430,7 @@ func (k *Bootstrapper) DeleteCluster(k8s config.KubernetesConfig) error { // PullImages downloads images that will be used by RestartCluster func (k *Bootstrapper) PullImages(k8s config.KubernetesConfig) error { - version, err := ParseKubernetesVersion(k8s.KubernetesVersion) + version, err := parseKubernetesVersion(k8s.KubernetesVersion) if err != nil { return errors.Wrap(err, "parsing kubernetes version") } @@ -448,7 +453,7 @@ func (k *Bootstrapper) SetupCerts(k8s config.KubernetesConfig) error { // NewKubeletConfig generates a new systemd unit containing a configured kubelet // based on the options present in the KubernetesConfig. func NewKubeletConfig(k8s config.KubernetesConfig, r cruntime.Manager) ([]byte, error) { - version, err := ParseKubernetesVersion(k8s.KubernetesVersion) + version, err := parseKubernetesVersion(k8s.KubernetesVersion) if err != nil { return nil, errors.Wrap(err, "parsing kubernetes version") } @@ -551,7 +556,7 @@ func (k *Bootstrapper) UpdateCluster(cfg config.KubernetesConfig) error { } // createExtraComponentConfig generates a map of component to extra args for all of the components except kubeadm -func createExtraComponentConfig(extraOptions util.ExtraOptionSlice, version semver.Version, componentFeatureArgs string) ([]ComponentExtraArgs, error) { +func createExtraComponentConfig(extraOptions config.ExtraOptionSlice, version semver.Version, componentFeatureArgs string) ([]ComponentExtraArgs, error) { extraArgsSlice, err := NewComponentExtraArgs(extraOptions, version, componentFeatureArgs) if err != nil { return nil, err @@ -571,7 +576,7 @@ func createExtraComponentConfig(extraOptions util.ExtraOptionSlice, version semv // generateConfig generates the kubeadm.yaml file func generateConfig(k8s config.KubernetesConfig, r cruntime.Manager) ([]byte, error) { - version, err := ParseKubernetesVersion(k8s.KubernetesVersion) + version, err := parseKubernetesVersion(k8s.KubernetesVersion) if err != nil { return nil, errors.Wrap(err, "parsing kubernetes version") } diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go index dd9e095eae..3718a6b5d1 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go @@ -26,7 +26,6 @@ import ( "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/cruntime" - "k8s.io/minikube/pkg/util" ) func TestGenerateKubeletConfig(t *testing.T) { @@ -145,29 +144,29 @@ ExecStart=/var/lib/minikube/binaries/v1.15.2/kubelet --authorization-mode=Webhoo } } -func getExtraOpts() []util.ExtraOption { - return util.ExtraOptionSlice{ - util.ExtraOption{ +func getExtraOpts() []config.ExtraOption { + return config.ExtraOptionSlice{ + config.ExtraOption{ Component: Apiserver, Key: "fail-no-swap", Value: "true", }, - util.ExtraOption{ + config.ExtraOption{ Component: ControllerManager, Key: "kube-api-burst", Value: "32", }, - util.ExtraOption{ + config.ExtraOption{ Component: Scheduler, Key: "scheduler-name", Value: "mini-scheduler", }, - util.ExtraOption{ + config.ExtraOption{ Component: Kubeadm, Key: "ignore-preflight-errors", Value: "true", }, - util.ExtraOption{ + config.ExtraOption{ Component: Kubeadm, Key: "dry-run", Value: "true", @@ -175,9 +174,9 @@ func getExtraOpts() []util.ExtraOption { } } -func getExtraOptsPodCidr() []util.ExtraOption { - return util.ExtraOptionSlice{ - util.ExtraOption{ +func getExtraOptsPodCidr() []config.ExtraOption { + return config.ExtraOptionSlice{ + config.ExtraOption{ Component: Kubeadm, Key: "pod-network-cidr", Value: "192.168.32.0/20", @@ -229,7 +228,7 @@ func TestGenerateConfig(t *testing.T) { {"crio", "crio", false, config.KubernetesConfig{}}, {"options", "docker", false, config.KubernetesConfig{ExtraOptions: extraOpts}}, {"crio-options-gates", "crio", false, config.KubernetesConfig{ExtraOptions: extraOpts, FeatureGates: "a=b"}}, - {"unknown-component", "docker", true, config.KubernetesConfig{ExtraOptions: util.ExtraOptionSlice{util.ExtraOption{Component: "not-a-real-component", Key: "killswitch", Value: "true"}}}}, + {"unknown-component", "docker", true, config.KubernetesConfig{ExtraOptions: config.ExtraOptionSlice{config.ExtraOption{Component: "not-a-real-component", Key: "killswitch", Value: "true"}}}}, {"containerd-api-port", "containerd", false, config.KubernetesConfig{NodePort: 12345}}, {"containerd-pod-network-cidr", "containerd", false, config.KubernetesConfig{ExtraOptions: extraOptsPodCidr}}, {"image-repository", "docker", false, config.KubernetesConfig{ImageRepository: "test/repo"}}, diff --git a/pkg/minikube/bootstrapper/kubeadm/versions.go b/pkg/minikube/bootstrapper/kubeadm/versions.go index 02d0de884f..c0ac79a56d 100644 --- a/pkg/minikube/bootstrapper/kubeadm/versions.go +++ b/pkg/minikube/bootstrapper/kubeadm/versions.go @@ -27,6 +27,7 @@ import ( "github.com/golang/glog" "github.com/pkg/errors" "k8s.io/kubernetes/cmd/kubeadm/app/features" + "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/util" ) @@ -43,7 +44,7 @@ const ( // ExtraConfigForComponent generates a map of flagname-value pairs for a k8s // component. -func ExtraConfigForComponent(component string, opts util.ExtraOptionSlice, version semver.Version) (map[string]string, error) { +func ExtraConfigForComponent(component string, opts config.ExtraOptionSlice, version semver.Version) (map[string]string, error) { versionedOpts, err := DefaultOptionsForComponentAndVersion(component, version) if err != nil { return nil, errors.Wrapf(err, "setting version specific options for %s", component) @@ -78,7 +79,7 @@ var componentToKubeadmConfigKey = map[string]string{ } // NewComponentExtraArgs creates a new ComponentExtraArgs -func NewComponentExtraArgs(opts util.ExtraOptionSlice, version semver.Version, featureGates string) ([]ComponentExtraArgs, error) { +func NewComponentExtraArgs(opts config.ExtraOptionSlice, version semver.Version, featureGates string) ([]ComponentExtraArgs, error) { var kubeadmExtraArgs []ComponentExtraArgs for _, extraOpt := range opts { if _, ok := componentToKubeadmConfigKey[extraOpt.Component]; !ok { @@ -158,8 +159,8 @@ func Supports(featureName string) bool { return false } -// ParseKubernetesVersion parses the kubernetes version -func ParseKubernetesVersion(version string) (semver.Version, error) { +// parseKubernetesVersion parses the kubernetes version +func parseKubernetesVersion(version string) (semver.Version, error) { // Strip leading 'v' prefix from version for semver parsing v, err := semver.Make(version[1:]) if err != nil { @@ -182,45 +183,9 @@ func convertToFlags(opts map[string]string) string { return strings.Join(flags, " ") } -// VersionedExtraOption holds information on flags to apply to a specific range -// of versions -type VersionedExtraOption struct { - // Special Cases: - // - // If LessThanOrEqual and GreaterThanOrEqual are both nil, the flag will be applied - // to all versions - // - // If LessThanOrEqual == GreaterThanOrEqual, the flag will only be applied to that - // specific version - - // The flag and component that will be set - Option util.ExtraOption - - // This flag will only be applied to versions before or equal to this version - // If it is the default value, it will have no upper bound on versions the - // flag is applied to - LessThanOrEqual semver.Version - - // The flag will only be applied to versions after or equal to this version - // If it is the default value, it will have no lower bound on versions the - // flag is applied to - GreaterThanOrEqual semver.Version -} - -// NewUnversionedOption returns a VersionedExtraOption that applies to all versions. -func NewUnversionedOption(component, k, v string) VersionedExtraOption { - return VersionedExtraOption{ - Option: util.ExtraOption{ - Component: component, - Key: k, - Value: v, - }, - } -} - -var versionSpecificOpts = []VersionedExtraOption{ +var versionSpecificOpts = []config.VersionedExtraOption{ { - Option: util.ExtraOption{ + Option: config.ExtraOption{ Component: Kubelet, Key: "fail-swap-on", Value: "false", @@ -228,22 +193,22 @@ var versionSpecificOpts = []VersionedExtraOption{ GreaterThanOrEqual: semver.MustParse("1.8.0-alpha.0"), }, // Kubeconfig args - NewUnversionedOption(Kubelet, "kubeconfig", "/etc/kubernetes/kubelet.conf"), - NewUnversionedOption(Kubelet, "bootstrap-kubeconfig", "/etc/kubernetes/bootstrap-kubelet.conf"), + config.NewUnversionedOption(Kubelet, "kubeconfig", "/etc/kubernetes/kubelet.conf"), + config.NewUnversionedOption(Kubelet, "bootstrap-kubeconfig", "/etc/kubernetes/bootstrap-kubelet.conf"), { - Option: util.ExtraOption{ + Option: config.ExtraOption{ Component: Kubelet, Key: "require-kubeconfig", Value: "true", }, LessThanOrEqual: semver.MustParse("1.9.10"), }, - NewUnversionedOption(Kubelet, "hostname-override", constants.DefaultNodeName), + config.NewUnversionedOption(Kubelet, "hostname-override", constants.DefaultNodeName), // System pods args - NewUnversionedOption(Kubelet, "pod-manifest-path", constants.GuestManifestsDir), + config.NewUnversionedOption(Kubelet, "pod-manifest-path", constants.GuestManifestsDir), { - Option: util.ExtraOption{ + Option: config.ExtraOption{ Component: Kubelet, Key: "allow-privileged", Value: "true", @@ -252,17 +217,17 @@ var versionSpecificOpts = []VersionedExtraOption{ }, // Network args - NewUnversionedOption(Kubelet, "cluster-dns", "10.96.0.10"), - NewUnversionedOption(Kubelet, "cluster-domain", "cluster.local"), + config.NewUnversionedOption(Kubelet, "cluster-dns", "10.96.0.10"), + config.NewUnversionedOption(Kubelet, "cluster-domain", "cluster.local"), // Auth args - NewUnversionedOption(Kubelet, "authorization-mode", "Webhook"), - NewUnversionedOption(Kubelet, "client-ca-file", path.Join(constants.GuestCertsDir, "ca.crt")), + config.NewUnversionedOption(Kubelet, "authorization-mode", "Webhook"), + config.NewUnversionedOption(Kubelet, "client-ca-file", path.Join(constants.GuestCertsDir, "ca.crt")), // Cgroup args - NewUnversionedOption(Kubelet, "cgroup-driver", "cgroupfs"), + config.NewUnversionedOption(Kubelet, "cgroup-driver", "cgroupfs"), { - Option: util.ExtraOption{ + Option: config.ExtraOption{ Component: Apiserver, Key: "admission-control", Value: strings.Join(util.DefaultLegacyAdmissionControllers, ","), @@ -271,7 +236,7 @@ var versionSpecificOpts = []VersionedExtraOption{ GreaterThanOrEqual: semver.MustParse("1.9.0-alpha.0"), }, { - Option: util.ExtraOption{ + Option: config.ExtraOption{ Component: Apiserver, Key: "enable-admission-plugins", Value: strings.Join(util.DefaultLegacyAdmissionControllers, ","), @@ -280,7 +245,7 @@ var versionSpecificOpts = []VersionedExtraOption{ LessThanOrEqual: semver.MustParse("1.13.1000"), }, { - Option: util.ExtraOption{ + Option: config.ExtraOption{ Component: Apiserver, Key: "enable-admission-plugins", Value: strings.Join(util.DefaultV114AdmissionControllers, ","), @@ -289,7 +254,7 @@ var versionSpecificOpts = []VersionedExtraOption{ }, { - Option: util.ExtraOption{ + Option: config.ExtraOption{ Component: Kubelet, Key: "cadvisor-port", Value: "0", diff --git a/pkg/minikube/bootstrapper/kubeadm/versions_test.go b/pkg/minikube/bootstrapper/kubeadm/versions_test.go index cc4ec232b6..40d46c077e 100644 --- a/pkg/minikube/bootstrapper/kubeadm/versions_test.go +++ b/pkg/minikube/bootstrapper/kubeadm/versions_test.go @@ -94,7 +94,7 @@ func TestVersionIsBetween(t *testing.T) { } func TestParseKubernetesVersion(t *testing.T) { - version, err := ParseKubernetesVersion("v1.8.0-alpha.5") + version, err := parseKubernetesVersion("v1.8.0-alpha.5") if err != nil { t.Fatalf("Error parsing version: %v", err) } diff --git a/pkg/minikube/cluster/cluster.go b/pkg/minikube/cluster/cluster.go index 036c6ffa24..07155aebd2 100644 --- a/pkg/minikube/cluster/cluster.go +++ b/pkg/minikube/cluster/cluster.go @@ -160,7 +160,7 @@ func configureHost(h *host.Host, e *engine.Options) error { if err != nil { return errors.Wrap(err, "detecting provisioner") } - glog.Infof("Provisioning: %+v", *h.HostOptions) + glog.Infof("Provisioning with %s: %+v", provisioner.String(), *h.HostOptions) if err := provisioner.Provision(*h.HostOptions.SwarmOptions, *h.HostOptions.AuthOptions, *h.HostOptions.EngineOptions); err != nil { return errors.Wrap(err, "provision") } @@ -338,6 +338,7 @@ func engineOptions(config cfg.MachineConfig) *engine.Options { InsecureRegistry: append([]string{pkgutil.DefaultServiceCIDR}, config.InsecureRegistry...), RegistryMirror: config.RegistryMirror, ArbitraryFlags: config.DockerOpt, + InstallURL: drivers.DefaultEngineInstallURL, } return &o } diff --git a/pkg/minikube/config/config.go b/pkg/minikube/config/config.go index d8a7636e7c..18d568fc0c 100644 --- a/pkg/minikube/config/config.go +++ b/pkg/minikube/config/config.go @@ -59,7 +59,7 @@ type MinikubeConfig map[string]interface{} // Get gets a named value from config func Get(name string) (string, error) { - m, err := ReadConfig() + m, err := ReadConfig(constants.ConfigFile) if err != nil { return "", err } @@ -88,11 +88,7 @@ func WriteConfig(configFile string, m MinikubeConfig) error { } // ReadConfig reads in the JSON minikube config -func ReadConfig() (MinikubeConfig, error) { - return readConfig(constants.ConfigFile) -} - -func readConfig(configFile string) (MinikubeConfig, error) { +func ReadConfig(configFile string) (MinikubeConfig, error) { f, err := os.Open(configFile) if err != nil { if os.IsNotExist(err) { diff --git a/pkg/minikube/config/config_test.go b/pkg/minikube/config/config_test.go index b6390a2ae3..d1c7254c1a 100644 --- a/pkg/minikube/config/config_test.go +++ b/pkg/minikube/config/config_test.go @@ -104,9 +104,9 @@ func Test_get(t *testing.T) { } } -func Test_readConfig(t *testing.T) { +func TestReadConfig(t *testing.T) { // non existing file - mkConfig, err := readConfig("non_existing_file") + mkConfig, err := ReadConfig("non_existing_file") if err != nil { t.Fatalf("Error not exepected but got %v", err) } @@ -116,7 +116,7 @@ func Test_readConfig(t *testing.T) { } // invalid config file - mkConfig, err = readConfig("./testdata/.minikube/config/invalid_config.json") + mkConfig, err = ReadConfig("./testdata/.minikube/config/invalid_config.json") if err == nil { t.Fatalf("Error expected but got none") } @@ -126,7 +126,7 @@ func Test_readConfig(t *testing.T) { } // valid config file - mkConfig, err = readConfig("./testdata/.minikube/config/valid_config.json") + mkConfig, err = ReadConfig("./testdata/.minikube/config/valid_config.json") if err != nil { t.Fatalf("Error not expected but got %v", err) } @@ -164,7 +164,7 @@ func TestWriteConfig(t *testing.T) { } defer os.Remove(configFile.Name()) - mkConfig, err := readConfig(configFile.Name()) + mkConfig, err := ReadConfig(configFile.Name()) if err != nil { t.Fatalf("Error not expected but got %v", err) } diff --git a/pkg/util/extra_options.go b/pkg/minikube/config/extra_options.go similarity index 80% rename from pkg/util/extra_options.go rename to pkg/minikube/config/extra_options.go index 2a60155261..0448750f39 100644 --- a/pkg/util/extra_options.go +++ b/pkg/minikube/config/extra_options.go @@ -1,5 +1,5 @@ /* -Copyright 2016 The Kubernetes Authors All rights reserved. +Copyright 2019 The Kubernetes Authors All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package util +package config import ( "fmt" @@ -76,7 +76,7 @@ func (es *ExtraOptionSlice) String() string { // component is not specified, all of the components are used. func (es *ExtraOptionSlice) Get(key string, component ...string) string { for _, opt := range *es { - if component == nil || ContainsString(component, opt.Component) { + if component == nil || ContainsParam(component, opt.Component) { if opt.Key == key { return opt.Value } @@ -107,3 +107,25 @@ func (es *ExtraOptionSlice) Type() string { func (cm ComponentExtraOptionMap) Get(component string) map[string]string { return cm[component] } + +// ContainsParam checks if a given slice of strings contains the provided string. +// If a modifier func is provided, it is called with the slice item before the comparation. +func ContainsParam(slice []string, s string) bool { + for _, item := range slice { + if item == s { + return true + } + } + return false +} + +// NewUnversionedOption returns a VersionedExtraOption that applies to all versions. +func NewUnversionedOption(component, k, v string) VersionedExtraOption { + return VersionedExtraOption{ + Option: ExtraOption{ + Component: component, + Key: k, + Value: v, + }, + } +} diff --git a/pkg/util/extra_options_test.go b/pkg/minikube/config/extra_options_test.go similarity index 99% rename from pkg/util/extra_options_test.go rename to pkg/minikube/config/extra_options_test.go index 4e54d99d13..cce3463ab8 100644 --- a/pkg/util/extra_options_test.go +++ b/pkg/minikube/config/extra_options_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package util +package config import ( "flag" diff --git a/pkg/minikube/config/types.go b/pkg/minikube/config/types.go index 0f7f7708e1..d2ed5e014c 100644 --- a/pkg/minikube/config/types.go +++ b/pkg/minikube/config/types.go @@ -19,6 +19,7 @@ package config import ( "net" + "github.com/blang/semver" "k8s.io/minikube/pkg/util" ) @@ -82,8 +83,33 @@ type KubernetesConfig struct { FeatureGates string ServiceCIDR string ImageRepository string - ExtraOptions util.ExtraOptionSlice + ExtraOptions ExtraOptionSlice ShouldLoadCachedImages bool EnableDefaultCNI bool } + +// VersionedExtraOption holds information on flags to apply to a specific range +// of versions +type VersionedExtraOption struct { + // Special Cases: + // + // If LessThanOrEqual and GreaterThanOrEqual are both nil, the flag will be applied + // to all versions + // + // If LessThanOrEqual == GreaterThanOrEqual, the flag will only be applied to that + // specific version + + // The flag and component that will be set + Option ExtraOption + + // This flag will only be applied to versions before or equal to this version + // If it is the default value, it will have no upper bound on versions the + // flag is applied to + LessThanOrEqual semver.Version + + // The flag will only be applied to versions after or equal to this version + // If it is the default value, it will have no lower bound on versions the + // flag is applied to + GreaterThanOrEqual semver.Version +} diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go index dfad95d1c0..88d9e17606 100644 --- a/pkg/minikube/cruntime/containerd.go +++ b/pkg/minikube/cruntime/containerd.go @@ -92,7 +92,7 @@ func (r *Containerd) Enable(disOthers bool) error { if err := enableIPForwarding(r.Runner); err != nil { return err } - // Oherwise, containerd will fail API requests with 'Unimplemented' + // Otherwise, containerd will fail API requests with 'Unimplemented' return r.Runner.Run("sudo systemctl restart containerd") } diff --git a/pkg/minikube/machine/client.go b/pkg/minikube/machine/client.go index 458dbf9edb..a970f741d9 100644 --- a/pkg/minikube/machine/client.go +++ b/pkg/minikube/machine/client.go @@ -123,7 +123,7 @@ func (api *LocalClient) NewHost(driverName string, rawDriver []byte) (*host.Host func (api *LocalClient) Load(name string) (*host.Host, error) { h, err := api.Filestore.Load(name) if err != nil { - return nil, errors.Wrap(err, "filestore") + return nil, errors.Wrapf(err, "filestore %q", name) } var def registry.DriverDef diff --git a/pkg/minikube/notify/notify.go b/pkg/minikube/notify/notify.go index 5d2b0e350a..55f09cdf9b 100644 --- a/pkg/minikube/notify/notify.go +++ b/pkg/minikube/notify/notify.go @@ -36,8 +36,6 @@ import ( "k8s.io/minikube/pkg/version" ) -const updateLinkPrefix = "https://github.com/kubernetes/minikube/releases/tag/v" - var ( timeLayout = time.RFC1123 lastUpdateCheckFilePath = constants.MakeMiniPath("last_update_check") @@ -67,7 +65,7 @@ func MaybePrintUpdateText(url string, lastUpdatePath string) { if err := writeTimeToFile(lastUpdateCheckFilePath, time.Now().UTC()); err != nil { glog.Errorf("write time failed: %v", err) } - url := fmt.Sprintf("%s/%s", updateLinkPrefix, latestVersion) + url := "https://github.com/kubernetes/minikube/releases/tag/v" + latestVersion.String() out.ErrT(out.WarningType, `minikube {{.version}} is available! Download it: {{.url}}`, out.V{"version": latestVersion, "url": url}) out.T(out.Tip, "To disable this notice, run: 'minikube config set WantUpdateNotification false'") } diff --git a/pkg/minikube/translate/translate.go b/pkg/minikube/translate/translate.go index f47d0930ec..960352da82 100644 --- a/pkg/minikube/translate/translate.go +++ b/pkg/minikube/translate/translate.go @@ -36,7 +36,7 @@ var ( Translations map[string]interface{} ) -// T translates the given string to the supplied language. +// T translates the given string to the preferred language. func T(s string) string { if preferredLanguage == defaultLanguage { return s diff --git a/pkg/provision/buildroot.go b/pkg/provision/buildroot.go index 3195c54fb9..9baa3791ca 100644 --- a/pkg/provision/buildroot.go +++ b/pkg/provision/buildroot.go @@ -69,6 +69,11 @@ func (p *BuildrootProvisioner) String() string { return "buildroot" } +// CompatibleWithHost checks if provisioner is compatible with host +func (p *BuildrootProvisioner) CompatibleWithHost() bool { + return p.OsReleaseInfo.ID == "buildroot" +} + // escapeSystemdDirectives escapes special characters in the input variables used to create the // systemd unit file, which would otherwise be interpreted as systemd directives. An example // are template specifiers (e.g. '%i') which are predefined variables that get evaluated dynamically diff --git a/pkg/util/lock/lock.go b/pkg/util/lock/lock.go index 564ff3a27c..370bfb662b 100644 --- a/pkg/util/lock/lock.go +++ b/pkg/util/lock/lock.go @@ -36,7 +36,7 @@ func WriteFile(filename string, data []byte, perm os.FileMode) (err error) { lockErr := lock.TryLock() if lockErr != nil { glog.Warningf("temporary error : %v", lockErr.Error()) - return errors.Wrapf(lockErr, "falied to acquire lock for %s > ", filename) + return errors.Wrapf(lockErr, "failed to acquire lock for %s > ", filename) } return nil } diff --git a/pkg/util/utils.go b/pkg/util/utils.go index 1cd596a736..c549ca4ebe 100644 --- a/pkg/util/utils.go +++ b/pkg/util/utils.go @@ -200,14 +200,3 @@ func ConcatStrings(src []string, prefix string, postfix string) []string { } return ret } - -// ContainsString checks if a given slice of strings contains the provided string. -// If a modifier func is provided, it is called with the slice item before the comparation. -func ContainsString(slice []string, s string) bool { - for _, item := range slice { - if item == s { - return true - } - } - return false -} diff --git a/site/content/en/docs/Overview/_index.md b/site/content/en/docs/Overview/_index.md index a31b1d417f..c500d9e27d 100644 --- a/site/content/en/docs/Overview/_index.md +++ b/site/content/en/docs/Overview/_index.md @@ -45,5 +45,5 @@ Then minikube is for you. ## Where should I go next? -* [Getting Started](/start/): Get started with minikube -* [Examples](/examples/): Check out some minikube examples! +* [Getting Started](/docs/start/): Get started with minikube +* [Examples](/docs/examples/): Check out some minikube examples! diff --git a/site/content/en/docs/Reference/Drivers/includes/kvm2_usage.inc b/site/content/en/docs/Reference/Drivers/includes/kvm2_usage.inc index 2be6136a17..fb8ee8aa12 100644 --- a/site/content/en/docs/Reference/Drivers/includes/kvm2_usage.inc +++ b/site/content/en/docs/Reference/Drivers/includes/kvm2_usage.inc @@ -27,3 +27,16 @@ virt-host-validate curl -LO https://storage.googleapis.com/minikube/releases/latest/docker-machine-driver-kvm2 \ && sudo install docker-machine-driver-kvm2 /usr/local/bin/ ``` + +## Usage + +To explicitly start minikube with KVM: + +```shell +minikube start --vm-driver=kvm2 +``` +To make KVM the default driver: + +```shell +minikube config set vm-driver kvm2 +``` diff --git a/site/content/en/docs/Start/macos.md b/site/content/en/docs/Start/macos.md index 29327b0eec..48555ff9ef 100644 --- a/site/content/en/docs/Start/macos.md +++ b/site/content/en/docs/Start/macos.md @@ -32,6 +32,21 @@ curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-darwin {{% /tab %}} {{% /tabs %}} +### Upgrading minikube + +{{% tabs %}} +{{% tab "Brew" %}} + +If the [Brew Package Manager](https://brew.sh/) is installed, use it to download and upgrade minikube: + +```shell +rm /usr/local/bin/minikube +brew cask reinstall minikube +``` + +{{% /tab %}} +{{% /tabs %}} + ## Hypervisor Setup {{% tabs %}} diff --git a/site/content/en/docs/Tasks/sync.md b/site/content/en/docs/Tasks/sync.md index fc66a5822a..fb39f5dc87 100644 --- a/site/content/en/docs/Tasks/sync.md +++ b/site/content/en/docs/Tasks/sync.md @@ -35,4 +35,4 @@ If you are looking for a solution tuned for iterative application development, c - [Draft](https://draft.sh): see specific [minikube instructions](https://github.com/Azure/draft/blob/master/docs/install-minikube.md) - [Okteto](https://github.com/okteto/okteto) -- [Skaffold](http://github.com/ContainerTools/skaffold) +- [Skaffold](https://github.com/GoogleContainerTools/skaffold) diff --git a/test/integration/config_test.go b/test/integration/config_test.go new file mode 100644 index 0000000000..626f84266f --- /dev/null +++ b/test/integration/config_test.go @@ -0,0 +1,81 @@ +// +build integration + +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "strings" + "testing" +) + +func TestConfig(t *testing.T) { + t.Parallel() + p := profileName(t) + mk := NewMinikubeRunner(t, p, "--wait=false") + + tests := []struct { + cmd string + stdout string + stderr string + }{ + { + cmd: "config unset cpus", + stdout: "", + stderr: "", + }, + { + cmd: "config get cpus", + stdout: "", + stderr: "Error: specified key could not be found in config", + }, + { + cmd: "config set cpus 2", + stdout: "! These changes will take effect upon a minikube delete and then a minikube start", + stderr: "", + }, + { + cmd: "config get cpus", + stdout: "2", + stderr: "", + }, + { + cmd: "config unset cpus", + stdout: "", + stderr: ""}, + { + cmd: "config get cpus", + stdout: "", + stderr: "Error: specified key could not be found in config", + }, + } + + for _, tc := range tests { + stdout, stderr, _ := mk.RunCommandRetriable(tc.cmd) + + if !compare(tc.stdout, stdout) { + t.Fatalf("Expected stdout to be: %s. Stdout was: %s Stderr: %s", tc.stdout, stdout, stderr) + } + if !compare(tc.stderr, stderr) { + t.Fatalf("Expected stderr to be: %s. Stdout was: %s Stderr: %s", tc.stderr, stdout, stderr) + } + } +} + +func compare(s1, s2 string) bool { + return strings.TrimSpace(s1) == strings.TrimSpace(s2) +} diff --git a/test/integration/containerd_test.go b/test/integration/containerd_test.go index 822631730e..7c72a13491 100644 --- a/test/integration/containerd_test.go +++ b/test/integration/containerd_test.go @@ -25,7 +25,7 @@ import ( "github.com/docker/machine/libmachine/state" "github.com/pkg/errors" "k8s.io/apimachinery/pkg/labels" - commonutil "k8s.io/minikube/pkg/util" + "k8s.io/minikube/pkg/kapi" "k8s.io/minikube/test/integration/util" ) @@ -107,13 +107,13 @@ func deleteUntrustedWorkload(t *testing.T, profile string) { // waitForGvisorControllerRunning waits for the gvisor controller pod to be running func waitForGvisorControllerRunning(p string) error { - client, err := commonutil.GetClient(p) + client, err := kapi.Client(p) if err != nil { return errors.Wrap(err, "getting kubernetes client") } selector := labels.SelectorFromSet(labels.Set(map[string]string{"kubernetes.io/minikube-addons": "gvisor"})) - if err := commonutil.WaitForPodsWithLabelRunning(client, "kube-system", selector); err != nil { + if err := kapi.WaitForPodsWithLabelRunning(client, "kube-system", selector); err != nil { return errors.Wrap(err, "waiting for gvisor controller pod to stabilize") } return nil @@ -121,13 +121,13 @@ func waitForGvisorControllerRunning(p string) error { // waitForUntrustedNginxRunning waits for the untrusted nginx pod to start running func waitForUntrustedNginxRunning(miniProfile string) error { - client, err := commonutil.GetClient(miniProfile) + client, err := kapi.Client(miniProfile) if err != nil { return errors.Wrap(err, "getting kubernetes client") } selector := labels.SelectorFromSet(labels.Set(map[string]string{"run": "nginx"})) - if err := commonutil.WaitForPodsWithLabelRunning(client, "default", selector); err != nil { + if err := kapi.WaitForPodsWithLabelRunning(client, "default", selector); err != nil { return errors.Wrap(err, "waiting for nginx pods") } return nil diff --git a/test/integration/flags.go b/test/integration/flags.go index f8ad39211e..d4b59f0ea7 100644 --- a/test/integration/flags.go +++ b/test/integration/flags.go @@ -32,7 +32,7 @@ func TestMain(m *testing.M) { os.Exit(m.Run()) } -var startTimeout = flag.Int("timeout", 25, "number of minutes to wait for minikube start") +var startTimeout = flag.Duration("timeout", 25*time.Minute, "max duration to wait for a full minikube start") var binaryPath = flag.String("binary", "../../out/minikube", "path to minikube binary") var globalArgs = flag.String("minikube-args", "", "Arguments to pass to minikube") var startArgs = flag.String("minikube-start-args", "", "Arguments to pass to minikube start") @@ -45,10 +45,10 @@ func NewMinikubeRunner(t *testing.T, profile string, extraStartArgs ...string) u return util.MinikubeRunner{ Profile: profile, BinaryPath: *binaryPath, - StartArgs: *startArgs + " " + strings.Join(extraStartArgs, " "), + StartArgs: *startArgs + " --wait-timeout=13m " + strings.Join(extraStartArgs, " "), // adding timeout per component GlobalArgs: *globalArgs, MountArgs: *mountArgs, - TimeOutStart: time.Duration(*startTimeout) * time.Minute, + TimeOutStart: *startTimeout, // timeout for all start T: t, } } @@ -73,7 +73,7 @@ func profileName(t *testing.T) string { return p } -// shouldRunInParallel deterimines if test should run in parallel or not +// shouldRunInParallel determines if test should run in parallel or not func shouldRunInParallel(t *testing.T) bool { t.Helper() if !*parallel { diff --git a/test/integration/fn_addons.go b/test/integration/fn_addons.go index 60af99b14f..0ecc1ecba4 100644 --- a/test/integration/fn_addons.go +++ b/test/integration/fn_addons.go @@ -34,8 +34,7 @@ import ( retryablehttp "github.com/hashicorp/go-retryablehttp" "k8s.io/apimachinery/pkg/labels" - commonutil "k8s.io/minikube/pkg/util" - pkgutil "k8s.io/minikube/pkg/util" + "k8s.io/minikube/pkg/kapi" "k8s.io/minikube/pkg/util/retry" "k8s.io/minikube/test/integration/util" ) @@ -43,12 +42,12 @@ import ( func testAddons(t *testing.T) { t.Parallel() p := profileName(t) - client, err := pkgutil.GetClient(p) + client, err := kapi.Client(p) if err != nil { t.Fatalf("Could not get kubernetes client: %v", err) } selector := labels.SelectorFromSet(labels.Set(map[string]string{"component": "kube-addon-manager"})) - if err := pkgutil.WaitForPodsWithLabelRunning(client, "kube-system", selector); err != nil { + if err := kapi.WaitForPodsWithLabelRunning(client, "kube-system", selector); err != nil { t.Errorf("Error waiting for addon manager to be up") } } @@ -193,22 +192,22 @@ func testRegistry(t *testing.T) { p := profileName(t) mk := NewMinikubeRunner(t, p) mk.RunCommand("addons enable registry", true) - client, err := pkgutil.GetClient(p) + client, err := kapi.Client(p) if err != nil { t.Fatalf("getting kubernetes client: %v", err) } - if err := pkgutil.WaitForRCToStabilize(client, "kube-system", "registry", time.Minute*5); err != nil { + if err := kapi.WaitForRCToStabilize(client, "kube-system", "registry", time.Minute*5); err != nil { t.Fatalf("waiting for registry replicacontroller to stabilize: %v", err) } rs := labels.SelectorFromSet(labels.Set(map[string]string{"actual-registry": "true"})) - if err := pkgutil.WaitForPodsWithLabelRunning(client, "kube-system", rs); err != nil { + if err := kapi.WaitForPodsWithLabelRunning(client, "kube-system", rs); err != nil { t.Fatalf("waiting for registry pods: %v", err) } ps, err := labels.Parse("kubernetes.io/minikube-addons=registry,actual-registry!=true") if err != nil { t.Fatalf("Unable to parse selector: %v", err) } - if err := pkgutil.WaitForPodsWithLabelRunning(client, "kube-system", ps); err != nil { + if err := kapi.WaitForPodsWithLabelRunning(client, "kube-system", ps); err != nil { t.Fatalf("waiting for registry-proxy pods: %v", err) } ip, stderr := mk.RunCommand("ip", true) @@ -265,18 +264,18 @@ func testRegistry(t *testing.T) { // waitForNginxRunning waits for nginx service to be up func waitForNginxRunning(t *testing.T, miniProfile string) error { - client, err := commonutil.GetClient(miniProfile) + client, err := kapi.Client(miniProfile) if err != nil { return errors.Wrap(err, "getting kubernetes client") } selector := labels.SelectorFromSet(labels.Set(map[string]string{"run": "nginx"})) - if err := commonutil.WaitForPodsWithLabelRunning(client, "default", selector); err != nil { + if err := kapi.WaitForPodsWithLabelRunning(client, "default", selector); err != nil { return errors.Wrap(err, "waiting for nginx pods") } - if err := commonutil.WaitForService(client, "default", "nginx", true, time.Millisecond*500, time.Minute*10); err != nil { + if err := kapi.WaitForService(client, "default", "nginx", true, time.Millisecond*500, time.Minute*10); err != nil { t.Errorf("Error waiting for nginx service to be up") } return nil @@ -284,17 +283,17 @@ func waitForNginxRunning(t *testing.T, miniProfile string) error { // waitForIngressControllerRunning waits until ingress controller pod to be running func waitForIngressControllerRunning(miniProfile string) error { - client, err := commonutil.GetClient(miniProfile) + client, err := kapi.Client(miniProfile) if err != nil { return errors.Wrap(err, "getting kubernetes client") } - if err := commonutil.WaitForDeploymentToStabilize(client, "kube-system", "nginx-ingress-controller", time.Minute*10); err != nil { + if err := kapi.WaitForDeploymentToStabilize(client, "kube-system", "nginx-ingress-controller", time.Minute*10); err != nil { return errors.Wrap(err, "waiting for ingress-controller deployment to stabilize") } selector := labels.SelectorFromSet(labels.Set(map[string]string{"app.kubernetes.io/name": "nginx-ingress-controller"})) - if err := commonutil.WaitForPodsWithLabelRunning(client, "kube-system", selector); err != nil { + if err := kapi.WaitForPodsWithLabelRunning(client, "kube-system", selector); err != nil { return errors.Wrap(err, "waiting for ingress-controller pods") } diff --git a/test/integration/fn_cluster_dns.go b/test/integration/fn_cluster_dns.go index d17f76e103..e27301e134 100644 --- a/test/integration/fn_cluster_dns.go +++ b/test/integration/fn_cluster_dns.go @@ -26,7 +26,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" - pkgutil "k8s.io/minikube/pkg/util" + "k8s.io/minikube/pkg/kapi" "k8s.io/minikube/pkg/util/retry" "k8s.io/minikube/test/integration/util" ) @@ -34,7 +34,7 @@ import ( func testClusterDNS(t *testing.T) { t.Parallel() p := profileName(t) - client, err := pkgutil.GetClient(p) + client, err := kapi.Client(p) if err != nil { t.Fatalf("Error getting kubernetes client %v", err) } diff --git a/test/integration/fn_mount.go b/test/integration/fn_mount.go index 5d68c18554..ce5d45f6c0 100644 --- a/test/integration/fn_mount.go +++ b/test/integration/fn_mount.go @@ -29,7 +29,7 @@ import ( "time" "k8s.io/apimachinery/pkg/labels" - pkgutil "k8s.io/minikube/pkg/util" + "k8s.io/minikube/pkg/kapi" "k8s.io/minikube/pkg/util/lock" "k8s.io/minikube/pkg/util/retry" "k8s.io/minikube/test/integration/util" @@ -132,12 +132,12 @@ func writeFilesFromHost(mountedDir string, files []string, content string) error } func waitForPods(s map[string]string, profile string) error { - client, err := pkgutil.GetClient(profile) + client, err := kapi.Client(profile) if err != nil { return fmt.Errorf("getting kubernetes client: %v", err) } selector := labels.SelectorFromSet(labels.Set(s)) - if err := pkgutil.WaitForPodsWithLabelRunning(client, "default", selector); err != nil { + if err := kapi.WaitForPodsWithLabelRunning(client, "default", selector); err != nil { return err } return nil diff --git a/test/integration/fn_pv.go b/test/integration/fn_pv.go index b1b34d8a34..c938400610 100644 --- a/test/integration/fn_pv.go +++ b/test/integration/fn_pv.go @@ -29,7 +29,7 @@ import ( core "k8s.io/api/core/v1" storage "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/labels" - commonutil "k8s.io/minikube/pkg/util" + "k8s.io/minikube/pkg/kapi" "k8s.io/minikube/pkg/util/retry" "k8s.io/minikube/test/integration/util" ) @@ -73,13 +73,13 @@ func testProvisioning(t *testing.T) { // Check that the storage provisioner pod is running checkPodRunning := func() error { - client, err := commonutil.GetClient(p) + client, err := kapi.Client(p) if err != nil { return errors.Wrap(err, "getting kubernetes client") } selector := labels.SelectorFromSet(labels.Set(map[string]string{"integration-test": "storage-provisioner"})) - if err := commonutil.WaitForPodsWithLabelRunning(client, "kube-system", selector); err != nil { + if err := kapi.WaitForPodsWithLabelRunning(client, "kube-system", selector); err != nil { return err } return nil diff --git a/test/integration/fn_tunnel.go b/test/integration/fn_tunnel.go index a5eed14773..d3721c71bf 100644 --- a/test/integration/fn_tunnel.go +++ b/test/integration/fn_tunnel.go @@ -31,8 +31,8 @@ import ( "github.com/pkg/errors" "k8s.io/apimachinery/pkg/labels" + "k8s.io/minikube/pkg/kapi" "k8s.io/minikube/pkg/minikube/tunnel" - commonutil "k8s.io/minikube/pkg/util" "k8s.io/minikube/pkg/util/retry" "k8s.io/minikube/test/integration/util" ) @@ -70,18 +70,18 @@ func testTunnel(t *testing.T) { t.Fatalf("creating nginx ingress resource: %s", err) } - client, err := commonutil.GetClient(p) + client, err := kapi.Client(p) if err != nil { t.Fatal(errors.Wrap(err, "getting kubernetes client")) } selector := labels.SelectorFromSet(labels.Set(map[string]string{"run": "nginx-svc"})) - if err := commonutil.WaitForPodsWithLabelRunning(client, "default", selector); err != nil { + if err := kapi.WaitForPodsWithLabelRunning(client, "default", selector); err != nil { t.Fatal(errors.Wrap(err, "waiting for nginx pods")) } - if err := commonutil.WaitForService(client, "default", "nginx-svc", true, 1*time.Second, 2*time.Minute); err != nil { + if err := kapi.WaitForService(client, "default", "nginx-svc", true, 1*time.Second, 2*time.Minute); err != nil { t.Fatal(errors.Wrap(err, "Error waiting for nginx service to be up")) } @@ -114,14 +114,14 @@ func testTunnel(t *testing.T) { func getIngress(kr *util.KubectlRunner) (string, error) { nginxIP := "" var ret error - err := wait.PollImmediate(1*time.Second, 1*time.Minute, func() (bool, error) { + err := wait.PollImmediate(1*time.Second, 2*time.Minute, func() (bool, error) { cmd := []string{"get", "svc", "nginx-svc", "-o", "jsonpath={.status.loadBalancer.ingress[0].ip}"} stdout, err := kr.RunCommand(cmd) switch { case err == nil: nginxIP = string(stdout) return len(stdout) != 0, nil - case !commonutil.IsRetryableAPIError(err): + case !kapi.IsRetryableAPIError(err): ret = fmt.Errorf("`%s` failed with non retriable error: %v", cmd, err) return false, err default: diff --git a/test/integration/none_test.go b/test/integration/none_test.go index 826026bed0..ec7f33ebc0 100644 --- a/test/integration/none_test.go +++ b/test/integration/none_test.go @@ -27,6 +27,8 @@ import ( "strings" "syscall" "testing" + + "k8s.io/minikube/pkg/minikube/constants" ) func TestNone(t *testing.T) { @@ -76,7 +78,7 @@ func testNoneMinikubeFolderPermissions(t *testing.T) { if err != nil { t.Errorf("Failed to convert uid to int: %v", err) } - info, err := os.Stat(filepath.Join(u.HomeDir, ".minikube")) + info, err := os.Stat(constants.GetMinipath()) if err != nil { t.Fatalf("Failed to get .minikube dir info, %v", err) } diff --git a/test/integration/util/common.go b/test/integration/util/common.go index b3b1143cdb..c313837155 100644 --- a/test/integration/util/common.go +++ b/test/integration/util/common.go @@ -23,17 +23,17 @@ import ( "github.com/pkg/errors" "k8s.io/apimachinery/pkg/labels" - commonutil "k8s.io/minikube/pkg/util" + "k8s.io/minikube/pkg/kapi" ) // WaitForBusyboxRunning waits until busybox pod to be running func WaitForBusyboxRunning(t *testing.T, namespace string, miniProfile string) error { - client, err := commonutil.GetClient(miniProfile) + client, err := kapi.Client(miniProfile) if err != nil { return errors.Wrap(err, "getting kubernetes client") } selector := labels.SelectorFromSet(labels.Set(map[string]string{"integration-test": "busybox"})) - return commonutil.WaitForPodsWithLabelRunning(client, namespace, selector) + return kapi.WaitForPodsWithLabelRunning(client, namespace, selector) } // Logf writes logs to stdout if -v is set. diff --git a/test/integration/util/minikube_runner.go b/test/integration/util/minikube_runner.go index 19c766e248..95bcfb603b 100644 --- a/test/integration/util/minikube_runner.go +++ b/test/integration/util/minikube_runner.go @@ -124,7 +124,7 @@ func (m *MinikubeRunner) RunCommand(cmdStr string, failError bool, waitForRun .. return stdout, stderr } -// RunCommandRetriable Error executes a command, returns error +// RunCommandRetriable executes a command, returns error // the purpose of this command is to make it retriable and // better logging for retrying func (m *MinikubeRunner) RunCommandRetriable(cmdStr string, waitForRun ...bool) (stdout string, stderr string, err error) { @@ -244,7 +244,7 @@ func (m *MinikubeRunner) TearDown(t *testing.T) { profileArg := fmt.Sprintf("-p=%s", m.Profile) path, _ := filepath.Abs(m.BinaryPath) cmd := exec.Command(path, profileArg, "delete") - err := cmd.Start() + err := cmd.Start() // don't wait for it to finish if err != nil { t.Errorf("error tearing down minikube %s : %v", profileArg, err) } diff --git a/test/integration/version_upgrade_test.go b/test/integration/version_upgrade_test.go index 700161ea31..7e45ec28fa 100644 --- a/test/integration/version_upgrade_test.go +++ b/test/integration/version_upgrade_test.go @@ -73,10 +73,14 @@ func TestVersionUpgrade(t *testing.T) { } defer os.Remove(fname) - mkCurrent := NewMinikubeRunner(t, p) - defer mkCurrent.TearDown(t) + mkHead := NewMinikubeRunner(t, p) // minikube from HEAD. + defer mkHead.TearDown(t) - mkRelease := NewMinikubeRunner(t, p) + mkRelease := NewMinikubeRunner(t, p) // lastest publicly released version minikbue. + + // because the --wait-timeout is a new flag and the current latest release (1.3.1) doesn't have it + // this won't be necessary after we release the change with --wait-timeout flag + mkRelease.StartArgs = strings.Replace("mkRelease.StartArgs", "--wait-timeout=13m", "", 1) mkRelease.BinaryPath = fname // For full coverage: also test upgrading from oldest to newest supported k8s release stdout, stderr, err := mkRelease.Start(fmt.Sprintf("--kubernetes-version=%s", constants.OldestKubernetesVersion)) @@ -89,9 +93,9 @@ func TestVersionUpgrade(t *testing.T) { mkRelease.CheckStatus(state.Stopped.String()) // Trim the leading "v" prefix to assert that we handle it properly. - stdout, stderr, err = mkCurrent.Start(fmt.Sprintf("--kubernetes-version=%s", strings.TrimPrefix(constants.NewestKubernetesVersion, "v"))) + stdout, stderr, err = mkHead.Start(fmt.Sprintf("--kubernetes-version=%s", strings.TrimPrefix(constants.NewestKubernetesVersion, "v"))) if err != nil { t.Fatalf("TestVersionUpgrade mkCurrent.Start start failed : %v\nstdout: %s\nstderr: %s", err, stdout, stderr) } - mkCurrent.CheckStatus(state.Running.String()) + mkHead.CheckStatus(state.Running.String()) }