From 39670fc06f5b59f5902ca3eee3bd2fd30aa7cf1d Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Wed, 4 Mar 2020 17:07:49 -0800 Subject: [PATCH 01/12] Add download package & support URL fallback --- cmd/minikube/cmd/start.go | 22 ++- go.mod | 3 - go.sum | 6 - pkg/minikube/bootstrapper/bsutil/binaries.go | 3 +- pkg/minikube/cluster/iso.go | 30 ---- pkg/minikube/config/types.go | 14 +- pkg/minikube/constants/constants.go | 6 - pkg/minikube/driver/install.go | 30 +--- pkg/minikube/machine/cache_binaries.go | 77 +-------- pkg/minikube/machine/cache_binaries_test.go | 150 ------------------ pkg/minikube/machine/cluster_test.go | 14 +- pkg/minikube/node/cache.go | 3 +- .../registry/drvs/hyperkit/hyperkit.go | 3 +- pkg/minikube/registry/drvs/hyperv/hyperv.go | 2 +- pkg/minikube/registry/drvs/kvm2/kvm2.go | 3 +- .../registry/drvs/parallels/parallels.go | 3 +- .../registry/drvs/virtualbox/virtualbox.go | 3 +- pkg/minikube/registry/drvs/vmware/vmware.go | 3 +- .../drvs/vmwarefusion/vmwarefusion.go | 3 +- pkg/util/downloader.go | 131 --------------- pkg/util/downloader_test.go | 117 -------------- pkg/util/progressbar.go | 74 --------- pkg/version/version.go | 4 - .../en/docs/Contributing/drivers.en.md | 2 +- 24 files changed, 42 insertions(+), 664 deletions(-) delete mode 100644 pkg/minikube/cluster/iso.go delete mode 100644 pkg/util/downloader.go delete mode 100644 pkg/util/downloader_test.go delete mode 100644 pkg/util/progressbar.go diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index c05d715ec4..e29b5f0418 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -43,10 +43,10 @@ import ( cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config" "k8s.io/minikube/pkg/minikube/bootstrapper/bsutil" "k8s.io/minikube/pkg/minikube/bootstrapper/images" - "k8s.io/minikube/pkg/minikube/cluster" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/cruntime" + "k8s.io/minikube/pkg/minikube/download" "k8s.io/minikube/pkg/minikube/driver" "k8s.io/minikube/pkg/minikube/exit" "k8s.io/minikube/pkg/minikube/kubeconfig" @@ -154,7 +154,7 @@ func initMinikubeFlags() { startCmd.Flags().String(humanReadableDiskSize, defaultDiskSize, "Disk size allocated to the minikube VM (format: [], where unit = b, k, m or g).") startCmd.Flags().Bool(downloadOnly, false, "If true, only download and cache files for later use - don't install or start anything.") startCmd.Flags().Bool(cacheImages, true, "If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --vm-driver=none.") - startCmd.Flags().String(isoURL, constants.DefaultISOURL, "Location of the minikube iso.") + startCmd.Flags().StringSlice(isoURL, download.DefaultISOURLs(), "Locations to fetch the minikube ISO from.") startCmd.Flags().Bool(keepContext, false, "This will keep the existing kubectl context and will create a minikube context.") startCmd.Flags().Bool(embedCerts, false, "if true, will embed the certs in kubeconfig.") startCmd.Flags().String(containerRuntime, "docker", "The container runtime to be used (docker, crio, containerd).") @@ -320,7 +320,13 @@ func runStart(cmd *cobra.Command, args []string) { return } - cacheISO(&mc, driverName) + if !driver.BareMetal(driverName) && !driver.IsKIC(driverName) { + url, err := download.ISO(viper.GetStringSlice(isoURL)) + if err != nil { + exit.WithError("Failed to cache ISO", err) + } + mc.MinikubeISO = url + } if viper.GetBool(nativeSSH) { ssh.SetDefaultClient(ssh.Native) @@ -354,14 +360,6 @@ func updateDriver(driverName string) { } } -func cacheISO(cfg *config.MachineConfig, driverName string) { - if !driver.BareMetal(driverName) && !driver.IsKIC(driverName) { - if err := cluster.CacheISO(*cfg); err != nil { - exit.WithError("Failed to cache ISO", err) - } - } -} - func displayVersion(version string) { prefix := "" if viper.GetString(config.MachineProfile) != constants.DefaultMachineName { @@ -782,7 +780,6 @@ func generateCfgFromFlags(cmd *cobra.Command, k8sVersion string, drvName string) Name: viper.GetString(config.MachineProfile), KeepContext: viper.GetBool(keepContext), EmbedCerts: viper.GetBool(embedCerts), - MinikubeISO: viper.GetString(isoURL), Memory: pkgutil.CalculateSizeInMB(viper.GetString(memory)), CPUs: viper.GetInt(cpus), DiskSize: pkgutil.CalculateSizeInMB(viper.GetString(humanReadableDiskSize)), @@ -803,7 +800,6 @@ func generateCfgFromFlags(cmd *cobra.Command, k8sVersion string, drvName string) KVMQemuURI: viper.GetString(kvmQemuURI), KVMGPU: viper.GetBool(kvmGPU), KVMHidden: viper.GetBool(kvmHidden), - Downloader: pkgutil.DefaultDownloader{}, DisableDriverMounts: viper.GetBool(disableDriverMounts), UUID: viper.GetString(uuid), NoVTXCheck: viper.GetBool(noVTXCheck), diff --git a/go.mod b/go.mod index e4798d896c..3618988a2a 100644 --- a/go.mod +++ b/go.mod @@ -23,15 +23,12 @@ require ( github.com/google/go-cmp v0.3.0 github.com/google/go-containerregistry v0.0.0-20200131185320-aec8da010de2 github.com/googleapis/gnostic v0.3.0 // indirect - github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce // indirect github.com/hashicorp/go-getter v1.4.0 - github.com/hashicorp/go-multierror v0.0.0-20160811015721-8c5f0ad93604 // indirect github.com/hashicorp/go-retryablehttp v0.5.4 github.com/hooklift/assert v0.0.0-20170704181755-9d1defd6d214 // indirect github.com/hooklift/iso9660 v0.0.0-20170318115843-1cf07e5970d8 github.com/imdario/mergo v0.3.8 // indirect github.com/intel-go/cpuid v0.0.0-20181003105527-1a4a6f06a1c6 // indirect - github.com/jimmidyson/go-download v0.0.0-20161028105827-7f9a90c8c95b github.com/johanneswuerbach/nfsexports v0.0.0-20181204082207-1aa528dcb345 github.com/juju/clock v0.0.0-20190205081909-9c5c9712527c github.com/juju/errors v0.0.0-20190806202954-0232dcc7464d // indirect diff --git a/go.sum b/go.sum index a5032159c7..42c9980d01 100644 --- a/go.sum +++ b/go.sum @@ -346,12 +346,8 @@ github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:Fecb github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce h1:prjrVgOk2Yg6w+PflHoszQNLTUh4kaByUcEWM/9uin4= -github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0 h1:wvCrVc9TjDls6+YGAF2hAifE1E5U1+b4tH6KdvN3Gig= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-multierror v0.0.0-20160811015721-8c5f0ad93604 h1:VIq8E7fMiC4h3agg0ya56L0jHn7QisZZcWZXVKJb9jQ= -github.com/hashicorp/go-multierror v0.0.0-20160811015721-8c5f0ad93604/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= github.com/hashicorp/go-retryablehttp v0.5.4 h1:1BZvpawXoJCWX6pNtow9+rpEj+3itIlutiqnntI6jOE= github.com/hashicorp/go-retryablehttp v0.5.4/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo= @@ -386,8 +382,6 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt github.com/intel-go/cpuid v0.0.0-20181003105527-1a4a6f06a1c6 h1:XboatR7lasl05yel5hNXF7kQBw2oFUGdMztcgisfhNU= github.com/intel-go/cpuid v0.0.0-20181003105527-1a4a6f06a1c6/go.mod h1:RmeVYf9XrPRbRc3XIx0gLYA8qOFvNoPOfaEZduRlEp4= github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= -github.com/jimmidyson/go-download v0.0.0-20161028105827-7f9a90c8c95b h1:3TknJxYSK1eDe21QorC3C2Yz8jylk6vlJG9YABnFzkU= -github.com/jimmidyson/go-download v0.0.0-20161028105827-7f9a90c8c95b/go.mod h1:I3WsAhNNoG7a/d8HMlYUywJJlfOs/+/83NEUjuDp4lc= github.com/jimstudt/http-authentication v0.0.0-20140401203705-3eca13d6893a/go.mod h1:wK6yTYYcgjHE1Z1QtXACPDjcFJyBskHEdagmnq3vsP8= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 h1:12VvqtR6Aowv3l/EQUlocDHW2Cp4G9WJVH7uyH8QFJE= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= diff --git a/pkg/minikube/bootstrapper/bsutil/binaries.go b/pkg/minikube/bootstrapper/bsutil/binaries.go index 532be8a7ad..6778599db7 100644 --- a/pkg/minikube/bootstrapper/bsutil/binaries.go +++ b/pkg/minikube/bootstrapper/bsutil/binaries.go @@ -27,6 +27,7 @@ import ( "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/constants" + "k8s.io/minikube/pkg/minikube/download" "k8s.io/minikube/pkg/minikube/machine" "k8s.io/minikube/pkg/minikube/vmpath" ) @@ -43,7 +44,7 @@ func TransferBinaries(cfg config.KubernetesConfig, c command.Runner) error { for _, name := range constants.KubernetesReleaseBinaries { name := name g.Go(func() error { - src, err := machine.CacheBinary(name, cfg.KubernetesVersion, "linux", runtime.GOARCH) + src, err := download.Binary(name, cfg.KubernetesVersion, "linux", runtime.GOARCH) if err != nil { return errors.Wrapf(err, "downloading %s", name) } diff --git a/pkg/minikube/cluster/iso.go b/pkg/minikube/cluster/iso.go deleted file mode 100644 index 253acbd370..0000000000 --- a/pkg/minikube/cluster/iso.go +++ /dev/null @@ -1,30 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cluster - -import ( - "k8s.io/minikube/pkg/minikube/config" - "k8s.io/minikube/pkg/minikube/driver" -) - -// CacheISO downloads and caches ISO. -func CacheISO(cfg config.MachineConfig) error { - if driver.BareMetal(cfg.Driver) { - return nil - } - return cfg.Downloader.CacheMinikubeISOFromURL(cfg.MinikubeISO) -} diff --git a/pkg/minikube/config/types.go b/pkg/minikube/config/types.go index 6513efd2d9..b953c514c1 100644 --- a/pkg/minikube/config/types.go +++ b/pkg/minikube/config/types.go @@ -20,7 +20,6 @@ import ( "net" "github.com/blang/semver" - "k8s.io/minikube/pkg/util" ) // Profile represents a minikube profile @@ -49,13 +48,12 @@ type MachineConfig struct { HypervVirtualSwitch string HypervUseExternalSwitch bool HypervExternalAdapter string - KVMNetwork string // Only used by the KVM driver - KVMQemuURI string // Only used by kvm2 - KVMGPU bool // Only used by kvm2 - KVMHidden bool // Only used by kvm2 - Downloader util.ISODownloader `json:"-"` - DockerOpt []string // Each entry is formatted as KEY=VALUE. - DisableDriverMounts bool // Only used by virtualbox + KVMNetwork string // Only used by the KVM driver + KVMQemuURI string // Only used by kvm2 + KVMGPU bool // Only used by kvm2 + KVMHidden bool // Only used by kvm2 + DockerOpt []string // Each entry is formatted as KEY=VALUE. + DisableDriverMounts bool // Only used by virtualbox NFSShare []string NFSSharesRoot string UUID string // Only used by hyperkit to restore the mac address diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index dbf74c4747..0bc95661f9 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -17,13 +17,11 @@ limitations under the License. package constants import ( - "fmt" "path/filepath" "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/util/homedir" "k8s.io/minikube/pkg/minikube/localpath" - minikubeVersion "k8s.io/minikube/pkg/version" ) const ( @@ -78,10 +76,6 @@ var ( var ( SHASuffix = ".sha256" - // DefaultISOURL is the default location of the minikube.iso file - DefaultISOURL = fmt.Sprintf("https://storage.googleapis.com/%s/minikube-%s.iso", minikubeVersion.GetISOPath(), minikubeVersion.GetISOVersion()) - // DefaultISOSHAURL is the default location of the minikube.iso.sha256 file - DefaultISOSHAURL = DefaultISOURL + SHASuffix // DockerDaemonEnvs is list of docker-daemon related environment variables. DockerDaemonEnvs = [3]string{DockerHostEnv, DockerTLSVerifyEnv, DockerCertPathEnv} diff --git a/pkg/minikube/driver/install.go b/pkg/minikube/driver/install.go index 8a49e833fa..ae17a917cd 100644 --- a/pkg/minikube/driver/install.go +++ b/pkg/minikube/driver/install.go @@ -27,12 +27,11 @@ import ( "github.com/blang/semver" "github.com/golang/glog" - "github.com/hashicorp/go-getter" "github.com/juju/mutex" "github.com/pkg/errors" + "k8s.io/minikube/pkg/minikube/download" "k8s.io/minikube/pkg/minikube/out" - "k8s.io/minikube/pkg/util" "k8s.io/minikube/pkg/util/lock" ) @@ -59,7 +58,7 @@ func InstallOrUpdate(name string, directory string, v semver.Version, interactiv if !exists || (err != nil && autoUpdate) { glog.Warningf("%s: %v", executable, err) path = filepath.Join(directory, executable) - derr := download(executable, path, v) + derr := download.Driver(executable, path, v) if derr != nil { return derr } @@ -139,31 +138,6 @@ func validateDriver(executable string, v semver.Version) (string, error) { return path, nil } -func driverWithChecksumURL(name string, v semver.Version) string { - base := fmt.Sprintf("https://github.com/kubernetes/minikube/releases/download/v%s/%s", v, name) - return fmt.Sprintf("%s?checksum=file:%s.sha256", base, base) -} - -// download an arbitrary driver -func download(name string, destination string, v semver.Version) error { - out.T(out.FileDownload, "Downloading driver {{.driver}}:", out.V{"driver": name}) - os.Remove(destination) - url := driverWithChecksumURL(name, v) - client := &getter.Client{ - Src: url, - Dst: destination, - Mode: getter.ClientModeFile, - Options: []getter.ClientOption{getter.WithProgress(util.DefaultProgressBar)}, - } - - glog.Infof("Downloading: %+v", client) - if err := client.Get(); err != nil { - return errors.Wrapf(err, "download failed: %s", url) - } - // Give downloaded drivers a baseline decent file permission - return os.Chmod(destination, 0755) -} - // extractDriverVersion extracts the driver version. // KVM and Hyperkit drivers support the 'version' command, that display the information as: // version: vX.X.X diff --git a/pkg/minikube/machine/cache_binaries.go b/pkg/minikube/machine/cache_binaries.go index 898540eab3..f702ee950c 100644 --- a/pkg/minikube/machine/cache_binaries.go +++ b/pkg/minikube/machine/cache_binaries.go @@ -17,22 +17,15 @@ limitations under the License. package machine import ( - "crypto" - "fmt" - "os" "path" "runtime" - "github.com/blang/semver" - "github.com/golang/glog" - "github.com/jimmidyson/go-download" "github.com/pkg/errors" "golang.org/x/sync/errgroup" "k8s.io/minikube/pkg/minikube/assets" "k8s.io/minikube/pkg/minikube/bootstrapper" "k8s.io/minikube/pkg/minikube/command" - "k8s.io/minikube/pkg/minikube/localpath" - "k8s.io/minikube/pkg/minikube/out" + "k8s.io/minikube/pkg/minikube/download" ) // CacheBinariesForBootstrapper will cache binaries for a bootstrapper @@ -43,7 +36,7 @@ func CacheBinariesForBootstrapper(version string, clusterBootstrapper string) er for _, bin := range binaries { bin := bin // https://golang.org/doc/faq#closures_and_goroutines g.Go(func() error { - if _, err := CacheBinary(bin, version, "linux", runtime.GOARCH); err != nil { + if _, err := download.Binary(bin, version, "linux", runtime.GOARCH); err != nil { return errors.Wrapf(err, "caching binary %s", bin) } return nil @@ -52,72 +45,6 @@ func CacheBinariesForBootstrapper(version string, clusterBootstrapper string) er return g.Wait() } -// releaseURL gets the location of a Kubernetes binary -func releaseURL(binaryName, version, osName, archName string) string { - return fmt.Sprintf("https://storage.googleapis.com/kubernetes-release/release/%s/bin/%s/%s/%s", version, osName, archName, binaryName) -} - -// downloadOptions returns appropriate download options for a -func downloadOptions(url string, version string) (download.FileOptions, error) { - fo := download.FileOptions{ - Mkdirs: download.MkdirAll, - Options: download.Options{ - ChecksumHash: crypto.SHA1, - Checksum: url + ".sha1", - }, - } - - v, err := semver.Make(version[1:]) - if err != nil { - return fo, err - } - - if v.GTE(semver.MustParse("1.17.0")) { - fo.ChecksumHash = crypto.SHA256 - fo.Checksum = url + ".sha256" - } - return fo, nil -} - -// CacheBinary will cache a binary on the host -func CacheBinary(binary, version, osName, archName string) (string, error) { - - targetDir := localpath.MakeMiniPath("cache", osName, version) - targetFilepath := path.Join(targetDir, binary) - - url := releaseURL(binary, version, osName, archName) - - _, err := os.Stat(targetFilepath) - // If it exists, do no verification and continue - if err == nil { - glog.Infof("Not caching binary, using %s", url) - return targetFilepath, nil - } - if !os.IsNotExist(err) { - return "", errors.Wrapf(err, "stat %s version %s at %s", binary, version, targetDir) - } - - if err = os.MkdirAll(targetDir, 0777); err != nil { - return "", errors.Wrapf(err, "mkdir %s", targetDir) - } - - options, err := downloadOptions(url, version) - if err != nil { - return "", errors.Wrap(err, "options") - } - glog.Infof("Downloading %s: options: %+v", url, options) - - out.T(out.FileDownload, "Downloading {{.name}} {{.version}}", out.V{"name": binary, "version": version}) - if err := download.ToFile(url, targetFilepath, options); err != nil { - return "", errors.Wrapf(err, url) - } - if osName == runtime.GOOS && archName == runtime.GOARCH { - if err = os.Chmod(targetFilepath, 0755); err != nil { - return "", errors.Wrapf(err, "chmod +x %s", targetFilepath) - } - } - return targetFilepath, nil -} // CopyBinary copies a locally cached binary to the guest VM func CopyBinary(cr command.Runner, src string, dest string) error { diff --git a/pkg/minikube/machine/cache_binaries_test.go b/pkg/minikube/machine/cache_binaries_test.go index b490974eac..500c0ea74d 100644 --- a/pkg/minikube/machine/cache_binaries_test.go +++ b/pkg/minikube/machine/cache_binaries_test.go @@ -17,15 +17,11 @@ limitations under the License. package machine import ( - "crypto" "fmt" "io/ioutil" "os" - "runtime" "testing" - "github.com/google/go-cmp/cmp" - "github.com/jimmidyson/go-download" "k8s.io/minikube/pkg/minikube/assets" "k8s.io/minikube/pkg/minikube/bootstrapper" "k8s.io/minikube/pkg/minikube/command" @@ -126,149 +122,3 @@ func TestCacheBinariesForBootstrapper(t *testing.T) { }) } } -func TestCacheBinary(t *testing.T) { - oldMinikubeHome := os.Getenv("MINIKUBE_HOME") - defer os.Setenv("MINIKUBE_HOME", oldMinikubeHome) - - minikubeHome, err := ioutil.TempDir("/tmp", "") - if err != nil { - t.Fatalf("error during creating tmp dir: %v", err) - } - defer os.RemoveAll(minikubeHome) - noWritePermDir, err := ioutil.TempDir("/tmp", "") - if err != nil { - t.Fatalf("error during creating tmp dir: %v", err) - } - defer os.RemoveAll(noWritePermDir) - err = os.Chmod(noWritePermDir, 0000) - if err != nil { - t.Fatalf("error (%v) during changing permissions of dir %v", err, noWritePermDir) - } - - var tc = []struct { - desc, version, osName, archName string - minikubeHome, binary, description string - err bool - }{ - { - desc: "ok kubeadm", - version: "v1.16.0", - osName: "linux", - archName: runtime.GOARCH, - binary: "kubeadm", - err: false, - minikubeHome: minikubeHome, - }, - { - desc: "minikube home in dir without perms and arm runtime", - version: "v1.16.0", - osName: runtime.GOOS, - archName: "arm", - binary: "kubectl", - err: true, - minikubeHome: noWritePermDir, - }, - { - desc: "minikube home in dir without perms", - version: "v1.16.0", - osName: runtime.GOOS, - archName: runtime.GOARCH, - binary: "kubectl", - err: true, - minikubeHome: noWritePermDir, - }, - { - desc: "binary foo", - version: "v1.16.0", - osName: runtime.GOOS, - archName: runtime.GOARCH, - binary: "foo", - err: true, - minikubeHome: minikubeHome, - }, - { - desc: "version 9000", - version: "v9000", - osName: runtime.GOOS, - archName: runtime.GOARCH, - binary: "foo", - err: true, - minikubeHome: minikubeHome, - }, - { - desc: "bad os", - version: "v1.16.0", - osName: "no-such-os", - archName: runtime.GOARCH, - binary: "kubectl", - err: true, - minikubeHome: minikubeHome, - }, - } - for _, test := range tc { - t.Run(test.desc, func(t *testing.T) { - os.Setenv("MINIKUBE_HOME", test.minikubeHome) - _, err := CacheBinary(test.binary, test.version, test.osName, test.archName) - if err != nil && !test.err { - t.Fatalf("Got unexpected error %v", err) - } - if err == nil && test.err { - t.Fatalf("Expected error but got %v", err) - } - }) - } -} - -func TestDownloadOptions(t *testing.T) { - var tc = []struct { - url string - version string - want download.FileOptions - }{ - { - url: "https://s/kubernetes-release/release/v1.16.0/bin/amd64/kubectl", - version: "v1.16.0", - want: download.FileOptions{ - Options: download.Options{ - Checksum: "https://s/kubernetes-release/release/v1.16.0/bin/amd64/kubectl.sha1", - ChecksumHash: crypto.SHA1, - }, - Mkdirs: download.MkdirAll, - }, - }, - { - url: "https://s/kubernetes-release/release/v1.10.0/bin/hp9k/kubeadm", - version: "v1.10.0", - want: download.FileOptions{ - Options: download.Options{ - Checksum: "https://s/kubernetes-release/release/v1.10.0/bin/hp9k/kubeadm.sha1", - ChecksumHash: crypto.SHA1, - }, - Mkdirs: download.MkdirAll, - }, - }, - { - url: "https://s/kubernetes-release/release/v1.18.0/bin/arm64/kubelet", - version: "v1.18.0", - want: download.FileOptions{ - Options: download.Options{ - Checksum: "https://s/kubernetes-release/release/v1.18.0/bin/arm64/kubelet.sha256", - ChecksumHash: crypto.SHA256, - }, - Mkdirs: download.MkdirAll, - }, - }, - } - for _, test := range tc { - t.Run(test.version, func(t *testing.T) { - got, err := downloadOptions(test.url, test.version) - if err != nil { - t.Fatalf("unexpected error %v", err) - } - - if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("unexpected options(-want +got):\n%s", diff) - } - }) - } -} diff --git a/pkg/minikube/machine/cluster_test.go b/pkg/minikube/machine/cluster_test.go index bf38656062..84c23ae889 100644 --- a/pkg/minikube/machine/cluster_test.go +++ b/pkg/minikube/machine/cluster_test.go @@ -30,7 +30,6 @@ import ( "github.com/docker/machine/libmachine/state" "github.com/spf13/viper" "k8s.io/minikube/pkg/minikube/config" - "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/driver" "k8s.io/minikube/pkg/minikube/registry" "k8s.io/minikube/pkg/minikube/tests" @@ -61,10 +60,8 @@ func RegisterMockDriver(t *testing.T) { } var defaultMachineConfig = config.MachineConfig{ - Driver: driver.Mock, - MinikubeISO: constants.DefaultISOURL, - Downloader: MockDownloader{}, - DockerEnv: []string{"MOCK_MAKE_IT_PROVISION=true"}, + Driver: driver.Mock, + DockerEnv: []string{"MOCK_MAKE_IT_PROVISION=true"}, } func TestCreateHost(t *testing.T) { @@ -262,10 +259,9 @@ func TestStartHostConfig(t *testing.T) { provision.SetDetector(md) config := config.MachineConfig{ - Driver: driver.Mock, - DockerEnv: []string{"FOO=BAR"}, - DockerOpt: []string{"param=value"}, - Downloader: MockDownloader{}, + Driver: driver.Mock, + DockerEnv: []string{"FOO=BAR"}, + DockerOpt: []string{"param=value"}, } h, err := StartHost(api, config) diff --git a/pkg/minikube/node/cache.go b/pkg/minikube/node/cache.go index b1d5ad8d12..e030b9942e 100644 --- a/pkg/minikube/node/cache.go +++ b/pkg/minikube/node/cache.go @@ -26,6 +26,7 @@ import ( cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/constants" + "k8s.io/minikube/pkg/minikube/download" "k8s.io/minikube/pkg/minikube/exit" "k8s.io/minikube/pkg/minikube/image" "k8s.io/minikube/pkg/minikube/localpath" @@ -71,7 +72,7 @@ func CacheKubectlBinary(k8sVerison string) (string, error) { binary = "kubectl.exe" } - return machine.CacheBinary(binary, k8sVerison, runtime.GOOS, runtime.GOARCH) + return download.Binary(binary, k8sVerison, runtime.GOOS, runtime.GOARCH) } // doCacheBinaries caches Kubernetes binaries in the foreground diff --git a/pkg/minikube/registry/drvs/hyperkit/hyperkit.go b/pkg/minikube/registry/drvs/hyperkit/hyperkit.go index 50a4e5a408..edde1da359 100644 --- a/pkg/minikube/registry/drvs/hyperkit/hyperkit.go +++ b/pkg/minikube/registry/drvs/hyperkit/hyperkit.go @@ -32,6 +32,7 @@ import ( "k8s.io/minikube/pkg/drivers/hyperkit" cfg "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/download" "k8s.io/minikube/pkg/minikube/driver" "k8s.io/minikube/pkg/minikube/localpath" "k8s.io/minikube/pkg/minikube/registry" @@ -69,7 +70,7 @@ func configure(config cfg.MachineConfig) (interface{}, error) { StorePath: localpath.MiniPath(), SSHUser: "docker", }, - Boot2DockerURL: config.Downloader.GetISOFileURI(config.MinikubeISO), + Boot2DockerURL: download.LocalISOResource(config.MinikubeISO), DiskSize: config.DiskSize, Memory: config.Memory, CPU: config.CPUs, diff --git a/pkg/minikube/registry/drvs/hyperv/hyperv.go b/pkg/minikube/registry/drvs/hyperv/hyperv.go index 9f15d0c470..db2f3c0144 100644 --- a/pkg/minikube/registry/drvs/hyperv/hyperv.go +++ b/pkg/minikube/registry/drvs/hyperv/hyperv.go @@ -54,7 +54,7 @@ func init() { func configure(config cfg.MachineConfig) (interface{}, error) { d := hyperv.NewDriver(config.Name, localpath.MiniPath()) - d.Boot2DockerURL = config.Downloader.GetISOFileURI(config.MinikubeISO) + d.Boot2DockerURL = download.LocalISOResource(config.MinikubeISO) d.VSwitch = config.HypervVirtualSwitch if d.VSwitch == "" && config.HypervUseExternalSwitch { switchName, adapter, err := chooseSwitch(config.HypervExternalAdapter) diff --git a/pkg/minikube/registry/drvs/kvm2/kvm2.go b/pkg/minikube/registry/drvs/kvm2/kvm2.go index dedad73bfb..0ae7d89754 100644 --- a/pkg/minikube/registry/drvs/kvm2/kvm2.go +++ b/pkg/minikube/registry/drvs/kvm2/kvm2.go @@ -30,6 +30,7 @@ import ( "github.com/docker/machine/libmachine/drivers" "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/download" "k8s.io/minikube/pkg/minikube/driver" "k8s.io/minikube/pkg/minikube/localpath" "k8s.io/minikube/pkg/minikube/registry" @@ -79,7 +80,7 @@ func configure(mc config.MachineConfig) (interface{}, error) { CPU: mc.CPUs, Network: mc.KVMNetwork, PrivateNetwork: "minikube-net", - Boot2DockerURL: mc.Downloader.GetISOFileURI(mc.MinikubeISO), + Boot2DockerURL: download.LocalISOResource(mc.MinikubeISO), DiskSize: mc.DiskSize, DiskPath: filepath.Join(localpath.MiniPath(), "machines", name, fmt.Sprintf("%s.rawdisk", name)), ISO: filepath.Join(localpath.MiniPath(), "machines", name, "boot2docker.iso"), diff --git a/pkg/minikube/registry/drvs/parallels/parallels.go b/pkg/minikube/registry/drvs/parallels/parallels.go index de319ec8fb..d5a6ef7169 100644 --- a/pkg/minikube/registry/drvs/parallels/parallels.go +++ b/pkg/minikube/registry/drvs/parallels/parallels.go @@ -25,6 +25,7 @@ import ( parallels "github.com/Parallels/docker-machine-parallels" "github.com/docker/machine/libmachine/drivers" cfg "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/download" "k8s.io/minikube/pkg/minikube/driver" "k8s.io/minikube/pkg/minikube/localpath" "k8s.io/minikube/pkg/minikube/registry" @@ -46,7 +47,7 @@ func init() { func configure(config cfg.MachineConfig) (interface{}, error) { d := parallels.NewDriver(config.Name, localpath.MiniPath()).(*parallels.Driver) - d.Boot2DockerURL = config.Downloader.GetISOFileURI(config.MinikubeISO) + d.Boot2DockerURL = download.LocalISOResource(config.MinikubeISO) d.Memory = config.Memory d.CPU = config.CPUs d.DiskSize = config.DiskSize diff --git a/pkg/minikube/registry/drvs/virtualbox/virtualbox.go b/pkg/minikube/registry/drvs/virtualbox/virtualbox.go index bfba0e42db..9fb264b8f6 100644 --- a/pkg/minikube/registry/drvs/virtualbox/virtualbox.go +++ b/pkg/minikube/registry/drvs/virtualbox/virtualbox.go @@ -27,6 +27,7 @@ import ( "github.com/docker/machine/libmachine/drivers" "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/download" "k8s.io/minikube/pkg/minikube/driver" "k8s.io/minikube/pkg/minikube/localpath" "k8s.io/minikube/pkg/minikube/registry" @@ -51,7 +52,7 @@ func init() { func configure(mc config.MachineConfig) (interface{}, error) { d := virtualbox.NewDriver(mc.Name, localpath.MiniPath()) - d.Boot2DockerURL = mc.Downloader.GetISOFileURI(mc.MinikubeISO) + d.Boot2DockerURL = download.LocalISOResource(mc.MinikubeISO) d.Memory = mc.Memory d.CPU = mc.CPUs d.DiskSize = mc.DiskSize diff --git a/pkg/minikube/registry/drvs/vmware/vmware.go b/pkg/minikube/registry/drvs/vmware/vmware.go index 885063cde2..f432916211 100644 --- a/pkg/minikube/registry/drvs/vmware/vmware.go +++ b/pkg/minikube/registry/drvs/vmware/vmware.go @@ -22,6 +22,7 @@ import ( vmwcfg "github.com/machine-drivers/docker-machine-driver-vmware/pkg/drivers/vmware/config" "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/download" "k8s.io/minikube/pkg/minikube/driver" "k8s.io/minikube/pkg/minikube/localpath" "k8s.io/minikube/pkg/minikube/registry" @@ -41,7 +42,7 @@ func init() { func configure(mc config.MachineConfig) (interface{}, error) { d := vmwcfg.NewConfig(mc.Name, localpath.MiniPath()) - d.Boot2DockerURL = mc.Downloader.GetISOFileURI(mc.MinikubeISO) + d.Boot2DockerURL = download.LocalISOResource(mc.MinikubeISO) d.Memory = mc.Memory d.CPU = mc.CPUs d.DiskSize = mc.DiskSize diff --git a/pkg/minikube/registry/drvs/vmwarefusion/vmwarefusion.go b/pkg/minikube/registry/drvs/vmwarefusion/vmwarefusion.go index bb5ed4196b..5c371d0dad 100644 --- a/pkg/minikube/registry/drvs/vmwarefusion/vmwarefusion.go +++ b/pkg/minikube/registry/drvs/vmwarefusion/vmwarefusion.go @@ -27,6 +27,7 @@ import ( "github.com/pkg/errors" cfg "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/download" "k8s.io/minikube/pkg/minikube/driver" "k8s.io/minikube/pkg/minikube/localpath" "k8s.io/minikube/pkg/minikube/registry" @@ -46,7 +47,7 @@ func init() { func configure(config cfg.MachineConfig) (interface{}, error) { d := vmwarefusion.NewDriver(config.Name, localpath.MiniPath()).(*vmwarefusion.Driver) - d.Boot2DockerURL = config.Downloader.GetISOFileURI(config.MinikubeISO) + d.Boot2DockerURL = download.LocalISOResource(config.MinikubeISO) d.Memory = config.Memory d.CPU = config.CPUs d.DiskSize = config.DiskSize diff --git a/pkg/util/downloader.go b/pkg/util/downloader.go deleted file mode 100644 index 72b4782565..0000000000 --- a/pkg/util/downloader.go +++ /dev/null @@ -1,131 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "net/url" - "os" - "path/filepath" - "time" - - "github.com/golang/glog" - "github.com/hashicorp/go-getter" - "github.com/juju/mutex" - "github.com/pkg/errors" - "k8s.io/minikube/pkg/minikube/constants" - "k8s.io/minikube/pkg/minikube/localpath" - "k8s.io/minikube/pkg/minikube/out" - "k8s.io/minikube/pkg/util/lock" -) - -const fileScheme = "file" - -// ISODownloader downloads an ISO -type ISODownloader interface { - GetISOFileURI(isoURL string) string - CacheMinikubeISOFromURL(isoURL string) error -} - -// DefaultDownloader is the default ISODownloader -type DefaultDownloader struct{} - -// GetISOFileURI gets the local destination for a remote source -func (f DefaultDownloader) GetISOFileURI(isoURL string) string { - urlObj, err := url.Parse(isoURL) - if err != nil { - return isoURL - } - if urlObj.Scheme == fileScheme { - return isoURL - } - isoPath := filepath.Join(localpath.MiniPath(), "cache", "iso", filepath.Base(isoURL)) - // As this is a file URL there should be no backslashes regardless of platform running on. - return "file://" + filepath.ToSlash(isoPath) -} - -// CacheMinikubeISOFromURL downloads the ISO, if it doesn't exist in cache -func (f DefaultDownloader) CacheMinikubeISOFromURL(url string) error { - dst := f.GetISOCacheFilepath(url) - - // Lock before we check for existence to avoid thundering herd issues - spec := lock.PathMutexSpec(dst) - spec.Timeout = 10 * time.Minute - glog.Infof("acquiring lock: %+v", spec) - releaser, err := mutex.Acquire(spec) - if err != nil { - return errors.Wrapf(err, "unable to acquire lock for %+v", spec) - } - defer releaser.Release() - - if !f.ShouldCacheMinikubeISO(url) { - glog.Infof("Not caching ISO, using %s", url) - return nil - } - - urlWithChecksum := url - if url == constants.DefaultISOURL { - urlWithChecksum = url + "?checksum=file:" + constants.DefaultISOSHAURL - } - - // Predictable temp destination so that resume can function - tmpDst := dst + ".download" - - opts := []getter.ClientOption{getter.WithProgress(DefaultProgressBar)} - client := &getter.Client{ - Src: urlWithChecksum, - Dst: tmpDst, - Mode: getter.ClientModeFile, - Options: opts, - } - - glog.Infof("full url: %s", urlWithChecksum) - out.T(out.ISODownload, "Downloading VM boot image ...") - if err := client.Get(); err != nil { - return errors.Wrap(err, url) - } - return os.Rename(tmpDst, dst) -} - -// ShouldCacheMinikubeISO returns if we need to download the ISO -func (f DefaultDownloader) ShouldCacheMinikubeISO(isoURL string) bool { - // store the minikube-iso inside the .minikube dir - - urlObj, err := url.Parse(isoURL) - if err != nil { - return false - } - if urlObj.Scheme == fileScheme { - return false - } - if f.IsMinikubeISOCached(isoURL) { - return false - } - return true -} - -// GetISOCacheFilepath returns the path of an ISO in the local cache -func (f DefaultDownloader) GetISOCacheFilepath(isoURL string) string { - return filepath.Join(localpath.MiniPath(), "cache", "iso", filepath.Base(isoURL)) -} - -// IsMinikubeISOCached returns if an ISO exists in the local cache -func (f DefaultDownloader) IsMinikubeISOCached(isoURL string) bool { - if _, err := os.Stat(f.GetISOCacheFilepath(isoURL)); os.IsNotExist(err) { - return false - } - return true -} diff --git a/pkg/util/downloader_test.go b/pkg/util/downloader_test.go deleted file mode 100644 index 3db8593296..0000000000 --- a/pkg/util/downloader_test.go +++ /dev/null @@ -1,117 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "bytes" - "io" - "io/ioutil" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "testing" - - "k8s.io/minikube/pkg/minikube/localpath" - "k8s.io/minikube/pkg/minikube/tests" -) - -func TestGetISOFileURI(t *testing.T) { - dler := DefaultDownloader{} - - tests := map[string]string{ - "file:///test/path/minikube-test.iso": "file:///test/path/minikube-test.iso", - "https://storage.googleapis.com/minikube/iso/minikube-test.iso": "file://" + filepath.ToSlash(filepath.Join(localpath.MiniPath(), "cache", "iso", "minikube-test.iso")), - } - - for input, expected := range tests { - if isoFileURI := dler.GetISOFileURI(input); isoFileURI != expected { - t.Fatalf("Expected GetISOFileURI with input %s to return %s but instead got: %s", input, expected, isoFileURI) - } - } - -} - -var testISOString = "hello" - -func TestCacheMinikubeISOFromURL(t *testing.T) { - tempDir := tests.MakeTempDir() - defer os.RemoveAll(tempDir) - dler := DefaultDownloader{} - isoPath := filepath.Join(localpath.MiniPath(), "cache", "iso", "minikube-test.iso") - - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if _, err := io.WriteString(w, testISOString); err != nil { - t.Fatalf("WriteString: %v", err) - } - })) - isoURL := server.URL + "/minikube-test.iso" - if err := dler.CacheMinikubeISOFromURL(isoURL); err != nil { - t.Fatalf("Unexpected error from CacheMinikubeISOFromURL: %v", err) - } - - transferred, err := ioutil.ReadFile(filepath.Join(isoPath)) - if err != nil { - t.Fatalf("File not copied. Could not open file at path: %s", isoPath) - } - - // test that the ISO is transferred properly - contents := []byte(testISOString) - if !bytes.Contains(transferred, contents) { - t.Fatalf("Expected transfers to contain: %s. It was: %s", contents, transferred) - } - -} - -func TestShouldCacheMinikubeISO(t *testing.T) { - dler := DefaultDownloader{} - - tests := map[string]bool{ - "file:///test/path/minikube-test.iso": false, - "https://storage.googleapis.com/minikube/iso/minikube-test.iso": true, - } - - for input, expected := range tests { - if out := dler.ShouldCacheMinikubeISO(input); out != expected { - t.Fatalf("Expected ShouldCacheMinikubeISO with input %s to return %t but instead got: %t", input, expected, out) - } - } -} - -func TestIsMinikubeISOCached(t *testing.T) { - tempDir := tests.MakeTempDir() - defer os.RemoveAll(tempDir) - - dler := DefaultDownloader{} - - testFileURI := "file:///test/path/minikube-test.iso" - expected := false - - if out := dler.IsMinikubeISOCached(testFileURI); out != expected { - t.Fatalf("Expected IsMinikubeISOCached with input %s to return %t but instead got: %t", testFileURI, expected, out) - } - - if err := ioutil.WriteFile(filepath.Join(localpath.MiniPath(), "cache", "iso", "minikube-test.iso"), []byte(testISOString), os.FileMode(int(0644))); err != nil { - t.Fatalf("WriteFile: %v", err) - } - - expected = true - if out := dler.IsMinikubeISOCached(testFileURI); out != expected { - t.Fatalf("Expected IsMinikubeISOCached with input %s to return %t but instead got: %t", testFileURI, expected, out) - } - -} diff --git a/pkg/util/progressbar.go b/pkg/util/progressbar.go deleted file mode 100644 index fb16f6773c..0000000000 --- a/pkg/util/progressbar.go +++ /dev/null @@ -1,74 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// This file implements a go-getter wrapper for cheggaaa progress bar -// based on: -// https://github.com/hashicorp/go-getter/blob/master/cmd/go-getter/progress_tracking.go - -package util - -import ( - "io" - "path/filepath" - "sync" - - "github.com/cheggaaa/pb/v3" - "github.com/hashicorp/go-getter" -) - -// DefaultProgressBar is the default cheggaaa progress bar -var DefaultProgressBar getter.ProgressTracker = &progressBar{} - -type progressBar struct { - lock sync.Mutex - progress *pb.ProgressBar -} - -// TrackProgress instantiates a new progress bar that will -// display the progress of stream until closed. -// total can be 0. -func (cpb *progressBar) TrackProgress(src string, currentSize, totalSize int64, stream io.ReadCloser) io.ReadCloser { - cpb.lock.Lock() - defer cpb.lock.Unlock() - if cpb.progress == nil { - cpb.progress = pb.New64(totalSize) - } - p := pb.Full.Start64(totalSize) - p.Set("prefix", " > "+filepath.Base(src+": ")) - p.SetCurrent(currentSize) - p.Set(pb.Bytes, true) - - // Just a hair less than 80 (standard terminal width) for aesthetics & pasting into docs - p.SetWidth(79) - barReader := p.NewProxyReader(stream) - - return &readCloser{ - Reader: barReader, - close: func() error { - cpb.lock.Lock() - defer cpb.lock.Unlock() - p.Finish() - return nil - }, - } -} - -type readCloser struct { - io.Reader - close func() error -} - -func (c *readCloser) Close() error { return c.close() } diff --git a/pkg/version/version.go b/pkg/version/version.go index 727035115b..6587f3f722 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -53,10 +53,6 @@ func GetISOVersion() string { return isoVersion } -// GetISOPath returns the remote path to the minikube.iso -func GetISOPath() string { - return isoPath -} // GetSemverVersion returns the current minikube semantic version (semver) func GetSemverVersion() (semver.Version, error) { diff --git a/site/content/en/docs/Contributing/drivers.en.md b/site/content/en/docs/Contributing/drivers.en.md index aecd0e1179..3641e6b766 100644 --- a/site/content/en/docs/Contributing/drivers.en.md +++ b/site/content/en/docs/Contributing/drivers.en.md @@ -87,7 +87,7 @@ func init() { func createVMwareFusionHost(config cfg.MachineConfig) interface{} { d := vmwarefusion.NewDriver(config.Name, localpath.MiniPath()).(*vmwarefusion.Driver) - d.Boot2DockerURL = config.Downloader.GetISOFileURI(config.MinikubeISO) + d.Boot2DockerURL = download.LocalISOResource(config.MinikubeISO) d.Memory = config.Memory d.CPU = config.CPUs d.DiskSize = config.DiskSize From 27cca764b57d89252f4a4184a3498845f4dc8cd6 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Wed, 4 Mar 2020 18:57:12 -0800 Subject: [PATCH 02/12] Fix integration test --- test/integration/aaa_download_only_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/integration/aaa_download_only_test.go b/test/integration/aaa_download_only_test.go index 08e42ed07b..2b9393c722 100644 --- a/test/integration/aaa_download_only_test.go +++ b/test/integration/aaa_download_only_test.go @@ -38,6 +38,7 @@ import ( "k8s.io/minikube/pkg/minikube/bootstrapper/images" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/constants" + "k8s.io/minikube/pkg/minikube/download" "k8s.io/minikube/pkg/minikube/localpath" ) @@ -181,15 +182,15 @@ func TestDownloadOnlyDocker(t *testing.T) { t.Errorf("%s failed: %v:\n%s", args, err, rr.Output()) } - // Make sure the preloaded image tarball exists - tarball := download.TarballFilepath(constants.DefaultKubernetesVersion) + // Make sure the downloaded image tarball exists + tarball := download.TarballPath(constants.DefaultKubernetesVersion) contents, err := ioutil.ReadFile(tarball) if err != nil { t.Errorf("reading tarball: %v", err) } // Make sure it has the correct checksum checksum := md5.Sum(contents) - remoteChecksum, err := ioutil.ReadFile(preload.ChecksumFilepath(constants.DefaultKubernetesVersion)) + remoteChecksum, err := ioutil.ReadFile(download.PreloadChecksumPath(constants.DefaultKubernetesVersion)) if err != nil { t.Errorf("reading checksum file: %v", err) } From fc7218424960bacb0d68615608621549026421ad Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Wed, 4 Mar 2020 20:08:49 -0800 Subject: [PATCH 03/12] go.mod update --- go.mod | 6 ++++-- go.sum | 8 ++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index f5a6c5170a..83b96bb6d4 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/cheggaaa/pb/v3 v3.0.1 github.com/cloudfoundry-attic/jibber_jabber v0.0.0-20151120183258-bcc4c8345a21 github.com/cloudfoundry/jibber_jabber v0.0.0-20151120183258-bcc4c8345a21 // indirect + github.com/docker/cli v0.0.0-20200303162255-7d407207c304 // indirect github.com/docker/docker v1.13.1 github.com/docker/go-units v0.4.0 github.com/docker/machine v0.7.1-0.20190718054102-a555e4f7a8f5 // version is 0.7.1 to pin to a555e4f7a8f5 @@ -63,7 +64,8 @@ require ( github.com/xeipuuv/gojsonschema v0.0.0-20160623135812-c539bca196be github.com/zchee/go-vmnet v0.0.0-20161021174912-97ebf9174097 golang.org/x/build v0.0.0-20190927031335-2835ba2e683f - golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 + golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 + golang.org/x/net v0.0.0-20200301022130-244492dfa37a // indirect golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e golang.org/x/sys v0.0.0-20191010194322-b09406accb47 golang.org/x/text v0.3.2 @@ -74,7 +76,7 @@ require ( k8s.io/client-go v0.17.3 k8s.io/kubectl v0.0.0 k8s.io/kubernetes v1.17.3 - k8s.io/utils v0.0.0-20200122174043-1e243dd1a584 // indirect + k8s.io/utils v0.0.0-20200229041039-0a110f9eb7ab // indirect sigs.k8s.io/sig-storage-lib-external-provisioner v4.0.0+incompatible ) diff --git a/go.sum b/go.sum index 1c873b1209..fa0a2efcd5 100644 --- a/go.sum +++ b/go.sum @@ -144,6 +144,8 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017 h1:2HQmlpI3yI9deH18Q6xiSOIjXD4sLI55Y/gfpa8/558= github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/cli v0.0.0-20200303162255-7d407207c304 h1:A7SYzidcyuQ/yS4wezWGYeUioUFJQk8HYWY9aMYTF4I= +github.com/docker/cli v0.0.0-20200303162255-7d407207c304/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7 h1:Cvj7S8I4Xpx78KAl6TwTmMHuHlZ/0SM60NUneGJQ7IE= @@ -729,6 +731,8 @@ golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 h1:ULYEB3JvPRE/IfO+9uO7vKV/xzVTO7XPAwm8xbf4w2g= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 h1:xMPOj6Pz6UipU1wXLkrtqpHbR0AVFnyPEQq/wRWz9lM= +golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -774,6 +778,8 @@ golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a h1:tImsplftrFpALCYumobsd0K86vlAs/eXGFms2txfJfA= @@ -992,6 +998,8 @@ k8s.io/utils v0.0.0-20191114184206-e782cd3c129f h1:GiPwtSzdP43eI1hpPCbROQCCIgCui k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20200122174043-1e243dd1a584 h1:3tT5mBZNurtd5BoYrPBII3Sa8n7T2w405qdTQvr3vmY= k8s.io/utils v0.0.0-20200122174043-1e243dd1a584/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= +k8s.io/utils v0.0.0-20200229041039-0a110f9eb7ab h1:I3f2hcBrepGRXI1z4sukzAb8w1R4eqbsHrAsx06LGYM= +k8s.io/utils v0.0.0-20200229041039-0a110f9eb7ab/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= From 832fb6b4ab56b09f099eaa61079763570f94f189 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Wed, 4 Mar 2020 20:42:05 -0800 Subject: [PATCH 04/12] preload: check local cache and status codes --- pkg/minikube/download/preload.go | 26 ++++++++++++++++++++++++-- pkg/minikube/node/cache.go | 1 + pkg/minikube/node/start.go | 11 +++++++---- 3 files changed, 32 insertions(+), 6 deletions(-) diff --git a/pkg/minikube/download/preload.go b/pkg/minikube/download/preload.go index 8161ec6820..05c0b4c490 100644 --- a/pkg/minikube/download/preload.go +++ b/pkg/minikube/download/preload.go @@ -77,9 +77,31 @@ func PreloadExists(k8sVersion, containerRuntime string) bool { if containerRuntime != "docker" { return false } + + // Omit remote check if tarball exists locally + targetPath := TarballPath(k8sVersion) + if _, err := os.Stat(targetPath); err == nil { + if err := verifyChecksum(k8sVersion); err == nil { + glog.Infof("Found %s in cache, no need to check remotely", targetPath) + return true + } + } + url := remoteTarballURL(k8sVersion) - _, err := http.Head(url) - return err == nil + resp, err := http.Head(url) + if err != nil { + glog.Warningf("%s fetch error: %v", url, err) + return false + } + + // note: err won't be set if it's a 404 + if resp.StatusCode != 200 { + glog.Warningf("%s status code: %d", url, resp.StatusCode) + return false + } + + glog.Infof("Goody! %s exists!", url) + return true } // Preload caches the preloaded images tarball on the host machine diff --git a/pkg/minikube/node/cache.go b/pkg/minikube/node/cache.go index 4336049af8..15dc7e4018 100644 --- a/pkg/minikube/node/cache.go +++ b/pkg/minikube/node/cache.go @@ -98,6 +98,7 @@ func beginDownloadKicArtifacts(g *errgroup.Group, k8sVersion, cRuntime string) { func waitDownloadKicArtifacts(g *errgroup.Group) { if err := g.Wait(); err != nil { glog.Errorln("Error downloading kic artifacts: ", err) + return } glog.Info("Successfully downloaded all kic artifacts") } diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index 7c148243cc..b1e72acd66 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -41,13 +41,16 @@ func Start(mc config.ClusterConfig, n config.Node, primary bool, existingAddons // See if we can create a volume of preloaded images // If not, pull images in the background while the VM boots. var kicGroup errgroup.Group + downloadImages := true if driver.IsKIC(driverName) { - beginDownloadKicArtifacts(&kicGroup, k8sVersion, mc.KubernetesConfig.ContainerRuntime) + if download.PreloadExists(k8sVersion, mc.KubernetesConfig.ContainerRuntime) { + beginDownloadKicArtifacts(&kicGroup, k8sVersion, mc.KubernetesConfig.ContainerRuntime) + downloadImages = false + } } - // Now that the ISO is downloaded, pull images in the background while the VM boots. + var cacheGroup errgroup.Group - skipCacheImages := driver.IsKIC(driverName) && download.PreloadExists(k8sVersion, mc.KubernetesConfig.ContainerRuntime) - if !skipCacheImages { + if downloadImages { beginCacheRequiredImages(&cacheGroup, mc.KubernetesConfig.ImageRepository, k8sVersion) } From 9adf54388ddccf73d75328dbfc54f6247866fa6e Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Wed, 4 Mar 2020 20:49:49 -0800 Subject: [PATCH 05/12] lint cleanup --- pkg/minikube/machine/cache_binaries.go | 1 - pkg/version/version.go | 5 ----- 2 files changed, 6 deletions(-) diff --git a/pkg/minikube/machine/cache_binaries.go b/pkg/minikube/machine/cache_binaries.go index f702ee950c..3bb9d4d998 100644 --- a/pkg/minikube/machine/cache_binaries.go +++ b/pkg/minikube/machine/cache_binaries.go @@ -45,7 +45,6 @@ func CacheBinariesForBootstrapper(version string, clusterBootstrapper string) er return g.Wait() } - // CopyBinary copies a locally cached binary to the guest VM func CopyBinary(cr command.Runner, src string, dest string) error { f, err := assets.NewFileAsset(src, path.Dir(dest), path.Base(dest), "0755") diff --git a/pkg/version/version.go b/pkg/version/version.go index 6587f3f722..a3b921c5ca 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -25,8 +25,6 @@ import ( // VersionPrefix is the prefix of the git tag for a version const VersionPrefix = "v" -// The current version of the minikube - // version is a private field and should be set when compiling with --ldflags="-X k8s.io/minikube/pkg/version.version=vX.Y.Z" var version = "v0.0.0-unset" @@ -36,8 +34,6 @@ var gitCommitID = "" // isoVersion is a private field and should be set when compiling with --ldflags="-X k8s.io/minikube/pkg/version.isoVersion=vX.Y.Z" var isoVersion = "v0.0.0-unset" -var isoPath = "minikube/iso" - // GetVersion returns the current minikube version func GetVersion() string { return version @@ -53,7 +49,6 @@ func GetISOVersion() string { return isoVersion } - // GetSemverVersion returns the current minikube semantic version (semver) func GetSemverVersion() (semver.Version, error) { return semver.Make(strings.TrimPrefix(GetVersion(), VersionPrefix)) From 883e108dce222e45e0a7a25f6895b6b5a6eaecfc Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Wed, 4 Mar 2020 21:05:38 -0800 Subject: [PATCH 06/12] Fix hyperv import --- pkg/minikube/registry/drvs/hyperv/hyperv.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/minikube/registry/drvs/hyperv/hyperv.go b/pkg/minikube/registry/drvs/hyperv/hyperv.go index b452c63d49..7b96d041d3 100644 --- a/pkg/minikube/registry/drvs/hyperv/hyperv.go +++ b/pkg/minikube/registry/drvs/hyperv/hyperv.go @@ -30,6 +30,7 @@ import ( "github.com/pkg/errors" cfg "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/download" "k8s.io/minikube/pkg/minikube/driver" "k8s.io/minikube/pkg/minikube/localpath" "k8s.io/minikube/pkg/minikube/registry" From 625592920ee5ec0db24be91ae0df158064e7c80e Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Thu, 5 Mar 2020 09:41:51 -0800 Subject: [PATCH 07/12] fallback to vm-driver --- .github/workflows/main.yml | 2 +- cmd/minikube/cmd/start.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0a8f55716d..96220ea5f6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -100,7 +100,7 @@ jobs: chmod a+x e2e-* chmod a+x minikube-* START_TIME=$(date -u +%s) - KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--driver=docker -test.timeout=70m -test.v -timeout-multiplier=3 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt + KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--vm-driver=docker -test.timeout=70m -test.v -timeout-multiplier=3 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt END_TIME=$(date -u +%s) TIME_ELAPSED=$(($END_TIME-$START_TIME)) min=$((${TIME_ELAPSED}/60)) diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index 605534885f..1f10d46b2f 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -452,8 +452,8 @@ func selectDriver(existing *config.ClusterConfig) registry.DriverState { } // Fallback to old driver parameter - if viper.GetString("driver") != "" { - ds := driver.Status(viper.GetString("driver")) + if viper.GetString("vm-driver") != "" { + ds := driver.Status(viper.GetString("vm-driver")) out.T(out.Sparkle, `Using the {{.driver}} driver based on user configuration`, out.V{"driver": ds.String()}) return ds } From 1a01b9450cc12dbe664d89f676a441c4762635df Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Thu, 5 Mar 2020 10:23:51 -0800 Subject: [PATCH 08/12] Clarify logic around beginDownloadKicArtifacts --- pkg/minikube/node/cache.go | 19 ++++++++++++------- pkg/minikube/node/start.go | 13 ++++++------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/pkg/minikube/node/cache.go b/pkg/minikube/node/cache.go index 15dc7e4018..67bf646ce7 100644 --- a/pkg/minikube/node/cache.go +++ b/pkg/minikube/node/cache.go @@ -35,8 +35,8 @@ import ( "k8s.io/minikube/pkg/minikube/out" ) -// beginCacheRequiredImages caches images required for kubernetes version in the background -func beginCacheRequiredImages(g *errgroup.Group, imageRepository string, k8sVersion string) { +// beginCacheKubernetesImages caches images required for kubernetes version in the background +func beginCacheKubernetesImages(g *errgroup.Group, imageRepository string, k8sVersion string) { if !viper.GetBool("cache-images") { return } @@ -82,17 +82,22 @@ func doCacheBinaries(k8sVersion string) error { return machine.CacheBinariesForBootstrapper(k8sVersion, viper.GetString(cmdcfg.Bootstrapper)) } -func beginDownloadKicArtifacts(g *errgroup.Group, k8sVersion, cRuntime string) { +// beginDownloadKicArtifacts downloads the kic image + preload tarball, returns true if preload is available +func beginDownloadKicArtifacts(g *errgroup.Group, k8sVersion, cRuntime string) bool { glog.Info("Beginning downloading kic artifacts") g.Go(func() error { glog.Infof("Downloading %s to local daemon", kic.BaseImage) return image.WriteImageToDaemon(kic.BaseImage) }) - g.Go(func() error { - glog.Info("Caching tarball of preloaded images") - return download.Preload(k8sVersion, cRuntime) - }) + if download.PreloadExists(k8sVersion, cRuntime) { + g.Go(func() error { + glog.Info("Caching tarball of preloaded images") + return download.Preload(k8sVersion, cRuntime) + }) + return true + } + return false } func waitDownloadKicArtifacts(g *errgroup.Group) { diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index f2a5507fc4..44a1667c46 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -23,7 +23,6 @@ import ( "golang.org/x/sync/errgroup" "k8s.io/minikube/pkg/addons" "k8s.io/minikube/pkg/minikube/config" - "k8s.io/minikube/pkg/minikube/download" "k8s.io/minikube/pkg/minikube/driver" "k8s.io/minikube/pkg/minikube/exit" "k8s.io/minikube/pkg/minikube/kubeconfig" @@ -41,17 +40,17 @@ func Start(mc config.ClusterConfig, n config.Node, primary bool, existingAddons // See if we can create a volume of preloaded images // If not, pull images in the background while the VM boots. var kicGroup errgroup.Group - downloadImages := true + needKubernetesImages := true if driver.IsKIC(driverName) { - if download.PreloadExists(k8sVersion, mc.KubernetesConfig.ContainerRuntime) { - beginDownloadKicArtifacts(&kicGroup, k8sVersion, mc.KubernetesConfig.ContainerRuntime) - downloadImages = false + // If we can download a preload tarball, it isn't necessary to pull Kubernetes images + if beginDownloadKicArtifacts(&kicGroup, k8sVersion, mc.KubernetesConfig.ContainerRuntime) { + needKubernetesImages = false } } var cacheGroup errgroup.Group - if downloadImages { - beginCacheRequiredImages(&cacheGroup, mc.KubernetesConfig.ImageRepository, k8sVersion) + if needKubernetesImages { + beginCacheKubernetesImages(&cacheGroup, mc.KubernetesConfig.ImageRepository, k8sVersion) } // Abstraction leakage alert: startHost requires the config to be saved, to satistfy pkg/provision/buildroot. From d0b0dce7a23e37163cfc858c7c567b4f17e3dbd0 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Thu, 5 Mar 2020 10:51:18 -0800 Subject: [PATCH 09/12] Add lz4 and tar to iso We can use preloaded tarballs for VMs as well, but this requires both lz4 and tar to be installed to that the tarball can be extracted correctly within the VM. --- deploy/iso/minikube-iso/configs/minikube_defconfig | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deploy/iso/minikube-iso/configs/minikube_defconfig b/deploy/iso/minikube-iso/configs/minikube_defconfig index 3652860308..381d88d2c7 100644 --- a/deploy/iso/minikube-iso/configs/minikube_defconfig +++ b/deploy/iso/minikube-iso/configs/minikube_defconfig @@ -68,3 +68,5 @@ BR2_PACKAGE_STRACE=y BR2_PACKAGE_SYSSTAT=y BR2_PACKAGE_HTOP=y BR2_PACKAGE_CONNTRACK_TOOLS=y +BR2_PACKAGE_TAR=y +BR2_PACKAGE_LZ4=y From 2aad1b2cb9e61861aae6f574a7dcf16860c8a43c Mon Sep 17 00:00:00 2001 From: Naveen Kumar Sangi Date: Thu, 5 Mar 2020 12:57:19 -0600 Subject: [PATCH 10/12] main.yml: Remove trailing whitespaces --- .github/workflows/main.yml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 96220ea5f6..02a56b84b7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -2,7 +2,7 @@ name: CI on: [pull_request] jobs: # Runs before all other jobs - # builds the minikube binaries + # builds the minikube binaries build_minikube: env: TIME_ELAPSED: time @@ -37,7 +37,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: install libvirt - run : | + run : | sudo apt-get update sudo apt-get install -y libvirt-dev - name: lint @@ -54,7 +54,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: install libvirt - run : | + run : | sudo apt-get update sudo apt-get install -y libvirt-dev - name: unit test @@ -170,7 +170,7 @@ jobs: START_TIME=$(date -u +%s) KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--driver=docker -test.timeout=70m -test.v -timeout-multiplier=3 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt END_TIME=$(date -u +%s) - TIME_ELAPSED=$(($END_TIME-$START_TIME)) + TIME_ELAPSED=$(($END_TIME-$START_TIME)) min=$((${TIME_ELAPSED}/60)) sec=$((${TIME_ELAPSED}%60)) TIME_ELAPSED="${min} min $sec seconds " @@ -184,7 +184,7 @@ jobs: STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}" -details "${GITHUB_SHA}") || true echo status: ${STAT} FailNum=$(echo $STAT | jq '.NumberOfFail') - TestsNum=$(echo $STAT | jq '.NumberOfTests') + TestsNum=$(echo $STAT | jq '.NumberOfTests') GOPOGH_RESULT="${JOB_NAME} : completed with ${FailNum} / ${TestsNum} failures in ${TIME_ELAPSED}" echo ::set-env name=GOPOGH_RESULT::${GOPOGH_RESULT} echo ::set-env name=STAT::${STAT} @@ -232,7 +232,7 @@ jobs: START_TIME=$(date -u +%s) KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome sudo -E ./e2e-linux-amd64 -minikube-start-args=--driver=none -test.timeout=70m -test.v -timeout-multiplier=3 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt END_TIME=$(date -u +%s) - TIME_ELAPSED=$(($END_TIME-$START_TIME)) + TIME_ELAPSED=$(($END_TIME-$START_TIME)) min=$((${TIME_ELAPSED}/60)) sec=$((${TIME_ELAPSED}%60)) TIME_ELAPSED="${min} min $sec seconds " @@ -246,7 +246,7 @@ jobs: STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}" -details "${GITHUB_SHA}") || true echo status: ${STAT} FailNum=$(echo $STAT | jq '.NumberOfFail') - TestsNum=$(echo $STAT | jq '.NumberOfTests') + TestsNum=$(echo $STAT | jq '.NumberOfTests') GOPOGH_RESULT="${JOB_NAME} : completed with ${FailNum} / ${TestsNum} failures in ${TIME_ELAPSED}" echo ::set-env name=GOPOGH_RESULT::${GOPOGH_RESULT} echo ::set-env name=STAT::${STAT} @@ -276,7 +276,7 @@ jobs: shell: bash run: | curl -LO https://github.com/medyagh/gopogh/releases/download/v0.1.16/gopogh-linux-amd64 - sudo install gopogh-linux-amd64 /usr/local/bin/gopogh + sudo install gopogh-linux-amd64 /usr/local/bin/gopogh - name: Download binaries uses: actions/download-artifact@v1 with: @@ -294,7 +294,7 @@ jobs: START_TIME=$(date -u +%s) KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome sudo -E ./e2e-linux-amd64 -minikube-start-args=--driver=none -test.timeout=70m -test.v -timeout-multiplier=3 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt END_TIME=$(date -u +%s) - TIME_ELAPSED=$(($END_TIME-$START_TIME)) + TIME_ELAPSED=$(($END_TIME-$START_TIME)) min=$((${TIME_ELAPSED}/60)) sec=$((${TIME_ELAPSED}%60)) TIME_ELAPSED="${min} min $sec seconds " @@ -308,7 +308,7 @@ jobs: STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}" -details "${GITHUB_SHA}") || true echo status: ${STAT} FailNum=$(echo $STAT | jq '.NumberOfFail') - TestsNum=$(echo $STAT | jq '.NumberOfTests') + TestsNum=$(echo $STAT | jq '.NumberOfTests') GOPOGH_RESULT="${JOB_NAME} : completed with ${FailNum} / ${TestsNum} failures in ${TIME_ELAPSED}" echo ::set-env name=GOPOGH_RESULT::${GOPOGH_RESULT} echo ::set-env name=STAT::${STAT} @@ -343,8 +343,8 @@ jobs: sudo apt-key add - < Release.key || true sudo apt-get update -qq sudo apt-get -qq -y install podman - sudo podman version || true - sudo podman info || true + sudo podman version || true + sudo podman info || true - name: Install gopogh shell: bash run: | @@ -367,7 +367,7 @@ jobs: START_TIME=$(date -u +%s) KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome sudo -E ./e2e-linux-amd64 -minikube-start-args=--driver=podman -test.timeout=70m -test.v -timeout-multiplier=3 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt END_TIME=$(date -u +%s) - TIME_ELAPSED=$(($END_TIME-$START_TIME)) + TIME_ELAPSED=$(($END_TIME-$START_TIME)) min=$((${TIME_ELAPSED}/60)) sec=$((${TIME_ELAPSED}%60)) TIME_ELAPSED="${min} min $sec seconds " @@ -381,7 +381,7 @@ jobs: STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}" -details "${GITHUB_SHA}") || true echo status: ${STAT} FailNum=$(echo $STAT | jq '.NumberOfFail') - TestsNum=$(echo $STAT | jq '.NumberOfTests') + TestsNum=$(echo $STAT | jq '.NumberOfTests') GOPOGH_RESULT="${JOB_NAME} : completed with ${FailNum} / ${TestsNum} failures in ${TIME_ELAPSED}" echo ::set-env name=GOPOGH_RESULT::${GOPOGH_RESULT} echo ::set-env name=STAT::${STAT} @@ -398,7 +398,7 @@ jobs: echo $STAT | jq '.FailedTests' || true echo "-------------------------------------------------------" if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi - # After all 4 integration tests finished + # After all 4 integration tests finished # collect all the reports and upload upload_all_reports: if: always() From 593bb4392fe0c8366e313bc1c0984b6ec9ce87cb Mon Sep 17 00:00:00 2001 From: Naveen Kumar Sangi Date: Thu, 5 Mar 2020 12:58:09 -0600 Subject: [PATCH 11/12] main.yml: Capitalize action names --- .github/workflows/main.yml | 52 +++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 02a56b84b7..0145e0ed6a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-18.04 steps: - uses: actions/checkout@v2 - - name: build binaries + - name: Build Binaries run : | make minikube-linux-amd64 make e2e-linux-amd64 @@ -36,11 +36,11 @@ jobs: runs-on: ubuntu-18.04 steps: - uses: actions/checkout@v2 - - name: install libvirt + - name: Install libvirt run : | sudo apt-get update sudo apt-get install -y libvirt-dev - - name: lint + - name: Lint env: TESTSUITE: lintall run : make test @@ -53,11 +53,11 @@ jobs: runs-on: ubuntu-18.04 steps: - uses: actions/checkout@v2 - - name: install libvirt + - name: Install libvirt run : | sudo apt-get update sudo apt-get install -y libvirt-dev - - name: unit test + - name: Unit Test env: TESTSUITE: unittest run : @@ -85,11 +85,11 @@ jobs: run: | curl -LO https://github.com/medyagh/gopogh/releases/download/v0.1.16/gopogh-linux-amd64 sudo install gopogh-linux-amd64 /usr/local/bin/gopogh - - name: Download binaries + - name: Download Binaries uses: actions/download-artifact@v1 with: name: minikube_binaries - - name: Run integration test + - name: Run Integration Test continue-on-error: true # bash {0} to allow test to continue to next step. in case of shell: bash {0} @@ -107,7 +107,7 @@ jobs: sec=$((${TIME_ELAPSED}%60)) TIME_ELAPSED="${min} min $sec seconds " echo ::set-env name=TIME_ELAPSED::${TIME_ELAPSED} - - name: Generate html report + - name: Generate HTML Report shell: bash run: | cd minikube_binaries @@ -153,11 +153,11 @@ jobs: run: | curl -LO https://github.com/medyagh/gopogh/releases/download/v0.1.16/gopogh-linux-amd64 sudo install gopogh-linux-amd64 /usr/local/bin/gopogh - - name: Download binaries + - name: Download Binaries uses: actions/download-artifact@v1 with: name: minikube_binaries - - name: Run integration test + - name: Run Integration Test continue-on-error: true # bash {0} to allow test to continue to next step. in case of shell: bash {0} @@ -175,7 +175,7 @@ jobs: sec=$((${TIME_ELAPSED}%60)) TIME_ELAPSED="${min} min $sec seconds " echo ::set-env name=TIME_ELAPSED::${TIME_ELAPSED} - - name: Generate html report + - name: Generate HTML Report shell: bash run: | cd minikube_binaries @@ -215,11 +215,11 @@ jobs: run: | curl -LO https://github.com/medyagh/gopogh/releases/download/v0.1.16/gopogh-linux-amd64 sudo install gopogh-linux-amd64 /usr/local/bin/gopogh - - name: Download binaries + - name: Download Binaries uses: actions/download-artifact@v1 with: name: minikube_binaries - - name: Run integration test + - name: Run Integration Test continue-on-error: true # bash {0} to allow test to continue to next step. in case of shell: bash {0} @@ -237,7 +237,7 @@ jobs: sec=$((${TIME_ELAPSED}%60)) TIME_ELAPSED="${min} min $sec seconds " echo ::set-env name=TIME_ELAPSED::${TIME_ELAPSED} - - name: Generate html report + - name: Generate HTML Report shell: bash run: | cd minikube_binaries @@ -277,11 +277,11 @@ jobs: run: | curl -LO https://github.com/medyagh/gopogh/releases/download/v0.1.16/gopogh-linux-amd64 sudo install gopogh-linux-amd64 /usr/local/bin/gopogh - - name: Download binaries + - name: Download Binaries uses: actions/download-artifact@v1 with: name: minikube_binaries - - name: Run integration test + - name: Run Integration Test continue-on-error: true # bash {0} to allow test to continue to next step. in case of shell: bash {0} @@ -299,7 +299,7 @@ jobs: sec=$((${TIME_ELAPSED}%60)) TIME_ELAPSED="${min} min $sec seconds " echo ::set-env name=TIME_ELAPSED::${TIME_ELAPSED} - - name: Generate html report + - name: Generate HTML Report shell: bash run: | cd minikube_binaries @@ -354,7 +354,7 @@ jobs: uses: actions/download-artifact@v1 with: name: minikube_binaries - - name: Run integration test + - name: Run Integration Test continue-on-error: true # bash {0} to allow test to continue to next step. in case of shell: bash {0} @@ -372,7 +372,7 @@ jobs: sec=$((${TIME_ELAPSED}%60)) TIME_ELAPSED="${min} min $sec seconds " echo ::set-env name=TIME_ELAPSED::${TIME_ELAPSED} - - name: Generate html report + - name: Generate HTML Report shell: bash run: | cd minikube_binaries @@ -405,7 +405,7 @@ jobs: needs: [docker_ubuntu_16_04,docker_ubuntu_18_04,none_ubuntu16_04,none_ubuntu18_04,podman_ubuntu_18_04] runs-on: ubuntu-18.04 steps: - - name: download results docker_ubuntu_16_04 + - name: Download Results docker_ubuntu_16_04 uses: actions/download-artifact@v1 with: name: docker_ubuntu_16_04 @@ -415,7 +415,7 @@ jobs: run: | mkdir -p all_reports cp -r docker_ubuntu_16_04 ./all_reports/ - - name: download results docker_ubuntu_18_04 + - name: Download Results docker_ubuntu_18_04 uses: actions/download-artifact@v1 with: name: docker_ubuntu_18_04 @@ -425,7 +425,7 @@ jobs: run: | mkdir -p all_reports cp -r docker_ubuntu_18_04 ./all_reports/ - - name: download results none_ubuntu16_04 + - name: Download Results none_ubuntu16_04 uses: actions/download-artifact@v1 with: name: none_ubuntu16_04 @@ -435,21 +435,21 @@ jobs: run: | mkdir -p all_reports cp -r none_ubuntu16_04 ./all_reports/ - - name: download results none_ubuntu18_04 + - name: Download Results none_ubuntu18_04 uses: actions/download-artifact@v1 with: name: none_ubuntu18_04 - - name: cp none_ubuntu18_04 to all_report + - name: Copy none_ubuntu18_04 to all_report continue-on-error: true shell: bash {0} run: | mkdir -p all_reports cp -r none_ubuntu18_04 ./all_reports/ - - name: download results podman_ubuntu_18_04 + - name: Download Results podman_ubuntu_18_04 uses: actions/download-artifact@v1 with: name: podman_ubuntu_18_04 - - name: cp podman_ubuntu_18_04 to all_report + - name: Copy podman_ubuntu_18_04 to all_report continue-on-error: true shell: bash {0} run: | From de4cb21d6a7e2194adb785c5087fb063453b7400 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Sangi Date: Thu, 5 Mar 2020 13:05:44 -0600 Subject: [PATCH 12/12] github actions: Download deps before running jobs --- .github/workflows/main.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0145e0ed6a..a3572ed151 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,5 +1,7 @@ name: CI on: [pull_request] +env: + GOPROXY: https://proxy.golang.org jobs: # Runs before all other jobs # builds the minikube binaries @@ -11,6 +13,8 @@ jobs: runs-on: ubuntu-18.04 steps: - uses: actions/checkout@v2 + - name: Download Dependencies + run : go mod download - name: Build Binaries run : | make minikube-linux-amd64 @@ -40,6 +44,8 @@ jobs: run : | sudo apt-get update sudo apt-get install -y libvirt-dev + - name: Download Dependencies + run : go mod download - name: Lint env: TESTSUITE: lintall @@ -57,6 +63,8 @@ jobs: run : | sudo apt-get update sudo apt-get install -y libvirt-dev + - name: Download Dependencies + run : go mod download - name: Unit Test env: TESTSUITE: unittest