diff --git a/Makefile b/Makefile index 60d5347ffb..850ad0204d 100755 --- a/Makefile +++ b/Makefile @@ -21,6 +21,7 @@ VERSION ?= v$(RAW_VERSION) KUBERNETES_VERSION ?= $(shell egrep "DefaultKubernetesVersion =" pkg/minikube/constants/constants.go | cut -d \" -f2) KIC_VERSION ?= $(shell egrep "Version =" pkg/drivers/kic/types.go | cut -d \" -f2) +PRELOADED_VOLUMES_GCS_BUCKET ?= $(shell egrep "PreloadedVolumeTarballsBucket =" pkg/minikube/constants/constants.go | cut -d \" -f2) # Default to .0 for higher cache hit rates, as build increments typically don't require new ISO versions ISO_VERSION ?= v$(VERSION_MAJOR).$(VERSION_MINOR).3 @@ -513,7 +514,7 @@ kic-base-image: ## builds the base image used for kic. .PHONY: upload-preloaded-images-tar upload-preloaded-images-tar: generate-preloaded-images-tar # Upload the preloaded images tar to the GCS bucket. Specify a specific kubernetes version to build via `KUBERNETES_VERSION=vx.y.z make upload-preloaded-images-tar`. - gsutil cp out/preloaded-images-k8s-${KUBERNETES_VERSION}.tar.lz4 gs://minikube-docker-volume-tarballs + gsutil cp out/preloaded-images-k8s-${KUBERNETES_VERSION}.tar.lz4 gs://${PRELOADED_VOLUMES_GCS_BUCKET} .PHONY: generate-preloaded-images-tar generate-preloaded-images-tar: out/minikube diff --git a/go.mod b/go.mod index e4798d896c..5442fdf4dc 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module k8s.io/minikube go 1.13 require ( + cloud.google.com/go v0.45.1 github.com/Parallels/docker-machine-parallels v1.3.0 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/blang/semver v3.5.0+incompatible diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go index 3e0ef46dfc..c9b8c246ec 100644 --- a/pkg/drivers/kic/kic.go +++ b/pkg/drivers/kic/kic.go @@ -30,10 +30,12 @@ import ( "github.com/docker/machine/libmachine/state" "github.com/golang/glog" "github.com/pkg/errors" + "github.com/spf13/viper" pkgdrivers "k8s.io/minikube/pkg/drivers" "k8s.io/minikube/pkg/drivers/kic/oci" "k8s.io/minikube/pkg/minikube/assets" "k8s.io/minikube/pkg/minikube/command" + "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/constants" ) @@ -91,7 +93,7 @@ func (d *Driver) Create() error { ) t := time.Now() glog.Infof("Starting creating preloaded images volume") - volumeName, err := oci.CreatePreloadedImagesVolume(d.NodeConfig.KubernetesVersion, BaseImage) + volumeName, err := oci.CreatePreloadedImagesVolume(d.NodeConfig.KubernetesVersion, BaseImage, viper.GetString(config.MachineProfile)) if err != nil { glog.Infof("Unable to create preloaded images volume: %v", err) } diff --git a/pkg/drivers/kic/oci/volumes.go b/pkg/drivers/kic/oci/volumes.go index dd8fd31707..36742c59d7 100644 --- a/pkg/drivers/kic/oci/volumes.go +++ b/pkg/drivers/kic/oci/volumes.go @@ -25,7 +25,7 @@ import ( "github.com/golang/glog" "github.com/pkg/errors" - "k8s.io/minikube/pkg/drivers/kic/preload" + "k8s.io/minikube/pkg/minikube/preload" ) // DeleteAllVolumesByLabel deletes all volumes that have a specific label @@ -90,18 +90,20 @@ func allVolumesByLabel(ociBin string, label string) ([]string, error) { } // CreatePreloadedImagesVolume creates a volume with preloaded images -func CreatePreloadedImagesVolume(k8sVersion, baseImage string) (string, error) { +// k8sVersion is used to name the volume and baseImage is the image that is run +// to extract the preloaded images to the volume +func CreatePreloadedImagesVolume(k8sVersion, baseImage, profile string) (string, error) { if err := PointToHostDockerDaemon(); err != nil { return "", errors.Wrap(err, "point host docker-daemon") } - volumeName := fmt.Sprintf("k8s-%s", k8sVersion) + volumeName := fmt.Sprintf("k8s-%s-%s", k8sVersion, profile) if dockerVolumeExists(volumeName) { return volumeName, nil } if err := createDockerVolume(volumeName); err != nil { return "", errors.Wrap(err, "creating docker volume") } - tarballPath := preload.TarballPath(k8sVersion) + tarballPath := preload.TarballFilepath(k8sVersion) if err := extractTarballToVolume(tarballPath, volumeName, baseImage); err != nil { return "", errors.Wrap(err, "extracting tarball to volume") @@ -109,6 +111,7 @@ func CreatePreloadedImagesVolume(k8sVersion, baseImage string) (string, error) { return volumeName, nil } +// dockerVolumeExists returns true if a docker volume with the passed in name exists func dockerVolumeExists(name string) bool { if err := PointToHostDockerDaemon(); err != nil { return false @@ -120,19 +123,20 @@ func dockerVolumeExists(name string) bool { } names := strings.Split(string(out), "\n") for _, n := range names { - if n == name { + if strings.TrimSpace(n) == name { return true } } return false } +// extractTarballToVolume runs a docker image imageName which extracts the tarball at tarballPath +// to the volume named volumeName func extractTarballToVolume(tarballPath, volumeName, imageName string) error { if err := PointToHostDockerDaemon(); err != nil { return errors.Wrap(err, "point host docker-daemon") } cmd := exec.Command(Docker, "run", "--rm", "--entrypoint", "/usr/bin/tar", "-v", fmt.Sprintf("%s:/preloaded.tar:ro", tarballPath), "-v", fmt.Sprintf("%s:/extractDir", volumeName), imageName, "-I", "lz4", "-xvf", "/preloaded.tar", "-C", "/extractDir") - fmt.Println(cmd.Args) if out, err := cmd.CombinedOutput(); err != nil { return errors.Wrapf(err, "output %s", string(out)) } diff --git a/pkg/drivers/kic/preload/preload.go b/pkg/drivers/kic/preload/preload.go deleted file mode 100644 index f95a7f092c..0000000000 --- a/pkg/drivers/kic/preload/preload.go +++ /dev/null @@ -1,55 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package preload - -import ( - "fmt" - "os" - "path" - - "github.com/golang/glog" - "github.com/jimmidyson/go-download" - "k8s.io/minikube/pkg/minikube/localpath" -) - -// TarballPath returns the path to the preloaded tarball -func TarballPath(k8sVersion string) string { - targetDir := localpath.MakeMiniPath("cache", "preloaded-tarball") - targetFilepath := path.Join(targetDir, fmt.Sprintf("preloaded-images-k8s-%s.tar.lz4", k8sVersion)) - return targetFilepath -} - -// CacheTarball caches the preloaded images tarball on the host machine -func CacheTarball(k8sVersion string) error { - targetFilepath := TarballPath(k8sVersion) - - if _, err := os.Stat(targetFilepath); err == nil { - glog.Infof("Found %s in cache, skipping downloading", targetFilepath) - return nil - } - - url := fmt.Sprintf("https://storage.googleapis.com/minikube-docker-volume-tarballs/preloaded-images-k8s-%s.tar", k8sVersion) - glog.Infof("Downloading %s to %s", url, targetFilepath) - return download.ToFile(url, targetFilepath, download.FileOptions{Mkdirs: download.MkdirAll}) -} - -// UsingPreloadedVolume returns true if the preloaded tarball exists -func UsingPreloadedVolume(k8sVersion string) bool { - path := TarballPath(k8sVersion) - _, err := os.Stat(path) - return err == nil -} diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index ac2922b539..091ed4a720 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -40,7 +40,6 @@ import ( kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/minikube/pkg/drivers/kic" "k8s.io/minikube/pkg/drivers/kic/oci" - "k8s.io/minikube/pkg/drivers/kic/preload" "k8s.io/minikube/pkg/kapi" "k8s.io/minikube/pkg/minikube/bootstrapper" "k8s.io/minikube/pkg/minikube/bootstrapper/bsutil" @@ -53,6 +52,7 @@ import ( "k8s.io/minikube/pkg/minikube/driver" "k8s.io/minikube/pkg/minikube/machine" "k8s.io/minikube/pkg/minikube/out" + "k8s.io/minikube/pkg/minikube/preload" "k8s.io/minikube/pkg/minikube/vmpath" "k8s.io/minikube/pkg/util/retry" ) diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index dbf74c4747..33a4892ebb 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -63,6 +63,9 @@ const ( MinikubeActiveDockerdEnv = "MINIKUBE_ACTIVE_DOCKERD" // PodmanVarlinkBridgeEnv is used for podman settings PodmanVarlinkBridgeEnv = "PODMAN_VARLINK_BRIDGE" + + // PreloadedVolumeTarballsBucket is the name of the GCS bucket where preloaded volume tarballs exist + PreloadedVolumeTarballsBucket = "minikube-preloaded-volume-tarballs" ) var ( diff --git a/pkg/minikube/node/cache.go b/pkg/minikube/node/cache.go index 633c02e406..73d08cb9fa 100644 --- a/pkg/minikube/node/cache.go +++ b/pkg/minikube/node/cache.go @@ -25,15 +25,14 @@ import ( "golang.org/x/sync/errgroup" cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config" "k8s.io/minikube/pkg/drivers/kic" - "k8s.io/minikube/pkg/drivers/kic/preload" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/constants" - "k8s.io/minikube/pkg/minikube/driver" "k8s.io/minikube/pkg/minikube/exit" "k8s.io/minikube/pkg/minikube/image" "k8s.io/minikube/pkg/minikube/localpath" "k8s.io/minikube/pkg/minikube/machine" "k8s.io/minikube/pkg/minikube/out" + "k8s.io/minikube/pkg/minikube/preload" ) // beginCacheRequiredImages caches images required for kubernetes version in the background @@ -47,16 +46,11 @@ func beginCacheRequiredImages(g *errgroup.Group, imageRepository string, k8sVers }) } -func handleDownloadOnly(cacheGroup *errgroup.Group, k8sVersion, driverName string) { +func handleDownloadOnly(cacheGroup, kicGroup *errgroup.Group, k8sVersion, driverName string) { // If --download-only, complete the remaining downloads and exit. if !viper.GetBool("download-only") { return } - var kicArtifactsGroup errgroup.Group - if driver.IsKIC(driverName) { // for kic we need to find what port docker/podman chose for us - // Download kic base image and preloaded images tarball - beginDownloadKicArtifacts(&kicArtifactsGroup, k8sVersion) - } if err := doCacheBinaries(k8sVersion); err != nil { exit.WithError("Failed to cache binaries", err) } @@ -64,7 +58,7 @@ func handleDownloadOnly(cacheGroup *errgroup.Group, k8sVersion, driverName strin exit.WithError("Failed to cache kubectl", err) } waitCacheRequiredImages(cacheGroup) - waitDownloadKicArtifacts(&kicArtifactsGroup) + waitDownloadKicArtifacts(kicGroup) if err := saveImagesToTarFromConfig(); err != nil { exit.WithError("Failed to cache images to tar", err) } diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index 8d2983a9ca..ed8cdc136f 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -22,7 +22,6 @@ import ( "github.com/spf13/viper" "golang.org/x/sync/errgroup" "k8s.io/minikube/pkg/addons" - "k8s.io/minikube/pkg/drivers/kic/preload" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/driver" "k8s.io/minikube/pkg/minikube/exit" @@ -38,13 +37,15 @@ func Start(mc config.MachineConfig, n config.Node, primary bool, existingAddons k8sVersion := mc.KubernetesConfig.KubernetesVersion driverName := mc.Driver - // Now that the ISO is downloaded, pull images in the background while the VM boots. - var cacheGroup errgroup.Group - if !preload.UsingPreloadedVolume(k8sVersion) && driver.IsKIC(driverName) { - beginCacheRequiredImages(&cacheGroup, mc.KubernetesConfig.ImageRepository, k8sVersion) - } else { + // See if we can create a volume of preloaded images + // If not, pull images in the background while the VM boots. + var kicGroup errgroup.Group + if driver.IsKIC(driverName) { + beginDownloadKicArtifacts(&kicGroup, k8sVersion) mc.KubernetesConfig.ShouldLoadCachedImages = false } + var cacheGroup errgroup.Group + beginCacheRequiredImages(&cacheGroup, mc.KubernetesConfig.ImageRepository, k8sVersion) // Abstraction leakage alert: startHost requires the config to be saved, to satistfy pkg/provision/buildroot. // Hence, saveConfig must be called before startHost, and again afterwards when we know the IP. @@ -53,13 +54,14 @@ func Start(mc config.MachineConfig, n config.Node, primary bool, existingAddons } // exits here in case of --download-only option. - handleDownloadOnly(&cacheGroup, k8sVersion, driverName) + handleDownloadOnly(&cacheGroup, &kicGroup, k8sVersion, driverName) mRunner, preExists, machineAPI, host := startMachine(&mc, &n) defer machineAPI.Close() // configure the runtime (docker, containerd, crio) cr := configureRuntimes(mRunner, driverName, mc.KubernetesConfig) showVersionInfo(k8sVersion, cr) waitCacheRequiredImages(&cacheGroup) + waitDownloadKicArtifacts(&kicGroup) //TODO(sharifelgamal): Part out the cluster-wide operations, perhaps using the "primary" param diff --git a/pkg/minikube/preload/preload.go b/pkg/minikube/preload/preload.go new file mode 100644 index 0000000000..cf64230ec2 --- /dev/null +++ b/pkg/minikube/preload/preload.go @@ -0,0 +1,157 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package preload + +import ( + "context" + "crypto/md5" + "fmt" + "io/ioutil" + "net/http" + "os" + "path" + + "cloud.google.com/go/storage" + "github.com/golang/glog" + "github.com/hashicorp/go-getter" + "github.com/pkg/errors" + "k8s.io/minikube/pkg/minikube/constants" + "k8s.io/minikube/pkg/minikube/localpath" + "k8s.io/minikube/pkg/minikube/out" + "k8s.io/minikube/pkg/util" +) + +// returns name of the tarball +func tarballName(k8sVersion string) string { + return fmt.Sprintf("preloaded-images-k8s-%s.tar.lz4", k8sVersion) +} + +// returns the name of the checksum file +func checksumName(k8sVersion string) string { + return fmt.Sprintf("preloaded-images-k8s-%s.tar.lz4.checksum", k8sVersion) +} + +// returns target dir for all cached items related to preloading +func targetDir() string { + return localpath.MakeMiniPath("cache", "preloaded-tarball") +} + +// returns path to checksum file +func checksumFilepath(k8sVersion string) string { + return path.Join(targetDir(), checksumName(k8sVersion)) +} + +// TarballFilepath returns the path to the preloaded tarball +func TarballFilepath(k8sVersion string) string { + return path.Join(targetDir(), tarballName(k8sVersion)) +} + +// remoteTarballURL returns the URL for the remote tarball in GCS +func remoteTarballURL(k8sVersion string) string { + return fmt.Sprintf("https://storage.googleapis.com/%s/%s", constants.PreloadedVolumeTarballsBucket, tarballName(k8sVersion)) +} + +// CacheTarball caches the preloaded images tarball on the host machine +func CacheTarball(k8sVersion string) error { + targetFilepath := TarballFilepath(k8sVersion) + + if _, err := os.Stat(targetFilepath); err == nil { + if err := verifyChecksum(k8sVersion); err == nil { + glog.Infof("Found %s in cache, skipping downloading", targetFilepath) + return nil + } + } + + url := remoteTarballURL(k8sVersion) + + // Make sure we support this k8s version + if _, err := http.Get(url); err != nil { + glog.Infof("Unable to get response from %s, skipping downloading: %v", err) + return nil + } + + out.T(out.FileDownload, "Downloading preloaded images tarball for k8s {{.version}}:", out.V{"version": k8sVersion}) + os.Remove(targetFilepath) + client := &getter.Client{ + Src: url, + Dst: targetFilepath, + Mode: getter.ClientModeFile, + Options: []getter.ClientOption{getter.WithProgress(util.DefaultProgressBar)}, + } + + glog.Infof("Downloading: %+v", client) + if err := client.Get(); err != nil { + return errors.Wrapf(err, "download failed: %s", url) + } + // Give downloaded drivers a baseline decent file permission + if err := os.Chmod(targetFilepath, 0755); err != nil { + return err + } + // Save checksum file locally + if err := saveChecksumFile(k8sVersion); err != nil { + return errors.Wrap(err, "saving checksum file") + } + return verifyChecksum(k8sVersion) +} + +func saveChecksumFile(k8sVersion string) error { + ctx := context.Background() + client, err := storage.NewClient(ctx) + if err != nil { + return err + } + attrs, err := client.Bucket(constants.PreloadedVolumeTarballsBucket).Object(tarballName(k8sVersion)).Attrs(ctx) + if err != nil { + return err + } + checksum := attrs.MD5 + return ioutil.WriteFile(checksumFilepath(k8sVersion), checksum, 0644) +} + +func cachedTarballExists(k8sVersion string) bool { + _, err := os.Stat(TarballFilepath(k8sVersion)) + if err != nil { + return false + } + return false +} + +// verifyChecksum returns true if the checksum of the local binary matches +// the checksum of the remote binary +func verifyChecksum(k8sVersion string) error { + // get md5 checksum of tarball path + contents, err := ioutil.ReadFile(TarballFilepath(k8sVersion)) + if err != nil { + return errors.Wrap(err, "reading tarball") + } + checksum := md5.Sum(contents) + + remoteChecksum, err := ioutil.ReadFile(checksumFilepath(k8sVersion)) + if err != nil { + return errors.Wrap(err, "reading checksum file") + } + + // create a slice of checksum, which is [16]byte + if string(remoteChecksum) != string(checksum[:]) { + return fmt.Errorf("checksum of %s does not match remote checksum", TarballFilepath(k8sVersion)) + } + return nil +} + +func UsingPreloadedVolume(k8sVersion string) bool { + return true +}