From 9c8102dd58f7403e1a18737d2b38bdc1ef715ea0 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Fri, 13 Mar 2020 16:05:46 -0700 Subject: [PATCH 01/10] Refactor preload_images.go to generate and upload a preloaded tarball This script is now responsible for making sure that the DefaultK8sVersion, NewestK8sVersion, and OldestK8sVersion supported by minikube have preloaded tarballs. It will be run on every PR as an automated release script for preloaded tarballs. --- Makefile | 10 +- hack/preload-images/generate.go | 125 +++++++++++++++++++++ hack/preload-images/preload_images.go | 151 ++++++-------------------- hack/preload-images/upload.go | 43 ++++++++ 4 files changed, 203 insertions(+), 126 deletions(-) create mode 100644 hack/preload-images/generate.go create mode 100644 hack/preload-images/upload.go diff --git a/Makefile b/Makefile index 9a4685affd..ac1787f480 100755 --- a/Makefile +++ b/Makefile @@ -526,14 +526,8 @@ kic-base-image: ## builds the base image used for kic. docker build -f ./hack/images/kicbase.Dockerfile -t $(REGISTRY)/kicbase:$(KIC_VERSION)-snapshot --build-arg COMMIT_SHA=${VERSION}-$(COMMIT) --target base . .PHONY: upload-preloaded-images-tar -upload-preloaded-images-tar: generate-preloaded-images-tar # Upload the preloaded images tar to the GCS bucket. Specify a specific kubernetes version to build via `KUBERNETES_VERSION=vx.y.z make upload-preloaded-images-tar`. - gsutil cp out/preloaded-images-k8s-${PRELOADED_TARBALL_VERSION}-${KUBERNETES_VERSION}-docker-overlay2.tar.lz4 gs://${PRELOADED_VOLUMES_GCS_BUCKET} - gsutil acl ch -u AllUsers:R gs://${PRELOADED_VOLUMES_GCS_BUCKET}/preloaded-images-k8s-${PRELOADED_TARBALL_VERSION}-${KUBERNETES_VERSION}-docker-overlay2.tar.lz4 - -.PHONY: generate-preloaded-images-tar -generate-preloaded-images-tar: - go run ./hack/preload-images/preload_images.go -kubernetes-version ${KUBERNETES_VERSION} -preloaded-tarball-version ${PRELOADED_TARBALL_VERSION} - +upload-preloaded-images-tar: out/minikube # Upload the preloaded images for oldest supported, newest supported, and default kubernetes versions to GCS. + go run ./hack/preload-images/*.go .PHONY: push-storage-provisioner-image push-storage-provisioner-image: storage-provisioner-image ## Push storage-provisioner docker image using gcloud diff --git a/hack/preload-images/generate.go b/hack/preload-images/generate.go new file mode 100644 index 0000000000..a3160bc5a1 --- /dev/null +++ b/hack/preload-images/generate.go @@ -0,0 +1,125 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + + "github.com/pkg/errors" + "k8s.io/minikube/pkg/drivers/kic" + "k8s.io/minikube/pkg/drivers/kic/oci" + "k8s.io/minikube/pkg/minikube/bootstrapper/bsutil" + "k8s.io/minikube/pkg/minikube/bootstrapper/images" + "k8s.io/minikube/pkg/minikube/command" + "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/driver" + "k8s.io/minikube/pkg/minikube/localpath" +) + +func generateTarball(kubernetesVersion, tarballFilename string) error { + defer func() { + if err := deleteMinikube(); err != nil { + fmt.Println(err) + } + }() + + driver := kic.NewDriver(kic.Config{ + KubernetesVersion: kubernetesVersion, + ContainerRuntime: driver.Docker, + OCIBinary: oci.Docker, + MachineName: profile, + ImageDigest: kic.BaseImage, + StorePath: localpath.MiniPath(), + CPU: 2, + Memory: 4000, + APIServerPort: 8080, + }) + + baseDir := filepath.Dir(driver.GetSSHKeyPath()) + defer os.Remove(baseDir) + + if err := os.MkdirAll(baseDir, 0755); err != nil { + return errors.Wrap(err, "mkdir") + } + if err := driver.Create(); err != nil { + return errors.Wrap(err, "creating kic driver") + } + + // Now, get images to pull + imgs, err := images.Kubeadm("", kubernetesVersion) + if err != nil { + return errors.Wrap(err, "kubeadm images") + } + + for _, img := range append(imgs, kic.OverlayImage) { + cmd := exec.Command("docker", "exec", profile, "docker", "pull", img) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return errors.Wrapf(err, "downloading %s", img) + } + } + + // Transfer in k8s binaries + kcfg := config.KubernetesConfig{ + KubernetesVersion: kubernetesVersion, + } + runner := command.NewKICRunner(profile, driver.OCIBinary) + if err := bsutil.TransferBinaries(kcfg, runner); err != nil { + return errors.Wrap(err, "transferring k8s binaries") + } + // Create image tarball + if err := createImageTarball(tarballFilename); err != nil { + return errors.Wrap(err, "create tarball") + } + return copyTarballToHost(tarballFilename) +} + +func createImageTarball(tarballFilename string) error { + dirs := []string{ + fmt.Sprintf("./lib/docker/%s", dockerStorageDriver), + "./lib/docker/image", + "./lib/minikube/binaries", + } + args := []string{"exec", profile, "sudo", "tar", "-I", "lz4", "-C", "/var", "-cvf", tarballFilename} + args = append(args, dirs...) + cmd := exec.Command("docker", args...) + cmd.Stdout = os.Stdout + if err := cmd.Run(); err != nil { + return errors.Wrapf(err, "tarball cmd: %s", cmd.Args) + } + return nil +} + +func copyTarballToHost(tarballFilename string) error { + dest := filepath.Join("out/", tarballFilename) + cmd := exec.Command("docker", "cp", fmt.Sprintf("%s:/%s", profile, tarballFilename), dest) + cmd.Stdout = os.Stdout + if err := cmd.Run(); err != nil { + return errors.Wrapf(err, "cp cmd: %s", cmd.Args) + } + return nil +} + +func deleteMinikube() error { + cmd := exec.Command(minikubePath, "delete", "-p", profile) + cmd.Stdout = os.Stdout + return cmd.Run() +} diff --git a/hack/preload-images/preload_images.go b/hack/preload-images/preload_images.go index 1c6c858331..17a66806a9 100644 --- a/hack/preload-images/preload_images.go +++ b/hack/preload-images/preload_images.go @@ -18,22 +18,13 @@ package main import ( "bytes" - "flag" "fmt" - "os" "os/exec" - "path/filepath" "strings" - "github.com/pkg/errors" - "k8s.io/minikube/pkg/drivers/kic" - "k8s.io/minikube/pkg/drivers/kic/oci" - "k8s.io/minikube/pkg/minikube/bootstrapper/bsutil" - "k8s.io/minikube/pkg/minikube/bootstrapper/images" - "k8s.io/minikube/pkg/minikube/command" - "k8s.io/minikube/pkg/minikube/config" - "k8s.io/minikube/pkg/minikube/driver" - "k8s.io/minikube/pkg/minikube/localpath" + "k8s.io/minikube/pkg/minikube/constants" + "k8s.io/minikube/pkg/minikube/download" + "k8s.io/minikube/pkg/minikube/exit" ) const ( @@ -42,123 +33,47 @@ const ( ) var ( - kubernetesVersion = "" - tarballFilename = "" - dockerStorageDriver = "" - preloadedTarballVersion = "" - containerRuntime = "" + dockerStorageDriver = "overlay2" + preloadedTarballVersion = "v1" + containerRuntimes = []string{"docker"} + kubernetesVersions = []string{ + constants.OldestKubernetesVersion, + constants.DefaultKubernetesVersion, + constants.NewestKubernetesVersion, + } ) -func init() { - flag.StringVar(&kubernetesVersion, "kubernetes-version", "", "desired kubernetes version, for example `v1.17.2`") - flag.StringVar(&dockerStorageDriver, "docker-storage-driver", "overlay2", "docker storage driver backend") - flag.StringVar(&preloadedTarballVersion, "preloaded-tarball-version", "", "preloaded tarball version") - flag.StringVar(&containerRuntime, "container-runtime", "docker", "container runtime") - - flag.Parse() - tarballFilename = fmt.Sprintf("preloaded-images-k8s-%s-%s-%s-%s.tar.lz4", preloadedTarballVersion, kubernetesVersion, containerRuntime, dockerStorageDriver) -} - func main() { if err := verifyDockerStorage(); err != nil { - fmt.Println(err) - os.Exit(1) + exit.WithError("Docker storage type is incompatible: %v\n", err) } - if err := executePreloadImages(); err != nil { - fmt.Println(err) - os.Exit(1) - } -} - -func executePreloadImages() error { - defer func() { - if err := deleteMinikube(); err != nil { - fmt.Println(err) - } - }() - - driver := kic.NewDriver(kic.Config{ - KubernetesVersion: kubernetesVersion, - ContainerRuntime: driver.Docker, - OCIBinary: oci.Docker, - MachineName: profile, - ImageDigest: kic.BaseImage, - StorePath: localpath.MiniPath(), - CPU: 2, - Memory: 4000, - APIServerPort: 8080, - }) - - baseDir := filepath.Dir(driver.GetSSHKeyPath()) - defer os.Remove(baseDir) - - if err := os.MkdirAll(baseDir, 0755); err != nil { - return errors.Wrap(err, "mkdir") - } - if err := driver.Create(); err != nil { - return errors.Wrap(err, "creating kic driver") - } - - // Now, get images to pull - imgs, err := images.Kubeadm("", kubernetesVersion) - if err != nil { - return errors.Wrap(err, "kubeadm images") - } - - for _, img := range append(imgs, kic.OverlayImage) { - cmd := exec.Command("docker", "exec", profile, "docker", "pull", img) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { - return errors.Wrapf(err, "downloading %s", img) + for _, kubernetesVersion := range kubernetesVersions { + for _, cr := range containerRuntimes { + tf := tarballFilename(kubernetesVersion, cr) + if tarballExists(tf) { + fmt.Printf("A preloaded tarball for k8s version %s already exists, skipping generation.\n", kubernetesVersion) + continue + } + fmt.Printf("A preloaded tarball for k8s version %s doesn't exist, generating now...\n", kubernetesVersion) + if err := generateTarball(kubernetesVersion, tf); err != nil { + exit.WithError(fmt.Sprintf("generating tarball for k8s version %s with %s", kubernetesVersion, cr), err) + } + if err := uploadTarball(tf); err != nil { + exit.WithError(fmt.Sprintf("uploading tarball for k8s version %s with %s", kubernetesVersion, cr), err) + } } } - - // Transfer in k8s binaries - kcfg := config.KubernetesConfig{ - KubernetesVersion: kubernetesVersion, - } - runner := command.NewKICRunner(profile, driver.OCIBinary) - if err := bsutil.TransferBinaries(kcfg, runner); err != nil { - return errors.Wrap(err, "transferring k8s binaries") - } - // Create image tarball - if err := createImageTarball(); err != nil { - return errors.Wrap(err, "create tarball") - } - return copyTarballToHost() } -func createImageTarball() error { - dirs := []string{ - fmt.Sprintf("./lib/docker/%s", dockerStorageDriver), - "./lib/docker/image", - "./lib/minikube/binaries", - } - args := []string{"exec", profile, "sudo", "tar", "-I", "lz4", "-C", "/var", "-cvf", tarballFilename} - args = append(args, dirs...) - cmd := exec.Command("docker", args...) - cmd.Stdout = os.Stdout - if err := cmd.Run(); err != nil { - return errors.Wrapf(err, "tarball cmd: %s", cmd.Args) - } - return nil +func tarballFilename(kubernetesVersion string, containerRuntime string) string { + return fmt.Sprintf("preloaded-images-k8s-%s-%s-%s-%s.tar.lz4", preloadedTarballVersion, kubernetesVersion, containerRuntime, dockerStorageDriver) } -func copyTarballToHost() error { - dest := filepath.Join("out/", tarballFilename) - cmd := exec.Command("docker", "cp", fmt.Sprintf("%s:/%s", profile, tarballFilename), dest) - cmd.Stdout = os.Stdout - if err := cmd.Run(); err != nil { - return errors.Wrapf(err, "cp cmd: %s", cmd.Args) - } - return nil -} - -func deleteMinikube() error { - cmd := exec.Command(minikubePath, "delete", "-p", profile) - cmd.Stdout = os.Stdout - return cmd.Run() +func tarballExists(tarballFilename string) bool { + fmt.Println("Checking if tarball already exists...") + gcsPath := fmt.Sprintf("gs://%s/%s", download.PreloadBucket, tarballFilename) + cmd := exec.Command("gsutil", "stat", gcsPath) + return cmd.Run() == nil } func verifyDockerStorage() error { diff --git a/hack/preload-images/upload.go b/hack/preload-images/upload.go new file mode 100644 index 0000000000..a2181294e6 --- /dev/null +++ b/hack/preload-images/upload.go @@ -0,0 +1,43 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "os/exec" + "path" + + "github.com/pkg/errors" + "k8s.io/minikube/pkg/minikube/download" +) + +func uploadTarball(tarballFilename string) error { + // Upload tarball to GCS + hostPath := path.Join("out/", tarballFilename) + gcsDest := fmt.Sprintf("gs://%s", download.PreloadBucket) + cmd := exec.Command("gsutil", "cp", hostPath, gcsDest) + if output, err := cmd.Output(); err != nil { + return errors.Wrapf(err, "uploading %s to GCS bucket: %v\n%s", hostPath, err, string(output)) + } + // Make tarball public to all users + gcsPath := fmt.Sprintf("%s/%s", gcsDest, tarballFilename) + cmd = exec.Command("gsutil", "acl", "ch", "-u", "AllUsers:R", gcsPath) + if output, err := cmd.Output(); err != nil { + return errors.Wrapf(err, "uploading %s to GCS bucket: %v\n%s", hostPath, err, string(output)) + } + return nil +} From a83478f2681f6a88f0ebd0ad24ff0896a7e330a6 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Fri, 13 Mar 2020 16:10:44 -0700 Subject: [PATCH 02/10] Add running preload release script to github actions --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a3572ed151..0d01eef0bd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,6 +19,7 @@ jobs: run : | make minikube-linux-amd64 make e2e-linux-amd64 + make upload-preloaded-images-tar cp -r test/integration/testdata ./out whoami echo github ref $GITHUB_REF From 8b364befc0046e94325de5b19f731993292ddc99 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Fri, 13 Mar 2020 16:10:55 -0700 Subject: [PATCH 03/10] Remove unused variables from Makefile --- Makefile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Makefile b/Makefile index ac1787f480..8d46355ada 100755 --- a/Makefile +++ b/Makefile @@ -21,8 +21,6 @@ VERSION ?= v$(RAW_VERSION) KUBERNETES_VERSION ?= $(shell egrep "DefaultKubernetesVersion =" pkg/minikube/constants/constants.go | cut -d \" -f2) KIC_VERSION ?= $(shell egrep "Version =" pkg/drivers/kic/types.go | cut -d \" -f2) -PRELOADED_TARBALL_VERSION ?= $(shell egrep "PreloadVersion =" pkg/minikube/download/preload.go | cut -d \" -f2) -PRELOADED_VOLUMES_GCS_BUCKET ?= $(shell egrep "PreloadBucket =" pkg/minikube/download/preload.go | cut -d \" -f2) # Default to .0 for higher cache hit rates, as build increments typically don't require new ISO versions ISO_VERSION ?= v$(VERSION_MAJOR).$(VERSION_MINOR).0 From 316eef44d1a67f0e140a0d04be079c9adcad6285 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Mon, 16 Mar 2020 10:44:01 -0700 Subject: [PATCH 04/10] Run preloaded images release script in jenkins --- .github/workflows/main.yml | 1 - hack/jenkins/minikube_cross_build_and_upload.sh | 6 +++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0d01eef0bd..a3572ed151 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,7 +19,6 @@ jobs: run : | make minikube-linux-amd64 make e2e-linux-amd64 - make upload-preloaded-images-tar cp -r test/integration/testdata ./out whoami echo github ref $GITHUB_REF diff --git a/hack/jenkins/minikube_cross_build_and_upload.sh b/hack/jenkins/minikube_cross_build_and_upload.sh index 382adacea7..9738cc08eb 100755 --- a/hack/jenkins/minikube_cross_build_and_upload.sh +++ b/hack/jenkins/minikube_cross_build_and_upload.sh @@ -39,7 +39,11 @@ declare -rx TAG="${ghprbActualCommit}" docker kill $(docker ps -q) || true docker rm $(docker ps -aq) || true -make -j 16 all && failed=$? || failed=$? +make -j 16 all && failed=$? || failed=$? + +echo "Running preloaded images release script..." +make upload-preloaded-images-tar + "out/minikube-$(go env GOOS)-$(go env GOARCH)" version From c9c597c2e1f23a1fedec6bb7a0cd93b1cbd8d112 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Mon, 16 Mar 2020 17:48:00 -0700 Subject: [PATCH 05/10] get last 30 releases of k8s from github --- go.mod | 2 + go.sum | 2 + .../minikube_cross_build_and_upload.sh | 4 -- hack/preload-images/kubernetes.go | 40 +++++++++++++++++++ hack/preload-images/preload_images.go | 11 +++-- 5 files changed, 49 insertions(+), 10 deletions(-) create mode 100644 hack/preload-images/kubernetes.go diff --git a/go.mod b/go.mod index 6777081f84..b1861de795 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,8 @@ require ( github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b github.com/google/go-cmp v0.3.1 github.com/google/go-containerregistry v0.0.0-20200131185320-aec8da010de2 + github.com/google/go-github v17.0.0+incompatible + github.com/google/go-github/v29 v29.0.3 // indirect github.com/googleapis/gnostic v0.3.0 // indirect github.com/hashicorp/go-getter v1.4.0 github.com/hashicorp/go-retryablehttp v0.5.4 diff --git a/go.sum b/go.sum index 0cd403ed47..112a60917c 100644 --- a/go.sum +++ b/go.sum @@ -335,6 +335,8 @@ github.com/google/go-containerregistry v0.0.0-20200131185320-aec8da010de2 h1:/z0 github.com/google/go-containerregistry v0.0.0-20200131185320-aec8da010de2/go.mod h1:Wtl/v6YdQxv397EREtzwgd9+Ud7Q5D8XMbi3Zazgkrs= github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= +github.com/google/go-github/v29 v29.0.3 h1:IktKCTwU//aFHnpA+2SLIi7Oo9uhAzgsdZNbcAqhgdc= +github.com/google/go-github/v29 v29.0.3/go.mod h1:CHKiKKPHJ0REzfwc14QMklvtHwCveD0PxlMjLlzAM5E= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= diff --git a/hack/jenkins/minikube_cross_build_and_upload.sh b/hack/jenkins/minikube_cross_build_and_upload.sh index 9738cc08eb..ff9dc97092 100755 --- a/hack/jenkins/minikube_cross_build_and_upload.sh +++ b/hack/jenkins/minikube_cross_build_and_upload.sh @@ -41,10 +41,6 @@ docker kill $(docker ps -q) || true docker rm $(docker ps -aq) || true make -j 16 all && failed=$? || failed=$? -echo "Running preloaded images release script..." -make upload-preloaded-images-tar - - "out/minikube-$(go env GOOS)-$(go env GOARCH)" version gsutil cp "gs://${bucket}/logs/index.html" \ diff --git a/hack/preload-images/kubernetes.go b/hack/preload-images/kubernetes.go new file mode 100644 index 0000000000..1114d39e12 --- /dev/null +++ b/hack/preload-images/kubernetes.go @@ -0,0 +1,40 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + + "github.com/golang/glog" + "github.com/google/go-github/github" +) + +// RecentK8sVersions returns the most recent k8s version, usually around 30 +func RecentK8sVersions() ([]string, error) { + client := github.NewClient(nil) + k8s := "kubernetes" + list, _, err := client.Repositories.ListReleases(context.Background(), k8s, k8s, &github.ListOptions{}) + if err != nil { + return nil, err + } + var releases []string + for _, r := range list { + releases = append(releases, r.GetTagName()) + } + glog.Infof("Got releases: %v", releases) + return releases, nil +} diff --git a/hack/preload-images/preload_images.go b/hack/preload-images/preload_images.go index 17a66806a9..8f25e58040 100644 --- a/hack/preload-images/preload_images.go +++ b/hack/preload-images/preload_images.go @@ -22,7 +22,6 @@ import ( "os/exec" "strings" - "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/download" "k8s.io/minikube/pkg/minikube/exit" ) @@ -36,17 +35,17 @@ var ( dockerStorageDriver = "overlay2" preloadedTarballVersion = "v1" containerRuntimes = []string{"docker"} - kubernetesVersions = []string{ - constants.OldestKubernetesVersion, - constants.DefaultKubernetesVersion, - constants.NewestKubernetesVersion, - } ) func main() { if err := verifyDockerStorage(); err != nil { exit.WithError("Docker storage type is incompatible: %v\n", err) } + kubernetesVersions, err := RecentK8sVersions() + if err != nil { + exit.WithError("Unable to get recent k8s versions: %v\n", err) + } + for _, kubernetesVersion := range kubernetesVersions { for _, cr := range containerRuntimes { tf := tarballFilename(kubernetesVersion, cr) From 0acb06bd34bf7ccdee14724329cfddcc443b5f87 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Mon, 16 Mar 2020 17:51:41 -0700 Subject: [PATCH 06/10] add flag for kubernetes version --- go.sum | 5 ++++ hack/preload-images/preload_images.go | 34 +++++++++++++++++++-------- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/go.sum b/go.sum index 112a60917c..dfeb89780e 100644 --- a/go.sum +++ b/go.sum @@ -68,6 +68,7 @@ github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYU github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/auth0/go-jwt-middleware v0.0.0-20170425171159-5493cabe49f7/go.mod h1:LWMyo4iOLWXHGdBki7NIht1kHru/0wM179h+d3g8ATM= github.com/aws/aws-sdk-go v1.15.78 h1:LaXy6lWR0YK7LKyuU0QWy2ws/LWTPfYV/UgfiBu4tvY= @@ -93,6 +94,7 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/bifurcation/mint v0.0.0-20180715133206-93c51c6ce115/go.mod h1:zVt7zX3K/aDCk9Tj+VM7YymsX66ERvzCJzw8rFCX2JU= github.com/blang/semver v3.5.0+incompatible h1:CGxCgetQ64DKk7rdZ++Vfnb1+ogGNnB17OJKJXD2Cfs= github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= github.com/c4milo/gotoolkit v0.0.0-20170318115440-bcc06269efa9 h1:+ziP/wVJWuAORkjv7386TRidVKY57X0bXBZFMeFlW+U= @@ -260,6 +262,7 @@ github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4= +github.com/go-ozzo/ozzo-validation v3.5.0+incompatible h1:sUy/in/P6askYr16XJgTKq/0SZhiWsdg4WZGaLsGQkM= github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4= @@ -393,7 +396,9 @@ github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uG github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/heketi/heketi v9.0.1-0.20190917153846-c2e2a4ab7ab9+incompatible h1:ysqc8k973k1lLJ4BOOHAkx14K2nt4cLjsIm+hwWDZDE= github.com/heketi/heketi v9.0.1-0.20190917153846-c2e2a4ab7ab9+incompatible/go.mod h1:bB9ly3RchcQqsQ9CpyaQwvva7RS5ytVoSoholZQON6o= +github.com/heketi/tests v0.0.0-20151005000721-f3775cbcefd6 h1:oJ/NLadJn5HoxvonA6VxG31lg0d6XOURNA09BTtM4fY= github.com/heketi/tests v0.0.0-20151005000721-f3775cbcefd6/go.mod h1:xGMAM8JLi7UkZt1i4FQeQy0R2T8GLUwQhOP5M1gBhy4= github.com/hooklift/assert v0.0.0-20170704181755-9d1defd6d214 h1:WgfvpuKg42WVLkxNwzfFraXkTXPK36bMqXvMFN67clI= github.com/hooklift/assert v0.0.0-20170704181755-9d1defd6d214/go.mod h1:kj6hFWqfwSjFjLnYW5PK1DoxZ4O0uapwHRmd9jhln4E= diff --git a/hack/preload-images/preload_images.go b/hack/preload-images/preload_images.go index 8f25e58040..7a9956d9c0 100644 --- a/hack/preload-images/preload_images.go +++ b/hack/preload-images/preload_images.go @@ -18,6 +18,7 @@ package main import ( "bytes" + "flag" "fmt" "os/exec" "strings" @@ -35,30 +36,43 @@ var ( dockerStorageDriver = "overlay2" preloadedTarballVersion = "v1" containerRuntimes = []string{"docker"} + k8sVersion string + k8sVersions []string ) +func init() { + flag.StringVar(&k8sVersion, "kubernetes-version", "", "desired kubernetes version, for example `v1.17.2`") + flag.Parse() + if k8sVersion != "" { + k8sVersions = append(k8sVersions, k8sVersion) + } +} + func main() { if err := verifyDockerStorage(); err != nil { exit.WithError("Docker storage type is incompatible: %v\n", err) } - kubernetesVersions, err := RecentK8sVersions() - if err != nil { - exit.WithError("Unable to get recent k8s versions: %v\n", err) + if k8sVersions == nil { + var err error + k8sVersions, err = RecentK8sVersions() + if err != nil { + exit.WithError("Unable to get recent k8s versions: %v\n", err) + } } - for _, kubernetesVersion := range kubernetesVersions { + for _, kv := range k8sVersions { for _, cr := range containerRuntimes { - tf := tarballFilename(kubernetesVersion, cr) + tf := tarballFilename(kv, cr) if tarballExists(tf) { - fmt.Printf("A preloaded tarball for k8s version %s already exists, skipping generation.\n", kubernetesVersion) + fmt.Printf("A preloaded tarball for k8s version %s already exists, skipping generation.\n", kv) continue } - fmt.Printf("A preloaded tarball for k8s version %s doesn't exist, generating now...\n", kubernetesVersion) - if err := generateTarball(kubernetesVersion, tf); err != nil { - exit.WithError(fmt.Sprintf("generating tarball for k8s version %s with %s", kubernetesVersion, cr), err) + fmt.Printf("A preloaded tarball for k8s version %s doesn't exist, generating now...\n", kv) + if err := generateTarball(kv, tf); err != nil { + exit.WithError(fmt.Sprintf("generating tarball for k8s version %s with %s", kv, cr), err) } if err := uploadTarball(tf); err != nil { - exit.WithError(fmt.Sprintf("uploading tarball for k8s version %s with %s", kubernetesVersion, cr), err) + exit.WithError(fmt.Sprintf("uploading tarball for k8s version %s with %s", kv, cr), err) } } } From c38cc9debef88afe1c0504b699dd439c4ed7fe66 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Tue, 17 Mar 2020 13:13:07 -0700 Subject: [PATCH 07/10] Add cloudbuild setup for preload generator image --- Makefile | 4 +++ deploy/preload/Dockerfile | 47 ++++++++++++++++++++++++++++++++++ deploy/preload/cloudbuild.yaml | 2 ++ 3 files changed, 53 insertions(+) create mode 100644 deploy/preload/Dockerfile create mode 100644 deploy/preload/cloudbuild.yaml diff --git a/Makefile b/Makefile index 8d46355ada..ad1deba4f6 100755 --- a/Makefile +++ b/Makefile @@ -523,6 +523,10 @@ kic-base-image: ## builds the base image used for kic. docker rmi -f $(REGISTRY)/kicbase:$(KIC_VERSION)-snapshot || true docker build -f ./hack/images/kicbase.Dockerfile -t $(REGISTRY)/kicbase:$(KIC_VERSION)-snapshot --build-arg COMMIT_SHA=${VERSION}-$(COMMIT) --target base . +.PHONY: preload-generator-image +preload-generator-image: + docker build -t gcr.io/k8s-minikube/preload-generator -f deploy/preload/Dockerfile deploy/preload + .PHONY: upload-preloaded-images-tar upload-preloaded-images-tar: out/minikube # Upload the preloaded images for oldest supported, newest supported, and default kubernetes versions to GCS. go run ./hack/preload-images/*.go diff --git a/deploy/preload/Dockerfile b/deploy/preload/Dockerfile new file mode 100644 index 0000000000..5159051106 --- /dev/null +++ b/deploy/preload/Dockerfile @@ -0,0 +1,47 @@ +# Copyright 2020 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Download gcloud +FROM alpine:3.10 as download-gcloud +ENV GCLOUD_VERSION 276.0.0 +ENV GCLOUD_URL https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${GCLOUD_VERSION}-linux-x86_64.tar.gz +RUN wget -O gcloud.tar.gz "${GCLOUD_URL}" +RUN tar -zxf gcloud.tar.gz + +FROM gcr.io/gcp-runtimes/ubuntu_16_0_4 + +# Install python and make +RUN apt-get update && \ + apt-get install --no-install-recommends --no-install-suggests -y \ + python make && \ + rm -rf /var/lib/apt/lists/* + +# Install docker +COPY --from=docker:18.09.6 /usr/local/bin/docker /usr/local/bin/ + +# Install gcloud dependencies +COPY --from=download-gcloud google-cloud-sdk/ /google-cloud-sdk/ +# Finish installation of gcloud +RUN CLOUDSDK_PYTHON="python2.7" /google-cloud-sdk/install.sh \ + --usage-reporting=false \ + --bash-completion=false \ + --disable-installation-options +ENV PATH=$PATH:/google-cloud-sdk/bin +RUN gcloud auth configure-docker + +# Install go +COPY --from=golang:1.14 /usr/local/go /usr/local/go +ENV PATH=$PATH:/usr/local/go/bin + +CMD make upload-preloaded-images-tar diff --git a/deploy/preload/cloudbuild.yaml b/deploy/preload/cloudbuild.yaml new file mode 100644 index 0000000000..7b53696447 --- /dev/null +++ b/deploy/preload/cloudbuild.yaml @@ -0,0 +1,2 @@ +steps: + - image: gcr.io/k8s-minikube/preload-generator From 2778bd1c99b8550d538a3830cceb70a315e84385 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Tue, 17 Mar 2020 13:14:38 -0700 Subject: [PATCH 08/10] Build preload generator image in cloudbuild --- deploy/preload/cloudbuild.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/deploy/preload/cloudbuild.yaml b/deploy/preload/cloudbuild.yaml index 7b53696447..750e8524d7 100644 --- a/deploy/preload/cloudbuild.yaml +++ b/deploy/preload/cloudbuild.yaml @@ -1,2 +1,4 @@ steps: - - image: gcr.io/k8s-minikube/preload-generator + - image: gcr.io/cloud-builders/docker + args: ["build", "-t", "preload-generator", "-f", "deploy/preload/Dockerfile", "deploy/preload"] + - image: preload-generator From 5659a1df6ea47ef37650d6fc153883b689d31ec2 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Tue, 17 Mar 2020 13:52:38 -0700 Subject: [PATCH 09/10] Remove cloudbuild stuff, run preload generation in jenkins --- Makefile | 4 -- deploy/preload/Dockerfile | 47 ------------------- deploy/preload/cloudbuild.yaml | 4 -- .../minikube_cross_build_and_upload.sh | 2 +- hack/jenkins/preload_generation.sh | 28 +++++++++++ 5 files changed, 29 insertions(+), 56 deletions(-) delete mode 100644 deploy/preload/Dockerfile delete mode 100644 deploy/preload/cloudbuild.yaml create mode 100644 hack/jenkins/preload_generation.sh diff --git a/Makefile b/Makefile index ad1deba4f6..8d46355ada 100755 --- a/Makefile +++ b/Makefile @@ -523,10 +523,6 @@ kic-base-image: ## builds the base image used for kic. docker rmi -f $(REGISTRY)/kicbase:$(KIC_VERSION)-snapshot || true docker build -f ./hack/images/kicbase.Dockerfile -t $(REGISTRY)/kicbase:$(KIC_VERSION)-snapshot --build-arg COMMIT_SHA=${VERSION}-$(COMMIT) --target base . -.PHONY: preload-generator-image -preload-generator-image: - docker build -t gcr.io/k8s-minikube/preload-generator -f deploy/preload/Dockerfile deploy/preload - .PHONY: upload-preloaded-images-tar upload-preloaded-images-tar: out/minikube # Upload the preloaded images for oldest supported, newest supported, and default kubernetes versions to GCS. go run ./hack/preload-images/*.go diff --git a/deploy/preload/Dockerfile b/deploy/preload/Dockerfile deleted file mode 100644 index 5159051106..0000000000 --- a/deploy/preload/Dockerfile +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2020 The Kubernetes Authors All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Download gcloud -FROM alpine:3.10 as download-gcloud -ENV GCLOUD_VERSION 276.0.0 -ENV GCLOUD_URL https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${GCLOUD_VERSION}-linux-x86_64.tar.gz -RUN wget -O gcloud.tar.gz "${GCLOUD_URL}" -RUN tar -zxf gcloud.tar.gz - -FROM gcr.io/gcp-runtimes/ubuntu_16_0_4 - -# Install python and make -RUN apt-get update && \ - apt-get install --no-install-recommends --no-install-suggests -y \ - python make && \ - rm -rf /var/lib/apt/lists/* - -# Install docker -COPY --from=docker:18.09.6 /usr/local/bin/docker /usr/local/bin/ - -# Install gcloud dependencies -COPY --from=download-gcloud google-cloud-sdk/ /google-cloud-sdk/ -# Finish installation of gcloud -RUN CLOUDSDK_PYTHON="python2.7" /google-cloud-sdk/install.sh \ - --usage-reporting=false \ - --bash-completion=false \ - --disable-installation-options -ENV PATH=$PATH:/google-cloud-sdk/bin -RUN gcloud auth configure-docker - -# Install go -COPY --from=golang:1.14 /usr/local/go /usr/local/go -ENV PATH=$PATH:/usr/local/go/bin - -CMD make upload-preloaded-images-tar diff --git a/deploy/preload/cloudbuild.yaml b/deploy/preload/cloudbuild.yaml deleted file mode 100644 index 750e8524d7..0000000000 --- a/deploy/preload/cloudbuild.yaml +++ /dev/null @@ -1,4 +0,0 @@ -steps: - - image: gcr.io/cloud-builders/docker - args: ["build", "-t", "preload-generator", "-f", "deploy/preload/Dockerfile", "deploy/preload"] - - image: preload-generator diff --git a/hack/jenkins/minikube_cross_build_and_upload.sh b/hack/jenkins/minikube_cross_build_and_upload.sh index ff9dc97092..382adacea7 100755 --- a/hack/jenkins/minikube_cross_build_and_upload.sh +++ b/hack/jenkins/minikube_cross_build_and_upload.sh @@ -39,7 +39,7 @@ declare -rx TAG="${ghprbActualCommit}" docker kill $(docker ps -q) || true docker rm $(docker ps -aq) || true -make -j 16 all && failed=$? || failed=$? +make -j 16 all && failed=$? || failed=$? "out/minikube-$(go env GOOS)-$(go env GOARCH)" version diff --git a/hack/jenkins/preload_generation.sh b/hack/jenkins/preload_generation.sh new file mode 100644 index 0000000000..66e2ba1125 --- /dev/null +++ b/hack/jenkins/preload_generation.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script builds all the minikube binary for all 3 platforms as well as Windows-installer and .deb +# This is intended to be run on a new release tag in order to build/upload the required files for a release + + +set -eux -o pipefail + +# Make sure the right golang version is installed based on Makefile +WANT_GOLANG_VERSION=$(grep '^GO_VERSION' Makefile | awk '{ print $3 }') +./hack/jenkins/installers/check_install_golang.sh $WANT_GOLANG_VERSION /usr/local + +make upload-preloaded-images-tar +make clean From 579ac4411cb26f0eb9089408e33732ffe10d46d5 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Tue, 17 Mar 2020 13:58:03 -0700 Subject: [PATCH 10/10] Use one tarballName function for consistency and add amd64 to the name --- hack/preload-images/preload_images.go | 6 +----- pkg/minikube/download/preload.go | 16 ++++++++-------- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/hack/preload-images/preload_images.go b/hack/preload-images/preload_images.go index 7a9956d9c0..ceb0e5021b 100644 --- a/hack/preload-images/preload_images.go +++ b/hack/preload-images/preload_images.go @@ -62,7 +62,7 @@ func main() { for _, kv := range k8sVersions { for _, cr := range containerRuntimes { - tf := tarballFilename(kv, cr) + tf := download.TarballName(kv) if tarballExists(tf) { fmt.Printf("A preloaded tarball for k8s version %s already exists, skipping generation.\n", kv) continue @@ -78,10 +78,6 @@ func main() { } } -func tarballFilename(kubernetesVersion string, containerRuntime string) string { - return fmt.Sprintf("preloaded-images-k8s-%s-%s-%s-%s.tar.lz4", preloadedTarballVersion, kubernetesVersion, containerRuntime, dockerStorageDriver) -} - func tarballExists(tarballFilename string) bool { fmt.Println("Checking if tarball already exists...") gcsPath := fmt.Sprintf("gs://%s/%s", download.PreloadBucket, tarballFilename) diff --git a/pkg/minikube/download/preload.go b/pkg/minikube/download/preload.go index 62893edf1b..08f7e307c4 100644 --- a/pkg/minikube/download/preload.go +++ b/pkg/minikube/download/preload.go @@ -42,14 +42,14 @@ const ( PreloadBucket = "minikube-preloaded-volume-tarballs" ) -// returns name of the tarball -func tarballName(k8sVersion string) string { - return fmt.Sprintf("preloaded-images-k8s-%s-%s-docker-overlay2.tar.lz4", PreloadVersion, k8sVersion) +// TarballName returns name of the tarball +func TarballName(k8sVersion string) string { + return fmt.Sprintf("preloaded-images-k8s-%s-%s-docker-overlay2-amd64.tar.lz4", PreloadVersion, k8sVersion) } // returns the name of the checksum file func checksumName(k8sVersion string) string { - return fmt.Sprintf("%s.checksum", tarballName(k8sVersion)) + return fmt.Sprintf("%s.checksum", TarballName(k8sVersion)) } // returns target dir for all cached items related to preloading @@ -64,12 +64,12 @@ func PreloadChecksumPath(k8sVersion string) string { // TarballPath returns the path to the preloaded tarball func TarballPath(k8sVersion string) string { - return path.Join(targetDir(), tarballName(k8sVersion)) + return path.Join(targetDir(), TarballName(k8sVersion)) } // remoteTarballURL returns the URL for the remote tarball in GCS func remoteTarballURL(k8sVersion string) string { - return fmt.Sprintf("https://storage.googleapis.com/%s/%s", PreloadBucket, tarballName(k8sVersion)) + return fmt.Sprintf("https://storage.googleapis.com/%s/%s", PreloadBucket, TarballName(k8sVersion)) } // PreloadExists returns true if there is a preloaded tarball that can be used @@ -147,13 +147,13 @@ func Preload(k8sVersion, containerRuntime string) error { } func saveChecksumFile(k8sVersion string) error { - glog.Infof("saving checksum for %s ...", tarballName(k8sVersion)) + glog.Infof("saving checksum for %s ...", TarballName(k8sVersion)) ctx := context.Background() client, err := storage.NewClient(ctx, option.WithoutAuthentication()) if err != nil { return errors.Wrap(err, "getting storage client") } - attrs, err := client.Bucket(PreloadBucket).Object(tarballName(k8sVersion)).Attrs(ctx) + attrs, err := client.Bucket(PreloadBucket).Object(TarballName(k8sVersion)).Attrs(ctx) if err != nil { return errors.Wrap(err, "getting storage object") }