Merge pull request #11095 from ilya-zuyev/ilyaz/preloads_jenkins_job

Move preload generation to Jenkins

Makefile

@@ -703,11 +703,16 @@ ifndef AUTOPUSH
 endif
 	env $(X_BUILD_ENV) docker buildx build --builder $(X_DOCKER_BUILDER) --platform $(KICBASE_ARCH) $(addprefix -t ,$(KICBASE_IMAGE_REGISTRIES)) --push  --build-arg COMMIT_SHA=${VERSION}-$(COMMIT) ./deploy/kicbase
 
-.PHONY: upload-preloaded-images-tar
+out/preload-tool:
-upload-preloaded-images-tar: out/minikube # Upload the preloaded images for oldest supported, newest supported, and default kubernetes versions to GCS.
+	go build -ldflags="$(MINIKUBE_LDFLAGS)" -o $@ ./hack/preload-images/*.go
-	go build -ldflags="$(MINIKUBE_LDFLAGS)" -o out/upload-preload ./hack/preload-images/*.go
-	./out/upload-preload
 
+.PHONY: upload-preloaded-images-tar
+upload-preloaded-images-tar: out/minikube out/preload-tool ## Upload the preloaded images for oldest supported, newest supported, and default kubernetes versions to GCS.
+	out/preload-tool
+
+.PHONY: generate-preloaded-images-tar
+generate-preloaded-images-tar: out/minikube out/preload-tool ## Generates the preloaded images for oldest supported, newest supported, and default kubernetes versions
+	out/preload-tool --no-upload
 
 ALL_ARCH = amd64 arm arm64 ppc64le s390x
 IMAGE = $(REGISTRY)/storage-provisioner

hack/preload-images/kubernetes.go

@@ -24,8 +24,8 @@ import (
 	"k8s.io/klog/v2"
 )
 
-// RecentK8sVersions returns the most recent k8s version, usually around 30
+// recentK8sVersions returns the most recent k8s version, usually around 30
-func RecentK8sVersions() ([]string, error) {
+func recentK8sVersions() ([]string, error) {
 	client := github.NewClient(nil)
 	k8s := "kubernetes"
 	list, _, err := client.Repositories.ListReleases(context.Background(), k8s, k8s, &github.ListOptions{})

hack/preload-images/preload_images.go

@@ -26,6 +26,8 @@ import (
 	"runtime/debug"
 	"strings"
 
+	"github.com/pkg/errors"
+
 	"github.com/spf13/viper"
 	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/download"
@@ -40,61 +42,113 @@ var (
 	dockerStorageDriver = "overlay2"
 	podmanStorageDriver = "overlay"
 	containerRuntimes   = []string{"docker", "containerd", "cri-o"}
-	k8sVersion          string
 	k8sVersions         []string
+	k8sVersion          = flag.String("kubernetes-version", "", "desired Kubernetes version, for example `v1.17.2`")
+	noUpload            = flag.Bool("no-upload", false, "Do not upload tarballs to GCS")
+	force               = flag.Bool("force", false, "Generate the preload tarball even if it's already exists")
+	limit               = flag.Int("limit", 0, "Limit the number of tarballs to generate")
 )
 
-func init() {
+type preloadCfg struct {
-	flag.StringVar(&k8sVersion, "kubernetes-version", "", "desired Kubernetes version, for example `v1.17.2`")
+	k8sVer  string
-	flag.Parse()
+	runtime string
-	if k8sVersion != "" {
+}
-		k8sVersions = append(k8sVersions, k8sVersion)
+
-	}
+func (p preloadCfg) String() string {
-	viper.Set("preload", "true")
+	return fmt.Sprintf("%q/%q", p.runtime, p.k8sVer)
 }
 
 func main() {
-	defer func() {
+	flag.Parse()
-		if err := deleteMinikube(); err != nil {
+
-			fmt.Printf("error cleaning up minikube: %v \n", err)
+	// used by pkg/minikube/download.PreloadExists()
-		}
+	viper.Set("preload", "true")
-	}()
+
+	if *k8sVersion != "" {
+		k8sVersions = []string{*k8sVersion}
+	}
 
 	if err := deleteMinikube(); err != nil {
 		fmt.Printf("error cleaning up minikube at start up: %v \n", err)
 	}
 
-	if k8sVersions == nil {
+	k8sVersions, err := collectK8sVers()
-		var err error
+	if err != nil {
-		k8sVersions, err = RecentK8sVersions()
+		exit("Unable to get recent k8s versions: %v\n", err)
-		if err != nil {
-			exit("Unable to get recent k8s versions: %v\n", err)
-		}
 	}
 
-	k8sVersions = append(k8sVersions, constants.DefaultKubernetesVersion, constants.NewestKubernetesVersion, constants.OldestKubernetesVersion)
+	var toGenerate []preloadCfg
+	var i int
 
+out:
 	for _, kv := range k8sVersions {
 		for _, cr := range containerRuntimes {
-			tf := download.TarballName(kv, cr)
+			if *limit > 0 && i >= *limit {
-			if download.PreloadExists(kv, cr) {
+				break out
+			}
+			if !download.PreloadExists(kv, cr) {
+				toGenerate = append(toGenerate, preloadCfg{kv, cr})
+				i++
+				fmt.Printf("[%d] A preloaded tarball for k8s version %s - runtime %q does not exist.\n", i, kv, cr)
+			} else if *force {
+				// the tarball already exists, but '--force' is passed. we need to overwrite the file
+				toGenerate = append(toGenerate, preloadCfg{kv, cr})
+				i++
+				fmt.Printf("[%d] A preloaded tarball for k8s version %s - runtime %q already exists. Going to overwrite it.\n", i, kv, cr)
+			} else {
 				fmt.Printf("A preloaded tarball for k8s version %s - runtime %q already exists, skipping generation.\n", kv, cr)
-				continue
 			}
-			fmt.Printf("A preloaded tarball for k8s version %s - runtime %q doesn't exist, generating now...\n", kv, cr)
-			if err := generateTarball(kv, cr, tf); err != nil {
-				exit(fmt.Sprintf("generating tarball for k8s version %s with %s", kv, cr), err)
-			}
-			if err := uploadTarball(tf); err != nil {
-				exit(fmt.Sprintf("uploading tarball for k8s version %s with %s", kv, cr), err)
-			}
-
-			if err := deleteMinikube(); err != nil {
-				fmt.Printf("error cleaning up minikube before finishing up: %v\n", err)
-			}
-
 		}
 	}
+
+	fmt.Printf("Going to generate preloads for %v\n", toGenerate)
+
+	for _, cfg := range toGenerate {
+		if err := makePreload(cfg); err != nil {
+			exit(err.Error(), err)
+		}
+	}
+}
+
+func collectK8sVers() ([]string, error) {
+	if k8sVersions == nil {
+		recent, err := recentK8sVersions()
+		if err != nil {
+			return nil, err
+		}
+		k8sVersions = recent
+	}
+	return append([]string{
+		constants.DefaultKubernetesVersion,
+		constants.NewestKubernetesVersion,
+		constants.OldestKubernetesVersion,
+	}, k8sVersions...), nil
+}
+
+func makePreload(cfg preloadCfg) error {
+	kv, cr := cfg.k8sVer, cfg.runtime
+
+	fmt.Printf("A preloaded tarball for k8s version %s - runtime %q doesn't exist, generating now...\n", kv, cr)
+	tf := download.TarballName(kv, cr)
+
+	defer func() {
+		if err := deleteMinikube(); err != nil {
+			fmt.Printf("error cleaning up minikube before finishing up: %v\n", err)
+		}
+	}()
+
+	if err := generateTarball(kv, cr, tf); err != nil {
+		return errors.Wrap(err, fmt.Sprintf("generating tarball for k8s version %s with %s", kv, cr))
+	}
+
+	if *noUpload {
+		fmt.Printf("skip upload of %q\n", tf)
+		return nil
+	}
+	if err := uploadTarball(tf); err != nil {
+		return errors.Wrap(err, fmt.Sprintf("uploading tarball for k8s version %s with %s", kv, cr))
+	}
+	return nil
 }
 
 func verifyDockerStorage() error {