From 1bf268c9f646490fe4156615a4c9cb25543fae6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sat, 27 Jun 2020 13:15:36 +0200 Subject: [PATCH 1/6] Generate cri-o container runtime preload tarball --- hack/preload-images/generate.go | 8 ++++++++ hack/preload-images/preload_images.go | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/hack/preload-images/generate.go b/hack/preload-images/generate.go index 572352cbe8..a8fc0a63ca 100644 --- a/hack/preload-images/generate.go +++ b/hack/preload-images/generate.go @@ -137,6 +137,10 @@ func imagePullCommand(containerRuntime, img string) *exec.Cmd { if containerRuntime == "containerd" { return exec.Command("docker", "exec", profile, "sudo", "crictl", "pull", img) } + + if containerRuntime == "cri-o" { + return exec.Command("docker", "exec", profile, "sudo", "crictl", "pull", img) + } return nil } @@ -154,6 +158,10 @@ func createImageTarball(tarballFilename, containerRuntime string) error { dirs = append(dirs, fmt.Sprintf("./lib/containerd")) } + if containerRuntime == "cri-o" { + dirs = append(dirs, fmt.Sprintf("./lib/containers")) + } + args := []string{"exec", profile, "sudo", "tar", "-I", "lz4", "-C", "/var", "-cvf", tarballFilename} args = append(args, dirs...) cmd := exec.Command("docker", args...) diff --git a/hack/preload-images/preload_images.go b/hack/preload-images/preload_images.go index 09d62e421b..88a480d445 100644 --- a/hack/preload-images/preload_images.go +++ b/hack/preload-images/preload_images.go @@ -37,7 +37,7 @@ const ( var ( dockerStorageDriver = "overlay2" - containerRuntimes = []string{"docker", "containerd"} + containerRuntimes = []string{"docker", "containerd", "cri-o"} k8sVersion string k8sVersions []string ) From b87a837f2ebb66b6f91d36aab06bc14f0284c163 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sun, 28 Jun 2020 09:49:50 +0200 Subject: [PATCH 2/6] Use the actual storage driver name for cri-o --- hack/preload-images/preload_images.go | 25 +++++++++++++++++++++++++ pkg/minikube/download/preload.go | 8 +++++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/hack/preload-images/preload_images.go b/hack/preload-images/preload_images.go index 88a480d445..33971aa334 100644 --- a/hack/preload-images/preload_images.go +++ b/hack/preload-images/preload_images.go @@ -18,6 +18,7 @@ package main import ( "bytes" + "encoding/json" "flag" "fmt" "os" @@ -37,6 +38,7 @@ const ( var ( dockerStorageDriver = "overlay2" + podmanStorageDriver = "overlay" containerRuntimes = []string{"docker", "containerd", "cri-o"} k8sVersion string k8sVersions []string @@ -65,6 +67,9 @@ func main() { if err := verifyDockerStorage(); err != nil { exit("Docker storage type is incompatible: %v \n", err) } + if err := verifyPodmanStorage(); err != nil { + exit("Podman storage type is incompatible: %v \n", err) + } if k8sVersions == nil { var err error k8sVersions, err = RecentK8sVersions() @@ -113,6 +118,26 @@ func verifyDockerStorage() error { return nil } +func verifyPodmanStorage() error { + cmd := exec.Command("sudo", "podman", "info", "-f", "json") + var stderr bytes.Buffer + cmd.Stderr = &stderr + output, err := cmd.Output() + if err != nil { + return fmt.Errorf("%v: %v:\n%s", cmd.Args, err, stderr.String()) + } + var info map[string]map[string]interface{} + err = json.Unmarshal(output, &info) + if err != nil { + return err + } + driver := info["store"]["graphDriverName"] + if driver != podmanStorageDriver { + return fmt.Errorf("podman storage driver %s does not match requested %s", driver, podmanStorageDriver) + } + return nil +} + // exit will exit and clean up minikube func exit(msg string, err error) { fmt.Printf("WithError(%s)=%v called from:\n%s", msg, err, debug.Stack()) diff --git a/pkg/minikube/download/preload.go b/pkg/minikube/download/preload.go index cf8b97d94b..75ae151288 100644 --- a/pkg/minikube/download/preload.go +++ b/pkg/minikube/download/preload.go @@ -47,7 +47,13 @@ const ( // TarballName returns name of the tarball func TarballName(k8sVersion, containerRuntime string) string { - return fmt.Sprintf("preloaded-images-k8s-%s-%s-%s-overlay2-%s.tar.lz4", PreloadVersion, k8sVersion, containerRuntime, runtime.GOARCH) + var storageDriver string + if containerRuntime == "cri-o" { + storageDriver = "overlay" + } else { + storageDriver = "overlay2" + } + return fmt.Sprintf("preloaded-images-k8s-%s-%s-%s-%s-%s.tar.lz4", PreloadVersion, k8sVersion, containerRuntime, storageDriver, runtime.GOARCH) } // returns the name of the checksum file From 51b4d2eb242e1fb6ec6ef8b88661e11916edfd77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sun, 28 Jun 2020 13:45:01 +0200 Subject: [PATCH 3/6] Normalize name of the container runtime param --- pkg/minikube/download/preload.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/minikube/download/preload.go b/pkg/minikube/download/preload.go index 75ae151288..67a34cbf81 100644 --- a/pkg/minikube/download/preload.go +++ b/pkg/minikube/download/preload.go @@ -47,6 +47,9 @@ const ( // TarballName returns name of the tarball func TarballName(k8sVersion, containerRuntime string) string { + if containerRuntime == "crio" { + containerRuntime = "cri-o" + } var storageDriver string if containerRuntime == "cri-o" { storageDriver = "overlay" From eaeb5f6060ef43cc4ad91696b23e54e2476b74fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sun, 28 Jun 2020 16:36:03 +0200 Subject: [PATCH 4/6] Use preloaded tarball for cri-o container runtime Pretty much the same as containerd, minus Restart (cherry picked from commit d9fb2e05420c8f3526d1b1079ee129842ea5522c) --- pkg/minikube/cruntime/containerd.go | 16 +---- pkg/minikube/cruntime/cri.go | 12 ++++ pkg/minikube/cruntime/crio.go | 103 +++++++++++++++++++++++++++- pkg/minikube/download/preload.go | 7 -- 4 files changed, 116 insertions(+), 22 deletions(-) diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go index 94bdc426af..50d59bbd57 100644 --- a/pkg/minikube/cruntime/containerd.go +++ b/pkg/minikube/cruntime/containerd.go @@ -370,7 +370,7 @@ func containerdImagesPreloaded(runner command.Runner, images []string) bool { if err != nil { return false } - type containerdImages struct { + type criImages struct { Images []struct { ID string `json:"id"` RepoTags []string `json:"repoTags"` @@ -381,7 +381,7 @@ func containerdImagesPreloaded(runner command.Runner, images []string) bool { } `json:"images"` } - var jsonImages containerdImages + var jsonImages criImages err = json.Unmarshal(rr.Stdout.Bytes(), &jsonImages) if err != nil { glog.Errorf("failed to unmarshal images, will assume images are not preloaded") @@ -412,15 +412,3 @@ func containerdImagesPreloaded(runner command.Runner, images []string) bool { glog.Infof("all images are preloaded for containerd runtime.") return true } - -// addRepoTagToImageName makes sure the image name has a repo tag in it. -// in crictl images list have the repo tag prepended to them -// for example "kubernetesui/dashboard:v2.0.0 will show up as "docker.io/kubernetesui/dashboard:v2.0.0" -// warning this is only meant for kuberentes images where we know the GCR addreses have .io in them -// not mean to be used for public images -func addRepoTagToImageName(imgName string) string { - if !strings.Contains(imgName, ".io/") { - return "docker.io/" + imgName - } // else it already has repo name dont add anything - return imgName -} diff --git a/pkg/minikube/cruntime/cri.go b/pkg/minikube/cruntime/cri.go index 6f05551235..d7b078f273 100644 --- a/pkg/minikube/cruntime/cri.go +++ b/pkg/minikube/cruntime/cri.go @@ -259,3 +259,15 @@ func criContainerLogCmd(cr CommandRunner, id string, len int, follow bool) strin cmd.WriteString(id) return cmd.String() } + +// addRepoTagToImageName makes sure the image name has a repo tag in it. +// in crictl images list have the repo tag prepended to them +// for example "kubernetesui/dashboard:v2.0.0 will show up as "docker.io/kubernetesui/dashboard:v2.0.0" +// warning this is only meant for kuberentes images where we know the GCR addreses have .io in them +// not mean to be used for public images +func addRepoTagToImageName(imgName string) string { + if !strings.Contains(imgName, ".io/") { + return "docker.io/" + imgName + } // else it already has repo name dont add anything + return imgName +} diff --git a/pkg/minikube/cruntime/crio.go b/pkg/minikube/cruntime/crio.go index f424ea4104..3283774299 100644 --- a/pkg/minikube/cruntime/crio.go +++ b/pkg/minikube/cruntime/crio.go @@ -17,14 +17,19 @@ limitations under the License. package cruntime import ( + "encoding/json" "fmt" "os/exec" + "path" "strings" + "time" "github.com/blang/semver" "github.com/golang/glog" "github.com/pkg/errors" + "k8s.io/minikube/pkg/minikube/assets" "k8s.io/minikube/pkg/minikube/bootstrapper/images" + "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/download" "k8s.io/minikube/pkg/minikube/out" @@ -222,5 +227,101 @@ func (r *CRIO) Preload(cfg config.KubernetesConfig) error { if !download.PreloadExists(cfg.KubernetesVersion, cfg.ContainerRuntime) { return nil } - return fmt.Errorf("not yet implemented for %s", r.Name()) + + k8sVersion := cfg.KubernetesVersion + cRuntime := cfg.ContainerRuntime + + // If images already exist, return + images, err := images.Kubeadm(cfg.ImageRepository, k8sVersion) + if err != nil { + return errors.Wrap(err, "getting images") + } + if crioImagesPreloaded(r.Runner, images) { + glog.Info("Images already preloaded, skipping extraction") + return nil + } + + tarballPath := download.TarballPath(k8sVersion, cRuntime) + targetDir := "/" + targetName := "preloaded.tar.lz4" + dest := path.Join(targetDir, targetName) + + c := exec.Command("which", "lz4") + if _, err := r.Runner.RunCmd(c); err != nil { + return NewErrISOFeature("lz4") + } + + // Copy over tarball into host + fa, err := assets.NewFileAsset(tarballPath, targetDir, targetName, "0644") + if err != nil { + return errors.Wrap(err, "getting file asset") + } + t := time.Now() + if err := r.Runner.Copy(fa); err != nil { + return errors.Wrap(err, "copying file") + } + glog.Infof("Took %f seconds to copy over tarball", time.Since(t).Seconds()) + + t = time.Now() + // extract the tarball to /var in the VM + if rr, err := r.Runner.RunCmd(exec.Command("sudo", "tar", "-I", "lz4", "-C", "/var", "-xvf", dest)); err != nil { + return errors.Wrapf(err, "extracting tarball: %s", rr.Output()) + } + glog.Infof("Took %f seconds t extract the tarball", time.Since(t).Seconds()) + + // remove the tarball in the VM + if err := r.Runner.Remove(fa); err != nil { + glog.Infof("error removing tarball: %v", err) + } + + return nil +} + +// crioImagesPreloaded returns true if all images have been preloaded +func crioImagesPreloaded(runner command.Runner, images []string) bool { + rr, err := runner.RunCmd(exec.Command("sudo", "crictl", "images", "--output", "json")) + if err != nil { + return false + } + type criImages struct { + Images []struct { + ID string `json:"id"` + RepoTags []string `json:"repoTags"` + RepoDigests []string `json:"repoDigests"` + Size string `json:"size"` + UID interface{} `json:"uid"` + Username string `json:"username"` + } `json:"images"` + } + + var jsonImages criImages + err = json.Unmarshal(rr.Stdout.Bytes(), &jsonImages) + if err != nil { + glog.Errorf("failed to unmarshal images, will assume images are not preloaded") + return false + } + + // Make sure images == imgs + for _, i := range images { + found := false + for _, ji := range jsonImages.Images { + for _, rt := range ji.RepoTags { + i = addRepoTagToImageName(i) + if i == rt { + found = true + break + } + } + if found { + break + } + + } + if !found { + glog.Infof("couldn't find preloaded image for %q. assuming images are not preloaded.", i) + return false + } + } + glog.Infof("all images are preloaded for crio runtime.") + return true } diff --git a/pkg/minikube/download/preload.go b/pkg/minikube/download/preload.go index 67a34cbf81..570a7d732b 100644 --- a/pkg/minikube/download/preload.go +++ b/pkg/minikube/download/preload.go @@ -87,13 +87,6 @@ func remoteTarballURL(k8sVersion, containerRuntime string) string { // PreloadExists returns true if there is a preloaded tarball that can be used func PreloadExists(k8sVersion, containerRuntime string, forcePreload ...bool) bool { - // and https://github.com/kubernetes/minikube/issues/6934 - // to track status of adding crio - if containerRuntime == "crio" { - glog.Info("crio is not supported yet, skipping preload") - return false - } - // TODO (#8166): Get rid of the need for this and viper at all force := false if len(forcePreload) > 0 { From b3e5e709fa1e82d28ab4795f1ede07affc8988d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Tue, 30 Jun 2020 18:56:30 +0200 Subject: [PATCH 5/6] Need to test storage driver of container runtime --- hack/preload-images/generate.go | 18 ++++++++++++++++++ hack/preload-images/preload_images.go | 10 ++-------- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/hack/preload-images/generate.go b/hack/preload-images/generate.go index a8fc0a63ca..fe2517e293 100644 --- a/hack/preload-images/generate.go +++ b/hack/preload-images/generate.go @@ -60,6 +60,10 @@ func generateTarball(kubernetesVersion, containerRuntime, tarballFilename string return errors.Wrap(err, "creating kic driver") } + if err := verifyStorage(containerRuntime); err != nil { + return errors.Wrap(err, "verifying storage") + } + // Now, get images to pull imgs, err := images.Kubeadm("", kubernetesVersion) if err != nil { @@ -128,6 +132,20 @@ func generateTarball(kubernetesVersion, containerRuntime, tarballFilename string return copyTarballToHost(tarballFilename) } +func verifyStorage(containerRuntime string) error { + if containerRuntime == "docker" || containerRuntime == "containerd" { + if err := verifyDockerStorage(); err != nil { + return errors.Wrap(err, "Docker storage type is incompatible") + } + } + if containerRuntime == "cri-o" { + if err := verifyPodmanStorage(); err != nil { + return errors.Wrap(err, "Podman storage type is incompatible") + } + } + return nil +} + // returns the right command to pull image for a specific runtime func imagePullCommand(containerRuntime, img string) *exec.Cmd { if containerRuntime == "docker" { diff --git a/hack/preload-images/preload_images.go b/hack/preload-images/preload_images.go index 33971aa334..2a34e0f7f4 100644 --- a/hack/preload-images/preload_images.go +++ b/hack/preload-images/preload_images.go @@ -64,12 +64,6 @@ func main() { fmt.Printf("error cleaning up minikube at start up: %v \n", err) } - if err := verifyDockerStorage(); err != nil { - exit("Docker storage type is incompatible: %v \n", err) - } - if err := verifyPodmanStorage(); err != nil { - exit("Podman storage type is incompatible: %v \n", err) - } if k8sVersions == nil { var err error k8sVersions, err = RecentK8sVersions() @@ -104,7 +98,7 @@ func main() { } func verifyDockerStorage() error { - cmd := exec.Command("docker", "info", "-f", "{{.Info.Driver}}") + cmd := exec.Command("docker", "exec", profile, "docker", "info", "-f", "{{.Info.Driver}}") var stderr bytes.Buffer cmd.Stderr = &stderr output, err := cmd.Output() @@ -119,7 +113,7 @@ func verifyDockerStorage() error { } func verifyPodmanStorage() error { - cmd := exec.Command("sudo", "podman", "info", "-f", "json") + cmd := exec.Command("docker", "exec", profile, "sudo", "podman", "info", "-f", "json") var stderr bytes.Buffer cmd.Stderr = &stderr output, err := cmd.Output() From b0b3a1ac8cf55708b5dc8896b5c8fb2cd17c6211 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Tue, 30 Jun 2020 19:12:11 +0200 Subject: [PATCH 6/6] Change indentation from gofmt to goimports --- pkg/minikube/cruntime/cri.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/minikube/cruntime/cri.go b/pkg/minikube/cruntime/cri.go index d7b078f273..32bbb0f566 100644 --- a/pkg/minikube/cruntime/cri.go +++ b/pkg/minikube/cruntime/cri.go @@ -266,8 +266,8 @@ func criContainerLogCmd(cr CommandRunner, id string, len int, follow bool) strin // warning this is only meant for kuberentes images where we know the GCR addreses have .io in them // not mean to be used for public images func addRepoTagToImageName(imgName string) string { - if !strings.Contains(imgName, ".io/") { - return "docker.io/" + imgName - } // else it already has repo name dont add anything - return imgName + if !strings.Contains(imgName, ".io/") { + return "docker.io/" + imgName + } // else it already has repo name dont add anything + return imgName }