From e59c41e6f2c9a78be4aff3d0a615ce9f93a49d2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sun, 8 Aug 2021 23:17:15 +0200 Subject: [PATCH 1/9] Add method to copy files from the VM as well Previously you could only Copy(To), not CopyFrom. Implies that some Assets can be written to instead. --- pkg/minikube/assets/vm_assets.go | 59 ++++++++++++++++++ pkg/minikube/command/command_runner.go | 3 + pkg/minikube/command/exec_runner.go | 18 ++++++ pkg/minikube/command/fake_runner.go | 13 ++++ pkg/minikube/command/kic_runner.go | 17 ++++++ pkg/minikube/command/ssh_runner.go | 82 ++++++++++++++++++++++++++ pkg/minikube/cruntime/cruntime.go | 2 + pkg/minikube/cruntime/cruntime_test.go | 4 ++ 8 files changed, 198 insertions(+) diff --git a/pkg/minikube/assets/vm_assets.go b/pkg/minikube/assets/vm_assets.go index b6ec89e9b8..752953e960 100644 --- a/pkg/minikube/assets/vm_assets.go +++ b/pkg/minikube/assets/vm_assets.go @@ -24,6 +24,7 @@ import ( "io" "os" "path" + "strconv" "time" "github.com/pkg/errors" @@ -37,8 +38,11 @@ const MemorySource = "memory" // CopyableFile is something that can be copied type CopyableFile interface { io.Reader + io.Writer GetLength() int + SetLength(int) GetSourcePath() string + GetTargetPath() string GetTargetDir() string GetTargetName() string @@ -62,6 +66,11 @@ func (b *BaseAsset) GetSourcePath() string { return b.SourcePath } +// GetTargetPath returns target path +func (b *BaseAsset) GetTargetPath() string { + return path.Join(b.GetTargetDir(), b.GetTargetName()) +} + // GetTargetDir returns target dir func (b *BaseAsset) GetTargetDir() string { return b.TargetDir @@ -86,6 +95,7 @@ func (b *BaseAsset) GetModTime() (time.Time, error) { type FileAsset struct { BaseAsset reader io.ReadSeeker + writer io.Writer file *os.File // Optional pointer to close file through FileAsset.Close() } @@ -134,6 +144,14 @@ func (f *FileAsset) GetLength() (flen int) { return int(fi.Size()) } +// SetLength sets the file length +func (f *FileAsset) SetLength(flen int) { + err := os.Truncate(f.SourcePath, int64(flen)) + if err != nil { + klog.Errorf("truncate(%q) failed: %v", f.SourcePath, err) + } +} + // GetModTime returns modification time of the file func (f *FileAsset) GetModTime() (time.Time, error) { fi, err := os.Stat(f.SourcePath) @@ -152,6 +170,23 @@ func (f *FileAsset) Read(p []byte) (int, error) { return f.reader.Read(p) } +// Write writes the asset +func (f *FileAsset) Write(p []byte) (int, error) { + if f.writer == nil { + f.file.Close() + perms, err := strconv.ParseUint(f.Permissions, 8, 32) + if err != nil || perms > 07777 { + return 0, err + } + f.file, err = os.OpenFile(f.SourcePath, os.O_RDWR|os.O_CREATE, os.FileMode(perms)) + if err != nil { + return 0, err + } + f.writer = io.Writer(f.file) + } + return f.writer.Write(p) +} + // Seek resets the reader to offset func (f *FileAsset) Seek(offset int64, whence int) (int64, error) { return f.reader.Seek(offset, whence) @@ -177,11 +212,23 @@ func (m *MemoryAsset) GetLength() int { return m.length } +// SetLength returns length +func (m *MemoryAsset) SetLength(len int) { + m.length = len +} + // Read reads the asset func (m *MemoryAsset) Read(p []byte) (int, error) { return m.reader.Read(p) } +// Writer writes the asset +func (m *MemoryAsset) Write(p []byte) (int, error) { + m.length = len(p) + m.reader = bytes.NewReader(p) + return len(p), nil +} + // Seek resets the reader to offset func (m *MemoryAsset) Seek(offset int64, whence int) (int64, error) { return m.reader.Seek(offset, whence) @@ -298,6 +345,11 @@ func (m *BinAsset) GetLength() int { return m.length } +// SetLength sets length +func (m *BinAsset) SetLength(len int) { + m.length = len +} + // Read reads the asset func (m *BinAsset) Read(p []byte) (int, error) { if m.GetLength() == 0 { @@ -306,6 +358,13 @@ func (m *BinAsset) Read(p []byte) (int, error) { return m.reader.Read(p) } +// Write writes the asset +func (m *BinAsset) Write(p []byte) (int, error) { + m.length = len(p) + m.reader = bytes.NewReader(p) + return len(p), nil +} + // Seek resets the reader to offset func (m *BinAsset) Seek(offset int64, whence int) (int64, error) { return m.reader.Seek(offset, whence) diff --git a/pkg/minikube/command/command_runner.go b/pkg/minikube/command/command_runner.go index 41619b1af8..3abd0dbdfc 100644 --- a/pkg/minikube/command/command_runner.go +++ b/pkg/minikube/command/command_runner.go @@ -75,6 +75,9 @@ type Runner interface { // Copy is a convenience method that runs a command to copy a file Copy(assets.CopyableFile) error + // CopyFrom is a convenience method that runs a command to copy a file back + CopyFrom(assets.CopyableFile) error + // Remove is a convenience method that runs a command to remove a file Remove(assets.CopyableFile) error } diff --git a/pkg/minikube/command/exec_runner.go b/pkg/minikube/command/exec_runner.go index b803b9b949..7d91519776 100644 --- a/pkg/minikube/command/exec_runner.go +++ b/pkg/minikube/command/exec_runner.go @@ -184,6 +184,24 @@ func (e *execRunner) Copy(f assets.CopyableFile) error { return writeFile(dst, f, os.FileMode(perms)) } +// CopyFrom copies a file +func (e *execRunner) CopyFrom(f assets.CopyableFile) error { + src := path.Join(f.GetTargetDir(), f.GetTargetName()) + + dst := f.GetSourcePath() + klog.Infof("cp: %s --> %s (%d bytes)", src, dst, f.GetLength()) + if f.GetLength() == 0 { + klog.Warningf("0 byte asset: %+v", f) + } + + perms, err := strconv.ParseInt(f.GetPermissions(), 8, 0) + if err != nil || perms > 07777 { + return errors.Wrapf(err, "error converting permissions %s to integer", f.GetPermissions()) + } + + return writeFile(dst, f, os.FileMode(perms)) +} + // Remove removes a file func (e *execRunner) Remove(f assets.CopyableFile) error { dst := filepath.Join(f.GetTargetDir(), f.GetTargetName()) diff --git a/pkg/minikube/command/fake_runner.go b/pkg/minikube/command/fake_runner.go index b663ff7a38..0e8521a99f 100644 --- a/pkg/minikube/command/fake_runner.go +++ b/pkg/minikube/command/fake_runner.go @@ -142,6 +142,19 @@ func (f *FakeCommandRunner) Copy(file assets.CopyableFile) error { return nil } +func (f *FakeCommandRunner) CopyFrom(file assets.CopyableFile) error { + v, ok := f.fileMap.Load(file.GetSourcePath()) + if !ok { + return fmt.Errorf("not found in map") + } + b := v.(bytes.Buffer) + _, err := io.Copy(file, &b) + if err != nil { + return errors.Wrapf(err, "error writing file: %+v", file) + } + return nil +} + // Remove removes the filename, file contents key value pair from the stored map func (f *FakeCommandRunner) Remove(file assets.CopyableFile) error { f.fileMap.Delete(file.GetSourcePath()) diff --git a/pkg/minikube/command/kic_runner.go b/pkg/minikube/command/kic_runner.go index 71e56feb77..7156246ff2 100644 --- a/pkg/minikube/command/kic_runner.go +++ b/pkg/minikube/command/kic_runner.go @@ -204,6 +204,15 @@ func (k *kicRunner) Copy(f assets.CopyableFile) error { return k.copy(tf.Name(), dst) } +// CopyFrom copies a file +func (k *kicRunner) CopyFrom(f assets.CopyableFile) error { + src := f.GetTargetPath() + dst := f.GetSourcePath() + + klog.Infof("%s (direct): %s --> %s", k.ociBin, src, dst) + return k.copyFrom(src, dst) +} + // tempDirectory returns the directory to use as the temp directory // or an empty string if it should use the os default temp directory. func tempDirectory(isMinikubeSnap bool, isDockerSnap bool) (string, error) { @@ -229,6 +238,14 @@ func (k *kicRunner) copy(src string, dst string) error { return copyToDocker(src, fullDest) } +func (k *kicRunner) copyFrom(src string, dst string) error { + fullSource := fmt.Sprintf("%s:%s", k.nameOrID, src) + if k.ociBin == oci.Podman { + return copyToPodman(fullSource, dst) + } + return copyToDocker(fullSource, dst) +} + func (k *kicRunner) chmod(dst string, perm string) error { _, err := k.RunCmd(exec.Command("sudo", "chmod", perm, dst)) return err diff --git a/pkg/minikube/command/ssh_runner.go b/pkg/minikube/command/ssh_runner.go index fe92bffe00..d0044f4c74 100644 --- a/pkg/minikube/command/ssh_runner.go +++ b/pkg/minikube/command/ssh_runner.go @@ -17,11 +17,14 @@ limitations under the License. package command import ( + "bufio" "bytes" "fmt" "io" "os/exec" "path" + "strconv" + "strings" "sync" "time" @@ -373,3 +376,82 @@ func (s *SSHRunner) Copy(f assets.CopyableFile) error { } return g.Wait() } + +// CopyFrom copies a file from the remote over SSH. +func (s *SSHRunner) CopyFrom(f assets.CopyableFile) error { + dst := path.Join(path.Join(f.GetTargetDir(), f.GetTargetName())) + + sess, err := s.session() + if err != nil { + return errors.Wrap(err, "NewSession") + } + defer func() { + if err := sess.Close(); err != nil { + if err != io.EOF { + klog.Errorf("session close: %v", err) + } + } + }() + + cmd := exec.Command("stat", "-c", "%s", dst) + rr, err := s.RunCmd(cmd) + if err != nil { + return fmt.Errorf("%s: %v", cmd, err) + } + length, err := strconv.Atoi(strings.TrimSuffix(rr.Stdout.String(), "\n")) + if err != nil { + return err + } + src := f.GetSourcePath() + klog.Infof("scp %s --> %s (%d bytes)", dst, src, length) + f.SetLength(length) + + r, err := sess.StdoutPipe() + if err != nil { + return errors.Wrap(err, "StdoutPipe") + } + w, err := sess.StdinPipe() + if err != nil { + return errors.Wrap(err, "StdinPipe") + } + // The scpcmd below *should not* return until all data is copied and the + // StdinPipe is closed. But let's use errgroup to make it explicit. + var g errgroup.Group + var copied int64 + + g.Go(func() error { + defer w.Close() + br := bufio.NewReader(r) + fmt.Fprint(w, "\x00") + b, err := br.ReadBytes('\n') + if err != nil { + return errors.Wrap(err, "ReadBytes") + } + if b[0] != 'C' { + return fmt.Errorf("unexpected: %v", b) + } + fmt.Fprint(w, "\x00") + + copied = 0 + for copied < int64(length) { + n, err := io.CopyN(f, br, int64(length)) + if err != nil { + return errors.Wrap(err, "io.CopyN") + } + copied += n + } + fmt.Fprint(w, "\x00") + err = sess.Wait() + if err != nil { + return err + } + return nil + }) + + scp := fmt.Sprintf("sudo scp -f %s", f.GetTargetPath()) + err = sess.Start(scp) + if err != nil { + return fmt.Errorf("%s: %s", scp, err) + } + return g.Wait() +} diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go index 95d9084839..e2df9d83a9 100644 --- a/pkg/minikube/cruntime/cruntime.go +++ b/pkg/minikube/cruntime/cruntime.go @@ -65,6 +65,8 @@ type CommandRunner interface { WaitCmd(sc *command.StartedCmd) (*command.RunResult, error) // Copy is a convenience method that runs a command to copy a file Copy(assets.CopyableFile) error + // CopyFrom is a convenience method that runs a command to copy a file back + CopyFrom(assets.CopyableFile) error // Remove is a convenience method that runs a command to remove a file Remove(assets.CopyableFile) error } diff --git a/pkg/minikube/cruntime/cruntime_test.go b/pkg/minikube/cruntime/cruntime_test.go index ca1c6cd54b..3a55059cc4 100644 --- a/pkg/minikube/cruntime/cruntime_test.go +++ b/pkg/minikube/cruntime/cruntime_test.go @@ -236,6 +236,10 @@ func (f *FakeRunner) Copy(assets.CopyableFile) error { return nil } +func (f *FakeRunner) CopyFrom(assets.CopyableFile) error { + return nil +} + func (f *FakeRunner) Remove(assets.CopyableFile) error { return nil } From 68197de86107fe8fae8ea790669daabf21f78c52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sun, 8 Aug 2021 23:21:10 +0200 Subject: [PATCH 2/9] Add command to save images from the cluster This is the opposite command of "minikube image load", and can be used after doing a "minikube image build". The default is to save images in the cache, but it is also possible to save to files or to standard output. --- cmd/minikube/cmd/image.go | 74 ++++++++ pkg/minikube/image/image.go | 58 ++++++ pkg/minikube/machine/cache_images.go | 171 ++++++++++++++++++ pkg/minikube/reason/reason.go | 2 + site/content/en/docs/commands/image.md | 48 +++++ site/content/en/docs/contrib/errorcodes.en.md | 3 + 6 files changed, 356 insertions(+) diff --git a/cmd/minikube/cmd/image.go b/cmd/minikube/cmd/image.go index 85616c09c4..8fdc870e57 100644 --- a/cmd/minikube/cmd/image.go +++ b/cmd/minikube/cmd/image.go @@ -144,6 +144,77 @@ var loadImageCmd = &cobra.Command{ }, } +func readFile(w io.Writer, tmp string) error { + r, err := os.Open(tmp) + if err != nil { + return err + } + _, err = io.Copy(w, r) + if err != nil { + return err + } + err = r.Close() + if err != nil { + return err + } + return nil +} + +// saveImageCmd represents the image load command +var saveImageCmd = &cobra.Command{ + Use: "save IMAGE [ARCHIVE | -]", + Short: "Save a image from minikube", + Long: "Save a image from minikube", + Example: "minikube image save image\nminikube image save image image.tar", + Run: func(cmd *cobra.Command, args []string) { + if len(args) == 0 { + exit.Message(reason.Usage, "Please provide an image in the container runtime to save from minikube via ") + } + // Save images from container runtime + profile, err := config.LoadProfile(viper.GetString(config.ProfileName)) + if err != nil { + exit.Error(reason.Usage, "loading profile", err) + } + + if len(args) > 1 { + output = args[1] + + if args[1] == "-" { + tmp, err := ioutil.TempFile("", "image.*.tar") + if err != nil { + exit.Error(reason.GuestImageSave, "Failed to get temp", err) + } + tmp.Close() + output = tmp.Name() + } + + if err := machine.DoSaveImages([]string{args[0]}, output, []*config.Profile{profile}, ""); err != nil { + exit.Error(reason.GuestImageSave, "Failed to save image", err) + } + + if args[1] == "-" { + err := readFile(os.Stdout, output) + if err != nil { + exit.Error(reason.GuestImageSave, "Failed to read temp", err) + } + os.Remove(output) + } + } else { + if err := machine.SaveAndCacheImages([]string{args[0]}, []*config.Profile{profile}); err != nil { + exit.Error(reason.GuestImageSave, "Failed to save image", err) + } + if imgDaemon || imgRemote { + image.UseDaemon(imgDaemon) + image.UseRemote(imgRemote) + err := image.UploadCachedImage(args[0]) + if err != nil { + exit.Error(reason.GuestImageSave, "Failed to save image", err) + } + } + } + }, +} + var removeImageCmd = &cobra.Command{ Use: "rm IMAGE [IMAGE...]", Short: "Remove one or more images", @@ -258,5 +329,8 @@ func init() { buildImageCmd.Flags().StringArrayVar(&buildEnv, "build-env", nil, "Environment variables to pass to the build. (format: key=value)") buildImageCmd.Flags().StringArrayVar(&buildOpt, "build-opt", nil, "Specify arbitrary flags to pass to the build. (format: key=value)") imageCmd.AddCommand(buildImageCmd) + saveImageCmd.Flags().BoolVar(&imgDaemon, "daemon", false, "Cache image to docker daemon") + saveImageCmd.Flags().BoolVar(&imgRemote, "remote", false, "Cache image to remote registry") + imageCmd.AddCommand(saveImageCmd) imageCmd.AddCommand(listImageCmd) } diff --git a/pkg/minikube/image/image.go b/pkg/minikube/image/image.go index 7814ce9abd..f10b4c16fd 100644 --- a/pkg/minikube/image/image.go +++ b/pkg/minikube/image/image.go @@ -33,10 +33,12 @@ import ( "github.com/google/go-containerregistry/pkg/v1/daemon" "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/tarball" "github.com/pkg/errors" "k8s.io/klog/v2" "k8s.io/minikube/pkg/minikube/constants" + "k8s.io/minikube/pkg/minikube/localpath" ) const ( @@ -191,6 +193,62 @@ func retrieveRemote(ref name.Reference, p v1.Platform) (v1.Image, error) { return img, err } +// imagePathInCache returns path in local cache directory +func imagePathInCache(img string) string { + f := filepath.Join(constants.ImageCacheDir, img) + f = localpath.SanitizeCacheDir(f) + return f +} + +func UploadCachedImage(imgName string) error { + tag, err := name.NewTag(imgName, name.WeakValidation) + if err != nil { + klog.Infof("error parsing image name %s tag %v ", imgName, err) + return err + } + return uploadImage(tag, imagePathInCache(imgName)) +} + +func uploadImage(tag name.Tag, p string) error { + var err error + var img v1.Image + + if !useDaemon && !useRemote { + return fmt.Errorf("neither daemon nor remote") + } + + img, err = tarball.ImageFromPath(p, &tag) + if err != nil { + return errors.Wrap(err, "tarball") + } + ref := name.Reference(tag) + + klog.Infof("uploading image: %+v from: %s", ref, p) + if useDaemon { + return uploadDaemon(ref, img) + } + if useRemote { + return uploadRemote(ref, img, defaultPlatform) + } + return nil +} + +func uploadDaemon(ref name.Reference, img v1.Image) error { + resp, err := daemon.Write(ref, img) + if err != nil { + klog.Warningf("daemon load for %s: %v\n%s", ref, err, resp) + } + return err +} + +func uploadRemote(ref name.Reference, img v1.Image, p v1.Platform) error { + err := remote.Write(ref, img, remote.WithAuthFromKeychain(authn.DefaultKeychain), remote.WithPlatform(p)) + if err != nil { + klog.Warningf("remote push for %s: %v", ref, err) + } + return err +} + // See https://github.com/kubernetes/minikube/issues/10402 // check if downloaded image Architecture field matches the requested and fix it otherwise func fixPlatform(ref name.Reference, img v1.Image, p v1.Platform) (v1.Image, error) { diff --git a/pkg/minikube/machine/cache_images.go b/pkg/minikube/machine/cache_images.go index ad97ddcf8d..f7ba9b8e3f 100644 --- a/pkg/minikube/machine/cache_images.go +++ b/pkg/minikube/machine/cache_images.go @@ -19,6 +19,7 @@ package machine import ( "fmt" "os" + "os/exec" "path" "path/filepath" "sort" @@ -48,6 +49,9 @@ var loadRoot = path.Join(vmpath.GuestPersistentDir, "images") // loadImageLock is used to serialize image loads to avoid overloading the guest VM var loadImageLock sync.Mutex +// saveRoot is where images should be saved from within the guest VM +var saveRoot = path.Join(vmpath.GuestPersistentDir, "images") + // CacheImagesForBootstrapper will cache images for a bootstrapper func CacheImagesForBootstrapper(imageRepository string, version string, clusterBootstrapper string) error { images, err := bootstrapper.GetCachedImageList(imageRepository, version, clusterBootstrapper) @@ -326,6 +330,173 @@ func removeExistingImage(r cruntime.Manager, src string, imgName string) error { return nil } +// SaveCachedImages saves from the container runtime to the cache +func SaveCachedImages(cc *config.ClusterConfig, runner command.Runner, images []string, cacheDir string) error { + klog.Infof("SaveImages start: %s", images) + start := time.Now() + + defer func() { + klog.Infof("SaveImages completed in %s", time.Since(start)) + }() + + var g errgroup.Group + + for _, image := range images { + image := image + g.Go(func() error { + return transferAndSaveCachedImage(runner, cc.KubernetesConfig, image, cacheDir) + }) + } + if err := g.Wait(); err != nil { + return errors.Wrap(err, "saving cached images") + } + klog.Infoln("Successfully saved all cached images") + return nil +} + +// SaveLocalImages saves images from the container runtime +func SaveLocalImages(cc *config.ClusterConfig, runner command.Runner, images []string, output string) error { + var g errgroup.Group + for _, image := range images { + image := image + g.Go(func() error { + return transferAndSaveImage(runner, cc.KubernetesConfig, output, image) + }) + } + if err := g.Wait(); err != nil { + return errors.Wrap(err, "saving images") + } + klog.Infoln("Successfully saved all images") + return nil +} + +// SaveAndCacheImages saves images from all profiles into the cache +func SaveAndCacheImages(images []string, profiles []*config.Profile) error { + if len(images) == 0 { + return nil + } + + return DoSaveImages(images, "", profiles, constants.ImageCacheDir) +} + +// DoSaveImages saves images from all profiles +func DoSaveImages(images []string, output string, profiles []*config.Profile, cacheDir string) error { + api, err := NewAPIClient() + if err != nil { + return errors.Wrap(err, "api") + } + defer api.Close() + + klog.Infof("Save images: %q", images) + + succeeded := []string{} + failed := []string{} + + for _, p := range profiles { // loading images to all running profiles + pName := p.Name // capture the loop variable + + c, err := config.Load(pName) + if err != nil { + // Non-fatal because it may race with profile deletion + klog.Errorf("Failed to load profile %q: %v", pName, err) + failed = append(failed, pName) + continue + } + + for _, n := range c.Nodes { + m := config.MachineName(*c, n) + + status, err := Status(api, m) + if err != nil { + klog.Warningf("error getting status for %s: %v", m, err) + failed = append(failed, m) + continue + } + + if status == state.Running.String() { // the not running hosts will load on next start + h, err := api.Load(m) + if err != nil { + klog.Warningf("Failed to load machine %q: %v", m, err) + failed = append(failed, m) + continue + } + cr, err := CommandRunner(h) + if err != nil { + return err + } + if cacheDir != "" { + // saving image names, to cache + err = SaveCachedImages(c, cr, images, cacheDir) + } else { + // saving mage files + err = SaveLocalImages(c, cr, images, output) + } + if err != nil { + failed = append(failed, m) + klog.Warningf("Failed to load cached images for profile %s. make sure the profile is running. %v", pName, err) + continue + } + succeeded = append(succeeded, m) + } + } + } + + klog.Infof("succeeded pulling from : %s", strings.Join(succeeded, " ")) + klog.Infof("failed pulling from : %s", strings.Join(failed, " ")) + // Live pushes are not considered a failure + return nil +} + +// transferAndSaveCachedImage transfers and loads a single image from the cache +func transferAndSaveCachedImage(cr command.Runner, k8s config.KubernetesConfig, imgName string, cacheDir string) error { + dst := filepath.Join(cacheDir, imgName) + dst = localpath.SanitizeCacheDir(dst) + return transferAndSaveImage(cr, k8s, dst, imgName) +} + +// transferAndSaveImage transfers and loads a single image +func transferAndSaveImage(cr command.Runner, k8s config.KubernetesConfig, dst string, imgName string) error { + r, err := cruntime.New(cruntime.Config{Type: k8s.ContainerRuntime, Runner: cr}) + if err != nil { + return errors.Wrap(err, "runtime") + } + + klog.Infof("Saving image to: %s", dst) + filename := filepath.Base(dst) + + _, err = os.OpenFile(dst, os.O_CREATE|os.O_WRONLY, 0777) + if err != nil { + return err + } + + f, err := assets.NewFileAsset(dst, saveRoot, filename, "0644") + if err != nil { + return errors.Wrapf(err, "creating copyable file asset: %s", filename) + } + defer func() { + if err := f.Close(); err != nil { + klog.Warningf("error closing the file %s: %v", f.GetSourcePath(), err) + } + }() + + src := path.Join(saveRoot, filename) + args := append([]string{"rm", "-f"}, src) + if _, err := cr.RunCmd(exec.Command("sudo", args...)); err != nil { + return err + } + err = r.SaveImage(imgName, src) + if err != nil { + return errors.Wrapf(err, "%s save %s", r.Name(), src) + } + + if err := cr.CopyFrom(f); err != nil { + return errors.Wrap(err, "transferring cached image") + } + + klog.Infof("Transferred and saved %s to cache", dst) + return nil +} + // pullImages pulls images to the container run time func pullImages(cruntime cruntime.Manager, images []string) error { klog.Infof("PullImages start: %s", images) diff --git a/pkg/minikube/reason/reason.go b/pkg/minikube/reason/reason.go index 9b044e45b8..dc01e2f3ae 100644 --- a/pkg/minikube/reason/reason.go +++ b/pkg/minikube/reason/reason.go @@ -317,6 +317,8 @@ var ( GuestImageRemove = Kind{ID: "GUEST_IMAGE_REMOVE", ExitCode: ExGuestError} // minikube failed to build an image GuestImageBuild = Kind{ID: "GUEST_IMAGE_BUILD", ExitCode: ExGuestError} + // minikube failed to push or save an image + GuestImageSave = Kind{ID: "GUEST_IMAGE_SAVE", ExitCode: ExGuestError} // minikube failed to load host GuestLoadHost = Kind{ID: "GUEST_LOAD_HOST", ExitCode: ExGuestError} // minkube failed to create a mount diff --git a/site/content/en/docs/commands/image.md b/site/content/en/docs/commands/image.md index 299e0c80ae..112794ff40 100644 --- a/site/content/en/docs/commands/image.md +++ b/site/content/en/docs/commands/image.md @@ -264,3 +264,51 @@ $ minikube image unload image busybox --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging ``` +## minikube image save + +Save a image from minikube + +### Synopsis + +Save a image from minikube + +```shell +minikube image save IMAGE [ARCHIVE | -] [flags] +``` + +### Examples + +``` +minikube image save image +minikube image save image image.tar +``` + +### Options + +``` + --daemon Cache image to docker daemon + --remote Cache image to remote registry +``` + +### Options inherited from parent commands + +``` + --add_dir_header If true, adds the file directory to the header of the log messages + --alsologtostderr log to standard error as well as files + -b, --bootstrapper string The name of the cluster bootstrapper that will set up the Kubernetes cluster. (default "kubeadm") + -h, --help + --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) + --log_dir string If non-empty, write log files in this directory + --log_file string If non-empty, use this log file + --log_file_max_size uint Defines the maximum size a log file can grow to. Unit is megabytes. If the value is 0, the maximum file size is unlimited. (default 1800) + --logtostderr log to standard error instead of files + --one_output If true, only write logs to their native severity level (vs also writing to each lower severity level) + -p, --profile string The name of the minikube VM being used. This can be set to allow having multiple instances of minikube independently. (default "minikube") + --skip_headers If true, avoid header prefixes in the log messages + --skip_log_headers If true, avoid headers when opening log files + --stderrthreshold severity logs at or above this threshold go to stderr (default 2) + --user string Specifies the user executing the operation. Useful for auditing operations executed by 3rd party tools. Defaults to the operating system username. + -v, --v Level number for the log level verbosity + --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging +``` + diff --git a/site/content/en/docs/contrib/errorcodes.en.md b/site/content/en/docs/contrib/errorcodes.en.md index d2589bf1e5..c3621b7ed4 100644 --- a/site/content/en/docs/contrib/errorcodes.en.md +++ b/site/content/en/docs/contrib/errorcodes.en.md @@ -381,6 +381,9 @@ minikube failed to remove an image "GUEST_IMAGE_BUILD" (Exit code ExGuestError) minikube failed to build an image +"GUEST_IMAGE_SAVE" (Exit code ExGuestError) +minikube failed to push or save an image + "GUEST_LOAD_HOST" (Exit code ExGuestError) minikube failed to load host From fa8ee4957e1969e04acd6023e087b65bc8f26b93 Mon Sep 17 00:00:00 2001 From: Predrag Rogic Date: Fri, 13 Aug 2021 23:28:48 +0100 Subject: [PATCH 3/9] fix original pr 12162 --- pkg/minikube/image/image.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/minikube/image/image.go b/pkg/minikube/image/image.go index f10b4c16fd..2bbcf919cb 100644 --- a/pkg/minikube/image/image.go +++ b/pkg/minikube/image/image.go @@ -225,7 +225,7 @@ func uploadImage(tag name.Tag, p string) error { klog.Infof("uploading image: %+v from: %s", ref, p) if useDaemon { - return uploadDaemon(ref, img) + return uploadDaemon(tag, img) } if useRemote { return uploadRemote(ref, img, defaultPlatform) @@ -233,10 +233,10 @@ func uploadImage(tag name.Tag, p string) error { return nil } -func uploadDaemon(ref name.Reference, img v1.Image) error { - resp, err := daemon.Write(ref, img) +func uploadDaemon(tag name.Tag, img v1.Image) error { + resp, err := daemon.Write(tag, img) if err != nil { - klog.Warningf("daemon load for %s: %v\n%s", ref, err, resp) + klog.Warningf("daemon load for %s: %v\n%s", tag, err, resp) } return err } From 9e588e6ba14f2ab68ab7284348fe507af6c43f1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sun, 15 Aug 2021 20:20:07 +0200 Subject: [PATCH 4/9] Add stand-alone image pull and image tag commands Needed for testing, but maybe elsewhere as well --- cmd/minikube/cmd/image.go | 42 +++++++++ pkg/minikube/cruntime/containerd.go | 10 +++ pkg/minikube/cruntime/crio.go | 10 +++ pkg/minikube/cruntime/cruntime.go | 2 + pkg/minikube/cruntime/docker.go | 10 +++ pkg/minikube/machine/cache_images.go | 57 ++++++++++++ pkg/minikube/reason/reason.go | 4 + site/content/en/docs/commands/image.md | 88 +++++++++++++++++++ site/content/en/docs/contrib/errorcodes.en.md | 6 ++ translations/strings.txt | 5 ++ 10 files changed, 234 insertions(+) diff --git a/cmd/minikube/cmd/image.go b/cmd/minikube/cmd/image.go index 8fdc870e57..c4ef1aab08 100644 --- a/cmd/minikube/cmd/image.go +++ b/cmd/minikube/cmd/image.go @@ -236,6 +236,24 @@ $ minikube image unload image busybox }, } +var pullImageCmd = &cobra.Command{ + Use: "pull", + Short: "Pull images", + Example: ` +$ minikube image pull busybox +`, + Run: func(cmd *cobra.Command, args []string) { + profile, err := config.LoadProfile(viper.GetString(config.ProfileName)) + if err != nil { + exit.Error(reason.Usage, "loading profile", err) + } + + if err := machine.PullImages(args, profile); err != nil { + exit.Error(reason.GuestImagePull, "Failed to pull images", err) + } + }, +} + func createTar(dir string) (string, error) { tar, err := docker.CreateTarStream(dir, dockerFile) if err != nil { @@ -316,6 +334,28 @@ $ minikube image ls }, } +var tagImageCmd = &cobra.Command{ + Use: "tag", + Short: "Tag images", + Example: ` +$ minikube image tag source target +`, + Aliases: []string{"list"}, + Run: func(cmd *cobra.Command, args []string) { + if len(args) != 2 { + exit.Message(reason.Usage, "Please provide source and target image") + } + profile, err := config.LoadProfile(viper.GetString(config.ProfileName)) + if err != nil { + exit.Error(reason.Usage, "loading profile", err) + } + + if err := machine.TagImage(profile, args[0], args[1]); err != nil { + exit.Error(reason.GuestImageTag, "Failed to tag images", err) + } + }, +} + func init() { loadImageCmd.Flags().BoolVarP(&pull, "pull", "", false, "Pull the remote image (no caching)") loadImageCmd.Flags().BoolVar(&imgDaemon, "daemon", false, "Cache image from docker daemon") @@ -323,6 +363,7 @@ func init() { loadImageCmd.Flags().BoolVar(&overwrite, "overwrite", true, "Overwrite image even if same image:tag name exists") imageCmd.AddCommand(loadImageCmd) imageCmd.AddCommand(removeImageCmd) + imageCmd.AddCommand(pullImageCmd) buildImageCmd.Flags().StringVarP(&tag, "tag", "t", "", "Tag to apply to the new image (optional)") buildImageCmd.Flags().BoolVarP(&push, "push", "", false, "Push the new image (requires tag)") buildImageCmd.Flags().StringVarP(&dockerFile, "file", "f", "", "Path to the Dockerfile to use (optional)") @@ -333,4 +374,5 @@ func init() { saveImageCmd.Flags().BoolVar(&imgRemote, "remote", false, "Cache image to remote registry") imageCmd.AddCommand(saveImageCmd) imageCmd.AddCommand(listImageCmd) + imageCmd.AddCommand(tagImageCmd) } diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go index edc22533f1..13b0834e9b 100644 --- a/pkg/minikube/cruntime/containerd.go +++ b/pkg/minikube/cruntime/containerd.go @@ -305,6 +305,16 @@ func (r *Containerd) RemoveImage(name string) error { return removeCRIImage(r.Runner, name) } +// TagImage tags an image in this runtime +func (r *Containerd) TagImage(source string, target string) error { + klog.Infof("Tagging image %s: %s", source, target) + c := exec.Command("sudo", "ctr", "-n=k8s.io", "images", "tag", source, target) + if _, err := r.Runner.RunCmd(c); err != nil { + return errors.Wrapf(err, "ctr images tag") + } + return nil +} + func gitClone(cr CommandRunner, src string) (string, error) { // clone to a temporary directory rr, err := cr.RunCmd(exec.Command("mktemp", "-d")) diff --git a/pkg/minikube/cruntime/crio.go b/pkg/minikube/cruntime/crio.go index 82d30647f6..ddb61ceb43 100644 --- a/pkg/minikube/cruntime/crio.go +++ b/pkg/minikube/cruntime/crio.go @@ -216,6 +216,16 @@ func (r *CRIO) RemoveImage(name string) error { return removeCRIImage(r.Runner, name) } +// TagImage tags an image in this runtime +func (r *CRIO) TagImage(source string, target string) error { + klog.Infof("Tagging image %s: %s", source, target) + c := exec.Command("sudo", "podman", "tag", source, target) + if _, err := r.Runner.RunCmd(c); err != nil { + return errors.Wrap(err, "crio tag image") + } + return nil +} + // BuildImage builds an image into this runtime func (r *CRIO) BuildImage(src string, file string, tag string, push bool, env []string, opts []string) error { klog.Infof("Building image: %s", src) diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go index e2df9d83a9..2388abc8ac 100644 --- a/pkg/minikube/cruntime/cruntime.go +++ b/pkg/minikube/cruntime/cruntime.go @@ -103,6 +103,8 @@ type Manager interface { BuildImage(string, string, string, bool, []string, []string) error // Save an image from the runtime on a host SaveImage(string, string) error + // Tag an image + TagImage(string, string) error // ImageExists takes image name and image sha checks if an it exists ImageExists(string, string) bool diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go index c25c6cc930..641775b78b 100644 --- a/pkg/minikube/cruntime/docker.go +++ b/pkg/minikube/cruntime/docker.go @@ -244,6 +244,16 @@ func (r *Docker) RemoveImage(name string) error { return nil } +// TagImage tags an image in this runtime +func (r *Docker) TagImage(source string, target string) error { + klog.Infof("Tagging image %s: %s", source, target) + c := exec.Command("docker", "tag", source, target) + if _, err := r.Runner.RunCmd(c); err != nil { + return errors.Wrap(err, "tag image docker.") + } + return nil +} + // BuildImage builds an image into this runtime func (r *Docker) BuildImage(src string, file string, tag string, push bool, env []string, opts []string) error { klog.Infof("Building image: %s", src) diff --git a/pkg/minikube/machine/cache_images.go b/pkg/minikube/machine/cache_images.go index f7ba9b8e3f..de3edefc4e 100644 --- a/pkg/minikube/machine/cache_images.go +++ b/pkg/minikube/machine/cache_images.go @@ -710,3 +710,60 @@ func ListImages(profile *config.Profile) error { return nil } + +// TagImage tags image in all nodes in profile +func TagImage(profile *config.Profile, source string, target string) error { + api, err := NewAPIClient() + if err != nil { + return errors.Wrap(err, "error creating api client") + } + defer api.Close() + + succeeded := []string{} + failed := []string{} + + pName := profile.Name + + c, err := config.Load(pName) + if err != nil { + klog.Errorf("Failed to load profile %q: %v", pName, err) + return errors.Wrapf(err, "error loading config for profile :%v", pName) + } + + for _, n := range c.Nodes { + m := config.MachineName(*c, n) + + status, err := Status(api, m) + if err != nil { + klog.Warningf("error getting status for %s: %v", m, err) + continue + } + + if status == state.Running.String() { + h, err := api.Load(m) + if err != nil { + klog.Warningf("Failed to load machine %q: %v", m, err) + continue + } + runner, err := CommandRunner(h) + if err != nil { + return err + } + cruntime, err := cruntime.New(cruntime.Config{Type: c.KubernetesConfig.ContainerRuntime, Runner: runner}) + if err != nil { + return errors.Wrap(err, "error creating container runtime") + } + err = cruntime.TagImage(source, target) + if err != nil { + failed = append(failed, m) + klog.Warningf("Failed to tag image for profile %s %v", pName, err.Error()) + continue + } + succeeded = append(succeeded, m) + } + } + + klog.Infof("succeeded tagging in: %s", strings.Join(succeeded, " ")) + klog.Infof("failed tagging in: %s", strings.Join(failed, " ")) + return nil +} diff --git a/pkg/minikube/reason/reason.go b/pkg/minikube/reason/reason.go index dc01e2f3ae..824cf98612 100644 --- a/pkg/minikube/reason/reason.go +++ b/pkg/minikube/reason/reason.go @@ -315,10 +315,14 @@ var ( GuestImageLoad = Kind{ID: "GUEST_IMAGE_LOAD", ExitCode: ExGuestError} // minikube failed to remove an image GuestImageRemove = Kind{ID: "GUEST_IMAGE_REMOVE", ExitCode: ExGuestError} + // minikube failed to pull an image + GuestImagePull = Kind{ID: "GUEST_IMAGE_PULL", ExitCode: ExGuestError} // minikube failed to build an image GuestImageBuild = Kind{ID: "GUEST_IMAGE_BUILD", ExitCode: ExGuestError} // minikube failed to push or save an image GuestImageSave = Kind{ID: "GUEST_IMAGE_SAVE", ExitCode: ExGuestError} + // minikube failed to tag an image + GuestImageTag = Kind{ID: "GUEST_IMAGE_TAG", ExitCode: ExGuestError} // minikube failed to load host GuestLoadHost = Kind{ID: "GUEST_LOAD_HOST", ExitCode: ExGuestError} // minkube failed to create a mount diff --git a/site/content/en/docs/commands/image.md b/site/content/en/docs/commands/image.md index 112794ff40..9e9478b1b4 100644 --- a/site/content/en/docs/commands/image.md +++ b/site/content/en/docs/commands/image.md @@ -216,6 +216,48 @@ $ minikube image ls --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging ``` +## minikube image pull + +Pull images + +### Synopsis + +Pull images + +```shell +minikube image pull [flags] +``` + +### Examples + +``` + +$ minikube image pull busybox + +``` + +### Options inherited from parent commands + +``` + --add_dir_header If true, adds the file directory to the header of the log messages + --alsologtostderr log to standard error as well as files + -b, --bootstrapper string The name of the cluster bootstrapper that will set up the Kubernetes cluster. (default "kubeadm") + -h, --help + --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) + --log_dir string If non-empty, write log files in this directory + --log_file string If non-empty, use this log file + --log_file_max_size uint Defines the maximum size a log file can grow to. Unit is megabytes. If the value is 0, the maximum file size is unlimited. (default 1800) + --logtostderr log to standard error instead of files + --one_output If true, only write logs to their native severity level (vs also writing to each lower severity level) + -p, --profile string The name of the minikube VM being used. This can be set to allow having multiple instances of minikube independently. (default "minikube") + --skip_headers If true, avoid header prefixes in the log messages + --skip_log_headers If true, avoid headers when opening log files + --stderrthreshold severity logs at or above this threshold go to stderr (default 2) + --user string Specifies the user executing the operation. Useful for auditing operations executed by 3rd party tools. Defaults to the operating system username. + -v, --v Level number for the log level verbosity + --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging +``` + ## minikube image rm Remove one or more images @@ -312,3 +354,49 @@ minikube image save image image.tar --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging ``` +## minikube image tag + +Tag images + +### Synopsis + +Tag images + +```shell +minikube image tag [flags] +``` + +### Aliases + +[list] + +### Examples + +``` + +$ minikube image tag source target + +``` + +### Options inherited from parent commands + +``` + --add_dir_header If true, adds the file directory to the header of the log messages + --alsologtostderr log to standard error as well as files + -b, --bootstrapper string The name of the cluster bootstrapper that will set up the Kubernetes cluster. (default "kubeadm") + -h, --help + --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) + --log_dir string If non-empty, write log files in this directory + --log_file string If non-empty, use this log file + --log_file_max_size uint Defines the maximum size a log file can grow to. Unit is megabytes. If the value is 0, the maximum file size is unlimited. (default 1800) + --logtostderr log to standard error instead of files + --one_output If true, only write logs to their native severity level (vs also writing to each lower severity level) + -p, --profile string The name of the minikube VM being used. This can be set to allow having multiple instances of minikube independently. (default "minikube") + --skip_headers If true, avoid header prefixes in the log messages + --skip_log_headers If true, avoid headers when opening log files + --stderrthreshold severity logs at or above this threshold go to stderr (default 2) + --user string Specifies the user executing the operation. Useful for auditing operations executed by 3rd party tools. Defaults to the operating system username. + -v, --v Level number for the log level verbosity + --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging +``` + diff --git a/site/content/en/docs/contrib/errorcodes.en.md b/site/content/en/docs/contrib/errorcodes.en.md index c3621b7ed4..6708070693 100644 --- a/site/content/en/docs/contrib/errorcodes.en.md +++ b/site/content/en/docs/contrib/errorcodes.en.md @@ -378,12 +378,18 @@ minikube failed to pull or load an image "GUEST_IMAGE_REMOVE" (Exit code ExGuestError) minikube failed to remove an image +"GUEST_IMAGE_PULL" (Exit code ExGuestError) +minikube failed to pull an image + "GUEST_IMAGE_BUILD" (Exit code ExGuestError) minikube failed to build an image "GUEST_IMAGE_SAVE" (Exit code ExGuestError) minikube failed to push or save an image +"GUEST_IMAGE_TAG" (Exit code ExGuestError) +minikube failed to tag an image + "GUEST_LOAD_HOST" (Exit code ExGuestError) minikube failed to load host diff --git a/translations/strings.txt b/translations/strings.txt index 9dfd494451..a88b3d4354 100644 --- a/translations/strings.txt +++ b/translations/strings.txt @@ -233,6 +233,7 @@ "Failed to load image": "", "Failed to persist images": "", "Failed to pull image": "", + "Failed to pull images": "", "Failed to reload cached images": "", "Failed to remove image": "", "Failed to save config {{.profile}}": "", @@ -243,6 +244,7 @@ "Failed to start container runtime": "", "Failed to start {{.driver}} {{.driver_type}}. Running \"{{.cmd}}\" may fix it: {{.error}}": "", "Failed to stop node {{.name}}": "", + "Failed to tag images": "", "Failed to update cluster": "", "Failed to update config": "", "Failed unmount: {{.error}}": "", @@ -406,6 +408,7 @@ "Please make sure the service you are looking for is deployed or is in the correct namespace.": "", "Please provide a path or url to build": "", "Please provide an image in your local daemon to load into minikube via \u003cminikube image load IMAGE_NAME\u003e": "", + "Please provide source and target image": "", "Please re-eval your docker-env, To ensure your environment variables have updated ports:\n\n\t'minikube -p {{.profile_name}} docker-env'\n\n\t": "", "Please re-eval your podman-env, To ensure your environment variables have updated ports:\n\n\t'minikube -p {{.profile_name}} podman-env'\n\n\t": "", "Please see {{.documentation_url}} for more details": "", @@ -430,6 +433,7 @@ "Profile name '{{.profilename}}' is not valid": "", "Profile name should be unique": "", "Provide VM UUID to restore MAC address (hyperkit driver only)": "", + "Pull images": "", "Pull the remote image (no caching)": "", "Pulling base image ...": "", "Push the new image (requires tag)": "", @@ -546,6 +550,7 @@ "Successfully stopped node {{.name}}": "", "Suggestion: {{.advice}}": "", "System only has {{.size}}MiB available, less than the required {{.req}}MiB for Kubernetes": "", + "Tag images": "", "Tag to apply to the new image (optional)": "", "Target directory {{.path}} must be an absolute path": "", "Target {{.path}} can not be empty": "", From ab61a07f9597dd5929e9edb818c41722d9e4c48f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sun, 15 Aug 2021 20:25:37 +0200 Subject: [PATCH 5/9] Make sure to use sudo for load/save docker images Add a test for SaveImage, similar to LoadImage --- pkg/minikube/cruntime/docker.go | 4 +- site/content/en/docs/contrib/tests.en.md | 6 ++ test/integration/functional_test.go | 99 +++++++++++++++++++++++- 3 files changed, 106 insertions(+), 3 deletions(-) diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go index 641775b78b..d805201da0 100644 --- a/pkg/minikube/cruntime/docker.go +++ b/pkg/minikube/cruntime/docker.go @@ -201,7 +201,7 @@ func (r *Docker) ListImages(ListImagesOptions) ([]string, error) { // LoadImage loads an image into this runtime func (r *Docker) LoadImage(path string) error { klog.Infof("Loading image: %s", path) - c := exec.Command("docker", "load", "-i", path) + c := exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo cat %s | docker load", path)) if _, err := r.Runner.RunCmd(c); err != nil { return errors.Wrap(err, "loadimage docker.") } @@ -224,7 +224,7 @@ func (r *Docker) PullImage(name string) error { // SaveImage saves an image from this runtime func (r *Docker) SaveImage(name string, path string) error { klog.Infof("Saving image %s: %s", name, path) - c := exec.Command("docker", "save", name, "-o", path) + c := exec.Command("/bin/bash", "-c", fmt.Sprintf("docker save '%s' | sudo tee %s >/dev/null", name, path)) if _, err := r.Runner.RunCmd(c); err != nil { return errors.Wrap(err, "saveimage docker.") } diff --git a/site/content/en/docs/contrib/tests.en.md b/site/content/en/docs/contrib/tests.en.md index f3c3efe625..119eef71ed 100644 --- a/site/content/en/docs/contrib/tests.en.md +++ b/site/content/en/docs/contrib/tests.en.md @@ -84,6 +84,12 @@ makes sure that `minikube image load` works from a local file #### validateRemoveImage makes sures that `minikube image rm` works as expected +#### validateSaveImage +makes sure that `minikube image save` works as expected + +#### validateSaveImageToFile +makes sure that `minikube image save` works to a local file + #### validateBuildImage makes sures that `minikube image build` works as expected diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go index 10bee2cbbf..cd784cba61 100644 --- a/test/integration/functional_test.go +++ b/test/integration/functional_test.go @@ -151,8 +151,10 @@ func TestFunctional(t *testing.T) { {"PodmanEnv", validatePodmanEnv}, {"NodeLabels", validateNodeLabels}, {"LoadImage", validateLoadImage}, + {"SaveImage", validateSaveImage}, {"RemoveImage", validateRemoveImage}, {"LoadImageFromFile", validateLoadImageFromFile}, + {"SaveImageToFile", validateSaveImageToFile}, {"BuildImage", validateBuildImage}, {"ListImages", validateListImages}, {"NonActiveRuntimeDisabled", validateNotActiveRuntimeDisabled}, @@ -249,7 +251,7 @@ func validateLoadImage(ctx context.Context, t *testing.T, profile string) { } // try to load the new image into minikube - rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "image", "load", newImage)) + rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "image", "load", "--daemon", newImage)) if err != nil { t.Fatalf("loading image into minikube: %v\n%s", err, rr.Output()) } @@ -363,6 +365,101 @@ func validateRemoveImage(ctx context.Context, t *testing.T, profile string) { } +// validateSaveImage makes sure that `minikube image save` works as expected +func validateSaveImage(ctx context.Context, t *testing.T, profile string) { + if NoneDriver() { + t.Skip("load image not available on none driver") + } + if GithubActionRunner() && runtime.GOOS == "darwin" { + t.Skip("skipping on github actions and darwin, as this test requires a running docker daemon") + } + defer PostMortemLogs(t, profile) + // pull busybox + busyboxImage := "docker.io/library/busybox:1.29" + rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "image", "pull", busyboxImage)) + if err != nil { + t.Fatalf("failed to setup test (pull image): %v\n%s", err, rr.Output()) + } + + // tag busybox + name := "busybox" + tag := fmt.Sprintf("save-%s", profile) + newImage := fmt.Sprintf("docker.io/library/%s:%s", name, tag) + rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "image", "tag", busyboxImage, newImage)) + if err != nil { + t.Fatalf("failed to setup test (tag image) : %v\n%s", err, rr.Output()) + } + + // try to save the new image from minikube + rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "image", "save", "--daemon", newImage)) + if err != nil { + t.Fatalf("loading image into minikube: %v\n%s", err, rr.Output()) + } + + // make sure the image was correctly loaded + rr, err = Run(t, exec.CommandContext(ctx, "docker", "images", name)) + if err != nil { + t.Fatalf("listing images: %v\n%s", err, rr.Output()) + } + if !strings.Contains(rr.Output(), fmt.Sprintf("save-%s", profile)) { + t.Fatalf("expected %s to be loaded into minikube but the image is not there", newImage) + } + +} + +// validateSaveImageToFile makes sure that `minikube image save` works to a local file +func validateSaveImageToFile(ctx context.Context, t *testing.T, profile string) { + if NoneDriver() { + t.Skip("save image not available on none driver") + } + if GithubActionRunner() && runtime.GOOS == "darwin" { + t.Skip("skipping on github actions and darwin, as this test requires a running docker daemon") + } + defer PostMortemLogs(t, profile) + // pull busybox + busyboxImage := "docker.io/library/busybox:1.30" + rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "image", "pull", busyboxImage)) + if err != nil { + t.Fatalf("failed to setup test (pull image): %v\n%s", err, rr.Output()) + } + + name := "busybox" + tag := fmt.Sprintf("save-to-file-%s", profile) + taggedImage := fmt.Sprintf("docker.io/library/%s:%s", name, tag) + rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "image", "tag", busyboxImage, taggedImage)) + if err != nil { + t.Fatalf("failed to setup test (tag image) : %v\n%s", err, rr.Output()) + } + + // try to save the new image from minikube + imageFile := "busybox.tar" + imagePath, err := filepath.Abs(imageFile) + if err != nil { + t.Fatalf("failed to get absolute path of file %q: %v", imageFile, err) + } + rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "image", "save", taggedImage, imagePath)) + if err != nil { + t.Fatalf("saving image from minikube: %v\n%s", err, rr.Output()) + } + + // load image from file + rr, err = Run(t, exec.CommandContext(ctx, "docker", "load", "-i", imagePath)) + if err != nil { + t.Fatalf("failed to load image to file: %v\n%s", err, rr.Output()) + } + defer os.Remove(imageFile) + + // make sure the image was correctly loaded + rr, err = Run(t, exec.CommandContext(ctx, "docker", "images", name)) + if err != nil { + t.Fatalf("listing images: %v\n%s", err, rr.Output()) + } + if !strings.Contains(rr.Output(), tag) { + t.Fatalf("expected %s to be loaded but the image is not there", taggedImage) + } + +} + func inspectImage(ctx context.Context, t *testing.T, profile string, image string) (*RunResult, error) { var cmd *exec.Cmd if ContainerRuntime() == "docker" { From 11d31dda140cb57acc94f5a11855f19187b8e440 Mon Sep 17 00:00:00 2001 From: Predrag Rogic Date: Mon, 16 Aug 2021 23:12:38 +0100 Subject: [PATCH 6/9] early check if image exists at all --- pkg/minikube/cruntime/containerd.go | 10 +++++++--- pkg/minikube/cruntime/crio.go | 4 ++-- pkg/minikube/cruntime/cruntime.go | 2 +- pkg/minikube/cruntime/docker.go | 4 ++-- pkg/minikube/machine/cache_images.go | 4 ++++ 5 files changed, 16 insertions(+), 8 deletions(-) diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go index 13b0834e9b..cb61f3b980 100644 --- a/pkg/minikube/cruntime/containerd.go +++ b/pkg/minikube/cruntime/containerd.go @@ -248,10 +248,14 @@ func (r *Containerd) Disable() error { return r.Init.ForceStop("containerd") } -// ImageExists checks if an image exists, expected input format +// ImageExists checks if image exists based on image name and optionally image sha func (r *Containerd) ImageExists(name string, sha string) bool { - c := exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo ctr -n=k8s.io images check | grep %s | grep %s", name, sha)) - if _, err := r.Runner.RunCmd(c); err != nil { + c := exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo ctr -n=k8s.io images check | grep %s", name)) + rr, err := r.Runner.RunCmd(c) + if err != nil { + return false + } + if sha != "" && !strings.Contains(rr.Output(), sha) { return false } return true diff --git a/pkg/minikube/cruntime/crio.go b/pkg/minikube/cruntime/crio.go index ddb61ceb43..820ade8e16 100644 --- a/pkg/minikube/cruntime/crio.go +++ b/pkg/minikube/cruntime/crio.go @@ -162,7 +162,7 @@ func (r *CRIO) Disable() error { return r.Init.ForceStop("crio") } -// ImageExists checks if an image exists +// ImageExists checks if image exists based on image name and optionally image sha func (r *CRIO) ImageExists(name string, sha string) bool { // expected output looks like [NAME@sha256:SHA] c := exec.Command("sudo", "podman", "image", "inspect", "--format", "{{.Id}}", name) @@ -170,7 +170,7 @@ func (r *CRIO) ImageExists(name string, sha string) bool { if err != nil { return false } - if !strings.Contains(rr.Output(), sha) { + if sha != "" && !strings.Contains(rr.Output(), sha) { return false } return true diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go index 2388abc8ac..1ad073a7aa 100644 --- a/pkg/minikube/cruntime/cruntime.go +++ b/pkg/minikube/cruntime/cruntime.go @@ -106,7 +106,7 @@ type Manager interface { // Tag an image TagImage(string, string) error - // ImageExists takes image name and image sha checks if an it exists + // ImageExists takes image name and optionally image sha to check if an image exists ImageExists(string, string) bool // ListImages returns a list of images managed by this container runtime ListImages(ListImagesOptions) ([]string, error) diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go index d805201da0..aace65297a 100644 --- a/pkg/minikube/cruntime/docker.go +++ b/pkg/minikube/cruntime/docker.go @@ -165,7 +165,7 @@ func (r *Docker) Disable() error { return r.Init.Mask("docker.service") } -// ImageExists checks if an image exists +// ImageExists checks if image exists based on image name and optionally image sha func (r *Docker) ImageExists(name string, sha string) bool { // expected output looks like [SHA_ALGO:SHA] c := exec.Command("docker", "image", "inspect", "--format", "{{.Id}}", name) @@ -173,7 +173,7 @@ func (r *Docker) ImageExists(name string, sha string) bool { if err != nil { return false } - if !strings.Contains(rr.Output(), sha) { + if sha != "" && !strings.Contains(rr.Output(), sha) { return false } return true diff --git a/pkg/minikube/machine/cache_images.go b/pkg/minikube/machine/cache_images.go index de3edefc4e..ce91789776 100644 --- a/pkg/minikube/machine/cache_images.go +++ b/pkg/minikube/machine/cache_images.go @@ -461,6 +461,10 @@ func transferAndSaveImage(cr command.Runner, k8s config.KubernetesConfig, dst st return errors.Wrap(err, "runtime") } + if !r.ImageExists(imgName, "") { + return errors.Errorf("image %s not found", imgName) + } + klog.Infof("Saving image to: %s", dst) filename := filepath.Base(dst) From c4bdef14ce0d21b48c1cfe9eeee9766f4d46d094 Mon Sep 17 00:00:00 2001 From: Predrag Rogic Date: Tue, 24 Aug 2021 20:02:19 +0100 Subject: [PATCH 7/9] fix TestFunctional/parallel/LoadImageFromFile --- test/integration/functional_test.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go index cd784cba61..a31120ab11 100644 --- a/test/integration/functional_test.go +++ b/test/integration/functional_test.go @@ -299,11 +299,7 @@ func validateLoadImageFromFile(ctx context.Context, t *testing.T, profile string defer os.Remove(imageFile) // try to load the new image into minikube - imagePath, err := filepath.Abs(imageFile) - if err != nil { - t.Fatalf("failed to get absolute path of file %q: %v", imageFile, err) - } - rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "image", "load", imagePath)) + rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "image", "load", imageFile)) if err != nil { t.Fatalf("loading image into minikube: %v\n%s", err, rr.Output()) } From c39712803af6fb666050d592cfcbe748dfa2175f Mon Sep 17 00:00:00 2001 From: Predrag Rogic Date: Tue, 24 Aug 2021 22:06:07 +0100 Subject: [PATCH 8/9] fix TestFunctional/parallel/LoadImageFromFile --- test/integration/functional_test.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go index a31120ab11..619765385f 100644 --- a/test/integration/functional_test.go +++ b/test/integration/functional_test.go @@ -65,6 +65,8 @@ var mitm *StartSession var runCorpProxy = GithubActionRunner() && runtime.GOOS == "linux" && !arm64Platform() +var imageFile = "busybox.tar" + // TestFunctional are functionality tests which can safely share a profile in parallel func TestFunctional(t *testing.T) { @@ -208,7 +210,7 @@ func cleanupUnwantedImages(ctx context.Context, t *testing.T, profile string) { } }) } - + os.Remove(imageFile) } // validateNodeLabels checks if minikube cluster is created with correct kubernetes's node label @@ -291,16 +293,18 @@ func validateLoadImageFromFile(ctx context.Context, t *testing.T, profile string } // save image to file - imageFile := "busybox.tar" rr, err = Run(t, exec.CommandContext(ctx, "docker", "save", "-o", imageFile, taggedImage)) if err != nil { t.Fatalf("failed to save image to file: %v\n%s", err, rr.Output()) } - defer os.Remove(imageFile) // try to load the new image into minikube - rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "image", "load", imageFile)) + imagePath, err := filepath.Abs(imageFile) if err != nil { + t.Fatalf("failed to get absolute path of file %q: %v", imageFile, err) + } + rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "image", "load", imagePath)) + if err != nil || rr.Stderr.String() != "" { t.Fatalf("loading image into minikube: %v\n%s", err, rr.Output()) } @@ -428,7 +432,6 @@ func validateSaveImageToFile(ctx context.Context, t *testing.T, profile string) } // try to save the new image from minikube - imageFile := "busybox.tar" imagePath, err := filepath.Abs(imageFile) if err != nil { t.Fatalf("failed to get absolute path of file %q: %v", imageFile, err) @@ -443,7 +446,6 @@ func validateSaveImageToFile(ctx context.Context, t *testing.T, profile string) if err != nil { t.Fatalf("failed to load image to file: %v\n%s", err, rr.Output()) } - defer os.Remove(imageFile) // make sure the image was correctly loaded rr, err = Run(t, exec.CommandContext(ctx, "docker", "images", name)) From f1ba75f436e477a541dbc2d1381ca40cdbf7fdee Mon Sep 17 00:00:00 2001 From: Predrag Rogic Date: Tue, 24 Aug 2021 23:18:54 +0100 Subject: [PATCH 9/9] fix TestFunctional/parallel/LoadImageFromFile - 2nd race condition --- test/integration/functional_test.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go index 619765385f..c758fb8fca 100644 --- a/test/integration/functional_test.go +++ b/test/integration/functional_test.go @@ -65,8 +65,6 @@ var mitm *StartSession var runCorpProxy = GithubActionRunner() && runtime.GOOS == "linux" && !arm64Platform() -var imageFile = "busybox.tar" - // TestFunctional are functionality tests which can safely share a profile in parallel func TestFunctional(t *testing.T) { @@ -210,7 +208,6 @@ func cleanupUnwantedImages(ctx context.Context, t *testing.T, profile string) { } }) } - os.Remove(imageFile) } // validateNodeLabels checks if minikube cluster is created with correct kubernetes's node label @@ -293,10 +290,12 @@ func validateLoadImageFromFile(ctx context.Context, t *testing.T, profile string } // save image to file + imageFile := "busybox-load.tar" rr, err = Run(t, exec.CommandContext(ctx, "docker", "save", "-o", imageFile, taggedImage)) if err != nil { t.Fatalf("failed to save image to file: %v\n%s", err, rr.Output()) } + defer os.Remove(imageFile) // try to load the new image into minikube imagePath, err := filepath.Abs(imageFile) @@ -314,7 +313,7 @@ func validateLoadImageFromFile(ctx context.Context, t *testing.T, profile string t.Fatalf("listing images: %v\n%s", err, rr.Output()) } if !strings.Contains(rr.Output(), tag) { - t.Fatalf("expected %s to be loaded into minikube but the image is not there", taggedImage) + t.Fatalf("expected %s to be loaded into minikube but the image is not there: %v", taggedImage, rr.Output()) } } @@ -432,6 +431,7 @@ func validateSaveImageToFile(ctx context.Context, t *testing.T, profile string) } // try to save the new image from minikube + imageFile := "busybox-save.tar" imagePath, err := filepath.Abs(imageFile) if err != nil { t.Fatalf("failed to get absolute path of file %q: %v", imageFile, err) @@ -446,6 +446,7 @@ func validateSaveImageToFile(ctx context.Context, t *testing.T, profile string) if err != nil { t.Fatalf("failed to load image to file: %v\n%s", err, rr.Output()) } + defer os.Remove(imageFile) // make sure the image was correctly loaded rr, err = Run(t, exec.CommandContext(ctx, "docker", "images", name))