//go:build integration /* Copyright 2019 The Kubernetes Authors All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package integration import ( "bufio" "bytes" "context" "crypto/md5" "crypto/sha256" "encoding/json" "fmt" "net/http" "net/http/httptest" "os" "os/exec" "path/filepath" "runtime" "strings" "testing" "k8s.io/minikube/pkg/minikube/bootstrapper/images" "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/download" "k8s.io/minikube/pkg/minikube/localpath" ) // TestDownloadOnly makes sure the --download-only parameter in minikube start caches the appropriate images and tarballs. func TestDownloadOnly(t *testing.T) { profile := UniqueProfileName("download-only") ctx, cancel := context.WithTimeout(context.Background(), Minutes(30)) defer Cleanup(t, profile, cancel) containerRuntime := ContainerRuntime() versions := []string{ constants.OldestKubernetesVersion, constants.DefaultKubernetesVersion, constants.NewestKubernetesVersion, } // Small optimization, don't run the exact same set of tests twice if constants.DefaultKubernetesVersion == constants.NewestKubernetesVersion { versions = versions[:len(versions)-1] } for _, v := range versions { t.Run(v, func(t *testing.T) { defer PostMortemLogs(t, profile) t.Run("json-events", func(t *testing.T) { // --force to avoid uid check args := append([]string{"start", "-o=json", "--download-only", "-p", profile, "--force", "--alsologtostderr", fmt.Sprintf("--kubernetes-version=%s", v), fmt.Sprintf("--container-runtime=%s", containerRuntime)}, StartArgs()...) rt, err := Run(t, exec.CommandContext(ctx, Target(), args...)) if err != nil { t.Errorf("failed to download only. args: %q %v", args, err) } s := bufio.NewScanner(bytes.NewReader(rt.Stdout.Bytes())) for s.Scan() { var rtObj map[string]interface{} err := json.Unmarshal(s.Bytes(), &rtObj) if err != nil { t.Errorf("failed to parse output: %v", err) } else if step, ok := rtObj["data"]; ok { if stepMap, ok := step.(map[string]interface{}); ok { if stepMap["currentstep"] == "" { t.Errorf("Empty step number for %v", stepMap["name"]) } } } } if err := s.Err(); err != nil { t.Errorf("failed to read output: %v", err) } }) preloadExists := false t.Run("preload-exists", func(t *testing.T) { // skip for none, as none driver does not have preload feature. if NoneDriver() { t.Skip("None driver does not have preload") } // Driver does not matter here, since the only exception is none driver, // which cannot occur here. if !download.PreloadExists(v, containerRuntime, "docker", true) { t.Skip("No preload image") } // Just make sure the tarball path exists if _, err := os.Stat(download.TarballPath(v, containerRuntime)); err != nil { t.Errorf("failed to verify preloaded tarball file exists: %v", err) } preloadExists = true }) t.Run("cached-images", func(t *testing.T) { // skip verify for cache images if --driver=none if NoneDriver() { t.Skip("None driver has no cache") } if preloadExists { t.Skip("Preload exists, images won't be cached") } imgs, err := images.Kubeadm("", v) if err != nil { t.Errorf("failed to get kubeadm images for %v: %+v", v, err) } for _, img := range imgs { img = strings.Replace(img, ":", "_", 1) // for example kube-scheduler:v1.15.2 --> kube-scheduler_v1.15.2 fp := filepath.Join(localpath.MiniPath(), "cache", "images", img) _, err := os.Stat(fp) if err != nil { t.Errorf("expected image file exist at %q but got error: %v", fp, err) } } }) t.Run("binaries", func(t *testing.T) { if preloadExists { t.Skip("Preload exists, binaries are present within.") } // checking binaries downloaded (kubelet,kubeadm) for _, bin := range constants.KubernetesReleaseBinaries { fp := filepath.Join(localpath.MiniPath(), "cache", "linux", runtime.GOARCH, v, bin) _, err := os.Stat(fp) if err != nil { t.Errorf("expected the file for binary exist at %q but got error %v", fp, err) } } }) t.Run("kubectl", func(t *testing.T) { // If we are on darwin/windows, check to make sure OS specific kubectl has been downloaded // as well for the `minikube kubectl` command if runtime.GOOS == "linux" { t.Skip("Test for darwin and windows") } binary := "kubectl" if runtime.GOOS == "windows" { binary = "kubectl.exe" } fp := filepath.Join(localpath.MiniPath(), "cache", runtime.GOOS, runtime.GOARCH, v, binary) if _, err := os.Stat(fp); err != nil { t.Errorf("expected the file for binary exist at %q but got error %v", fp, err) } }) // checks if the duration of `minikube logs` takes longer than 5 seconds t.Run("LogsDuration", func(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), Seconds(5)) defer cancel() args := []string{"logs", "-p", profile} if _, err := Run(t, exec.CommandContext(ctx, Target(), args...)); err != nil { t.Logf("minikube logs failed with error: %v", err) } if err := ctx.Err(); err == context.DeadlineExceeded { t.Error("minikube logs expected to finish by 5 seconds, but took longer") } }) }) } // This is a weird place to test profile deletion, but this test is serial, and we have a profile to delete! t.Run("DeleteAll", func(t *testing.T) { defer PostMortemLogs(t, profile) if !CanCleanup() { t.Skip("skipping, as cleanup is disabled") } rr, err := Run(t, exec.CommandContext(ctx, Target(), "delete", "--all")) if err != nil { t.Errorf("failed to delete all. args: %q : %v", rr.Command(), err) } }) // Delete should always succeed, even if previously partially or fully deleted. t.Run("DeleteAlwaysSucceeds", func(t *testing.T) { defer PostMortemLogs(t, profile) if !CanCleanup() { t.Skip("skipping, as cleanup is disabled") } rr, err := Run(t, exec.CommandContext(ctx, Target(), "delete", "-p", profile)) if err != nil { t.Errorf("failed to delete. args: %q: %v", rr.Command(), err) } }) } // TestDownloadOnlyKic makes sure --download-only caches the docker driver images as well. func TestDownloadOnlyKic(t *testing.T) { if !KicDriver() { t.Skip("skipping, only for docker or podman driver") } profile := UniqueProfileName("download-docker") ctx, cancel := context.WithTimeout(context.Background(), Minutes(15)) defer Cleanup(t, profile, cancel) cRuntime := ContainerRuntime() args := []string{"start", "--download-only", "-p", profile, "--alsologtostderr"} args = append(args, StartArgs()...) cmd := exec.CommandContext(ctx, Target(), args...) if _, err := Run(t, cmd); err != nil { t.Errorf("start with download only failed %q : %v", args, err) } // Make sure the downloaded image tarball exists tarball := download.TarballPath(constants.DefaultKubernetesVersion, cRuntime) contents, err := os.ReadFile(tarball) if err != nil { t.Errorf("failed to read tarball file %q: %v", tarball, err) } if arm64Platform() { t.Skip("Skip for arm64 platform. See https://github.com/kubernetes/minikube/issues/10144") } // Make sure it has the correct checksum checksum := md5.Sum(contents) remoteChecksum, err := os.ReadFile(download.PreloadChecksumPath(constants.DefaultKubernetesVersion, cRuntime)) if err != nil { t.Errorf("failed to read checksum file %q : %v", download.PreloadChecksumPath(constants.DefaultKubernetesVersion, cRuntime), err) } if string(remoteChecksum) != string(checksum[:]) { t.Errorf("failed to verify checksum. checksum of %q does not match remote checksum (%q != %q)", tarball, string(remoteChecksum), string(checksum[:])) } } // createSha256File is a helper function which creates sha256 checksum file from given file func createSha256File(filePath string) error { dat, _ := os.ReadFile(filePath) sum := sha256.Sum256(dat) f, err := os.Create(filePath + ".sha256") if err != nil { return err } defer f.Close() _, err = f.WriteString(fmt.Sprintf("%x", sum[:])) if err != nil { return err } return nil } // TestBinaryMirror tests functionality of --binary-mirror flag func TestBinaryMirror(t *testing.T) { profile := UniqueProfileName("binary-mirror") ctx, cancel := context.WithTimeout(context.Background(), Minutes(10)) defer Cleanup(t, profile, cancel) tmpDir := t.TempDir() // Start test server which will serve binary files ts := httptest.NewServer( http.FileServer(http.Dir(tmpDir)), ) defer ts.Close() binaryName := "kubectl" if runtime.GOOS == "windows" { binaryName = "kubectl.exe" } binaryPath, err := download.Binary(binaryName, constants.DefaultKubernetesVersion, runtime.GOOS, runtime.GOARCH, "") if err != nil { t.Errorf("Failed to download binary: %+v", err) } newBinaryDir := filepath.Join(tmpDir, constants.DefaultKubernetesVersion, "bin", runtime.GOOS, runtime.GOARCH) if err := os.MkdirAll(newBinaryDir, os.ModePerm); err != nil { t.Errorf("Failed to create %s directories", newBinaryDir) } newBinaryPath := filepath.Join(newBinaryDir, binaryName) if err := os.Rename(binaryPath, newBinaryPath); err != nil { t.Errorf("Failed to move binary file: %+v", err) } if err := createSha256File(newBinaryPath); err != nil { t.Errorf("Failed to generate sha256 checksum file: %+v", err) } args := append([]string{"start", "--download-only", "-p", profile, "--alsologtostderr", "--binary-mirror", ts.URL}, StartArgs()...) cmd := exec.CommandContext(ctx, Target(), args...) if _, err := Run(t, cmd); err != nil { t.Errorf("start with --binary-mirror failed %q : %v", args, err) } }