kubernetes
/
minikube
mirror of https://github.com/kubernetes/minikube.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #5315 from tstromberg/norunner2 

Refactor parallel integration tests for clarity and reliability
pull/5341/head
Thomas Strömberg 2019-09-12 19:59:25 -07:00 committed by GitHub
parent 38123cd49d ad9debaec4
commit d3cee23f2d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
56 changed files with 1959 additions and 2750 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -197,7 +197,7 @@ iso_in_docker:
$(ISO_BUILD_IMAGE) /bin/bash
test-iso: pkg/minikube/assets/assets.go pkg/minikube/translate/translations.go
go test -v ./test/integration --tags=iso --minikube-args="--iso-url=file://$(shell pwd)/out/buildroot/output/images/rootfs.iso9660"
go test -v ./test/integration --tags=iso --minikube-start-args="--iso-url=file://$(shell pwd)/out/buildroot/output/images/rootfs.iso9660"
.PHONY: test-pkg
test-pkg/%: pkg/minikube/assets/assets.go pkg/minikube/translate/translations.go

8
cmd/minikube/cmd/config/get.go
View File

@ -30,9 +30,13 @@ var configGetCmd = &cobra.Command{
Short: "Gets the value of PROPERTY_NAME from the minikube config file",
Long: "Returns the value of PROPERTY_NAME from the minikube config file. Can be overwritten at runtime by flags or environmental variables.",
RunE: func(cmd *cobra.Command, args []string) error {
if len(args) != 1 {
if len(args) == 0 {
cmd.SilenceErrors = true
return errors.New("usage: minikube config get PROPERTY_NAME")
return errors.New("not enough arguments.\nusage: minikube config get PROPERTY_NAME")
}
if len(args) > 1 {
cmd.SilenceErrors = true
return fmt.Errorf("too many arguments (%d)\nusage: minikube config get PROPERTY_NAME", len(args))
}
cmd.SilenceUsage = true

8
cmd/minikube/cmd/config/set.go
View File

@ -21,6 +21,7 @@ import (
pkgConfig "k8s.io/minikube/pkg/minikube/config"
"k8s.io/minikube/pkg/minikube/constants"
"k8s.io/minikube/pkg/minikube/exit"
"k8s.io/minikube/pkg/minikube/out"
)
var configSetCmd = &cobra.Command{
@ -29,8 +30,11 @@ var configSetCmd = &cobra.Command{
Long: `Sets the PROPERTY_NAME config value to PROPERTY_VALUE
These values can be overwritten by flags or environment variables at runtime.`,
Run: func(cmd *cobra.Command, args []string) {
if len(args) != 2 {
exit.UsageT("usage: minikube config set PROPERTY_NAME PROPERTY_VALUE")
if len(args) < 2 {
exit.UsageT("not enough arguments ({{.ArgCount}}).\nusage: minikube config set PROPERTY_NAME PROPERTY_VALUE", out.V{"ArgCount": len(args)})
}
if len(args) > 2 {
exit.UsageT("toom any arguments ({{.ArgCount}}).\nusage: minikube config set PROPERTY_NAME PROPERTY_VALUE", out.V{"ArgCount": len(args)})
}
err := Set(args[0], args[1])
if err != nil {

2
cmd/minikube/cmd/dashboard.go
View File

@ -112,7 +112,7 @@ var dashboardCmd = &cobra.Command{
svc := "kubernetes-dashboard"
out.ErrT(out.Verifying, "Verifying dashboard health ...")
checkSVC := func() error { return service.CheckService(ns, svc) }
if err = retry.Expo(checkSVC, 1*time.Second, time.Minute*3); err != nil {
if err = retry.Expo(checkSVC, 1*time.Second, time.Minute*5); err != nil {
exit.WithCodeT(exit.Unavailable, "dashboard service is not running: {{.error}}", out.V{"error": err})
}

10
cmd/minikube/cmd/start.go
View File

@ -158,7 +158,7 @@ func initMinikubeFlags() {
startCmd.Flags().String(networkPlugin, "", "The name of the network plugin.")
startCmd.Flags().Bool(enableDefaultCNI, false, "Enable the default CNI plugin (/etc/cni/net.d/k8s.conf). Used in conjunction with \"--network-plugin=cni\".")
startCmd.Flags().Bool(waitUntilHealthy, true, "Wait until Kubernetes core services are healthy before exiting.")
startCmd.Flags().Duration(waitTimeout, 3*time.Minute, "max time to wait per Kubernetes core services to be healthy.")
startCmd.Flags().Duration(waitTimeout, 6*time.Minute, "max time to wait per Kubernetes core services to be healthy.")
startCmd.Flags().Bool(nativeSSH, true, "Use native Golang SSH client (default true). Set to 'false' to use the command line 'ssh' command when accessing the docker machine. Useful for the machine drivers when they will not start with 'Waiting for SSH'.")
}
@ -328,15 +328,15 @@ func runStart(cmd *cobra.Command, args []string) {
showVersionInfo(k8sVersion, cr)
waitCacheImages(&cacheGroup)
// setup kube adm and certs and return bootstrapperx
bs := setupKubeAdm(machineAPI, config.KubernetesConfig)
// The kube config must be update must come before bootstrapping, otherwise health checks may use a stale IP
// Must be written before bootstrap, otherwise health checks may flake due to stale IP
kubeconfig, err := setupKubeconfig(host, &config)
if err != nil {
exit.WithError("Failed to setup kubeconfig", err)
}
// setup kubeadm (must come after setupKubeconfig)
bs := setupKubeAdm(machineAPI, config.KubernetesConfig)
// pull images or restart cluster
bootstrapCluster(bs, cr, mRunner, config.KubernetesConfig, preExists, isUpgrade)
configureMounts()

1
go.mod
View File

@ -67,6 +67,7 @@ require (
github.com/xeipuuv/gojsonreference v0.0.0-20150808065054-e02fc20de94c // indirect
github.com/xeipuuv/gojsonschema v0.0.0-20160623135812-c539bca196be
github.com/zchee/go-vmnet v0.0.0-20161021174912-97ebf9174097
golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d
golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8
golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a
golang.org/x/sync v0.0.0-20190423024810-112230192c58

3
go.sum
View File

@ -114,8 +114,6 @@ github.com/docker/go-connections v0.3.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5Xh
github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk=
github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/docker/libnetwork v0.0.0-20180830151422-a9cd636e3789/go.mod h1:93m0aTqz6z+g32wla4l4WxTrdtvBRmVzYRkYvasA5Z8=
github.com/docker/machine v0.7.1-0.20190718054102-a555e4f7a8f5 h1:/2G2PrqxKga8hAVGPri/5NEv24rvDwXoH5pjPXUBCpA=
github.com/docker/machine v0.7.1-0.20190718054102-a555e4f7a8f5/go.mod h1:I8mPNDeK1uH+JTcUU7X0ZW8KiYz0jyAgNaeSJ1rCfDI=
github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 h1:cenwrSVm+Z7QLSV/BsnenAOcDXdX4cMv4wP0B/5QbPg=
github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
@ -519,6 +517,7 @@ go.uber.org/atomic v0.0.0-20181018215023-8dc6146f7569/go.mod h1:gD2HeocX3+yG+ygL
go.uber.org/multierr v0.0.0-20180122172545-ddea229ff1df/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/zap v0.0.0-20180814183419-67bc79d13d15/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d h1:E2M5QgjZ/Jg+ObCQAudsXxuTsLj7Nl5RV/lZcQZmKSo=
golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181025213731-e84da0312774/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=

13
hack/jenkins/common.sh
View File

@ -260,7 +260,6 @@ cleanup_stale_routes || true
mkdir -p "${TEST_HOME}"
export MINIKUBE_HOME="${TEST_HOME}/.minikube"
export MINIKUBE_WANTREPORTERRORPROMPT=False
export KUBECONFIG="${TEST_HOME}/kubeconfig"
# Build the gvisor image. This will be copied into minikube and loaded by ctr.
@ -272,18 +271,13 @@ if [ "$(uname)" != "Darwin" ]; then
docker build -t gcr.io/k8s-minikube/gvisor-addon:latest -f testdata/gvisor-addon-Dockerfile ./testdata
fi
# Display the default image URL
echo ""
echo ">> ISO URL"
"${MINIKUBE_BIN}" start -h | grep iso-url || true
echo ""
echo ">> Starting ${E2E_BIN} at $(date)"
${SUDO_PREFIX}${E2E_BIN} \
-minikube-start-args="--vm-driver=${VM_DRIVER} ${EXTRA_START_ARGS}" \
-minikube-args="--v=10 --logtostderr ${EXTRA_ARGS}" \
-test.v -test.timeout=100m -test.parallel=${PARALLEL_COUNT} -binary="${MINIKUBE_BIN}" && result=$? || result=$?
-test.timeout=60m \
-test.parallel=${PARALLEL_COUNT} \
-binary="${MINIKUBE_BIN}" && result=$? || result=$?
echo ">> ${E2E_BIN} exited with ${result} at $(date)"
echo ""
@ -293,7 +287,6 @@ if [[ $result -eq 0 ]]; then
else
status="failure"
echo "minikube: FAIL"
source print-debug-info.sh
fi
echo ">> Cleaning up after ourselves ..."

68
hack/jenkins/print-debug-info.sh
View File

@ -1,68 +0,0 @@
#!/bin/bash
# Copyright 2016 The Kubernetes Authors All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Prints some debug info about the state of the cluster
#
# We don't check the error on these commands, since they might fail depending on
# the cluster state.
set +e
echo ""
echo ">>> print-debug-info at $(date):"
echo ""
${SUDO_PREFIX} cat "${KUBECONFIG}"
kubectl version \
&& kubectl get pods --all-namespaces \
&& kubectl cluster-info dump
# minikube has probably been shut down, so iterate forward each command rather than spamming.
MINIKUBE=${SUDO_PREFIX}out/minikube-${OS_ARCH}
${MINIKUBE} status
${MINIKUBE} ip && ${MINIKUBE} logs
if [[ "${VM_DRIVER}" == "none" ]]; then
run=""
else
run="${MINIKUBE} ssh --"
fi
echo "Local date: $(date)"
${run} date
${run} uptime
${run} docker ps
${run} env TERM=dumb systemctl list-units --state=failed
${run} env TERM=dumb journalctl --no-tail --no-pager -p notice
${run} free
${run} cat /etc/VERSION
if type -P virsh; then
virsh -c qemu:///system list --all
fi
if type -P vboxmanage; then
vboxmanage list vms
fi
if type -P hdiutil; then
hdiutil info | grep -E "/dev/disk[1-9][^s]"
fi
netstat -rn -f inet
echo ""
echo ">>> end print-debug-info"
echo ""
set -e

2
hack/jenkins/windows_integration_test_hyperv.ps1
View File

@ -19,7 +19,7 @@ gsutil.cmd -m cp -r gs://minikube-builds/$env:MINIKUBE_LOCATION/testdata .
./out/minikube-windows-amd64.exe delete
out/e2e-windows-amd64.exe -minikube-start-args="--vm-driver=hyperv --hyperv-virtual-switch=primary-virtual-switch --bootstrapper=kubeadm" -minikube-args="--v=10 --logtostderr" -binary=out/minikube-windows-amd64.exe -test.v -test.timeout=65m
out/e2e-windows-amd64.exe -minikube-start-args="--vm-driver=hyperv --hyperv-virtual-switch=primary-virtual-switch" -binary=out/minikube-windows-amd64.exe -test.v -test.timeout=65m
$env:result=$lastexitcode
# If the last exit code was 0->success, x>0->error
If($env:result -eq 0){$env:status="success"}

2
hack/jenkins/windows_integration_test_virtualbox.ps1
View File

@ -19,7 +19,7 @@ gsutil.cmd -m cp -r gs://minikube-builds/$env:MINIKUBE_LOCATION/testdata .
./out/minikube-windows-amd64.exe delete
out/e2e-windows-amd64.exe -minikube-start-args="--vm-driver=virtualbox --bootstrapper=kubeadm" -minikube-args="--v=10 --logtostderr" -binary=out/minikube-windows-amd64.exe -test.v -test.timeout=30m
out/e2e-windows-amd64.exe -minikube-start-args="--vm-driver=virtualbox" -binary=out/minikube-windows-amd64.exe -test.v -test.timeout=30m
$env:result=$lastexitcode
# If the last exit code was 0->success, x>0->error
If($env:result -eq 0){$env:status="success"}

34
pkg/kapi/kapi.go
View File

@ -22,7 +22,6 @@ import (
"time"
"github.com/golang/glog"
"github.com/pkg/errors"
apps "k8s.io/api/apps/v1"
core "k8s.io/api/core/v1"
apierr "k8s.io/apimachinery/pkg/api/errors"
@ -33,6 +32,7 @@ import (
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
watchtools "k8s.io/client-go/tools/watch"
kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants"
@ -46,26 +46,26 @@ var (
ReasonableStartTime = time.Minute * 5
)
// Client gets the kubernetes client from default kubeconfig
func Client(kubectlContext ...string) (kubernetes.Interface, error) {
loadingRules := clientcmd.NewDefaultClientConfigLoadingRules()
configOverrides := &clientcmd.ConfigOverrides{}
if kubectlContext != nil {
configOverrides = &clientcmd.ConfigOverrides{
CurrentContext: kubectlContext[0],
}
}
kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides)
config, err := kubeConfig.ClientConfig()
// ClientConfig returns the client configuration for a kubectl context
func ClientConfig(context string) (*rest.Config, error) {
loader := clientcmd.NewDefaultClientConfigLoadingRules()
cc := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loader, &clientcmd.ConfigOverrides{CurrentContext: context})
c, err := cc.ClientConfig()
if err != nil {
return nil, fmt.Errorf("error creating kubeConfig: %v", err)
return nil, fmt.Errorf("client config: %v", err)
}
config = proxy.UpdateTransport(config)
client, err := kubernetes.NewForConfig(config)
c = proxy.UpdateTransport(c)
glog.V(1).Infof("client config for %s: %+v", context, c)
return c, nil
}
// Client gets the kubernetes client for a kubectl context name
func Client(context string) (*kubernetes.Clientset, error) {
c, err := ClientConfig(context)
if err != nil {
return nil, errors.Wrap(err, "error creating new client from kubeConfig.ClientConfig()")
return nil, err
}
return client, nil
return kubernetes.NewForConfig(c)
}
// WaitForPodsWithLabelRunning waits for all matching pods to become Running and at least one matching pod exists.

2
pkg/minikube/assets/vm_assets.go
View File

@ -201,7 +201,7 @@ func (m *BinAsset) loadData(isTemplate bool) error {
m.length = len(contents)
m.reader = bytes.NewReader(contents)
glog.Infof("Created asset %s with %d bytes", m.AssetName, m.length)
glog.V(1).Infof("Created asset %s with %d bytes", m.AssetName, m.length)
if m.length == 0 {
return fmt.Errorf("%s is an empty asset", m.AssetName)
}

18
pkg/minikube/bootstrapper/certs.go
View File

@ -25,6 +25,7 @@ import (
"path"
"path/filepath"
"strings"
"time"
"github.com/golang/glog"
"github.com/pkg/errors"
@ -37,6 +38,9 @@ import (
"k8s.io/minikube/pkg/minikube/constants"
"k8s.io/minikube/pkg/minikube/kubeconfig"
"k8s.io/minikube/pkg/util"
"github.com/juju/clock"
"github.com/juju/mutex"
)
const (
@ -122,13 +126,25 @@ func SetupCerts(cmd command.Runner, k8s config.KubernetesConfig) error {
}
func generateCerts(k8s config.KubernetesConfig) error {
// TODO: Instead of racey manipulation of a shared certificate, use per-profile certs
spec := mutex.Spec{
Name: "generateCerts",
Clock: clock.WallClock,
Delay: 10 * time.Second,
}
glog.Infof("acquiring lock: %+v", spec)
releaser, err := mutex.Acquire(spec)
if err != nil {
return errors.Wrapf(err, "unable to acquire lock for %+v", spec)
}
defer releaser.Release()
serviceIP, err := util.GetServiceClusterIP(k8s.ServiceCIDR)
if err != nil {
return errors.Wrap(err, "getting service cluster ip")
}
localPath := constants.GetMinipath()
caCertPath := filepath.Join(localPath, "ca.crt")
caKeyPath := filepath.Join(localPath, "ca.key")

29
pkg/minikube/bootstrapper/kubeadm/kubeadm.go
View File

@ -38,6 +38,7 @@ import (
"golang.org/x/sync/errgroup"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes"
kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/minikube/pkg/kapi"
"k8s.io/minikube/pkg/minikube/assets"
@ -105,12 +106,14 @@ var SkipAdditionalPreflights = map[string][]string{}
// Bootstrapper is a bootstrapper using kubeadm
type Bootstrapper struct {
c command.Runner
c command.Runner
contextName string
}
// NewKubeadmBootstrapper creates a new kubeadm.Bootstrapper
func NewKubeadmBootstrapper(api libmachine.API) (*Bootstrapper, error) {
h, err := api.Load(config.GetMachineName())
name := config.GetMachineName()
h, err := api.Load(name)
if err != nil {
return nil, errors.Wrap(err, "getting api client")
}
@ -118,7 +121,7 @@ func NewKubeadmBootstrapper(api libmachine.API) (*Bootstrapper, error) {
if err != nil {
return nil, errors.Wrap(err, "command runner")
}
return &Bootstrapper{c: runner}, nil
return &Bootstrapper{c: runner, contextName: name}, nil
}
// GetKubeletStatus returns the kubelet status
@ -339,10 +342,6 @@ func (k *Bootstrapper) WaitCluster(k8s config.KubernetesConfig, timeout time.Dur
// up. Otherwise, minikube won't start, as "k8s-app" pods are not ready.
componentsOnly := k8s.NetworkPlugin == "cni"
out.T(out.WaitingPods, "Waiting for:")
client, err := kapi.Client()
if err != nil {
return errors.Wrap(err, "k8s client")
}
// Wait until the apiserver can answer queries properly. We don't care if the apiserver
// pod shows up as registered, but need the webserver for all subsequent queries.
@ -351,6 +350,22 @@ func (k *Bootstrapper) WaitCluster(k8s config.KubernetesConfig, timeout time.Dur
return errors.Wrap(err, "waiting for apiserver")
}
// Catch case if WaitCluster was called with a stale ~/.kube/config
config, err := kapi.ClientConfig(k.contextName)
if err != nil {
return errors.Wrap(err, "client config")
}
endpoint := fmt.Sprintf("https://%s:%d", k8s.NodeIP, k8s.NodePort)
if config.Host != endpoint {
glog.Errorf("Overriding stale ClientConfig host %s with %s", config.Host, endpoint)
config.Host = endpoint
}
client, err := kubernetes.NewForConfig(config)
if err != nil {
return errors.Wrap(err, "k8s client")
}
for _, p := range PodsByLayer {
if componentsOnly && p.key != "component" { // skip component check if network plugin is cni
continue

25
pkg/minikube/cluster/mount.go
View File

@ -61,10 +61,11 @@ func Mount(r mountRunner, source string, target string, c *MountConfig) error {
cmd := fmt.Sprintf("sudo mkdir -m %o -p %s && %s", c.Mode, target, mntCmd(source, target, c))
glog.Infof("Will run: %s", cmd)
out, err := r.CombinedOutput(cmd)
glog.Infof("mount err=%s, out=%s", err, out)
if err != nil {
glog.Infof("%s failed: err=%s, output: %q", cmd, err, out)
return errors.Wrap(err, out)
}
glog.Infof("%s output: %q", cmd, out)
return nil
}
@ -131,32 +132,16 @@ func mntCmd(source string, target string, c *MountConfig) string {
}
// umountCmd returns a command for unmounting
func umountCmd(target string, force bool) string {
// Call fuser before unmount, for killing the processes using the mount point. Notes: don't use 'lsof' to avoid the innocents
flag1 := fmt.Sprintf("sudo fuser -km %s;", target)
flag2 := ""
if force {
flag1 = ""
flag2 = "-f "
}
func umountCmd(target string) string {
// grep because findmnt will also display the parent!
return fmt.Sprintf("[ \"x$(findmnt -T %s | grep %s)\" != \"x\" ] && { %s sudo umount %s%s; } || echo ", target, target, flag1, flag2, target)
return fmt.Sprintf("[ \"x$(findmnt -T %s | grep %s)\" != \"x\" ] && sudo umount -f %s || echo ", target, target, target)
}
// Unmount unmounts a path
func Unmount(r mountRunner, target string) error {
cmd := umountCmd(target, false)
cmd := umountCmd(target)
glog.Infof("Will run: %s", cmd)
out, err := r.CombinedOutput(cmd)
if err == nil {
return nil
}
glog.Warningf("initial unmount error: %v, out: %s", err, out)
// Try again, using force if needed.
cmd = umountCmd(target, true)
glog.Infof("Will run: %s", cmd)
out, err = r.CombinedOutput(cmd)
glog.Infof("unmount force err=%v, out=%s", err, out)
if err != nil {
return errors.Wrap(err, out)

10
pkg/minikube/cluster/mount_test.go
View File

@ -54,7 +54,7 @@ func TestMount(t *testing.T) {
target: "target",
cfg: &MountConfig{Type: "9p", Mode: os.FileMode(0700)},
want: []string{
"[ \"x$(findmnt -T target | grep target)\" != \"x\" ] && { sudo fuser -km target; sudo umount target; } || echo ",
"[ \"x$(findmnt -T target | grep target)\" != \"x\" ] && sudo umount -f target || echo ",
"sudo mkdir -m 700 -p target && sudo mount -t 9p -o dfltgid=0,dfltuid=0 src target",
},
},
@ -64,7 +64,7 @@ func TestMount(t *testing.T) {
target: "target",
cfg: &MountConfig{Type: "9p", Mode: os.FileMode(0700), UID: "docker", GID: "docker"},
want: []string{
"[ \"x$(findmnt -T target | grep target)\" != \"x\" ] && { sudo fuser -km target; sudo umount target; } || echo ",
"[ \"x$(findmnt -T target | grep target)\" != \"x\" ] && sudo umount -f target || echo ",
"sudo mkdir -m 700 -p target && sudo mount -t 9p -o dfltgid=$(grep ^docker: /etc/group | cut -d: -f3),dfltuid=$(id -u docker) src target",
},
},
@ -77,7 +77,7 @@ func TestMount(t *testing.T) {
"cache": "fscache",
}},
want: []string{
"[ \"x$(findmnt -T /target | grep /target)\" != \"x\" ] && { sudo fuser -km /target; sudo umount /target; } || echo ",
"[ \"x$(findmnt -T /target | grep /target)\" != \"x\" ] && sudo umount -f /target || echo ",
"sudo mkdir -m 777 -p /target && sudo mount -t 9p -o cache=fscache,dfltgid=72,dfltuid=82,noextend,version=9p2000.u 10.0.0.1 /target",
},
},
@ -89,7 +89,7 @@ func TestMount(t *testing.T) {
"version": "9p2000.L",
}},
want: []string{
"[ \"x$(findmnt -T tgt | grep tgt)\" != \"x\" ] && { sudo fuser -km tgt; sudo umount tgt; } || echo ",
"[ \"x$(findmnt -T tgt | grep tgt)\" != \"x\" ] && sudo umount -f tgt || echo ",
"sudo mkdir -m 700 -p tgt && sudo mount -t 9p -o dfltgid=0,dfltuid=0,version=9p2000.L src tgt",
},
},
@ -115,7 +115,7 @@ func TestUnmount(t *testing.T) {
t.Fatalf("Unmount(/mnt): %v", err)
}
want := []string{"[ \"x$(findmnt -T /mnt | grep /mnt)\" != \"x\" ] && { sudo fuser -km /mnt; sudo umount /mnt; } || echo "}
want := []string{"[ \"x$(findmnt -T /mnt | grep /mnt)\" != \"x\" ] && sudo umount -f /mnt || echo "}
if diff := cmp.Diff(r.cmds, want); diff != "" {
t.Errorf("command diff (-want +got): %s", diff)
}

4
pkg/util/crypto.go
View File

@ -30,6 +30,7 @@ import (
"path/filepath"
"time"
"github.com/golang/glog"
"github.com/pkg/errors"
"k8s.io/minikube/pkg/util/lock"
)
@ -65,6 +66,7 @@ func GenerateCACert(certPath, keyPath string, name string) error {
// GenerateSignedCert generates a signed certificate and key
func GenerateSignedCert(certPath, keyPath, cn string, ips []net.IP, alternateDNS []string, signerCertPath, signerKeyPath string) error {
glog.Infof("Generating cert %s with IP's: %s", certPath, ips)
signerCertBytes, err := ioutil.ReadFile(signerCertPath)
if err != nil {
return errors.Wrap(err, "Error reading file: signerCertPath")
@ -152,6 +154,7 @@ func writeCertsAndKeys(template *x509.Certificate, certPath string, signeeKey *r
if err := os.MkdirAll(filepath.Dir(certPath), os.FileMode(0755)); err != nil {
return errors.Wrap(err, "Error creating certificate directory")
}
glog.Infof("Writing cert to %s ...", certPath)
if err := lock.WriteFile(certPath, certBuffer.Bytes(), os.FileMode(0644)); err != nil {
return errors.Wrap(err, "Error writing certificate to cert path")
}
@ -159,6 +162,7 @@ func writeCertsAndKeys(template *x509.Certificate, certPath string, signeeKey *r
if err := os.MkdirAll(filepath.Dir(keyPath), os.FileMode(0755)); err != nil {
return errors.Wrap(err, "Error creating key directory")
}
glog.Infof("Writing key to %s ...", keyPath)
if err := lock.WriteFile(keyPath, keyBuffer.Bytes(), os.FileMode(0600)); err != nil {
return errors.Wrap(err, "Error writing key file")
}

24
test/integration/README.md Normal file
View File

@ -0,0 +1,24 @@
# Integration tests
## The basics
To run all tests from the minikube root directory:
`make integration`
## Quickly iterating on a single test
Run a single test on an active cluster:
`make integration -e TEST_ARGS="-test.v -test.run TestFunctional/parallel/MountCmd --profile=minikube --cleanup=false"`
WARNING: For this to work repeatedly, the test must be written so that it cleans up after itself.
See `main.go` for details.
## Disabling parallelism
`make integration -e TEST_ARGS="-test.parallel=1"`
## Testing philosophy

89
test/integration/a_download_only_test.go
View File

@ -20,89 +20,62 @@ limitations under the License.
package integration
import (
"context"
"fmt"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
"github.com/hashicorp/go-getter"
"github.com/pkg/errors"
"k8s.io/minikube/pkg/minikube/constants"
pkgutil "k8s.io/minikube/pkg/util"
"k8s.io/minikube/pkg/util/retry"
)
// Note this test runs before all because filename is alphabetically first
// is used to cache images and binaries used by other parallel tests to avoid redownloading.
// TestDownloadOnly tests the --download-only option
func TestDownloadOnly(t *testing.T) {
p := profileName(t)
mk := NewMinikubeRunner(t, p)
if !isTestNoneDriver(t) { // none driver doesnt need to be deleted
defer mk.TearDown(t)
}
t.Run("group", func(t *testing.T) {
t.Run("CacheOldestNewest", func(t *testing.T) {
if isTestNoneDriver(t) { // don't cache images
t.Skip("skipping test for none driver as it doesn't cache images")
}
profile := UniqueProfileName("download")
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
defer Cleanup(t, profile, cancel)
minHome := constants.GetMinipath()
for _, v := range []string{constants.OldestKubernetesVersion, constants.NewestKubernetesVersion} {
mk.MustStart("--download-only", fmt.Sprintf("--kubernetes-version=%s", v))
// checking if cached images are downloaded for example (kube-apiserver_v1.15.2, kube-scheduler_v1.15.2, ...)
_, imgs := constants.GetKubeadmCachedImages("", v)
for _, img := range imgs {
img = strings.Replace(img, ":", "_", 1) // for example kube-scheduler:v1.15.2 --> kube-scheduler_v1.15.2
fp := filepath.Join(minHome, "cache", "images", img)
_, err := os.Stat(fp)
if err != nil {
t.Errorf("expected image file exist at %q but got error: %v", fp, err)
t.Run("group", func(t *testing.T) {
versions := []string{
constants.OldestKubernetesVersion,
constants.DefaultKubernetesVersion,
constants.NewestKubernetesVersion,
}
for _, v := range versions {
t.Run(v, func(t *testing.T) {
args := append([]string{"start", "--download-only", "-p", profile, fmt.Sprintf("--kubernetes-version=%s", v)}, StartArgs()...)
_, err := Run(t, exec.CommandContext(ctx, Target(), args...))
if err != nil {
t.Errorf("%s failed: %v", args, err)
}
// None driver does not cache images, so this test will fail
if !NoneDriver() {
_, imgs := constants.GetKubeadmCachedImages("", v)
for _, img := range imgs {
img = strings.Replace(img, ":", "_", 1) // for example kube-scheduler:v1.15.2 --> kube-scheduler_v1.15.2
fp := filepath.Join(constants.GetMinipath(), "cache", "images", img)
_, err := os.Stat(fp)
if err != nil {
t.Errorf("expected image file exist at %q but got error: %v", fp, err)
}
}
}
// checking binaries downloaded (kubelet,kubeadm)
for _, bin := range constants.KubeadmBinaries {
fp := filepath.Join(minHome, "cache", v, bin)
fp := filepath.Join(constants.GetMinipath(), "cache", v, bin)
_, err := os.Stat(fp)
if err != nil {
t.Errorf("expected the file for binary exist at %q but got error %v", fp, err)
}
}
}
})
})
// this downloads the latest published binary from where we publish the minikube binary
t.Run("DownloadLatestRelease", func(t *testing.T) {
dest := filepath.Join(*testdataDir, fmt.Sprintf("minikube-%s-%s-latest-stable", runtime.GOOS, runtime.GOARCH))
err := downloadMinikubeBinary(t, dest, "latest")
if err != nil {
t.Errorf("erorr downloading the latest minikube release %v", err)
})
}
})
}
// downloadMinikubeBinary downloads the minikube binary from github used by TestVersionUpgrade
// acts as a test setup for TestVersionUpgrade
func downloadMinikubeBinary(t *testing.T, dest string, version string) error {
t.Helper()
// Grab latest release binary
url := pkgutil.GetBinaryDownloadURL(version, runtime.GOOS)
download := func() error {
return getter.GetFile(dest, url)
}
if err := retry.Expo(download, 3*time.Second, 3*time.Minute); err != nil {
return errors.Wrap(err, "Failed to get latest release binary")
}
if runtime.GOOS != "windows" {
if err := os.Chmod(dest, 0700); err != nil {
return err
}
}
return nil
}

206
test/integration/addons_test.go Normal file
View File

@ -0,0 +1,206 @@
// +build integration
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package integration
import (
"context"
"fmt"
"net/http"
"net/url"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
"github.com/hashicorp/go-retryablehttp"
"k8s.io/minikube/pkg/kapi"
"k8s.io/minikube/pkg/util/retry"
)
// TestAddons tests addons that require no special environment -- in parallel
func TestAddons(t *testing.T) {
MaybeParallel(t)
profile := UniqueProfileName("addons")
ctx, cancel := context.WithTimeout(context.Background(), 40*time.Minute)
defer CleanupWithLogs(t, profile, cancel)
args := append([]string{"start", "-p", profile, "--wait=false", "--memory=2600"}, StartArgs()...)
rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
if err != nil {
t.Fatalf("%s failed: %v", rr.Args, err)
}
// Parallelized tests
t.Run("parallel", func(t *testing.T) {
tests := []struct {
name string
validator validateFunc
}{
{"Registry", validateRegistryAddon},
{"Ingress", validateIngressAddon},
}
for _, tc := range tests {
tc := tc
t.Run(tc.name, func(t *testing.T) {
MaybeParallel(t)
tc.validator(ctx, t, profile)
})
}
})
}
func validateIngressAddon(ctx context.Context, t *testing.T, profile string) {
if NoneDriver() {
t.Skipf("skipping: ssh unsupported by none")
}
rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "addons", "enable", "ingress"))
if err != nil {
t.Fatalf("%s failed: %v", rr.Args, err)
}
client, err := kapi.Client(profile)
if err != nil {
t.Fatalf("kubernetes client: %v", client)
}
if err := kapi.WaitForDeploymentToStabilize(client, "kube-system", "nginx-ingress-controller", time.Minute*5); err != nil {
t.Errorf("waiting for ingress-controller deployment to stabilize: %v", err)
}
if _, err := PodWait(ctx, t, profile, "kube-system", "app.kubernetes.io/name=nginx-ingress-controller", 3*time.Minute); err != nil {
t.Fatalf("wait: %v", err)
}
rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "replace", "--force", "-f", filepath.Join(*testdataDir, "nginx-ing.yaml")))
if err != nil {
t.Errorf("%s failed: %v", rr.Args, err)
}
rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "replace", "--force", "-f", filepath.Join(*testdataDir, "nginx-pod-svc.yaml")))
if err != nil {
t.Errorf("%s failed: %v", rr.Args, err)
}
if _, err := PodWait(ctx, t, profile, "default", "run=nginx", 2*time.Minute); err != nil {
t.Fatalf("wait: %v", err)
}
if err := kapi.WaitForService(client, "default", "nginx", true, time.Millisecond*500, time.Minute*10); err != nil {
t.Errorf("Error waiting for nginx service to be up")
}
want := "Welcome to nginx!"
checkIngress := func() error {
rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", fmt.Sprintf("curl http://127.0.0.1:80 -H 'Host: nginx.example.com'")))
if err != nil {
return err
}
if rr.Stderr.String() != "" {
t.Logf("%v: unexpected stderr: %s", rr.Args, rr.Stderr)
}
if !strings.Contains(rr.Stdout.String(), want) {
return fmt.Errorf("%v stdout = %q, want %q", rr.Args, rr.Stdout, want)
}
return nil
}
if err := retry.Expo(checkIngress, 500*time.Millisecond, time.Minute); err != nil {
t.Errorf("ingress never responded as expected on 127.0.0.1:80: %v", err)
}
rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "addons", "disable", "ingress", "--alsologtostderr", "-v=1"))
if err != nil {
t.Errorf("%s failed: %v", rr.Args, err)
}
}
func validateRegistryAddon(ctx context.Context, t *testing.T, profile string) {
rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "addons", "enable", "registry"))
if err != nil {
t.Fatalf("%s failed: %v", rr.Args, err)
}
client, err := kapi.Client(profile)
if err != nil {
t.Fatalf("kubernetes client: %v", client)
}
start := time.Now()
if err := kapi.WaitForRCToStabilize(client, "kube-system", "registry", 3*time.Minute); err != nil {
t.Errorf("waiting for registry replicacontroller to stabilize: %v", err)
}
t.Logf("registry stabilized in %s", time.Since(start))
if _, err := PodWait(ctx, t, profile, "kube-system", "actual-registry=true", 3*time.Minute); err != nil {
t.Fatalf("wait: %v", err)
}
if _, err := PodWait(ctx, t, profile, "kube-system", "registry-proxy=true", 3*time.Minute); err != nil {
t.Fatalf("wait: %v", err)
}
// Test from inside the cluster (no curl available on busybox)
rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "delete", "po", "-l", "run=registry-test", "--now"))
if err != nil {
t.Logf("pre-cleanup %s failed: %v (not a problem)", rr.Args, err)
}
rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "run", "--rm", "registry-test", "--restart=Never", "--image=busybox", "-it", "--", "sh", "-c", "wget --spider -S http://registry.kube-system.svc.cluster.local"))
if err != nil {
t.Errorf("%s failed: %v", rr.Args, err)
}
want := "HTTP/1.1 200"
if !strings.Contains(rr.Stdout.String(), want) {
t.Errorf("curl = %q, want *%s*", rr.Stdout.String(), want)
}
// Test from outside the cluster
rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ip"))
if err != nil {
t.Fatalf("%s failed: %v", rr.Args, err)
}
if rr.Stderr.String() != "" {
t.Errorf("%s: unexpected stderr: %s", rr.Args, rr.Stderr)
}
endpoint := fmt.Sprintf("http://%s:%d", strings.TrimSpace(rr.Stdout.String()), 5000)
u, err := url.Parse(endpoint)
if err != nil {
t.Fatalf("failed to parse %q: %v", endpoint, err)
}
checkExternalAccess := func() error {
resp, err := retryablehttp.Get(u.String())
if err != nil {
return err
}
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("%s = status code %d, want %d", u, resp.StatusCode, http.StatusOK)
}
return nil
}
if err := retry.Expo(checkExternalAccess, 500*time.Millisecond, 2*time.Minute); err != nil {
t.Errorf(err.Error())
}
rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "addons", "disable", "registry", "--alsologtostderr", "-v=1"))
if err != nil {
t.Errorf("%s failed: %v", rr.Args, err)
}
}

83
test/integration/config_test.go
View File

@ -1,83 +0,0 @@
// +build integration
/*
Copyright 2019 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package integration
import (
"strings"
"testing"
)
func TestConfig(t *testing.T) {
t.Parallel()
p := profileName(t)
mk := NewMinikubeRunner(t, p, "--wait=false")
tests := []struct {
cmd string
stdout string
stderr string
}{
{
cmd: "config unset cpus",
stdout: "",
stderr: "",
},
{
cmd: "config get cpus",
stdout: "",
stderr: "Error: specified key could not be found in config",
},
{
cmd: "config set cpus 2",
stdout: "! These changes will take effect upon a minikube delete and then a minikube start",
stderr: "",
},
{
cmd: "config get cpus",
stdout: "2",
stderr: "",
},
{
cmd: "config unset cpus",
stdout: "",
stderr: ""},
{
cmd: "config get cpus",
stdout: "",
stderr: "Error: specified key could not be found in config",
},
}
for _, tc := range tests {
stdout, stderr, err := mk.RunCommand(tc.cmd, false)
if err != nil {
t.Logf("error running config test command (might be okay): %v ", err)
}
if !compare(tc.stdout, stdout) {
t.Fatalf("Expected stdout to be: %q. Stdout was: %q Stderr: %q", tc.stdout, stdout, stderr)
}
if !compare(tc.stderr, stderr) {
t.Fatalf("Expected stderr to be: %q. Stdout was: %s Stderr: %q", tc.stderr, stdout, stderr)
}
}
}
func compare(s1, s2 string) bool {
return strings.TrimSpace(s1) == strings.TrimSpace(s2)
}

190
test/integration/containerd_test.go
View File

@ -1,190 +0,0 @@
// +build integration
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package integration
import (
"path/filepath"
"testing"
"github.com/docker/machine/libmachine/state"
"github.com/pkg/errors"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/minikube/pkg/kapi"
"k8s.io/minikube/test/integration/util"
)
// TestGvisorWorkload tests the gVisor addon for pods with the untrusted
// workload annotation and gvisor runtime class.
func TestGvisorWorkload(t *testing.T) {
if isTestNoneDriver(t) {
t.Skip("Can't run containerd backend with none driver")
}
p := profileName(t)
mk := NewMinikubeRunner(t, p, "--wait=false")
defer mk.TearDown(t)
mk.MustStart("--container-runtime=containerd", "--docker-opt containerd=/var/run/containerd/containerd.sock")
mk.MustRun("cache add gcr.io/k8s-minikube/gvisor-addon:latest")
// NOTE: addons are enabled globally.
mk.MustRun("addons enable gvisor")
t.Log("waiting for gvisor controller to come up")
if err := waitForGvisorControllerRunning(p); err != nil {
t.Fatalf("waiting for gvisor controller to be up: %v", err)
}
// Test a pod with the untrusted workload annotation.
createUntrustedWorkload(t, p)
t.Log("making sure untrusted workload is Running")
if err := waitForUntrustedNginxRunning(p); err != nil {
t.Fatalf("waiting for untrusted-workload nginx to be up: %v", err)
}
deleteUntrustedWorkload(t, p)
// Test a pod with the gvisor runtime class.
createGvisorWorkload(t, p)
t.Log("making sure gvisor workload is Running")
if err := waitForGvisorNginxRunning(p); err != nil {
t.Fatalf("waiting for gvisor nginx to be up: %v", err)
}
deleteGvisorWorkload(t, p)
}
// TestGvisorRestart tests the gVisor addon is functional after a VM restart.
func TestGvisorRestart(t *testing.T) {
if isTestNoneDriver(t) {
t.Skip("Can't run containerd backend with none driver")
}
p := profileName(t)
mk := NewMinikubeRunner(t, p, "--wait=false")
defer mk.TearDown(t)
mk.MustStart("--container-runtime=containerd", "--docker-opt containerd=/var/run/containerd/containerd.sock")
mk.MustRun("cache add gcr.io/k8s-minikube/gvisor-addon:latest")
// NOTE: addons are enabled globally.
mk.MustRun("addons enable gvisor")
t.Log("waiting for gvisor controller to come up")
if err := waitForGvisorControllerRunning(p); err != nil {
t.Errorf("waiting for gvisor controller to be up: %v", err)
}
createUntrustedWorkload(t, p)
t.Log("making sure untrusted workload is Running")
if err := waitForUntrustedNginxRunning(p); err != nil {
t.Errorf("waiting for nginx to be up: %v", err)
}
deleteUntrustedWorkload(t, p)
mk.MustRun("delete")
mk.MustStart("--container-runtime=containerd", "--docker-opt containerd=/var/run/containerd/containerd.sock")
mk.CheckStatus(state.Running.String())
t.Log("waiting for gvisor controller to come up")
if err := waitForGvisorControllerRunning(p); err != nil {
t.Errorf("waiting for gvisor controller to be up: %v", err)
}
createUntrustedWorkload(t, p)
t.Log("making sure untrusted workload is Running")
if err := waitForUntrustedNginxRunning(p); err != nil {
t.Errorf("waiting for nginx to be up: %v", err)
}
deleteUntrustedWorkload(t, p)
}
func createUntrustedWorkload(t *testing.T, profile string) {
kr := util.NewKubectlRunner(t, profile)
untrustedPath := filepath.Join(*testdataDir, "nginx-untrusted.yaml")
t.Log("creating pod with untrusted workload annotation")
if _, err := kr.RunCommand([]string{"replace", "-f", untrustedPath, "--force"}); err != nil {
t.Fatalf("creating untrusted nginx resource: %v", err)
}
}
func deleteUntrustedWorkload(t *testing.T, profile string) {
kr := util.NewKubectlRunner(t, profile)
untrustedPath := filepath.Join(*testdataDir, "nginx-untrusted.yaml")
if _, err := kr.RunCommand([]string{"delete", "-f", untrustedPath}); err != nil {
t.Logf("error deleting untrusted nginx resource: %v", err)
}
}
func createGvisorWorkload(t *testing.T, profile string) {
kr := util.NewKubectlRunner(t, profile)
gvisorPath := filepath.Join(*testdataDir, "nginx-gvisor.yaml")
t.Log("creating pod with gvisor workload annotation")
if _, err := kr.RunCommand([]string{"replace", "-f", gvisorPath, "--force"}); err != nil {
t.Fatalf("creating gvisor nginx resource: %v", err)
}
}
func deleteGvisorWorkload(t *testing.T, profile string) {
kr := util.NewKubectlRunner(t, profile)
gvisorPath := filepath.Join(*testdataDir, "nginx-gvisor.yaml")
if _, err := kr.RunCommand([]string{"delete", "-f", gvisorPath}); err != nil {
t.Logf("error deleting gvisor nginx resource: %v", err)
}
}
// waitForGvisorControllerRunning waits for the gvisor controller pod to be running.
func waitForGvisorControllerRunning(p string) error {
client, err := kapi.Client(p)
if err != nil {
return errors.Wrap(err, "getting kubernetes client")
}
selector := labels.SelectorFromSet(labels.Set(map[string]string{"kubernetes.io/minikube-addons": "gvisor"}))
if err := kapi.WaitForPodsWithLabelRunning(client, "kube-system", selector); err != nil {
return errors.Wrap(err, "waiting for gvisor controller pod to stabilize")
}
return nil
}
// waitForUntrustedNginxRunning waits for the untrusted nginx pod to start
// running.
func waitForUntrustedNginxRunning(miniProfile string) error {
client, err := kapi.Client(miniProfile)
if err != nil {
return errors.Wrap(err, "getting kubernetes client")
}
selector := labels.SelectorFromSet(labels.Set(map[string]string{"run": "nginx", "untrusted": "true"}))
if err := kapi.WaitForPodsWithLabelRunning(client, "default", selector); err != nil {
return errors.Wrap(err, "waiting for nginx pods")
}
return nil
}
// waitForGvisorNginxRunning waits for the nginx pod with gvisor runtime class
// to start running.
func waitForGvisorNginxRunning(miniProfile string) error {
client, err := kapi.Client(miniProfile)
if err != nil {
return errors.Wrap(err, "getting kubernetes client")
}
selector := labels.SelectorFromSet(labels.Set(map[string]string{"run": "nginx", "runtime": "gvisor"}))
if err := kapi.WaitForPodsWithLabelRunning(client, "default", selector); err != nil {
return errors.Wrap(err, "waiting for nginx pods")
}
return nil
}

52
test/integration/docker_test.go
View File

@ -20,54 +20,46 @@ package integration
import (
"context"
"os/exec"
"strings"
"testing"
"time"
"github.com/docker/machine/libmachine/state"
)
func TestDocker(t *testing.T) {
if isTestNoneDriver(t) {
t.Skip("skipping test as none driver does not bundle docker")
func TestDockerFlags(t *testing.T) {
if NoneDriver() {
t.Skip("skipping: none driver does not support ssh or bundle docker")
}
p := profileName(t)
if shouldRunInParallel(t) {
t.Parallel()
}
mk := NewMinikubeRunner(t, p, "--wait=false")
defer mk.TearDown(t)
MaybeParallel(t)
// Start a timer for all remaining commands, to display failure output before a panic.
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
defer cancel()
profile := UniqueProfileName("docker-flags")
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute)
defer Cleanup(t, profile, cancel)
if _, _, err := mk.RunWithContext(ctx, "delete"); err != nil {
t.Logf("pre-delete failed (probably ok): %v", err)
}
mk.MustStart("--docker-env=FOO=BAR", "--docker-env=BAZ=BAT", "--docker-opt=debug", " --docker-opt=icc=true")
mk.CheckStatus(state.Running.String())
stdout, stderr, err := mk.RunWithContext(ctx, "ssh -- systemctl show docker --property=Environment --no-pager")
args := append([]string{"start", "-p", profile, "--wait=false", "--docker-env=FOO=BAR", "--docker-env=BAZ=BAT", "--docker-opt=debug", "--docker-opt=icc=true", "--alsologtostderr", "-v=8"}, StartArgs()...)
rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
if err != nil {
t.Errorf("docker env: %v\nstderr: %s", err, stderr)
t.Errorf("%s failed: %v", rr.Args, err)
}
rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", "systemctl show docker --property=Environment --no-pager"))
if err != nil {
t.Errorf("%s failed: %v", rr.Args, err)
}
for _, envVar := range []string{"FOO=BAR", "BAZ=BAT"} {
if !strings.Contains(stdout, envVar) {
t.Errorf("Env var %s missing: %s.", envVar, stdout)
if !strings.Contains(rr.Stdout.String(), envVar) {
t.Errorf("env var %s missing: %s.", envVar, rr.Stdout)
}
}
stdout, stderr, err = mk.RunWithContext(ctx, "ssh -- systemctl show docker --property=ExecStart --no-pager")
rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", "systemctl show docker --property=ExecStart --no-pager"))
if err != nil {
t.Errorf("ssh show docker: %v\nstderr: %s", err, stderr)
t.Errorf("%s failed: %v", rr.Args, err)
}
for _, opt := range []string{"--debug", "--icc=true"} {
if !strings.Contains(stdout, opt) {
t.Fatalf("Option %s missing from ExecStart: %s.", opt, stdout)
if !strings.Contains(rr.Stdout.String(), opt) {
t.Fatalf("%s = %q, want *%s*", rr.Command(), rr.Stdout, opt)
}
}
}

3
test/integration/driver_install_or_update_test.go
View File

@ -30,13 +30,14 @@ import (
)
func TestDriverInstallOrUpdate(t *testing.T) {
if isTestNoneDriver(t) {
if NoneDriver() {
t.Skip("Skip none driver.")
}
if runtime.GOOS != "linux" {
t.Skip("Skip if not linux.")
}
MaybeParallel(t)
tests := []struct {
name string

87
test/integration/flags.go
View File

@ -1,87 +0,0 @@
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package integration
import (
"flag"
"os"
"strings"
"testing"
"time"
"k8s.io/minikube/test/integration/util"
)
// TestMain is the test main
func TestMain(m *testing.M) {
flag.Parse()
os.Exit(m.Run())
}
var startTimeout = flag.Duration("timeout", 25*time.Minute, "max duration to wait for a full minikube start")
var binaryPath = flag.String("binary", "../../out/minikube", "path to minikube binary")
var globalArgs = flag.String("minikube-args", "", "Arguments to pass to minikube")
var startArgs = flag.String("minikube-start-args", "", "Arguments to pass to minikube start")
var mountArgs = flag.String("minikube-mount-args", "", "Arguments to pass to minikube mount")
var testdataDir = flag.String("testdata-dir", "testdata", "the directory relative to test/integration where the testdata lives")
var parallel = flag.Bool("parallel", true, "run the tests in parallel, set false for run sequentially")
// NewMinikubeRunner creates a new MinikubeRunner
func NewMinikubeRunner(t *testing.T, profile string, extraStartArgs ...string) util.MinikubeRunner {
return util.MinikubeRunner{
Profile: profile,
BinaryPath: *binaryPath,
StartArgs: *startArgs + " --wait-timeout=13m " + strings.Join(extraStartArgs, " "), // adding timeout per component
GlobalArgs: *globalArgs,
MountArgs: *mountArgs,
TimeOutStart: *startTimeout, // timeout for all start
T: t,
}
}
// isTestNoneDriver checks if the current test is for none driver
func isTestNoneDriver(t *testing.T) bool {
t.Helper()
return strings.Contains(*startArgs, "--vm-driver=none")
}
// profileName chooses a profile name based on the test name
// to be used in minikube and kubecontext across that test
func profileName(t *testing.T) string {
t.Helper()
if isTestNoneDriver(t) {
return "minikube"
}
p := strings.Split(t.Name(), "/")[0] // for i.e, TestFunctional/SSH returns TestFunctional
if p == "TestFunctional" {
return "minikube"
}
return p
}
// shouldRunInParallel determines if test should run in parallel or not
func shouldRunInParallel(t *testing.T) bool {
t.Helper()
if !*parallel {
return false
}
if isTestNoneDriver(t) {
return false
}
p := strings.Split(t.Name(), "/")[0] // for i.e, TestFunctional/SSH returns TestFunctional
return p != "TestFunctional" // gosimple lint: https://staticcheck.io/docs/checks#S1008
}

301
test/integration/fn_addons.go
View File

@ -1,301 +0,0 @@
// +build integration
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package integration
import (
"bufio"
"fmt"
"io/ioutil"
"net"
"net/http"
"net/url"
"path/filepath"
"strings"
"testing"
"time"
"github.com/pkg/errors"
retryablehttp "github.com/hashicorp/go-retryablehttp"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/minikube/pkg/kapi"
"k8s.io/minikube/pkg/util/retry"
"k8s.io/minikube/test/integration/util"
)
func testAddons(t *testing.T) {
t.Parallel()
p := profileName(t)
client, err := kapi.Client(p)
if err != nil {
t.Fatalf("Could not get kubernetes client: %v", err)
}
selector := labels.SelectorFromSet(labels.Set(map[string]string{"component": "kube-addon-manager"}))
if err := kapi.WaitForPodsWithLabelRunning(client, "kube-system", selector); err != nil {
t.Errorf("Error waiting for addon manager to be up")
}
}
func readLineWithTimeout(b *bufio.Reader, timeout time.Duration) (string, error) {
s := make(chan string)
e := make(chan error)
go func() {
read, err := b.ReadString('\n')
if err != nil {
e <- err
} else {
s <- read
}
close(s)
close(e)
}()
select {
case line := <-s:
return line, nil
case err := <-e:
return "", err
case <-time.After(timeout):
return "", fmt.Errorf("timeout after %s", timeout)
}
}
func testDashboard(t *testing.T) {
t.Parallel()
p := profileName(t)
mk := NewMinikubeRunner(t, p, "--wait=false")
cmd, out := mk.RunDaemon("dashboard --url")
defer func() {
err := util.KillProcess(cmd.Process.Pid, t)
if err != nil {
t.Logf("Failed to kill dashboard command: %v", err)
}
}()
s, err := readLineWithTimeout(out, 240*time.Second)
if err != nil {
t.Fatalf("failed to read url: %v", err)
}
u, err := url.Parse(strings.TrimSpace(s))
if err != nil {
t.Fatalf("failed to parse %q: %v", s, err)
}
if u.Scheme != "http" {
t.Errorf("got Scheme %s, expected http", u.Scheme)
}
host, _, err := net.SplitHostPort(u.Host)
if err != nil {
t.Fatalf("failed SplitHostPort: %v", err)
}
if host != "127.0.0.1" {
t.Errorf("got host %s, expected 127.0.0.1", host)
}
resp, err := retryablehttp.Get(u.String())
if err != nil {
t.Fatalf("failed get: %v", err)
}
if resp.StatusCode != http.StatusOK {
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
t.Fatalf("Unable to read http response body: %v", err)
}
t.Errorf("%s returned status code %d, expected %d.\nbody:\n%s", u, resp.StatusCode, http.StatusOK, body)
}
}
func testIngressController(t *testing.T) {
t.Parallel()
p := profileName(t)
mk := NewMinikubeRunner(t, p, "--wait=false")
kr := util.NewKubectlRunner(t, p)
mk.MustRun("addons enable ingress")
if err := waitForIngressControllerRunning(p); err != nil {
t.Fatalf("Failed waiting for ingress-controller to be up: %v", err)
}
ingressPath := filepath.Join(*testdataDir, "nginx-ing.yaml")
if _, err := kr.RunCommand([]string{"create", "-f", ingressPath}); err != nil {
t.Fatalf("Failed creating nginx ingress resource: %v", err)
}
podPath := filepath.Join(*testdataDir, "nginx-pod-svc.yaml")
if _, err := kr.RunCommand([]string{"create", "-f", podPath}); err != nil {
t.Fatalf("Failed creating nginx ingress resource: %v", err)
}
if err := waitForNginxRunning(t, p); err != nil {
t.Fatalf("Failed waiting for nginx to be up: %v", err)
}
checkIngress := func() error {
expectedStr := "Welcome to nginx!"
runCmd := fmt.Sprintf("curl http://127.0.0.1:80 -H 'Host: nginx.example.com'")
sshCmdOutput, _ := mk.SSH(runCmd)
if !strings.Contains(sshCmdOutput, expectedStr) {
return fmt.Errorf("ExpectedStr sshCmdOutput to be: %s. Output was: %s", expectedStr, sshCmdOutput)
}
return nil
}
if err := retry.Expo(checkIngress, 500*time.Millisecond, time.Minute); err != nil {
t.Fatalf(err.Error())
}
defer func() {
for _, p := range []string{podPath, ingressPath} {
if out, err := kr.RunCommand([]string{"delete", "-f", p}); err != nil {
t.Logf("delete -f %s failed: %v\noutput: %s\n", p, err, out)
}
}
}()
mk.MustRun("addons disable ingress")
}
func testServicesList(t *testing.T) {
t.Parallel()
p := profileName(t)
mk := NewMinikubeRunner(t, p)
checkServices := func() error {
output, stderr, err := mk.RunCommand("service list", false)
if err != nil {
return err
}
if !strings.Contains(output, "kubernetes") {
return fmt.Errorf("error, kubernetes service missing from output: %s, \n stderr: %s", output, stderr)
}
return nil
}
if err := retry.Expo(checkServices, 500*time.Millisecond, time.Minute); err != nil {
t.Fatalf(err.Error())
}
}
func testRegistry(t *testing.T) {
t.Parallel()
p := profileName(t)
mk := NewMinikubeRunner(t, p)
mk.MustRun("addons enable registry")
client, err := kapi.Client(p)
if err != nil {
t.Fatalf("getting kubernetes client: %v", err)
}
if err := kapi.WaitForRCToStabilize(client, "kube-system", "registry", time.Minute*5); err != nil {
t.Fatalf("waiting for registry replicacontroller to stabilize: %v", err)
}
rs := labels.SelectorFromSet(labels.Set(map[string]string{"actual-registry": "true"}))
if err := kapi.WaitForPodsWithLabelRunning(client, "kube-system", rs); err != nil {
t.Fatalf("waiting for registry pods: %v", err)
}
ps := labels.SelectorFromSet(labels.Set(map[string]string{"registry-proxy": "true"}))
if err := kapi.WaitForPodsWithLabelRunning(client, "kube-system", ps); err != nil {
t.Fatalf("waiting for registry-proxy pods: %v", err)
}
ip, stderr := mk.MustRun("ip")
ip = strings.TrimSpace(ip)
endpoint := fmt.Sprintf("http://%s:%d", ip, 5000)
u, err := url.Parse(endpoint)
if err != nil {
t.Fatalf("failed to parse %q: %v stderr : %s", endpoint, err, stderr)
}
t.Log("checking registry access from outside cluster")
// Check access from outside the cluster on port 5000, validing connectivity via registry-proxy
checkExternalAccess := func() error {
resp, err := retryablehttp.Get(u.String())
if err != nil {
t.Errorf("failed get: %v", err)
}
if resp.StatusCode != http.StatusOK {
t.Errorf("%s returned status code %d, expected %d.\n", u, resp.StatusCode, http.StatusOK)
}
return nil
}
if err := retry.Expo(checkExternalAccess, 500*time.Millisecond, 2*time.Minute); err != nil {
t.Fatalf(err.Error())
}
t.Log("checking registry access from inside cluster")
kr := util.NewKubectlRunner(t, p)
// TODO: Fix this
out, _ := kr.RunCommand([]string{
"run",
"registry-test",
"--restart=Never",
"--image=busybox",
"-it",
"--",
"sh",
"-c",
"wget --spider -S 'http://registry.kube-system.svc.cluster.local' 2>&1 | grep 'HTTP/' | awk '{print $2}'"})
internalCheckOutput := string(out)
expectedStr := "200"
if !strings.Contains(internalCheckOutput, expectedStr) {
t.Errorf("ExpectedStr internalCheckOutput to be: %s. Output was: %s", expectedStr, internalCheckOutput)
}
defer func() {
if _, err := kr.RunCommand([]string{"delete", "pod", "registry-test"}); err != nil {
t.Errorf("failed to delete pod registry-test")
}
}()
mk.MustRun("addons disable registry")
}
// waitForNginxRunning waits for nginx service to be up
func waitForNginxRunning(t *testing.T, miniProfile string) error {
client, err := kapi.Client(miniProfile)
if err != nil {
return errors.Wrap(err, "getting kubernetes client")
}
selector := labels.SelectorFromSet(labels.Set(map[string]string{"run": "nginx"}))
if err := kapi.WaitForPodsWithLabelRunning(client, "default", selector); err != nil {
return errors.Wrap(err, "waiting for nginx pods")
}
if err := kapi.WaitForService(client, "default", "nginx", true, time.Millisecond*500, time.Minute*10); err != nil {
t.Errorf("Error waiting for nginx service to be up")
}
return nil
}
// waitForIngressControllerRunning waits until ingress controller pod to be running
func waitForIngressControllerRunning(miniProfile string) error {
client, err := kapi.Client(miniProfile)
if err != nil {
return errors.Wrap(err, "getting kubernetes client")
}
if err := kapi.WaitForDeploymentToStabilize(client, "kube-system", "nginx-ingress-controller", time.Minute*10); err != nil {
return errors.Wrap(err, "waiting for ingress-controller deployment to stabilize")
}
selector := labels.SelectorFromSet(labels.Set(map[string]string{"app.kubernetes.io/name": "nginx-ingress-controller"}))
if err := kapi.WaitForPodsWithLabelRunning(client, "kube-system", selector); err != nil {
return errors.Wrap(err, "waiting for ingress-controller pods")
}
return nil
}

84
test/integration/fn_cluster_dns.go
View File

@ -1,84 +0,0 @@
// +build integration
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package integration
import (
"bytes"
"path/filepath"
"testing"
"time"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/minikube/pkg/kapi"
"k8s.io/minikube/pkg/util/retry"
"k8s.io/minikube/test/integration/util"
)
func testClusterDNS(t *testing.T) {
t.Parallel()
p := profileName(t)
client, err := kapi.Client(p)
if err != nil {
t.Fatalf("Error getting kubernetes client %v", err)
}
kr := util.NewKubectlRunner(t, p)
busybox := busyBoxPod(t, client, kr, p)
defer func() {
if _, err := kr.RunCommand([]string{"delete", "po", busybox}); err != nil {
t.Errorf("delete failed: %v", err)
}
}()
out := []byte{}
nslookup := func() error {
out, err = kr.RunCommand([]string{"exec", busybox, "nslookup", "kubernetes.default"})
return err
}
if err = retry.Expo(nslookup, 500*time.Millisecond, time.Minute); err != nil {
t.Fatalf(err.Error())
}
clusterIP := []byte("10.96.0.1")
if !bytes.Contains(out, clusterIP) {
t.Errorf("output did not contain expected IP:\n%s", out)
}
}
func busyBoxPod(t *testing.T, c kubernetes.Interface, kr *util.KubectlRunner, profile string) string {
if _, err := kr.RunCommand([]string{"create", "-f", filepath.Join(*testdataDir, "busybox.yaml")}); err != nil {
t.Fatalf("creating busybox pod: %s", err)
}
// TODO(tstromberg): Refactor WaitForBusyboxRunning to return name of pod.
if err := util.WaitForBusyboxRunning(t, "default", profile); err != nil {
t.Fatalf("Waiting for busybox pod to be up: %v", err)
}
pods, err := c.CoreV1().Pods("default").List(metav1.ListOptions{LabelSelector: "integration-test=busybox"})
if err != nil {
t.Fatalf("list error: %v", err)
}
if len(pods.Items) == 0 {
t.Fatal("Expected a busybox pod to be running")
}
return pods.Items[0].Name
}

67
test/integration/fn_cluster_env.go
View File

@ -1,67 +0,0 @@
// +build integration
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package integration
import (
"os"
"os/exec"
"testing"
"time"
"k8s.io/minikube/pkg/util/retry"
)
// Assert that docker-env subcommand outputs usable information for "docker ps"
func testClusterEnv(t *testing.T) {
t.Parallel()
p := profileName(t)
mk := NewMinikubeRunner(t, p, "--wait=false")
// Set a specific shell syntax so that we don't have to handle every possible user shell
envOut, stderr := mk.MustRun("docker-env --shell=bash")
vars := mk.ParseEnvCmdOutput(envOut)
if len(vars) == 0 {
t.Fatalf("Failed to parse env vars:\n%s, \n stderr: %s ", envOut, stderr)
}
for k, v := range vars {
t.Logf("Found: %s=%s", k, v)
if err := os.Setenv(k, v); err != nil {
t.Errorf("failed to set %s=%s: %v", k, v, err)
}
}
path, err := exec.LookPath("docker")
if err != nil {
t.Fatalf("Unable to complete test: docker is not installed in PATH")
}
t.Logf("Using docker installed at %s", path)
var output []byte
dockerPs := func() error {
cmd := exec.Command(path, "ps")
output, err = cmd.CombinedOutput()
if err != nil {
return err
}
return nil
}
if err := retry.Expo(dockerPs, 500*time.Millisecond, time.Minute); err != nil {
t.Fatalf("Error running command: %s. Error: %v Output: %s", "docker ps", err, output)
}
}

39
test/integration/fn_cluster_logs.go
View File

@ -1,39 +0,0 @@
// +build integration
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package integration
import (
"strings"
"testing"
)
func testClusterLogs(t *testing.T) {
t.Parallel()
p := profileName(t)
mk := NewMinikubeRunner(t, p)
logsCmdStdout, _ := mk.GetLogs()
// check for # of lines or check for strings
logWords := []string{"minikube", ".go"}
for _, logWord := range logWords {
if !strings.Contains(logsCmdStdout, logWord) {
t.Fatalf("Error in logsCmdOutput, expected to find: %s. Output: %s", logWord, logsCmdStdout)
}
}
}

35
test/integration/fn_cluster_ssh.go
View File

@ -1,35 +0,0 @@
// +build integration
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package integration
import (
"strings"
"testing"
)
func testClusterSSH(t *testing.T) {
t.Parallel()
p := profileName(t)
mk := NewMinikubeRunner(t, p, "--wait=false")
expectedStr := "hello"
sshCmdOutput, stderr := mk.MustRun("ssh echo " + expectedStr)
if !strings.Contains(sshCmdOutput, expectedStr) {
t.Fatalf("ExpectedStr sshCmdOutput to be: %s. Output was: %s Stderr: %s", expectedStr, sshCmdOutput, stderr)
}
}

61
test/integration/fn_cluster_status.go
View File

@ -1,61 +0,0 @@
// +build integration
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package integration
import (
"fmt"
"testing"
"time"
api "k8s.io/kubernetes/pkg/apis/core"
"k8s.io/minikube/pkg/util/retry"
"k8s.io/minikube/test/integration/util"
)
func testClusterStatus(t *testing.T) {
p := profileName(t)
kr := util.NewKubectlRunner(t, p)
cs := api.ComponentStatusList{}
healthy := func() error {
t.Log("Checking if cluster is healthy.")
if err := kr.RunCommandParseOutput([]string{"get", "cs"}, &cs); err != nil {
return err
}
for _, i := range cs.Items {
status := api.ConditionFalse
for _, c := range i.Conditions {
if c.Type != api.ComponentHealthy {
continue
}
status = c.Status
}
if status != api.ConditionTrue {
err := fmt.Errorf("component %s is not Healthy! Status: %s", i.GetName(), status)
t.Logf("Retrying, %v", err)
return err
}
}
return nil
}
if err := retry.Expo(healthy, 500*time.Millisecond, time.Minute); err != nil {
t.Errorf("Cluster is not healthy: %v", err)
}
}

198
test/integration/fn_mount.go
View File

@ -1,198 +0,0 @@
// +build integration
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package integration
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/minikube/pkg/kapi"
"k8s.io/minikube/pkg/util/lock"
"k8s.io/minikube/pkg/util/retry"
"k8s.io/minikube/test/integration/util"
)
func testMounting(t *testing.T) {
if runtime.GOOS == "darwin" {
t.Skip("mount tests disabled in darwin due to timeout (issue#3200)")
}
if isTestNoneDriver(t) {
t.Skip("skipping test for none driver as it does not need mount")
}
t.Parallel()
p := profileName(t)
mk := NewMinikubeRunner(t, p, "--wait=false")
tempDir, err := ioutil.TempDir("", "mounttest")
if err != nil {
t.Fatalf("Unexpected error while creating tempDir: %v", err)
}
defer os.RemoveAll(tempDir)
mountCmd := getMountCmd(mk, tempDir)
cmd, _, _ := mk.RunDaemon2(mountCmd)
defer func() {
err = util.KillProcess(cmd.Process.Pid, t)
if err != nil {
t.Logf("Failed to kill mount command: %v", err)
}
}()
kr := util.NewKubectlRunner(t, p)
podName := "busybox-mount"
podPath := filepath.Join(*testdataDir, "busybox-mount-test.yaml")
// Write file in mounted dir from host
expected := "test\n"
if err := writeFilesFromHost(tempDir, []string{"fromhost", "fromhostremove"}, expected); err != nil {
t.Fatalf(err.Error())
}
// Create the pods we need outside the main test loop.
setupTest := func() error {
t.Logf("Deploying pod from: %s", podPath)
if _, err := kr.RunCommand([]string{"create", "-f", podPath}); err != nil {
return err
}
return nil
}
defer func() {
t.Logf("Deleting pod from: %s", podPath)
if out, err := kr.RunCommand([]string{"delete", "-f", podPath}); err != nil {
t.Logf("delete -f %s failed: %v\noutput: %s\n", podPath, err, out)
}
}()
if err = retry.Expo(setupTest, 500*time.Millisecond, 4*time.Minute); err != nil {
t.Fatal("mountTest failed with error:", err)
}
if err := waitForPods(map[string]string{"integration-test": "busybox-mount"}, p); err != nil {
t.Fatalf("Error waiting for busybox mount pod to be up: %v", err)
}
t.Logf("Pods appear to be running")
mountTest := func() error {
if err := verifyFiles(mk, kr, tempDir, podName, expected); err != nil {
t.Fatalf(err.Error())
}
return nil
}
if err = retry.Expo(mountTest, 500*time.Millisecond, 4*time.Minute); err != nil {
t.Fatalf("mountTest failed with error: %v", err)
}
}
func getMountCmd(mk util.MinikubeRunner, mountDir string) string {
var mountCmd string
if len(mk.MountArgs) > 0 {
mountCmd = fmt.Sprintf("mount %s %s:/mount-9p", mk.MountArgs, mountDir)
} else {
mountCmd = fmt.Sprintf("mount %s:/mount-9p", mountDir)
}
return mountCmd
}
func writeFilesFromHost(mountedDir string, files []string, content string) error {
for _, file := range files {
path := filepath.Join(mountedDir, file)
err := lock.WriteFile(path, []byte(content), 0644)
if err != nil {
return fmt.Errorf("unexpected error while writing file %s: %v", path, err)
}
}
return nil
}
func waitForPods(s map[string]string, profile string) error {
client, err := kapi.Client(profile)
if err != nil {
return fmt.Errorf("getting kubernetes client: %v", err)
}
selector := labels.SelectorFromSet(labels.Set(s))
if err := kapi.WaitForPodsWithLabelRunning(client, "default", selector); err != nil {
return err
}
return nil
}
func verifyFiles(mk util.MinikubeRunner, kr *util.KubectlRunner, tempDir string, podName string, expected string) error {
path := filepath.Join(tempDir, "frompod")
out, err := ioutil.ReadFile(path)
if err != nil {
return err
}
// test that file written from pod can be read from host echo test > /mount-9p/frompod; in pod
if string(out) != expected {
return fmt.Errorf("expected file %s to contain text %q, was %q", path, expected, out)
}
// test that file written from host was read in by the pod via cat /mount-9p/fromhost;
if out, err = kr.RunCommand([]string{"logs", podName}); err != nil {
return err
}
if string(out) != expected {
return fmt.Errorf("expected file %s to contain text %q, was %q", path, expected, out)
}
// test file timestamps are correct
files := []string{"fromhost", "frompod"}
for _, file := range files {
statCmd := fmt.Sprintf("stat /mount-9p/%s", file)
statOutput, err := mk.SSH(statCmd)
if err != nil {
return fmt.Errorf("inable to stat %s via SSH. error %v, %s", file, err, statOutput)
}
if runtime.GOOS == "windows" {
if strings.Contains(statOutput, "Access: 1970-01-01") {
return fmt.Errorf("invalid access time\n%s", statOutput)
}
}
if strings.Contains(statOutput, "Modify: 1970-01-01") {
return fmt.Errorf("invalid modify time\n%s", statOutput)
}
}
// test that fromhostremove was deleted by the pod from the mount via rm /mount-9p/fromhostremove
path = filepath.Join(tempDir, "fromhostremove")
if _, err := os.Stat(path); err == nil {
return fmt.Errorf("expected file %s to be removed", path)
}
// test that frompodremove can be deleted on the host
path = filepath.Join(tempDir, "frompodremove")
if err := os.Remove(path); err != nil {
return fmt.Errorf("unexpected error removing file %s: %v", path, err)
}
return nil
}

195
test/integration/fn_mount_cmd.go Normal file
View File

@ -0,0 +1,195 @@
// +build integration
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package integration
import (
"bytes"
"context"
"fmt"
"io/ioutil"
"os"
"os/exec"
"path"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
"k8s.io/minikube/pkg/util/retry"
)
const (
guestMount = "/mount-9p"
createdByPod = "created-by-pod"
createdByTest = "created-by-test"
createdByTestRemovedByPod = "created-by-test-removed-by-pod"
createdByPodRemovedByTest = "created-by-pod-removed-by-test"
)
func validateMountCmd(ctx context.Context, t *testing.T, profile string) {
if NoneDriver() {
t.Skip("skipping: none driver does not support mount")
}
if HyperVDriver() {
t.Skip("skipping: mount broken on hyperv: https://github.com/kubernetes/minikube/issues/5029")
}
tempDir, err := ioutil.TempDir("", "mounttest")
if err != nil {
t.Fatalf("Unexpected error while creating tempDir: %v", err)
}
ctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
args := []string{"mount", "-p", profile, fmt.Sprintf("%s:%s", tempDir, guestMount), "--alsologtostderr", "-v=1"}
ss, err := Start(t, exec.CommandContext(ctx, Target(), args...))
if err != nil {
t.Fatalf("%v failed: %v", args, err)
}
defer func() {
if t.Failed() {
t.Logf("%s failed, getting debug info...", t.Name())
rr, err := Run(t, exec.Command(Target(), "-p", profile, "ssh", "mount | grep 9p; ls -la /mount-9p; cat /mount-9p/pod-dates"))
if err != nil {
t.Logf("%s: %v", rr.Command(), err)
} else {
t.Logf("(debug) %s:\n%s", rr.Command(), rr.Stdout)
}
}
// Cleanup in advance of future tests
rr, err := Run(t, exec.Command(Target(), "-p", profile, "ssh", "sudo umount -f /mount-9p"))
if err != nil {
t.Logf("%s: %v", rr.Command(), err)
}
ss.Stop(t)
cancel()
if *cleanup {
os.RemoveAll(tempDir)
}
}()
// Write local files
testMarker := fmt.Sprintf("test-%d", time.Now().UnixNano())
wantFromTest := []byte(testMarker)
for _, name := range []string{createdByTest, createdByTestRemovedByPod, testMarker} {
p := filepath.Join(tempDir, name)
err := ioutil.WriteFile(p, wantFromTest, 0644)
t.Logf("wrote %q to %s", wantFromTest, p)
if err != nil {
t.Errorf("WriteFile %s: %v", p, err)
}
}
// Block until the mount succeeds to avoid file race
checkMount := func() error {
_, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", "findmnt -T /mount-9p | grep 9p"))
return err
}
start := time.Now()
if err := retry.Expo(checkMount, time.Second, 15*time.Second); err != nil {
// For local testing, allow macOS users to click prompt. If they don't, skip the test.
if runtime.GOOS == "darwin" {
t.Skip("skipping: mount did not appear, likely because macOS requires prompt to allow non-codesigned binaries to listen on non-localhost port")
}
t.Fatalf("/mount-9p did not appear within %s: %v", time.Since(start), err)
}
// Assert that we can access the mount without an error. Display for debugging.
rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", "--", "ls", "-la", guestMount))
if err != nil {
t.Fatalf("%s failed: %v", rr.Args, err)
}
t.Logf("guest mount directory contents\n%s", rr.Stdout)
// Assert that the mount contains our unique test marker, as opposed to a stale mount
tp := filepath.Join("/mount-9p", testMarker)
rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", "cat", tp))
if err != nil {
t.Fatalf("%s failed: %v", rr.Args, err)
}
if !bytes.Equal(rr.Stdout.Bytes(), wantFromTest) {
// The mount is hosed, exit fast before wasting time launching pods.
t.Fatalf("%s = %q, want %q", tp, rr.Stdout.Bytes(), wantFromTest)
}
// Start the "busybox-mount" pod.
rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "replace", "--force", "-f", filepath.Join(*testdataDir, "busybox-mount-test.yaml")))
if err != nil {
t.Fatalf("%s failed: %v", rr.Args, err)
}
if _, err := PodWait(ctx, t, profile, "default", "integration-test=busybox-mount", 2*time.Minute); err != nil {
t.Fatalf("wait: %v", err)
}
// Read the file written by pod startup
p := filepath.Join(tempDir, createdByPod)
got, err := ioutil.ReadFile(p)
if err != nil {
t.Errorf("readfile %s: %v", p, err)
}
wantFromPod := []byte("test\n")
if !bytes.Equal(got, wantFromPod) {
t.Errorf("%s = %q, want %q", p, got, wantFromPod)
}
// test that file written from host was read in by the pod via cat /mount-9p/written-by-host;
rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "logs", "busybox-mount"))
if err != nil {
t.Errorf("%s failed: %v", rr.Args, err)
}
if !bytes.Equal(rr.Stdout.Bytes(), wantFromTest) {
t.Errorf("busybox-mount logs = %q, want %q", rr.Stdout.Bytes(), wantFromTest)
}
// test file timestamps are correct
for _, name := range []string{createdByTest, createdByPod} {
gp := path.Join(guestMount, name)
// test that file written from host was read in by the pod via cat /mount-9p/fromhost;
rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", "stat", gp))
if err != nil {
t.Errorf("%s failed: %v", rr.Args, err)
}
if runtime.GOOS == "windows" {
if strings.Contains(rr.Stdout.String(), "Access: 1970-01-01") {
t.Errorf("invalid access time: %v", rr.Stdout)
}
}
if strings.Contains(rr.Stdout.String(), "Modify: 1970-01-01") {
t.Errorf("invalid modify time: %v", rr.Stdout)
}
}
p = filepath.Join(tempDir, createdByTestRemovedByPod)
if _, err := os.Stat(p); err == nil {
t.Errorf("expected file %s to be removed", p)
}
p = filepath.Join(tempDir, createdByPodRemovedByTest)
if err := os.Remove(p); err != nil {
t.Errorf("unexpected error removing file %s: %v", p, err)
}
}

35
test/integration/fn_profile.go
View File

@ -1,35 +0,0 @@
// +build integration
/*
Copyright 2019 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package integration
import (
"strings"
"testing"
)
// testProfileList tests the `minikube profile list` command
func testProfileList(t *testing.T) {
p := profileName(t)
t.Parallel()
mk := NewMinikubeRunner(t, p, "--wait=false")
out, stderr := mk.MustRun("profile list")
if !strings.Contains(out, p) {
t.Errorf("Error , failed to read profile name (%s) in `profile list` command output : \n %q : \n stderr: %s ", p, out, stderr)
}
}

115
test/integration/fn_pv.go
View File

@ -1,115 +0,0 @@
// +build integration
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package integration
import (
"fmt"
"path/filepath"
"testing"
"time"
"github.com/pkg/errors"
core "k8s.io/api/core/v1"
storage "k8s.io/api/storage/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/minikube/pkg/kapi"
"k8s.io/minikube/pkg/util/retry"
"k8s.io/minikube/test/integration/util"
)
var (
pvcName = "testpvc"
pvcCmd = []string{"get", "pvc", pvcName}
)
func testProvisioning(t *testing.T) {
p := profileName(t)
t.Parallel()
kr := util.NewKubectlRunner(t, p)
defer func() {
if out, err := kr.RunCommand([]string{"delete", "pvc", pvcName}); err != nil {
t.Logf("delete pvc %s failed: %v\noutput: %s\n", pvcName, err, out)
}
}()
// We have to make sure the addon-manager has created the StorageClass before creating
// a claim. Otherwise it will never get bound.
checkStorageClass := func() error {
scl := storage.StorageClassList{}
if err := kr.RunCommandParseOutput([]string{"get", "storageclass"}, &scl); err != nil {
return fmt.Errorf("get storageclass: %v", err)
}
if len(scl.Items) > 0 {
return nil
}
return fmt.Errorf("no default StorageClass yet")
}
if err := retry.Expo(checkStorageClass, time.Second, 90*time.Second); err != nil {
t.Fatalf("no default storage class after retry: %v", err)
}
// Check that the storage provisioner pod is running
checkPodRunning := func() error {
client, err := kapi.Client(p)
if err != nil {
return errors.Wrap(err, "getting kubernetes client")
}
selector := labels.SelectorFromSet(labels.Set(map[string]string{"integration-test": "storage-provisioner"}))
if err := kapi.WaitForPodsWithLabelRunning(client, "kube-system", selector); err != nil {
return err
}
return nil
}
if err := retry.Expo(checkPodRunning, 2*time.Second, 2*time.Minute); err != nil {
t.Fatalf("Check storage-provisioner pod running failed with error: %v", err)
}
// Now create the PVC
pvcPath := filepath.Join(*testdataDir, "pvc.yaml")
if _, err := kr.RunCommand([]string{"create", "-f", pvcPath}); err != nil {
t.Fatalf("Error creating pvc: %v", err)
}
// And check that it gets bound to a PV.
checkStorage := func() error {
pvc := core.PersistentVolumeClaim{}
if err := kr.RunCommandParseOutput(pvcCmd, &pvc); err != nil {
return err
}
// The test passes if the volume claim gets bound.
if pvc.Status.Phase == "Bound" {
return nil
}
return fmt.Errorf("PV not attached to PVC: %v", pvc)
}
if err := retry.Expo(checkStorage, 2*time.Second, 2*time.Minute); err != nil {
t.Fatalf("PV Creation failed with error: %v", err)
}
}

89
test/integration/fn_pvc.go Normal file
View File

@ -0,0 +1,89 @@
// +build integration
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package integration
import (
"bytes"
"context"
"encoding/json"
"fmt"
"os/exec"
"path/filepath"
"testing"
"time"
core "k8s.io/api/core/v1"
storage "k8s.io/api/storage/v1"
"k8s.io/minikube/pkg/util/retry"
)
func validatePersistentVolumeClaim(ctx context.Context, t *testing.T, profile string) {
ctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
defer cancel()
if _, err := PodWait(ctx, t, profile, "kube-system", "integration-test=storage-provisioner", 2*time.Minute); err != nil {
t.Fatalf("wait: %v", err)
}
checkStorageClass := func() error {
rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "get", "storageclass", "-o=json"))
if err != nil {
return err
}
scl := storage.StorageClassList{}
if err := json.NewDecoder(bytes.NewReader(rr.Stdout.Bytes())).Decode(&scl); err != nil {
return err
}
if len(scl.Items) == 0 {
return fmt.Errorf("no storageclass yet")
}
return nil
}
// Ensure the addon-manager has created the StorageClass before creating a claim, otherwise it won't be bound
if err := retry.Expo(checkStorageClass, time.Second, 90*time.Second); err != nil {
t.Errorf("no default storage class after retry: %v", err)
}
// Now create a testpvc
rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "apply", "-f", filepath.Join(*testdataDir, "pvc.yaml")))
if err != nil {
t.Fatalf("%s failed: %v", rr.Args, err)
}
checkStoragePhase := func() error {
rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "get", "pvc", "testpvc", "-o=json"))
if err != nil {
return err
}
pvc := core.PersistentVolumeClaim{}
if err := json.NewDecoder(bytes.NewReader(rr.Stdout.Bytes())).Decode(&pvc); err != nil {
return err
}
// The test passes if the volume claim gets bound.
if pvc.Status.Phase == "Bound" {
return nil
}
return fmt.Errorf("testpvc phase = %q, want %q (msg=%+v)", pvc.Status.Phase, "Bound", pvc)
}
if err := retry.Expo(checkStoragePhase, 2*time.Second, 2*time.Minute); err != nil {
t.Fatalf("PV Creation failed with error: %v", err)
}
}

171
test/integration/fn_tunnel.go
View File

@ -1,171 +0,0 @@
/*
Copyright 2018 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package integration
import (
"fmt"
"io/ioutil"
"net/http"
"os/exec"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
"k8s.io/apimachinery/pkg/util/wait"
"github.com/pkg/errors"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/minikube/pkg/kapi"
"k8s.io/minikube/pkg/minikube/tunnel"
"k8s.io/minikube/pkg/util/retry"
"k8s.io/minikube/test/integration/util"
)
func testTunnel(t *testing.T) {
if runtime.GOOS != "windows" {
// Otherwise minikube fails waiting for a password.
if err := exec.Command("sudo", "-n", "route").Run(); err != nil {
t.Skipf("password required to execute 'route', skipping testTunnel: %v", err)
}
}
t.Log("starting tunnel test...")
p := profileName(t)
mk := NewMinikubeRunner(t, p, "--wait=false")
go func() {
output, stderr := mk.MustRun("tunnel --alsologtostderr -v 8 --logtostderr")
if t.Failed() {
t.Logf("tunnel stderr : %s", stderr)
t.Logf("tunnel output : %s", output)
}
}()
err := tunnel.NewManager().CleanupNotRunningTunnels()
if err != nil {
t.Fatal(errors.Wrap(err, "cleaning up tunnels"))
}
kr := util.NewKubectlRunner(t, p)
t.Log("deploying nginx...")
podPath := filepath.Join(*testdataDir, "testsvc.yaml")
if _, err := kr.RunCommand([]string{"apply", "-f", podPath}); err != nil {
t.Fatalf("creating nginx ingress resource: %s", err)
}
client, err := kapi.Client(p)
if err != nil {
t.Fatal(errors.Wrap(err, "getting kubernetes client"))
}
selector := labels.SelectorFromSet(labels.Set(map[string]string{"run": "nginx-svc"}))
if err := kapi.WaitForPodsWithLabelRunning(client, "default", selector); err != nil {
t.Fatal(errors.Wrap(err, "waiting for nginx pods"))
}
if err := kapi.WaitForService(client, "default", "nginx-svc", true, 1*time.Second, 2*time.Minute); err != nil {
t.Fatal(errors.Wrap(err, "Error waiting for nginx service to be up"))
}
t.Log("getting nginx ingress...")
nginxIP, err := getIngress(kr)
if err != nil {
t.Errorf("error getting ingress IP for nginx: %s", err)
}
if len(nginxIP) == 0 {
stdout, err := describeIngress(kr)
if err != nil {
t.Errorf("error debugging nginx service: %s", err)
}
t.Fatalf("svc should have ingress after tunnel is created, but it was empty! Result of `kubectl describe svc nginx-svc`:\n %s", string(stdout))
}
responseBody, err := getResponseBody(nginxIP)
if err != nil {
t.Fatalf("error reading from nginx at address(%s): %s", nginxIP, err)
}
if !strings.Contains(responseBody, "Welcome to nginx!") {
t.Fatalf("response body doesn't seem like an nginx response:\n%s", responseBody)
}
}
func getIngress(kr *util.KubectlRunner) (string, error) {
nginxIP := ""
var ret error
err := wait.PollImmediate(1*time.Second, 2*time.Minute, func() (bool, error) {
cmd := []string{"get", "svc", "nginx-svc", "-o", "jsonpath={.status.loadBalancer.ingress[0].ip}"}
stdout, err := kr.RunCommand(cmd)
switch {
case err == nil:
nginxIP = string(stdout)
return len(stdout) != 0, nil
case !kapi.IsRetryableAPIError(err):
ret = fmt.Errorf("`%s` failed with non retriable error: %v", cmd, err)
return false, err
default:
ret = fmt.Errorf("`%s` failed: %v", cmd, err)
return false, nil
}
})
if err != nil {
return "", err
}
return nginxIP, ret
}
func describeIngress(kr *util.KubectlRunner) ([]byte, error) {
return kr.RunCommand([]string{"get", "svc", "nginx-svc", "-o", "jsonpath={.status}"})
}
// getResponseBody returns the contents of a URL
func getResponseBody(address string) (string, error) {
httpClient := http.DefaultClient
httpClient.Timeout = 5 * time.Second
var resp *http.Response
var err error
req := func() error {
resp, err = httpClient.Get(fmt.Sprintf("http://%s", address))
if err != nil {
retriable := &retry.RetriableError{Err: err}
return retriable
}
return nil
}
if err = retry.Expo(req, time.Millisecond*500, 2*time.Minute, 6); err != nil {
return "", err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil || len(body) == 0 {
return "", errors.Wrapf(err, "error reading body, len bytes read: %d", len(body))
}
return string(body), nil
}

130
test/integration/fn_tunnel_cmd.go Normal file
View File

@ -0,0 +1,130 @@
/*
Copyright 2018 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package integration
import (
"context"
"fmt"
"io/ioutil"
"net/http"
"os/exec"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
"k8s.io/apimachinery/pkg/util/wait"
"github.com/pkg/errors"
"k8s.io/minikube/pkg/kapi"
"k8s.io/minikube/pkg/minikube/tunnel"
"k8s.io/minikube/pkg/util/retry"
)
func validateTunnelCmd(ctx context.Context, t *testing.T, profile string) {
ctx, cancel := context.WithTimeout(ctx, 20*time.Minute)
defer cancel()
if runtime.GOOS != "windows" {
// Otherwise minikube fails waiting for a password.
if err := exec.Command("sudo", "-n", "route").Run(); err != nil {
t.Skipf("password required to execute 'route', skipping testTunnel: %v", err)
}
}
client, err := kapi.Client(profile)
if err != nil {
t.Fatalf("client: %v", err)
}
// Pre-Cleanup
if err := tunnel.NewManager().CleanupNotRunningTunnels(); err != nil {
t.Errorf("CleanupNotRunningTunnels: %v", err)
}
// Start the tunnel
args := []string{"-p", profile, "tunnel", "--alsologtostderr", "-v=1"}
ss, err := Start(t, exec.CommandContext(ctx, Target(), args...))
if err != nil {
t.Errorf("%s failed: %v", args, err)
}
defer ss.Stop(t)
// Start the "nginx" pod.
rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "apply", "-f", filepath.Join(*testdataDir, "testsvc.yaml")))
if err != nil {
t.Fatalf("%s failed: %v", rr.Args, err)
}
if _, err := PodWait(ctx, t, profile, "default", "run=nginx-svc", 2*time.Minute); err != nil {
t.Fatalf("wait: %v", err)
}
if err := kapi.WaitForService(client, "default", "nginx-svc", true, 1*time.Second, 2*time.Minute); err != nil {
t.Fatal(errors.Wrap(err, "Error waiting for nginx service to be up"))
}
// Wait until the nginx-svc has a loadbalancer ingress IP
nginxIP := ""
err = wait.PollImmediate(1*time.Second, 3*time.Minute, func() (bool, error) {
rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "get", "svc", "nginx-svc", "-o", "jsonpath={.status.loadBalancer.ingress[0].ip}"))
if err != nil {
return false, err
}
if len(rr.Stdout.String()) > 0 {
nginxIP = rr.Stdout.String()
return true, nil
}
return false, nil
})
if err != nil {
t.Errorf("nginx-svc svc.status.loadBalancer.ingress never got an IP")
rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "get", "svc", "nginx-svc"))
if err != nil {
t.Errorf("%s failed: %v", rr.Args, err)
}
t.Logf("kubectl get svc nginx-svc:\n%s", rr.Stdout)
}
got := []byte{}
fetch := func() error {
h := &http.Client{Timeout: time.Second * 10}
resp, err := h.Get(fmt.Sprintf("http://%s", nginxIP))
if err != nil {
return &retry.RetriableError{Err: err}
}
if resp.Body == nil {
return &retry.RetriableError{Err: fmt.Errorf("no body")}
}
defer resp.Body.Close()
got, err = ioutil.ReadAll(resp.Body)
if err != nil {
return &retry.RetriableError{Err: err}
}
return nil
}
if err = retry.Expo(fetch, time.Millisecond*500, 2*time.Minute, 6); err != nil {
t.Errorf("failed to contact nginx at %s: %v", nginxIP, err)
}
want := "Welcome to nginx!"
if !strings.Contains(string(got), want) {
t.Errorf("body = %q, want *%s*", got, want)
}
}

345
test/integration/functional_test.go
View File

@ -19,41 +19,330 @@ limitations under the License.
package integration
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
"github.com/elazarl/goproxy"
"github.com/hashicorp/go-retryablehttp"
"github.com/phayes/freeport"
"github.com/pkg/errors"
"golang.org/x/build/kubernetes/api"
"k8s.io/minikube/pkg/util/retry"
)
// validateFunc are for subtests that share a single setup
type validateFunc func(context.Context, *testing.T, string)
// TestFunctional are functionality tests which can safely share a profile in parallel
func TestFunctional(t *testing.T) {
p := profileName(t)
mk := NewMinikubeRunner(t, p)
mk.MustStart()
if !isTestNoneDriver(t) { // none driver doesn't need to be deleted
defer mk.TearDown(t)
}
// group is needed to make sure tear down runs after parallel runs
// https://github.com/golang/go/issues/17791#issuecomment-258476786
t.Run("group", func(t *testing.T) {
// This one is not parallel, and ensures the cluster comes up
// before we run any other tests.
t.Run("Status", testClusterStatus)
t.Run("ProfileList", testProfileList)
t.Run("DNS", testClusterDNS)
t.Run("Logs", testClusterLogs)
t.Run("Addons", testAddons)
t.Run("Registry", testRegistry)
t.Run("Dashboard", testDashboard)
t.Run("ServicesList", testServicesList)
t.Run("Provisioning", testProvisioning)
t.Run("Tunnel", testTunnel)
t.Run("kubecontext", testKubeConfigCurrentCtx)
profile := UniqueProfileName("functional")
ctx, cancel := context.WithTimeout(context.Background(), 40*time.Minute)
defer CleanupWithLogs(t, profile, cancel)
if !isTestNoneDriver(t) {
t.Run("EnvVars", testClusterEnv)
t.Run("SSH", testClusterSSH)
t.Run("IngressController", testIngressController)
t.Run("Mounting", testMounting)
// Serial tests
t.Run("serial", func(t *testing.T) {
tests := []struct {
name string
validator validateFunc
}{
{"StartWithProxy", validateStartWithProxy}, // Set everything else up for success
{"KubeContext", validateKubeContext}, // Racy: must come immediately after "minikube start"
{"CacheCmd", validateCacheCmd}, // Caches images needed for subsequent tests because of proxy
}
for _, tc := range tests {
tc := tc
t.Run(tc.name, func(t *testing.T) {
tc.validator(ctx, t, profile)
})
}
})
// Now that we are out of the woods, lets go.
MaybeParallel(t)
// Parallelized tests
t.Run("parallel", func(t *testing.T) {
tests := []struct {
name string
validator validateFunc
}{
{"AddonManager", validateAddonManager},
{"ComponentHealth", validateComponentHealth},
{"ConfigCmd", validateConfigCmd},
{"DashboardCmd", validateDashboardCmd},
{"DNS", validateDNS},
{"LogsCmd", validateLogsCmd},
{"MountCmd", validateMountCmd},
{"ProfileCmd", validateProfileCmd},
{"ServicesCmd", validateServicesCmd},
{"PersistentVolumeClaim", validatePersistentVolumeClaim},
{"TunnelCmd", validateTunnelCmd},
{"SSHCmd", validateSSHCmd},
}
for _, tc := range tests {
tc := tc
t.Run(tc.name, func(t *testing.T) {
MaybeParallel(t)
tc.validator(ctx, t, profile)
})
}
})
}
func validateStartWithProxy(ctx context.Context, t *testing.T, profile string) {
srv, err := startHTTPProxy(t)
if err != nil {
t.Fatalf("Failed to set up the test proxy: %s", err)
}
startArgs := append([]string{"start", "-p", profile, "--wait=false"}, StartArgs()...)
c := exec.CommandContext(ctx, Target(), startArgs...)
env := os.Environ()
env = append(env, fmt.Sprintf("HTTP_PROXY=%s", srv.Addr))
env = append(env, "NO_PROXY=")
c.Env = env
rr, err := Run(t, c)
if err != nil {
t.Errorf("%s failed: %v", rr.Args, err)
}
want := "Found network options:"
if !strings.Contains(rr.Stdout.String(), want) {
t.Errorf("start stdout=%s, want: *%s*", rr.Stdout.String(), want)
}
want = "You appear to be using a proxy"
if !strings.Contains(rr.Stderr.String(), want) {
t.Errorf("start stderr=%s, want: *%s*", rr.Stderr.String(), want)
}
}
// validateKubeContext asserts that kubectl is properly configured (race-condition prone!)
func validateKubeContext(ctx context.Context, t *testing.T, profile string) {
rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "config", "current-context"))
if err != nil {
t.Errorf("%s failed: %v", rr.Args, err)
}
if !strings.Contains(rr.Stdout.String(), profile) {
t.Errorf("current-context = %q, want %q", rr.Stdout.String(), profile)
}
}
// validateAddonManager asserts that the kube-addon-manager pod is deployed properly
func validateAddonManager(ctx context.Context, t *testing.T, profile string) {
// If --wait=false, this may take a couple of minutes
if _, err := PodWait(ctx, t, profile, "kube-system", "component=kube-addon-manager", 3*time.Minute); err != nil {
t.Errorf("wait: %v", err)
}
}
// validateComponentHealth asserts that all Kubernetes components are healthy
func validateComponentHealth(ctx context.Context, t *testing.T, profile string) {
rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "get", "cs", "-o=json"))
if err != nil {
t.Fatalf("%s failed: %v", rr.Args, err)
}
cs := api.ComponentStatusList{}
d := json.NewDecoder(bytes.NewReader(rr.Stdout.Bytes()))
if err := d.Decode(&cs); err != nil {
t.Fatalf("decode: %v", err)
}
for _, i := range cs.Items {
status := api.ConditionFalse
for _, c := range i.Conditions {
if c.Type != api.ComponentHealthy {
continue
}
status = c.Status
}
if status != api.ConditionTrue {
t.Errorf("unexpected status: %v - item: %+v", status, i)
}
}
}
// validateDashboardCmd asserts that the dashboard command works
func validateDashboardCmd(ctx context.Context, t *testing.T, profile string) {
args := []string{"dashboard", "--url", "-p", profile, "--alsologtostderr", "-v=1"}
ss, err := Start(t, exec.CommandContext(ctx, Target(), args...))
if err != nil {
t.Errorf("%s failed: %v", args, err)
}
defer func() {
ss.Stop(t)
}()
start := time.Now()
s, err := ReadLineWithTimeout(ss.Stdout, 300*time.Second)
if err != nil {
t.Fatalf("failed to read url within %s: %v\n", time.Since(start), err)
}
u, err := url.Parse(strings.TrimSpace(s))
if err != nil {
t.Fatalf("failed to parse %q: %v", s, err)
}
resp, err := retryablehttp.Get(u.String())
if err != nil {
t.Errorf("failed get: %v", err)
}
if resp.StatusCode != http.StatusOK {
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
t.Errorf("Unable to read http response body: %v", err)
}
t.Errorf("%s returned status code %d, expected %d.\nbody:\n%s", u, resp.StatusCode, http.StatusOK, body)
}
}
// validateDNS asserts that all Kubernetes DNS is healthy
func validateDNS(ctx context.Context, t *testing.T, profile string) {
rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "replace", "--force", "-f", filepath.Join(*testdataDir, "busybox.yaml")))
if err != nil {
t.Fatalf("%s failed: %v", rr.Args, err)
}
names, err := PodWait(ctx, t, profile, "default", "integration-test=busybox", 3*time.Minute)
if err != nil {
t.Fatalf("wait: %v", err)
}
nslookup := func() error {
rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "exec", names[0], "nslookup", "kubernetes.default"))
return err
}
// If the coredns process was stable, this retry wouldn't be necessary.
if err = retry.Expo(nslookup, 1*time.Second, 1*time.Minute); err != nil {
t.Errorf("nslookup failing: %v", err)
}
want := []byte("10.96.0.1")
if !bytes.Contains(rr.Stdout.Bytes(), want) {
t.Errorf("nslookup: got=%q, want=*%q*", rr.Stdout.Bytes(), want)
}
}
// validateCacheCmd asserts basic "ssh" command functionality
func validateCacheCmd(ctx context.Context, t *testing.T, profile string) {
if NoneDriver() {
t.Skipf("skipping: cache unsupported by none")
}
for _, img := range []string{"busybox", "busybox:1.28.4-glibc"} {
rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "cache", "add", img))
if err != nil {
t.Errorf("%s failed: %v", rr.Args, err)
}
}
}
// validateConfigCmd asserts basic "config" command functionality
func validateConfigCmd(ctx context.Context, t *testing.T, profile string) {
tests := []struct {
args []string
wantOut string
wantErr string
}{
{[]string{"unset", "cpus"}, "", ""},
{[]string{"get", "cpus"}, "", "Error: specified key could not be found in config"},
{[]string{"set", "cpus", "2"}, "! These changes will take effect upon a minikube delete and then a minikube start", ""},
{[]string{"get", "cpus"}, "2", ""},
{[]string{"unset", "cpus"}, "", ""},
{[]string{"get", "cpus"}, "", "Error: specified key could not be found in config"},
}
for _, tc := range tests {
args := append([]string{"-p", profile, "config"}, tc.args...)
rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
if err != nil && tc.wantErr == "" {
t.Errorf("unexpected failure: %s failed: %v", rr.Args, err)
}
got := strings.TrimSpace(rr.Stdout.String())
if got != tc.wantOut {
t.Errorf("%s stdout got: %q, want: %q", rr.Command(), got, tc.wantOut)
}
got = strings.TrimSpace(rr.Stderr.String())
if got != tc.wantErr {
t.Errorf("%s stderr got: %q, want: %q", rr.Command(), got, tc.wantErr)
}
}
}
// validateLogsCmd asserts basic "logs" command functionality
func validateLogsCmd(ctx context.Context, t *testing.T, profile string) {
rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "logs"))
if err != nil {
t.Errorf("%s failed: %v", rr.Args, err)
}
for _, word := range []string{"Docker", "apiserver", "Linux", "kubelet"} {
if !strings.Contains(rr.Stdout.String(), word) {
t.Errorf("minikube logs missing expected word: %q", word)
}
}
}
// validateProfileCmd asserts basic "profile" command functionality
func validateProfileCmd(ctx context.Context, t *testing.T, profile string) {
rr, err := Run(t, exec.CommandContext(ctx, Target(), "profile", "list"))
if err != nil {
t.Errorf("%s failed: %v", rr.Args, err)
}
}
// validateServiceCmd asserts basic "service" command functionality
func validateServicesCmd(ctx context.Context, t *testing.T, profile string) {
rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "service", "list"))
if err != nil {
t.Errorf("%s failed: %v", rr.Args, err)
}
if !strings.Contains(rr.Stdout.String(), "kubernetes") {
t.Errorf("service list got %q, wanted *kubernetes*", rr.Stdout.String())
}
}
// validateSSHCmd asserts basic "ssh" command functionality
func validateSSHCmd(ctx context.Context, t *testing.T, profile string) {
if NoneDriver() {
t.Skipf("skipping: ssh unsupported by none")
}
want := "hello\r\n"
rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", fmt.Sprintf("echo hello")))
if err != nil {
t.Errorf("%s failed: %v", rr.Args, err)
}
if rr.Stdout.String() != want {
t.Errorf("%v = %q, want = %q", rr.Args, rr.Stdout.String(), want)
}
}
// startHTTPProxy runs a local http proxy and sets the env vars for it.
func startHTTPProxy(t *testing.T) (*http.Server, error) {
port, err := freeport.GetFreePort()
if err != nil {
return nil, errors.Wrap(err, "Failed to get an open port")
}
addr := fmt.Sprintf("localhost:%d", port)
proxy := goproxy.NewProxyHttpServer()
srv := &http.Server{Addr: addr, Handler: proxy}
go func(s *http.Server, t *testing.T) {
if err := s.ListenAndServe(); err != http.ErrServerClosed {
t.Errorf("Failed to start http server for proxy mock")
}
}(srv, t)
return srv, nil
}

75
test/integration/guest_env_test.go Normal file
View File

@ -0,0 +1,75 @@
// +build iso
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package integration
import (
"context"
"fmt"
"os/exec"
"testing"
"time"
)
func TestGuestEnvironment(t *testing.T) {
MaybeParallel(t)
profile := UniqueProfileName("guest")
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
defer CleanupWithLogs(t, profile, cancel)
args := append([]string{"start", "-p", profile, "--wait=false"}, StartArgs()...)
rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
if err != nil {
t.Errorf("%s failed: %v", rr.Args, err)
}
// Run as a group so that our defer doesn't happen as tests are runnings
t.Run("Binaries", func(t *testing.T) {
for _, pkg := range []string{"git", "rsync", "curl", "wget", "socat", "iptables", "VBoxControl", "VBoxService"} {
pkg := pkg
t.Run(pkg, func(t *testing.T) {
t.Parallel()
rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", fmt.Sprintf("which %s", pkg)))
if err != nil {
t.Errorf("%s failed: %v", rr.Args, err)
}
})
}
})
t.Run("PersistentMounts", func(t *testing.T) {
for _, mount := range []string{
"/data",
"/var/lib/docker",
"/var/lib/cni",
"/var/lib/kubelet",
"/var/lib/minikube",
"/var/lib/toolbox",
"/var/lib/boot2docker",
} {
mount := mount
t.Run(mount, func(t *testing.T) {
t.Parallel()
rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", fmt.Sprintf("df -t ext4 %s | grep %s", mount, mount)))
if err != nil {
t.Errorf("%s failed: %v", rr.Args, err)
}
})
}
})
}

111
test/integration/gvisor_addon_test.go Normal file
View File

@ -0,0 +1,111 @@
// +build integration
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package integration
import (
"context"
"os/exec"
"path/filepath"
"testing"
"time"
)
func TestGvisorAddon(t *testing.T) {
// TODO(tstromberg): Fix or remove addon.
t.Skip("SKIPPING: Currently broken (gvisor-containerd-shim.toml CrashLoopBackoff): https://github.com/kubernetes/minikube/issues/5305")
if NoneDriver() {
t.Skip("Can't run containerd backend with none driver")
}
MaybeParallel(t)
profile := UniqueProfileName("gvisor")
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute)
defer func() {
CleanupWithLogs(t, profile, cancel)
}()
startArgs := append([]string{"start", "-p", profile, "--container-runtime=containerd", "--docker-opt", "containerd=/var/run/containerd/containerd.sock", "--wait=false"}, StartArgs()...)
rr, err := Run(t, exec.CommandContext(ctx, Target(), startArgs...))
if err != nil {
t.Fatalf("%s failed: %v", rr.Args, err)
}
// TODO: Re-examine if we should be pulling in an image which users don't normally invoke
rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "cache", "add", "gcr.io/k8s-minikube/gvisor-addon:latest"))
if err != nil {
t.Errorf("%s failed: %v", rr.Args, err)
}
// NOTE: addons are global, but the addon must assert that the runtime is containerd
rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "addons", "enable", "gvisor"))
if err != nil {
t.Fatalf("%s failed: %v", rr.Args, err)
}
// Because addons are persistent across profiles :(
defer func() {
rr, err := Run(t, exec.Command(Target(), "-p", profile, "addons", "disable", "gvisor"))
if err != nil {
t.Logf("%s failed: %v", rr.Args, err)
}
}()
if _, err := PodWait(ctx, t, profile, "kube-system", "kubernetes.io/minikube-addons=gvisor", 2*time.Minute); err != nil {
t.Fatalf("waiting for gvisor controller to be up: %v", err)
}
// Create an untrusted workload
rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "replace", "--force", "-f", filepath.Join(*testdataDir, "nginx-untrusted.yaml")))
if err != nil {
t.Fatalf("%s failed: %v", rr.Args, err)
}
// Create gvisor workload
rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "replace", "--force", "-f", filepath.Join(*testdataDir, "nginx-gvisor.yaml")))
if err != nil {
t.Fatalf("%s failed: %v", rr.Args, err)
}
if _, err := PodWait(ctx, t, profile, "default", "run=nginx,untrusted=true", 2*time.Minute); err != nil {
t.Errorf("nginx: %v", err)
}
if _, err := PodWait(ctx, t, profile, "default", "run=nginx,runtime=gvisor", 2*time.Minute); err != nil {
t.Errorf("nginx: %v", err)
}
// Ensure that workloads survive a restart
rr, err = Run(t, exec.CommandContext(ctx, Target(), "stop", "-p", profile))
if err != nil {
t.Fatalf("%s failed: %v", rr.Args, err)
}
rr, err = Run(t, exec.CommandContext(ctx, Target(), startArgs...))
if err != nil {
t.Fatalf("%s failed: %v", rr.Args, err)
}
if _, err := PodWait(ctx, t, profile, "kube-system", "kubernetes.io/minikube-addons=gvisor", 2*time.Minute); err != nil {
t.Errorf("waiting for gvisor controller to be up: %v", err)
}
if _, err := PodWait(ctx, t, profile, "default", "run=nginx,untrusted=true", 2*time.Minute); err != nil {
t.Errorf("nginx: %v", err)
}
if _, err := PodWait(ctx, t, profile, "default", "run=nginx,runtime=gvisor", 2*time.Minute); err != nil {
t.Errorf("nginx: %v", err)
}
}

355
test/integration/helpers.go Normal file
View File

@ -0,0 +1,355 @@
/*
Copyright 2019 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package integration
// These are test helpers that:
//
// - Accept *testing.T arguments (see helpers.go)
// - Are used in multiple tests
// - Must not compare test values
import (
"bufio"
"bytes"
"context"
"fmt"
"io/ioutil"
"os/exec"
"strings"
"testing"
"time"
"github.com/shirou/gopsutil/process"
core "k8s.io/api/core/v1"
meta "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/minikube/pkg/kapi"
)
// RunResult stores the result of an cmd.Run call
type RunResult struct {
Stdout *bytes.Buffer
Stderr *bytes.Buffer
ExitCode int
Args []string
}
// Command returns a human readable command string that does not induce eye fatigue
func (rr RunResult) Command() string {
var sb strings.Builder
sb.WriteString(strings.TrimPrefix(rr.Args[0], "../../"))
for _, a := range rr.Args[1:] {
if strings.Contains(a, " ") {
sb.WriteString(fmt.Sprintf(` "%s"`, a))
continue
}
sb.WriteString(fmt.Sprintf(" %s", a))
}
return sb.String()
}
func (rr RunResult) String() string {
var sb strings.Builder
sb.WriteString(fmt.Sprintf("Command: %v\n", rr.Command()))
if rr.Stdout.Len() > 0 {
sb.WriteString(fmt.Sprintf("\n-- stdout -- \n%s\n", rr.Stdout.Bytes()))
}
if rr.Stderr.Len() > 0 {
sb.WriteString(fmt.Sprintf("\n** stderr ** \n%s\n", rr.Stderr.Bytes()))
}
return sb.String()
}
// Run is a test helper to log a command being executed \_(ツ)_/¯
func Run(t *testing.T, cmd *exec.Cmd) (*RunResult, error) {
t.Helper()
rr := &RunResult{Args: cmd.Args}
t.Logf("(dbg) Run: %v", rr.Command())
var outb, errb bytes.Buffer
cmd.Stdout, rr.Stdout = &outb, &outb
cmd.Stderr, rr.Stderr = &errb, &errb
start := time.Now()
err := cmd.Run()
elapsed := time.Since(start)
if err == nil {
// Reduce log spam
if elapsed > (1 * time.Second) {
t.Logf("(dbg) Done: %v: (%s)", rr.Command(), elapsed)
}
} else {
if exitError, ok := err.(*exec.ExitError); ok {
rr.ExitCode = exitError.ExitCode()
}
t.Logf("(dbg) Non-zero exit: %v: %v (%s)", rr.Command(), err, elapsed)
t.Logf("(dbg) %s", rr.String())
}
return rr, err
}
// StartSession stores the result of an cmd.Start call
type StartSession struct {
Stdout *bufio.Reader
Stderr *bufio.Reader
cmd *exec.Cmd
}
// Start starts a process in the background, streaming output
func Start(t *testing.T, cmd *exec.Cmd) (*StartSession, error) {
t.Helper()
t.Logf("Daemon: %v", cmd.Args)
stdoutPipe, err := cmd.StdoutPipe()
if err != nil {
t.Fatalf("stdout pipe failed: %v %v", cmd.Args, err)
}
stderrPipe, err := cmd.StderrPipe()
if err != nil {
t.Fatalf("stderr pipe failed: %v %v", cmd.Args, err)
}
sr := &StartSession{Stdout: bufio.NewReader(stdoutPipe), Stderr: bufio.NewReader(stderrPipe), cmd: cmd}
return sr, cmd.Start()
}
// Stop stops the started process
func (ss *StartSession) Stop(t *testing.T) {
t.Helper()
t.Logf("Stopping %s ...", ss.cmd.Args)
if ss.cmd.Process == nil {
t.Logf("%s has a nil Process. Maybe it's dead? How weird!", ss.cmd.Args)
return
}
killProcessFamily(t, ss.cmd.Process.Pid)
if t.Failed() {
if ss.Stdout.Size() > 0 {
stdout, err := ioutil.ReadAll(ss.Stdout)
if err != nil {
t.Logf("read stdout failed: %v", err)
}
t.Logf("(dbg) %s stdout:\n%s", ss.cmd.Args, stdout)
}
if ss.Stderr.Size() > 0 {
stderr, err := ioutil.ReadAll(ss.Stderr)
if err != nil {
t.Logf("read stderr failed: %v", err)
}
t.Logf("(dbg) %s stderr:\n%s", ss.cmd.Args, stderr)
}
}
}
// Cleanup cleans up after a test run
func Cleanup(t *testing.T, profile string, cancel context.CancelFunc) {
// No helper because it makes the call log confusing.
if *cleanup {
_, err := Run(t, exec.Command(Target(), "delete", "-p", profile))
if err != nil {
t.Logf("failed cleanup: %v", err)
}
} else {
t.Logf("Skipping cleanup of %s (--cleanup=false)", profile)
}
cancel()
}
// CleanupWithLogs cleans up after a test run, fetching logs and deleting the profile
func CleanupWithLogs(t *testing.T, profile string, cancel context.CancelFunc) {
t.Helper()
if t.Failed() && *postMortemLogs {
t.Logf("%s failed, collecting logs ...", t.Name())
rr, err := Run(t, exec.Command(Target(), "-p", profile, "logs", "-n", "10"))
if err != nil {
t.Logf("failed logs error: %v", err)
}
t.Logf("%s logs: %s\n", t.Name(), rr)
t.Logf("Sorry that %s failed :(", t.Name())
}
Cleanup(t, profile, cancel)
}
// podStatusMsg returns a human-readable pod status, for generating debug status
func podStatusMsg(pod core.Pod) string {
var sb strings.Builder
sb.WriteString(fmt.Sprintf("%q [%s] %s", pod.ObjectMeta.GetName(), pod.ObjectMeta.GetUID(), pod.Status.Phase))
for i, c := range pod.Status.Conditions {
if c.Reason != "" {
if i == 0 {
sb.WriteString(": ")
} else {
sb.WriteString(" / ")
}
sb.WriteString(fmt.Sprintf("%s:%s", c.Type, c.Reason))
}
if c.Message != "" {
sb.WriteString(fmt.Sprintf(" (%s)", c.Message))
}
}
return sb.String()
}
// PodWait waits for pods to achieve a running state.
func PodWait(ctx context.Context, t *testing.T, profile string, ns string, selector string, timeout time.Duration) ([]string, error) {
t.Helper()
client, err := kapi.Client(profile)
if err != nil {
return nil, err
}
// For example: kubernetes.io/minikube-addons=gvisor
listOpts := meta.ListOptions{LabelSelector: selector}
minUptime := 5 * time.Second
podStart := time.Time{}
foundNames := map[string]bool{}
lastMsg := ""
start := time.Now()
t.Logf("Waiting for pods with labels %q in namespace %q ...", selector, ns)
f := func() (bool, error) {
pods, err := client.CoreV1().Pods(ns).List(listOpts)
if err != nil {
t.Logf("Pod(%s).List(%v) returned error: %v", ns, selector, err)
// Don't bother to retry: something is very wrong.
return true, err
}
if len(pods.Items) == 0 {
podStart = time.Time{}
return false, nil
}
for _, pod := range pods.Items {
foundNames[pod.ObjectMeta.Name] = true
msg := podStatusMsg(pod)
// Prevent spamming logs with identical messages
if msg != lastMsg {
t.Log(msg)
lastMsg = msg
}
// Successful termination of a short-lived process, will not be restarted
if pod.Status.Phase == core.PodSucceeded {
return true, nil
}
// Long-running process state
if pod.Status.Phase != core.PodRunning {
if !podStart.IsZero() {
t.Logf("WARNING: %s was running %s ago - may be unstable", selector, time.Since(podStart))
}
podStart = time.Time{}
return false, nil
}
if podStart.IsZero() {
podStart = time.Now()
}
if time.Since(podStart) > minUptime {
return true, nil
}
}
return false, nil
}
err = wait.PollImmediate(500*time.Millisecond, timeout, f)
names := []string{}
for n := range foundNames {
names = append(names, n)
}
if err == nil {
t.Logf("pods %s up and healthy within %s", selector, time.Since(start))
return names, nil
}
t.Logf("pods %q: %v", selector, err)
showPodLogs(ctx, t, profile, ns, names)
return names, fmt.Errorf("%s: %v", fmt.Sprintf("%s within %s", selector, timeout), err)
}
// showPodLogs logs debug info for pods
func showPodLogs(ctx context.Context, t *testing.T, profile string, ns string, names []string) {
rr, rerr := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "get", "po", "-A", "--show-labels"))
if rerr != nil {
t.Logf("%s: %v", rr.Command(), rerr)
// return now, because kubectl is hosed
return
}
t.Logf("(dbg) %s:\n%s", rr.Command(), rr.Stdout)
for _, name := range names {
rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "describe", "po", name, "-n", ns))
if err != nil {
t.Logf("%s: %v", rr.Command(), err)
} else {
t.Logf("(dbg) %s:\n%s", rr.Command(), rr.Stdout)
}
rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "logs", name, "-n", ns))
if err != nil {
t.Logf("%s: %v", rr.Command(), err)
} else {
t.Logf("(dbg) %s:\n%s", rr.Command(), rr.Stdout)
}
}
}
// Status returns the minikube cluster status as a string
func Status(ctx context.Context, t *testing.T, path string, profile string) string {
t.Helper()
rr, err := Run(t, exec.CommandContext(ctx, path, "status", "--format={{.Host}}", "-p", profile))
if err != nil {
t.Logf("status error: %v (may be ok)", err)
}
return strings.TrimSpace(rr.Stdout.String())
}
// MaybeParallel sets that the test should run in parallel
func MaybeParallel(t *testing.T) {
t.Helper()
// TODO: Allow paralellized tests on "none" that do not require independent clusters
if NoneDriver() {
return
}
t.Parallel()
}
// killProcessFamily kills a pid and all of its children
func killProcessFamily(t *testing.T, pid int) {
parent, err := process.NewProcess(int32(pid))
if err != nil {
t.Logf("unable to find parent, assuming dead: %v", err)
return
}
procs := []*process.Process{}
children, err := parent.Children()
if err == nil {
procs = append(procs, children...)
}
procs = append(procs, parent)
for _, p := range procs {
if err := p.Terminate(); err != nil {
t.Logf("unable to terminate pid %d: %v", p.Pid, err)
continue
}
// Allow process a chance to cleanup before instant death.
time.Sleep(100 * time.Millisecond)
if err := p.Kill(); err != nil {
t.Logf("unable to kill pid %d: %v", p.Pid, err)
continue
}
}
}

115
test/integration/iso_test.go
View File

@ -1,115 +0,0 @@
// +build iso
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package integration
import (
"fmt"
"strings"
"testing"
)
func TestISO(t *testing.T) {
p := profileName(t)
if shouldRunInParallel(t) {
t.Parallel()
}
mk := NewMinikubeRunner(t, p, "--wait=false")
mk.MustRun("delete")
stdout, stderr := mk.MustStart()
if err != nil {
t.Fatalf("failed to start minikube (for profile %s) %s) failed : %v\nstdout: %s\nstderr: %s", t.Name(), err, stdout, stderr)
}
if !isTestNoneDriver(t) { // none driver doesn't need to be deleted
defer mk.TearDown(t)
}
t.Run("permissions", testMountPermissions)
t.Run("packages", testPackages)
t.Run("persistence", testPersistence)
}
func testMountPermissions(t *testing.T) {
p := profileName(t)
mk := NewMinikubeRunner(t, p, "--wait=false")
// test mount permissions
mountPoints := []string{"/Users", "/hosthome"}
perms := "drwxr-xr-x"
foundMount := false
for _, dir := range mountPoints {
output, err := mk.SSH(fmt.Sprintf("ls -l %s", dir))
if err != nil {
continue
}
foundMount = true
if !strings.Contains(output, perms) {
t.Fatalf("Incorrect permissions. Expected %s, got %s.", perms, output)
}
}
if !foundMount {
t.Fatalf("No shared mount found. Checked %s", mountPoints)
}
}
func testPackages(t *testing.T) {
p := profileName(t)
mk := NewMinikubeRunner(t, p, "--wait=false")
packages := []string{
"git",
"rsync",
"curl",
"wget",
"socat",
"iptables",
"VBoxControl",
"VBoxService",
}
for _, pkg := range packages {
if output, err := mk.SSH(fmt.Sprintf("which %s", pkg)); err != nil {
t.Errorf("Error finding package: %s. Error: %v. Output: %s", pkg, err, output)
}
}
}
func testPersistence(t *testing.T) {
p := profileName(t)
mk := NewMinikubeRunner(t, p, "--wait=false")
for _, dir := range []string{
"/data",
"/var/lib/docker",
"/var/lib/cni",
"/var/lib/kubelet",
"/var/lib/minikube",
"/var/lib/toolbox",
"/var/lib/boot2docker",
} {
output, err := mk.SSH(fmt.Sprintf("df %s | tail -n 1 | awk '{print $1}'", dir))
if err != nil {
t.Errorf("Error checking device for %s. Error: %v", dir, err)
}
if !strings.Contains(output, "/dev/sda1") {
t.Errorf("Path %s is not mounted persistently. %s", dir, output)
}
}
}

62
test/integration/main.go Normal file
View File

@ -0,0 +1,62 @@
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package integration
import (
"flag"
"os"
"strings"
"testing"
)
// General configuration: used to set the VM Driver
var startArgs = flag.String("minikube-start-args", "", "Arguments to pass to minikube start")
// Flags for faster local integration testing
var forceProfile = flag.String("profile", "", "force tests to run against a particular profile")
var cleanup = flag.Bool("cleanup", true, "cleanup failed test run")
var postMortemLogs = flag.Bool("postmortem-logs", true, "show logs after a failed test run")
// Paths to files - normally set for CI
var binaryPath = flag.String("binary", "../../out/minikube", "path to minikube binary")
var testdataDir = flag.String("testdata-dir", "testdata", "the directory relative to test/integration where the testdata lives")
// TestMain is the test main
func TestMain(m *testing.M) {
flag.Parse()
os.Exit(m.Run())
}
// StartArgs returns the arguments normally used for starting minikube
func StartArgs() []string {
return strings.Split(*startArgs, " ")
}
// Target returns where the minikube binary can be found
func Target() string {
return *binaryPath
}
// NoneDriver returns whether or not this test is using the none driver
func NoneDriver() bool {
return strings.Contains(*startArgs, "--vm-driver=none")
}
// HyperVDriver returns whether or not this test is using the Hyper-V driver
func HyperVDriver() bool {
return strings.Contains(*startArgs, "--vm-driver=hyperv")
}

90
test/integration/none_test.go
View File

@ -20,49 +20,51 @@ limitations under the License.
package integration
import (
"context"
"os"
"os/exec"
"os/user"
"path/filepath"
"strconv"
"strings"
"syscall"
"testing"
"time"
"k8s.io/minikube/pkg/minikube/constants"
)
func TestNone(t *testing.T) {
if !isTestNoneDriver(t) {
// None-driver specific test for CHANGE_MINIKUBE_NONE_USER
func TestChangeNoneUser(t *testing.T) {
if !NoneDriver() {
t.Skip("Only test none driver.")
}
if shouldRunInParallel(t) {
t.Parallel()
}
MaybeParallel(t)
err := os.Setenv("CHANGE_MINIKUBE_NONE_USER", "true")
profile := UniqueProfileName("none")
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
defer CleanupWithLogs(t, profile, cancel)
startArgs := append([]string{"CHANGE_MINIKUBE_NONE_USER=true", Target(), "start", "--wait=false"}, StartArgs()...)
rr, err := Run(t, exec.CommandContext(ctx, "/usr/bin/env", startArgs...))
if err != nil {
t.Fatalf("Failed to setup TestNone: set env: %v", err)
t.Errorf("%s failed: %v", rr.Args, err)
}
p := profileName(t)
mk := NewMinikubeRunner(t, p, "--wait=false")
mk.MustRun("delete")
stdout, stderr := mk.MustStart()
msg := "Configuring local host environment"
if !strings.Contains(stdout, msg) {
t.Errorf("Expected: stdout to contain %q, got: %s", msg, stdout)
}
msg = "may reduce system security and reliability."
if !strings.Contains(stderr, msg) {
t.Errorf("Expected: stderr to contain %q, got: %s", msg, stderr)
rr, err = Run(t, exec.CommandContext(ctx, Target(), "delete"))
if err != nil {
t.Errorf("%s failed: %v", rr.Args, err)
}
t.Run("minikube permissions", testNoneMinikubeFolderPermissions)
t.Run("kubeconfig permissions", testNoneKubeConfigPermissions)
rr, err = Run(t, exec.CommandContext(ctx, "/usr/bin/env", startArgs...))
if err != nil {
t.Errorf("%s failed: %v", rr.Args, err)
}
}
rr, err = Run(t, exec.CommandContext(ctx, Target(), "status"))
if err != nil {
t.Errorf("%s failed: %v", rr.Args, err)
}
func testNoneMinikubeFolderPermissions(t *testing.T) {
username := os.Getenv("SUDO_USER")
if username == "" {
t.Fatal("Expected $SUDO_USER env to not be empty")
@ -75,39 +77,15 @@ func testNoneMinikubeFolderPermissions(t *testing.T) {
if err != nil {
t.Errorf("Failed to convert uid to int: %v", err)
}
info, err := os.Stat(constants.GetMinipath())
if err != nil {
t.Fatalf("Failed to get .minikube dir info, %v", err)
}
fileUID := info.Sys().(*syscall.Stat_t).Uid
if fileUID != uint32(uid) {
t.Errorf("Expected .minikube folder user: %d, got: %d", uint32(uid), fileUID)
for _, p := range []string{constants.GetMinipath(), filepath.Join(u.HomeDir, ".kube/config")} {
info, err := os.Stat(p)
if err != nil {
t.Errorf("stat(%s): %v", p, err)
}
got := info.Sys().(*syscall.Stat_t).Uid
if got != uint32(uid) {
t.Errorf("uid(%s) = %d, want %d", p, got, uint32(uid))
}
}
}
func testNoneKubeConfigPermissions(t *testing.T) {
username := os.Getenv("SUDO_USER")
if username == "" {
t.Fatal("Expected $SUDO_USER env to not be empty")
}
u, err := user.Lookup(username)
if err != nil {
t.Fatalf("Getting user failed: %v", err)
}
uid, err := strconv.Atoi(u.Uid)
if err != nil {
t.Errorf("Failed to convert uid to int: %v", err)
}
info, err := os.Stat(filepath.Join(u.HomeDir, ".kube/config"))
if err != nil {
t.Errorf("Failed to get .minikube dir info, %v", err)
}
fileUID := info.Sys().(*syscall.Stat_t).Uid
if fileUID != uint32(uid) {
t.Errorf("Expected .minikube folder user: %d, got: %d", uint32(uid), fileUID)
}
}

63
test/integration/persistence_test.go
View File

@ -1,63 +0,0 @@
// +build integration
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package integration
import (
"path/filepath"
"testing"
"github.com/docker/machine/libmachine/state"
"k8s.io/minikube/test/integration/util"
)
func TestPersistence(t *testing.T) {
if isTestNoneDriver(t) {
t.Skip("skipping test as none driver does not support persistence")
}
p := profileName(t)
if shouldRunInParallel(t) {
t.Parallel()
}
mk := NewMinikubeRunner(t, p, "--wait=false")
defer mk.TearDown(t)
mk.MustStart()
kr := util.NewKubectlRunner(t, p)
if _, err := kr.RunCommand([]string{"create", "-f", filepath.Join(*testdataDir, "busybox.yaml")}); err != nil {
t.Fatalf("creating busybox pod: %s", err)
}
verifyBusybox := func(t *testing.T) {
if err := util.WaitForBusyboxRunning(t, "default", p); err != nil {
t.Fatalf("waiting for busybox to be up: %v", err)
}
}
// Make sure everything is up before we stop.
verifyBusybox(t)
mk.MustRun("stop")
mk.CheckStatus(state.Stopped.String())
mk.MustStart()
mk.CheckStatus(state.Running.String())
// Make sure the same things come up after we've restarted.
verifyBusybox(t)
}

105
test/integration/start_stop_delete_test.go
View File

@ -19,36 +19,36 @@ limitations under the License.
package integration
import (
"context"
"fmt"
"net"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
"github.com/docker/machine/libmachine/state"
"k8s.io/minikube/pkg/minikube/constants"
)
func TestStartStop(t *testing.T) {
p := profileName(t) // gets profile name used for minikube and kube context
if shouldRunInParallel(t) {
t.Parallel()
}
MaybeParallel(t)
t.Run("group", func(t *testing.T) {
if shouldRunInParallel(t) {
t.Parallel()
}
tests := []struct {
name string
args []string
}{
{"oldest", []string{
{"docker", []string{
"--cache-images=false",
fmt.Sprintf("--kubernetes-version=%s", constants.OldestKubernetesVersion),
// default is the network created by libvirt, if we change the name minikube won't boot
// because the given network doesn't exist
"--kvm-network=default",
"--kvm-qemu-uri=qemu:///system",
"--disable-driver-mounts",
"--keep-context=false",
"--container-runtime=docker",
}},
{"cni", []string{
"--feature-gates",
@ -60,7 +60,8 @@ func TestStartStop(t *testing.T) {
}},
{"containerd", []string{
"--container-runtime=containerd",
"--docker-opt containerd=/var/run/containerd/containerd.sock",
"--docker-opt",
"containerd=/var/run/containerd/containerd.sock",
"--apiserver-port=8444",
}},
{"crio", []string{
@ -71,39 +72,73 @@ func TestStartStop(t *testing.T) {
}
for _, tc := range tests {
n := tc.name // because similar to https://golang.org/doc/faq#closures_and_goroutines
tc := tc
t.Run(tc.name, func(t *testing.T) {
if shouldRunInParallel(t) {
t.Parallel()
}
MaybeParallel(t)
pn := p + n // TestStartStopoldest
mk := NewMinikubeRunner(t, pn, "--wait=false")
// TODO : redundant first clause ? never happens?
if !strings.Contains(pn, "docker") && isTestNoneDriver(t) {
if !strings.Contains(tc.name, "docker") && NoneDriver() {
t.Skipf("skipping %s - incompatible with none driver", t.Name())
}
mk.MustRun("config set WantReportErrorPrompt false")
mk.MustStart(tc.args...)
profile := UniqueProfileName(tc.name)
ctx, cancel := context.WithTimeout(context.Background(), 40*time.Minute)
defer CleanupWithLogs(t, profile, cancel)
mk.CheckStatus(state.Running.String())
ip, stderr := mk.MustRun("ip")
ip = strings.TrimRight(ip, "\n")
if net.ParseIP(ip) == nil {
t.Fatalf("IP command returned an invalid address: %s \n %s", ip, stderr)
}
mk.MustRun("stop")
err := mk.CheckStatusNoFail(state.Stopped.String())
// Use copious amounts of debugging for this stress test: we will need it.
startArgs := append([]string{"start", "-p", profile, "--alsologtostderr", "-v=8"}, tc.args...)
startArgs = append(startArgs, StartArgs()...)
rr, err := Run(t, exec.CommandContext(ctx, Target(), startArgs...))
if err != nil {
t.Errorf("expected status to be %s but got error %v ", state.Stopped.String(), err)
t.Errorf("%s failed: %v", rr.Args, err)
}
// SADNESS: 0/1 nodes are available: 1 node(s) had taints that the pod didn't tolerate.
if strings.Contains(tc.name, "cni") {
t.Logf("WARNING: cni mode requires additional setup before pods can schedule :(")
} else {
// schedule a pod to assert persistence
rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "create", "-f", filepath.Join(*testdataDir, "busybox.yaml")))
if err != nil {
t.Fatalf("%s failed: %v", rr.Args, err)
}
if _, err := PodWait(ctx, t, profile, "default", "integration-test=busybox", 2*time.Minute); err != nil {
t.Fatalf("wait: %v", err)
}
}
rr, err = Run(t, exec.CommandContext(ctx, Target(), "stop", "-p", profile))
if err != nil {
t.Errorf("%s failed: %v", rr.Args, err)
}
got := Status(ctx, t, Target(), profile)
if got != state.Stopped.String() {
t.Errorf("status = %q; want = %q", got, state.Stopped)
}
rr, err = Run(t, exec.CommandContext(ctx, Target(), startArgs...))
if err != nil {
// Explicit fatal so that failures don't move directly to deletion
t.Fatalf("%s failed: %v", rr.Args, err)
}
if strings.Contains(tc.name, "cni") {
t.Logf("WARNING: cni mode requires additional setup before pods can schedule :(")
} else if _, err := PodWait(ctx, t, profile, "default", "integration-test=busybox", 2*time.Minute); err != nil {
t.Fatalf("wait: %v", err)
}
got = Status(ctx, t, Target(), profile)
if got != state.Running.String() {
t.Errorf("status = %q; want = %q", got, state.Running)
}
// Normally handled by cleanuprofile, but not fatal there
rr, err = Run(t, exec.CommandContext(ctx, Target(), "delete", "-p", profile))
if err != nil {
t.Errorf("%s failed: %v", rr.Args, err)
}
mk.MustStart(tc.args...)
mk.CheckStatus(state.Running.String())
mk.MustRun("delete")
mk.CheckStatus(state.None.String())
})
}
})

33
test/integration/testdata/busybox-mount-test.yaml vendored
View File

@ -4,18 +4,23 @@ metadata:
name: busybox-mount
labels:
integration-test: busybox-mount
spec:
containers:
- image: busybox:1.28.4-glibc
command: [ "/bin/sh", "-c", "--" ]
args: [ "cat /mount-9p/fromhost; echo test > /mount-9p/frompod; rm /mount-9p/fromhostremove; echo test > /mount-9p/frompodremove;" ]
name: busybox
volumeMounts:
- mountPath: /mount-9p
name: test-volume
volumes:
- name: test-volume
hostPath:
# directory location on host
path: /mount-9p
spec:
restartPolicy: Never
containers:
- name: mount-munger
image: busybox:1.28.4-glibc
command: ["/bin/sh", "-c", "--"]
args:
- cat /mount-9p/created-by-test;
echo test > /mount-9p/created-by-pod;
rm /mount-9p/created-by-test-removed-by-pod;
echo test > /mount-9p/created-by-pod-removed-by-test
date >> /mount-9p/pod-dates
volumeMounts:
- mountPath: /mount-9p
name: test-volume
volumes:
- name: test-volume
hostPath:
path: /mount-9p

14
test/integration/testdata/busybox.yaml vendored
View File

@ -1,15 +1,15 @@
apiVersion: v1
kind: Pod
metadata:
generateName: busybox-
name: busybox
labels:
integration-test: busybox
spec:
containers:
- image: busybox:1.28.4-glibc
command:
- sleep
- "3600"
imagePullPolicy: IfNotPresent
name: busybox
- image: busybox:1.28.4-glibc
command:
- sleep
- "3600"
imagePullPolicy: IfNotPresent
name: busybox
restartPolicy: Always

60
test/integration/util.go Normal file
View File

@ -0,0 +1,60 @@
/*
Copyright 2019 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package integration
import (
"bufio"
"fmt"
"os"
"time"
)
// ReadLineWithTimeout reads a line of text from a buffer with a timeout
func ReadLineWithTimeout(b *bufio.Reader, timeout time.Duration) (string, error) {
s := make(chan string)
e := make(chan error)
go func() {
read, err := b.ReadString('\n')
if err != nil {
e <- err
} else {
s <- read
}
close(s)
close(e)
}()
select {
case line := <-s:
return line, nil
case err := <-e:
return "", err
case <-time.After(timeout):
return "", fmt.Errorf("timeout after %s", timeout)
}
}
// UniqueProfileName returns a reasonably unique profile name
func UniqueProfileName(prefix string) string {
if *forceProfile != "" {
return *forceProfile
}
if NoneDriver() {
return "minikube"
}
return fmt.Sprintf("%s-%s-%d", prefix, time.Now().Format("20060102T150405.999999999"), os.Getpid())
}

74
test/integration/util/common.go
View File

@ -1,74 +0,0 @@
/*
Copyright 2019 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
"fmt"
"testing"
"time"
"github.com/pkg/errors"
"github.com/shirou/gopsutil/process"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/minikube/pkg/kapi"
)
// WaitForBusyboxRunning waits until busybox pod to be running
func WaitForBusyboxRunning(t *testing.T, namespace string, miniProfile string) error {
client, err := kapi.Client(miniProfile)
if err != nil {
return errors.Wrap(err, "getting kubernetes client")
}
selector := labels.SelectorFromSet(labels.Set(map[string]string{"integration-test": "busybox"}))
return kapi.WaitForPodsWithLabelRunning(client, namespace, selector)
}
// Logf writes logs to stdout if -v is set.
func Logf(str string, args ...interface{}) {
if !testing.Verbose() {
return
}
fmt.Printf(" %s | ", time.Now().Format("15:04:05"))
fmt.Println(fmt.Sprintf(str, args...))
}
// KillProcess kills the process associated with the given pid and all its children
func KillProcess(pid int, t *testing.T) error {
p, err := process.NewProcess(int32(pid))
if err != nil {
// Process doesn't exist
return err
}
children, err := p.Children()
if err != nil {
// No children, log the error, don't exit
t.Log(err)
}
for _, c := range children {
err = c.Kill()
if err != nil {
// Log the error, but don't exit
t.Log(err)
}
}
err = p.Kill()
if err != nil {
return err
}
return nil
}

117
test/integration/util/kubectl_runner.go
View File

@ -1,117 +0,0 @@
/*
Copyright 2019 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
"bytes"
"encoding/json"
"fmt"
"math/rand"
"os/exec"
"testing"
"time"
"k8s.io/minikube/pkg/util/retry"
)
const kubectlBinary = "kubectl"
// KubectlRunner runs a command using kubectl
type KubectlRunner struct {
Profile string // kube-context maps to a minikube profile
T *testing.T
BinaryPath string
}
// NewKubectlRunner creates a new KubectlRunner
func NewKubectlRunner(t *testing.T, profile ...string) *KubectlRunner {
if profile == nil {
profile = []string{"minikube"}
}
p, err := exec.LookPath(kubectlBinary)
if err != nil {
t.Fatalf("Couldn't find kubectl on path.")
}
return &KubectlRunner{Profile: profile[0], BinaryPath: p, T: t}
}
// RunCommandParseOutput runs a command and parses the JSON output
func (k *KubectlRunner) RunCommandParseOutput(args []string, outputObj interface{}, useKubeContext ...bool) error {
args = append(args, "-o=json")
output, err := k.RunCommand(args, useKubeContext...)
if err != nil {
return err
}
d := json.NewDecoder(bytes.NewReader(output))
if err := d.Decode(outputObj); err != nil {
return err
}
return nil
}
// RunCommand runs a command, returning stdout
func (k *KubectlRunner) RunCommand(args []string, useKubeContext ...bool) (stdout []byte, err error) {
if useKubeContext == nil {
useKubeContext = []bool{true}
}
if useKubeContext[0] {
kubecContextArg := fmt.Sprintf("--context=%s", k.Profile)
args = append([]string{kubecContextArg}, args...) // prepending --context so it can be with with -- space
}
inner := func() error {
cmd := exec.Command(k.BinaryPath, args...)
stdout, err = cmd.CombinedOutput()
if err != nil {
retriable := &retry.RetriableError{Err: fmt.Errorf("error running command %s: %v. Stdout: \n %s", args, err, stdout)}
k.T.Log(retriable)
return retriable
}
return nil
}
err = retry.Expo(inner, time.Millisecond*500, 1*time.Minute, 5)
return stdout, err
}
// CreateRandomNamespace creates a random namespace
func (k *KubectlRunner) CreateRandomNamespace() string {
const strLen = 20
name := genRandString(strLen)
if _, err := k.RunCommand([]string{"create", "namespace", name}); err != nil {
k.T.Fatalf("Error creating namespace: %v", err)
}
return name
}
func genRandString(strLen int) string {
const chars = "abcdefghijklmnopqrstuvwxyz0123456789"
rand.Seed(time.Now().UTC().UnixNano())
result := make([]byte, strLen)
for i := 0; i < strLen; i++ {
result[i] = chars[rand.Intn(len(chars))]
}
return string(result)
}
// DeleteNamespace deletes the namespace
func (k *KubectlRunner) DeleteNamespace(namespace string) error {
_, err := k.RunCommand([]string{"delete", "namespace", namespace})
return err
}

315
test/integration/util/minikube_runner.go
View File

@ -1,315 +0,0 @@
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
"bufio"
"bytes"
"context"
"fmt"
"os/exec"
"path"
"path/filepath"
"regexp"
"strings"
"sync"
"testing"
"time"
"github.com/docker/machine/libmachine/state"
"github.com/pkg/errors"
"k8s.io/minikube/pkg/minikube/assets"
commonutil "k8s.io/minikube/pkg/util"
"k8s.io/minikube/pkg/util/retry"
)
// MinikubeRunner runs a command
type MinikubeRunner struct {
Profile string
T *testing.T
BinaryPath string
GlobalArgs string
StartArgs string
MountArgs string
Runtime string
TimeOutStart time.Duration // time to wait for minikube start before killing it
}
// Remove removes a file
func (m *MinikubeRunner) Remove(f assets.CopyableFile) error {
_, err := m.SSH(fmt.Sprintf("rm -rf %s", path.Join(f.GetTargetDir(), f.GetTargetName())))
return err
}
// teeRun runs a command, streaming stdout, stderr to console
func (m *MinikubeRunner) teeRun(cmd *exec.Cmd, waitForRun ...bool) (string, string, error) {
w := true
if waitForRun != nil {
w = waitForRun[0]
}
errPipe, err := cmd.StderrPipe()
if err != nil {
return "", "", err
}
outPipe, err := cmd.StdoutPipe()
if err != nil {
return "", "", err
}
if err := cmd.Start(); err != nil {
return "", "", err
}
if w {
var outB bytes.Buffer
var errB bytes.Buffer
var wg sync.WaitGroup
wg.Add(2)
go func() {
if err := commonutil.TeePrefix(commonutil.ErrPrefix, errPipe, &errB, Logf); err != nil {
m.T.Logf("tee: %v", err)
}
wg.Done()
}()
go func() {
if err := commonutil.TeePrefix(commonutil.OutPrefix, outPipe, &outB, Logf); err != nil {
m.T.Logf("tee: %v", err)
}
wg.Done()
}()
err = cmd.Wait()
wg.Wait()
return outB.String(), errB.String(), err
}
return "", "", err
}
// MustRun executes a command and fails if error, and and unless waitForRun is set to false it waits for it finish.
func (m *MinikubeRunner) MustRun(cmdStr string, waitForRun ...bool) (string, string) {
stdout, stderr, err := m.RunCommand(cmdStr, true, waitForRun...)
if err != nil {
m.T.Logf("MusRun error: %v", err)
}
return stdout, stderr
}
// RunCommand executes a command, optionally checking for error and by default waits for run to finish
func (m *MinikubeRunner) RunCommand(cmdStr string, failError bool, waitForRun ...bool) (string, string, error) {
profileArg := fmt.Sprintf("-p=%s ", m.Profile)
cmdStr = profileArg + cmdStr
cmdArgs := strings.Split(cmdStr, " ")
path, _ := filepath.Abs(m.BinaryPath)
cmd := exec.Command(path, cmdArgs...)
Logf("Run: %s", cmd.Args)
stdout, stderr, err := m.teeRun(cmd, waitForRun...)
if err != nil {
exitCode := ""
if exitError, ok := err.(*exec.ExitError); ok {
exitCode = string(exitError.Stderr)
}
errMsg := fmt.Sprintf("Error RunCommand : %s \n\t Begin RunCommand log block ---> \n\t With Profile: %s \n\t With ExitCode: %q \n\t With STDOUT %s \n\t With STDERR %s \n\t <--- End of RunCommand log block", cmdStr, m.Profile, exitCode, stdout, stderr)
if failError {
m.T.Fatalf(errMsg)
} else {
m.T.Logf(errMsg)
}
}
return stdout, stderr, err
}
// RunWithContext calls the minikube command with a context, useful for timeouts.
func (m *MinikubeRunner) RunWithContext(ctx context.Context, cmdStr string, wait ...bool) (string, string, error) {
profileArg := fmt.Sprintf("-p=%s ", m.Profile)
cmdStr = profileArg + cmdStr
cmdArgs := strings.Split(cmdStr, " ")
path, _ := filepath.Abs(m.BinaryPath)
cmd := exec.CommandContext(ctx, path, cmdArgs...)
Logf("RunWithContext: %s", cmd.Args)
return m.teeRun(cmd, wait...)
}
// RunDaemon executes a command, returning the stdout
func (m *MinikubeRunner) RunDaemon(cmdStr string) (*exec.Cmd, *bufio.Reader) {
profileArg := fmt.Sprintf("-p=%s ", m.Profile)
cmdStr = profileArg + cmdStr
cmdArgs := strings.Split(cmdStr, " ")
path, _ := filepath.Abs(m.BinaryPath)
cmd := exec.Command(path, cmdArgs...)
stdoutPipe, err := cmd.StdoutPipe()
if err != nil {
m.T.Fatalf("stdout pipe failed: %s %v", cmdStr, err)
}
stderrPipe, err := cmd.StderrPipe()
if err != nil {
m.T.Fatalf("stderr pipe failed: %s %v", cmdStr, err)
}
var errB bytes.Buffer
go func() {
if err := commonutil.TeePrefix(commonutil.ErrPrefix, stderrPipe, &errB, Logf); err != nil {
m.T.Logf("tee: %v", err)
}
}()
err = cmd.Start()
if err != nil {
m.T.Fatalf("Error running command: %s %v", cmdStr, err)
}
return cmd, bufio.NewReader(stdoutPipe)
}
// RunDaemon2 executes a command, returning the stdout and stderr
func (m *MinikubeRunner) RunDaemon2(cmdStr string) (*exec.Cmd, *bufio.Reader, *bufio.Reader) {
profileArg := fmt.Sprintf("-p=%s ", m.Profile)
cmdStr = profileArg + cmdStr
cmdArgs := strings.Split(cmdStr, " ")
path, _ := filepath.Abs(m.BinaryPath)
cmd := exec.Command(path, cmdArgs...)
stdoutPipe, err := cmd.StdoutPipe()
if err != nil {
m.T.Fatalf("stdout pipe failed: %s %v", cmdStr, err)
}
stderrPipe, err := cmd.StderrPipe()
if err != nil {
m.T.Fatalf("stderr pipe failed: %s %v", cmdStr, err)
}
err = cmd.Start()
if err != nil {
m.T.Fatalf("Error running command: %s %v", cmdStr, err)
}
return cmd, bufio.NewReader(stdoutPipe), bufio.NewReader(stderrPipe)
}
// SSH returns the output of running a command using SSH
func (m *MinikubeRunner) SSH(cmdStr string) (string, error) {
profileArg := fmt.Sprintf("-p=%s", m.Profile)
path, _ := filepath.Abs(m.BinaryPath)
cmd := exec.Command(path, profileArg, "ssh", cmdStr)
Logf("SSH: %s", cmdStr)
stdout, err := cmd.CombinedOutput()
Logf("Output: %s", stdout)
if err, ok := err.(*exec.ExitError); ok {
return string(stdout), err
}
return string(stdout), nil
}
// Start starts the cluster
func (m *MinikubeRunner) start(opts ...string) (stdout string, stderr string, err error) {
cmd := fmt.Sprintf("start %s %s %s", m.StartArgs, m.GlobalArgs, strings.Join(opts, " "))
ctx := context.Background()
ctx, cancel := context.WithTimeout(ctx, m.TimeOutStart)
defer cancel()
return m.RunWithContext(ctx, cmd, true)
}
// MustStart starts the cluster and fail the test if error
func (m *MinikubeRunner) MustStart(opts ...string) (stdout string, stderr string) {
stdout, stderr, err := m.start(opts...)
// the reason for this formatting is, the logs are very big but useful and also in parallel testing logs are harder to identify
if err != nil {
m.T.Fatalf("%s Failed to start minikube With error: %v \n\t begin Start log block ------------> \n\t With Profile: %s \n\t With Args: %v \n\t With Global Args: %s \n\t With Driver Args: %s \n\t With STDOUT: \n \t %s \n\t With STDERR: \n \t %s \n\t <------------ End of Start (%s) log block", m.T.Name(), err, m.Profile, strings.Join(opts, " "), m.GlobalArgs, m.StartArgs, stdout, stderr, m.Profile)
}
return stdout, stderr
}
// TearDown deletes minikube without waiting for it. used to free up ram/cpu after each test
func (m *MinikubeRunner) TearDown(t *testing.T) {
profileArg := fmt.Sprintf("-p=%s", m.Profile)
path, _ := filepath.Abs(m.BinaryPath)
cmd := exec.Command(path, profileArg, "delete")
err := cmd.Start() // don't wait for it to finish
if err != nil {
t.Errorf("error tearing down minikube %s : %v", profileArg, err)
}
}
// EnsureRunning makes sure the container runtime is running
func (m *MinikubeRunner) EnsureRunning(opts ...string) {
s, _, err := m.Status()
if err != nil {
m.T.Errorf("error getting status for ensure running: %v", err)
}
if s != state.Running.String() {
stdout, stderr, err := m.start(opts...)
if err != nil {
m.T.Errorf("error starting while running EnsureRunning : %v , stdout %s stderr %s", err, stdout, stderr)
}
}
m.CheckStatus(state.Running.String())
}
// ParseEnvCmdOutput parses the output of `env` (assumes bash)
func (m *MinikubeRunner) ParseEnvCmdOutput(out string) map[string]string {
env := map[string]string{}
re := regexp.MustCompile(`(\w+?) ?= ?"?(.+?)"?\n`)
for _, m := range re.FindAllStringSubmatch(out, -1) {
env[m[1]] = m[2]
}
return env
}
// Status returns the status of a service
func (m *MinikubeRunner) Status() (status string, stderr string, err error) {
s := func() error {
status, stderr, err = m.RunCommand("status --format={{.Host}} %s", false)
status = strings.TrimRight(status, "\n")
if err != nil && (status == state.None.String() || status == state.Stopped.String()) {
err = nil // because https://github.com/kubernetes/minikube/issues/4932
}
return err
}
err = retry.Expo(s, 3*time.Second, 2*time.Minute)
return status, stderr, err
}
// GetLogs returns the logs of a service
func (m *MinikubeRunner) GetLogs() (string, string) {
stdout, stderr, err := m.RunCommand(fmt.Sprintf("logs %s", m.GlobalArgs), true)
if err != nil {
m.T.Logf("Error in GetLogs %v", err)
}
return stdout, stderr
}
// CheckStatus makes sure the service has the desired status, or cause fatal error
func (m *MinikubeRunner) CheckStatus(desired string) {
err := m.CheckStatusNoFail(desired)
if err != nil { // none status returns 1 exit code
m.T.Fatalf("%v", err)
}
}
// CheckStatusNoFail makes sure the service has the desired status, returning error
func (m *MinikubeRunner) CheckStatusNoFail(desired string) error {
s, stderr, err := m.Status()
if s != desired {
return fmt.Errorf("got state: %q, expected %q : stderr: %s err: %v ", s, desired, stderr, err)
}
if err != nil {
return errors.Wrapf(err, stderr)
}
return nil
}

105
test/integration/version_upgrade_test.go
View File

@ -17,80 +17,81 @@ limitations under the License.
package integration
import (
"context"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"os/exec"
"runtime"
"strings"
"testing"
"time"
"github.com/docker/machine/libmachine/state"
"github.com/pkg/errors"
"k8s.io/minikube/pkg/minikube/constants"
"k8s.io/minikube/pkg/util/retry"
"github.com/hashicorp/go-getter"
pkgutil "k8s.io/minikube/pkg/util"
)
func fileExists(fname string) error {
check := func() error {
info, err := os.Stat(fname)
if os.IsNotExist(err) {
return err
}
if info.IsDir() {
return fmt.Errorf("error expect file got dir")
}
return nil
}
if err := retry.Expo(check, 1*time.Second, 3); err != nil {
return errors.Wrap(err, fmt.Sprintf("Failed check if file (%q) exists,", fname))
}
return nil
}
// TestVersionUpgrade downloads latest version of minikube and runs with
// the odlest supported k8s version and then runs the current head minikube
// and it tries to upgrade from the older supported k8s to news supported k8s
func TestVersionUpgrade(t *testing.T) {
p := profileName(t)
if shouldRunInParallel(t) {
t.Parallel()
profile := UniqueProfileName("vupgrade")
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
MaybeParallel(t)
defer CleanupWithLogs(t, profile, cancel)
tf, err := ioutil.TempFile("", "minikube-release.*.exe")
if err != nil {
t.Fatalf("tempfile: %v", err)
}
// fname is the filename for the minikube's latetest binary. this file been pre-downloaded before test by hacks/jenkins/common.sh
fname := filepath.Join(*testdataDir, fmt.Sprintf("minikube-%s-%s-latest-stable", runtime.GOOS, runtime.GOARCH))
err := fileExists(fname)
if err != nil { // download file if it is not downloaded by other test
dest := filepath.Join(*testdataDir, fmt.Sprintf("minikube-%s-%s-latest-stable", runtime.GOOS, runtime.GOARCH))
if runtime.GOOS == "windows" {
dest += ".exe"
}
err := downloadMinikubeBinary(t, dest, "latest")
if err != nil {
// binary is needed for the test
t.Fatalf("erorr downloading the latest minikube release %v", err)
defer os.Remove(tf.Name())
tf.Close()
url := pkgutil.GetBinaryDownloadURL("latest", runtime.GOOS)
if err := retry.Expo(func() error { return getter.GetFile(tf.Name(), url) }, 3*time.Second, 3*time.Minute); err != nil {
t.Fatalf("get failed: %v", err)
}
if runtime.GOOS != "windows" {
if err := os.Chmod(tf.Name(), 0700); err != nil {
t.Errorf("chmod: %v", err)
}
}
defer os.Remove(fname)
mkHead := NewMinikubeRunner(t, p) // minikube from HEAD.
defer mkHead.TearDown(t)
args := append([]string{"start", "-p", profile, fmt.Sprintf("--kubernetes-version=%s", constants.OldestKubernetesVersion)}, StartArgs()...)
rr := &RunResult{}
releaseStart := func() error {
rr, err = Run(t, exec.CommandContext(ctx, tf.Name(), args...))
return err
}
mkRelease := NewMinikubeRunner(t, p) // lastest publicly released version minikbue.
// Retry to allow flakiness for the previous release
if err := retry.Expo(releaseStart, 1*time.Second, 20*time.Minute); err != nil {
t.Fatalf("release start failed: %v", err)
}
// because the --wait-timeout is a new flag and the current latest release (1.3.1) doesn't have it
// this won't be necessary after we release the change with --wait-timeout flag
mkRelease.StartArgs = strings.Replace(mkRelease.StartArgs, "--wait-timeout=13m", "", 1)
mkRelease.BinaryPath = fname
// For full coverage: also test upgrading from oldest to newest supported k8s release
mkRelease.MustStart(fmt.Sprintf("--kubernetes-version=%s", constants.OldestKubernetesVersion))
rr, err = Run(t, exec.CommandContext(ctx, tf.Name(), "stop", "-p", profile))
if err != nil {
t.Fatalf("%s failed: %v", rr.Args, err)
}
mkRelease.CheckStatus(state.Running.String())
mkRelease.MustRun("stop")
mkRelease.CheckStatus(state.Stopped.String())
rr, err = Run(t, exec.CommandContext(ctx, tf.Name(), "-p", profile, "status", "--format={{.Host}}"))
if err != nil {
t.Logf("status error: %v (may be ok)", err)
}
got := strings.TrimSpace(rr.Stdout.String())
if got != state.Stopped.String() {
t.Errorf("status = %q; want = %q", got, state.Stopped.String())
}
// Trim the leading "v" prefix to assert that we handle it properly.
mkHead.MustStart(fmt.Sprintf("--kubernetes-version=%s", strings.TrimPrefix(constants.NewestKubernetesVersion, "v")))
mkHead.CheckStatus(state.Running.String())
args = append([]string{"start", "-p", profile, fmt.Sprintf("--kubernetes-version=%s", constants.NewestKubernetesVersion)}, StartArgs()...)
rr, err = Run(t, exec.CommandContext(ctx, Target(), args...))
if err != nil {
t.Errorf("%s failed: %v", rr.Args, err)
}
}

175
test/integration/z_proxy_test.go
View File

@ -1,175 +0,0 @@
// +build integration
/*
Copyright 2019 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// the name of this file starts with z intentionally to make it run last after all other tests
// the intent is to make sure os env proxy settings be done after all other tests.
// for example in the case the test proxy clean up gets killed or fails
package integration
import (
"context"
"fmt"
"io/ioutil"
"os"
"strings"
"testing"
"time"
"net/http"
"net/url"
"github.com/elazarl/goproxy"
retryablehttp "github.com/hashicorp/go-retryablehttp"
"github.com/phayes/freeport"
"github.com/pkg/errors"
"k8s.io/minikube/test/integration/util"
)
// setUpProxy runs a local http proxy and sets the env vars for it.
func setUpProxy(t *testing.T) (*http.Server, error) {
port, err := freeport.GetFreePort()
if err != nil {
return nil, errors.Wrap(err, "Failed to get an open port")
}
addr := fmt.Sprintf("localhost:%d", port)
err = os.Setenv("NO_PROXY", "")
if err != nil {
return nil, errors.Wrap(err, "Failed to set no proxy env")
}
err = os.Setenv("HTTP_PROXY", addr)
if err != nil {
return nil, errors.Wrap(err, "Failed to set http proxy env")
}
proxy := goproxy.NewProxyHttpServer()
srv := &http.Server{Addr: addr, Handler: proxy}
go func(s *http.Server, t *testing.T) {
if err := s.ListenAndServe(); err != http.ErrServerClosed {
t.Errorf("Failed to start http server for proxy mock")
}
}(srv, t)
return srv, nil
}
func TestProxy(t *testing.T) {
origHP := os.Getenv("HTTP_PROXY")
origNP := os.Getenv("NO_PROXY")
p := profileName(t) // profile name
if isTestNoneDriver(t) {
// TODO fix this later
t.Skip("Skipping proxy warning for none")
}
srv, err := setUpProxy(t)
if err != nil {
t.Fatalf("Failed to set up the test proxy: %s", err)
}
// making sure there is no running minikube to avoid https://github.com/kubernetes/minikube/issues/4132
mk := NewMinikubeRunner(t, p)
// Clean up after setting up proxy
defer func(t *testing.T) {
err = os.Setenv("HTTP_PROXY", origHP)
if err != nil {
t.Errorf("Error reverting the HTTP_PROXY env")
}
err = os.Setenv("NO_PROXY", origNP)
if err != nil {
t.Errorf("Error reverting the NO_PROXY env")
}
err := srv.Shutdown(context.TODO()) // shutting down the http proxy after tests
if err != nil {
t.Errorf("Error shutting down the http proxy")
}
if !isTestNoneDriver(t) {
mk.TearDown(t)
}
}(t)
t.Run("ProxyConsoleWarnning", testProxyWarning)
t.Run("ProxyDashboard", testProxyDashboard)
t.Run("KubeconfigContext", testKubeConfigCurrentCtx)
}
// testProxyWarning checks user is warned correctly about the proxy related env vars
func testProxyWarning(t *testing.T) {
p := profileName(t) // profile name
mk := NewMinikubeRunner(t, p)
stdout, stderr := mk.MustStart("--wait=false")
msg := "Found network options:"
if !strings.Contains(stdout, msg) {
t.Errorf("Proxy wranning (%s) is missing from the output: %s", msg, stderr)
}
msg = "You appear to be using a proxy"
if !strings.Contains(stderr, msg) {
t.Errorf("Proxy wranning (%s) is missing from the output: %s", msg, stderr)
}
}
// testProxyDashboard checks if dashboard URL is accessible if proxy is set
func testProxyDashboard(t *testing.T) {
p := profileName(t) // profile name
mk := NewMinikubeRunner(t, p)
cmd, out := mk.RunDaemon("dashboard --url")
defer func() {
err := util.KillProcess(cmd.Process.Pid, t)
if err != nil {
t.Logf("Failed to kill dashboard command: %v", err)
}
}()
s, err := readLineWithTimeout(out, 180*time.Second)
if err != nil {
t.Fatalf("failed to read url: %v", err)
}
u, err := url.Parse(strings.TrimSpace(s))
if err != nil {
t.Fatalf("failed to parse %q: %v", s, err)
}
resp, err := retryablehttp.Get(u.String())
if err != nil {
t.Fatalf("failed get: %v", err)
}
if resp.StatusCode != http.StatusOK {
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
t.Fatalf("Unable to read http response body: %v", err)
}
t.Errorf("%s returned status code %d, expected %d.\nbody:\n%s", u, resp.StatusCode, http.StatusOK, body)
}
}
// testKubeConfigCurrentCtx checks weather the current-context is set after star
func testKubeConfigCurrentCtx(t *testing.T) {
p := profileName(t) // profile name
kr := util.NewKubectlRunner(t, p)
ctxAfter, err := kr.RunCommand([]string{"config", "current-context"}, false)
if err != nil {
t.Errorf("expected not to get error for kubectl config current-context but got error: %v", err)
}
if !strings.Contains(string(ctxAfter), p) {
t.Errorf("expected kubecontext after start to be %s but got %s", p, ctxAfter)
}
}