From 574e81d9bb8cde02488367164c6e0ecf23eeeb05 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Tue, 26 Jan 2021 12:07:49 -0800 Subject: [PATCH 01/16] Turn on TestPause for containerd --- test/integration/main_test.go | 4 ---- test/integration/pause_test.go | 3 --- 2 files changed, 7 deletions(-) diff --git a/test/integration/main_test.go b/test/integration/main_test.go index 58a041eb0b..922be0a969 100644 --- a/test/integration/main_test.go +++ b/test/integration/main_test.go @@ -152,10 +152,6 @@ func GithubActionRunner() bool { return os.Getenv("GITHUB_ACTIONS") == "true" } -func ContainerdContainerRuntime() bool { - return strings.Contains(*startArgs, "--container-runtime=containerd") -} - // arm64Platform returns true if running on arm64/* platform func arm64Platform() bool { return runtime.GOARCH == "arm64" diff --git a/test/integration/pause_test.go b/test/integration/pause_test.go index cd202745ee..c412d62942 100644 --- a/test/integration/pause_test.go +++ b/test/integration/pause_test.go @@ -29,9 +29,6 @@ import ( ) func TestPause(t *testing.T) { - if ContainerdContainerRuntime() { - t.Skip("skipping as this test currently times out on containerd") - } MaybeParallel(t) type validateFunc func(context.Context, *testing.T, string) From 2f78eb7771fcab90529b02f4fc80bdd7387ce652 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Tue, 26 Jan 2021 14:26:56 -0800 Subject: [PATCH 02/16] Docker_Linux_containerd is timing out when this test is run, give the test 10 minutes to run. It completes in under 5 on Docker_linux so this should be enough time --- test/integration/pause_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/pause_test.go b/test/integration/pause_test.go index c412d62942..e3ceb7b61d 100644 --- a/test/integration/pause_test.go +++ b/test/integration/pause_test.go @@ -33,7 +33,7 @@ func TestPause(t *testing.T) { type validateFunc func(context.Context, *testing.T, string) profile := UniqueProfileName("pause") - ctx, cancel := context.WithTimeout(context.Background(), Minutes(30)) + ctx, cancel := context.WithTimeout(context.Background(), Minutes(10)) defer Cleanup(t, profile, cancel) // Serial tests From 6d1c03c6aae88be9ed2af2dae77ff126cbce93cd Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Tue, 26 Jan 2021 16:08:55 -0800 Subject: [PATCH 03/16] Give less time to TestNetworkPlugins --- test/integration/net_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/net_test.go b/test/integration/net_test.go index 9fff54dc56..57f327d61d 100644 --- a/test/integration/net_test.go +++ b/test/integration/net_test.go @@ -70,10 +70,10 @@ func TestNetworkPlugins(t *testing.T) { MaybeParallel(t) profile := UniqueProfileName(tc.name) - ctx, cancel := context.WithTimeout(context.Background(), Minutes(40)) + ctx, cancel := context.WithTimeout(context.Background(), Minutes(30)) defer CleanupWithLogs(t, profile, cancel) - startArgs := append([]string{"start", "-p", profile, "--memory=1800", "--alsologtostderr", "--wait=true", "--wait-timeout=5m"}, tc.args...) + startArgs := append([]string{"start", "-p", profile, "--memory=1800", "--alsologtostderr", "--wait=true", "--wait-timeout=3m"}, tc.args...) startArgs = append(startArgs, StartArgs()...) t.Run("Start", func(t *testing.T) { From 3e59bf74ae1ddf402cd5fe7713cf935c31aa3dde Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Tue, 26 Jan 2021 17:39:56 -0800 Subject: [PATCH 04/16] don't run pause test in parallel --- test/integration/pause_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/pause_test.go b/test/integration/pause_test.go index e3ceb7b61d..e3194b0282 100644 --- a/test/integration/pause_test.go +++ b/test/integration/pause_test.go @@ -29,7 +29,7 @@ import ( ) func TestPause(t *testing.T) { - MaybeParallel(t) + // MaybeParallel(t) type validateFunc func(context.Context, *testing.T, string) profile := UniqueProfileName("pause") From b1a6808b7aebaf948baa6af358b3fbe4650c3f7d Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Tue, 26 Jan 2021 19:32:58 -0800 Subject: [PATCH 05/16] Reintroduce parallelization, add wait-timeout to start --- test/integration/pause_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/integration/pause_test.go b/test/integration/pause_test.go index e3194b0282..0672462e85 100644 --- a/test/integration/pause_test.go +++ b/test/integration/pause_test.go @@ -29,7 +29,7 @@ import ( ) func TestPause(t *testing.T) { - // MaybeParallel(t) + MaybeParallel(t) type validateFunc func(context.Context, *testing.T, string) profile := UniqueProfileName("pause") @@ -71,7 +71,7 @@ func TestPause(t *testing.T) { func validateFreshStart(ctx context.Context, t *testing.T, profile string) { defer PostMortemLogs(t, profile) - args := append([]string{"start", "-p", profile, "--memory=1800", "--install-addons=false", "--wait=all"}, StartArgs()...) + args := append([]string{"start", "-p", profile, "--memory=1800", "--install-addons=false", "--wait=all", "--wait-timeout=2m"}, StartArgs()...) rr, err := Run(t, exec.CommandContext(ctx, Target(), args...)) if err != nil { t.Fatalf("failed to start minikube with args: %q : %v", rr.Command(), err) @@ -82,7 +82,7 @@ func validateFreshStart(ctx context.Context, t *testing.T, profile string) { func validateStartNoReconfigure(ctx context.Context, t *testing.T, profile string) { defer PostMortemLogs(t, profile) - args := []string{"start", "-p", profile, "--alsologtostderr", "-v=1"} + args := []string{"start", "-p", profile, "--alsologtostderr", "-v=1", "--wait-timeout=2m"} args = append(args, StartArgs()...) rr, err := Run(t, exec.CommandContext(ctx, Target(), args...)) if err != nil { From 77fa8ab39767795702212d9a78dd477e8ec61bf6 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Tue, 26 Jan 2021 23:01:47 -0800 Subject: [PATCH 06/16] reduce retry to 1 minute, when this test passes it takes <1s --- test/integration/net_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/net_test.go b/test/integration/net_test.go index 57f327d61d..fa78212b03 100644 --- a/test/integration/net_test.go +++ b/test/integration/net_test.go @@ -158,7 +158,7 @@ func TestNetworkPlugins(t *testing.T) { } // If the coredns process was stable, this retry wouldn't be necessary. - if err := retry.Expo(nslookup, 1*time.Second, Minutes(6)); err != nil { + if err := retry.Expo(nslookup, 1*time.Second, Minutes(1)); err != nil { t.Errorf("failed to do nslookup on kubernetes.default: %v", err) } From 843ae893964009848c75bb62539cbc1755ff54cc Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Wed, 27 Jan 2021 11:12:12 -0800 Subject: [PATCH 07/16] Add --wait-timeout to TestStartStop --- test/integration/start_stop_delete_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/start_stop_delete_test.go b/test/integration/start_stop_delete_test.go index 8154b53449..c7666566c0 100644 --- a/test/integration/start_stop_delete_test.go +++ b/test/integration/start_stop_delete_test.go @@ -94,7 +94,7 @@ func TestStartStop(t *testing.T) { waitFlag = "--wait=apiserver,system_pods,default_sa" } - startArgs := append([]string{"start", "-p", profile, "--memory=2200", "--alsologtostderr", waitFlag}, tc.args...) + startArgs := append([]string{"start", "-p", profile, "--memory=2200", "--alsologtostderr", waitFlag, "--wait-timeout=2m"}, tc.args...) startArgs = append(startArgs, StartArgs()...) startArgs = append(startArgs, fmt.Sprintf("--kubernetes-version=%s", tc.version)) From 95f9c5c0a398c66cdb3cb7c1904ab398c7c9e544 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Wed, 27 Jan 2021 13:12:14 -0800 Subject: [PATCH 08/16] UP timeout to 90m to see if containerd tests pass then --- hack/jenkins/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh index 0182427673..15d28e16e3 100755 --- a/hack/jenkins/common.sh +++ b/hack/jenkins/common.sh @@ -31,7 +31,7 @@ export GOPATH="$HOME/go" export KUBECONFIG="${TEST_HOME}/kubeconfig" export PATH=$PATH:"/usr/local/bin/:/usr/local/go/bin/:$GOPATH/bin" -readonly TIMEOUT=${1:-70m} +readonly TIMEOUT=${1:-90m} if [ "$(uname)" != "Darwin" ]; then # install lsof for finding none driver procs, psmisc to use pstree in cronjobs From 10d5d0915f8d77f12a5f75e9314c791be7c8fd40 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Wed, 27 Jan 2021 20:32:41 -0800 Subject: [PATCH 09/16] Increase timeout --- hack/jenkins/common.sh | 4 ++-- test/integration/pause_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh index 15d28e16e3..357110ecde 100755 --- a/hack/jenkins/common.sh +++ b/hack/jenkins/common.sh @@ -31,7 +31,7 @@ export GOPATH="$HOME/go" export KUBECONFIG="${TEST_HOME}/kubeconfig" export PATH=$PATH:"/usr/local/bin/:/usr/local/go/bin/:$GOPATH/bin" -readonly TIMEOUT=${1:-90m} +readonly TIMEOUT=${1:-100m} if [ "$(uname)" != "Darwin" ]; then # install lsof for finding none driver procs, psmisc to use pstree in cronjobs @@ -308,7 +308,7 @@ fi ${SUDO_PREFIX}${E2E_BIN} \ -minikube-start-args="--driver=${VM_DRIVER} ${EXTRA_START_ARGS}" \ - -test.timeout=${TIMEOUT} -test.v \ + -test.timeout=100m -test.v \ ${EXTRA_TEST_ARGS} \ -binary="${MINIKUBE_BIN}" 2>&1 | tee "${TEST_OUT}" diff --git a/test/integration/pause_test.go b/test/integration/pause_test.go index 0672462e85..81043d4cd8 100644 --- a/test/integration/pause_test.go +++ b/test/integration/pause_test.go @@ -33,7 +33,7 @@ func TestPause(t *testing.T) { type validateFunc func(context.Context, *testing.T, string) profile := UniqueProfileName("pause") - ctx, cancel := context.WithTimeout(context.Background(), Minutes(10)) + ctx, cancel := context.WithTimeout(context.Background(), Minutes(15)) defer Cleanup(t, profile, cancel) // Serial tests From fe32e66aa75ca18bcafa92be5d2485cb85f955ae Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Thu, 28 Jan 2021 12:47:47 -0800 Subject: [PATCH 10/16] double timeout time --- hack/jenkins/common.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh index 357110ecde..af8623cae4 100755 --- a/hack/jenkins/common.sh +++ b/hack/jenkins/common.sh @@ -31,7 +31,7 @@ export GOPATH="$HOME/go" export KUBECONFIG="${TEST_HOME}/kubeconfig" export PATH=$PATH:"/usr/local/bin/:/usr/local/go/bin/:$GOPATH/bin" -readonly TIMEOUT=${1:-100m} +readonly TIMEOUT=${1:-200m} if [ "$(uname)" != "Darwin" ]; then # install lsof for finding none driver procs, psmisc to use pstree in cronjobs @@ -308,7 +308,7 @@ fi ${SUDO_PREFIX}${E2E_BIN} \ -minikube-start-args="--driver=${VM_DRIVER} ${EXTRA_START_ARGS}" \ - -test.timeout=100m -test.v \ + -test.timeout=200m -test.v \ ${EXTRA_TEST_ARGS} \ -binary="${MINIKUBE_BIN}" 2>&1 | tee "${TEST_OUT}" From 6fd7f208674c69d42a4189e07afad43aaa436e94 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Fri, 29 Jan 2021 11:29:34 -0800 Subject: [PATCH 11/16] Change time to 180m --- hack/jenkins/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh index af8623cae4..6bd7233bc3 100755 --- a/hack/jenkins/common.sh +++ b/hack/jenkins/common.sh @@ -308,7 +308,7 @@ fi ${SUDO_PREFIX}${E2E_BIN} \ -minikube-start-args="--driver=${VM_DRIVER} ${EXTRA_START_ARGS}" \ - -test.timeout=200m -test.v \ + -test.timeout=180m -test.v \ ${EXTRA_TEST_ARGS} \ -binary="${MINIKUBE_BIN}" 2>&1 | tee "${TEST_OUT}" From 8b858290e0ab71c3771107fb397a3f17b05ecc0d Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Fri, 29 Jan 2021 13:52:05 -0800 Subject: [PATCH 12/16] Add timeout to cleaning up tests --- hack/jenkins/common.sh | 2 +- test/integration/helpers_test.go | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh index 6bd7233bc3..fba415cdc5 100755 --- a/hack/jenkins/common.sh +++ b/hack/jenkins/common.sh @@ -308,7 +308,7 @@ fi ${SUDO_PREFIX}${E2E_BIN} \ -minikube-start-args="--driver=${VM_DRIVER} ${EXTRA_START_ARGS}" \ - -test.timeout=180m -test.v \ + -test.timeout=120m -test.v \ ${EXTRA_TEST_ARGS} \ -binary="${MINIKUBE_BIN}" 2>&1 | tee "${TEST_OUT}" diff --git a/test/integration/helpers_test.go b/test/integration/helpers_test.go index 74bfdac5ed..c41a20a209 100644 --- a/test/integration/helpers_test.go +++ b/test/integration/helpers_test.go @@ -169,7 +169,9 @@ func Cleanup(t *testing.T, profile string, cancel context.CancelFunc) { // No helper because it makes the call log confusing. if *cleanup { t.Logf("Cleaning up %q profile ...", profile) - _, err := Run(t, exec.Command(Target(), "delete", "-p", profile)) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + _, err := Run(t, exec.CommandContext(ctx, Target(), "delete", "-p", profile)) if err != nil { t.Logf("failed cleanup: %v", err) } From 49210bd8b580c1464360c7db0c16ef409cc36ad9 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Fri, 29 Jan 2021 15:07:03 -0800 Subject: [PATCH 13/16] Revert timings back to how they are at HEAD --- hack/jenkins/common.sh | 4 ++-- test/integration/net_test.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh index fba415cdc5..0182427673 100755 --- a/hack/jenkins/common.sh +++ b/hack/jenkins/common.sh @@ -31,7 +31,7 @@ export GOPATH="$HOME/go" export KUBECONFIG="${TEST_HOME}/kubeconfig" export PATH=$PATH:"/usr/local/bin/:/usr/local/go/bin/:$GOPATH/bin" -readonly TIMEOUT=${1:-200m} +readonly TIMEOUT=${1:-70m} if [ "$(uname)" != "Darwin" ]; then # install lsof for finding none driver procs, psmisc to use pstree in cronjobs @@ -308,7 +308,7 @@ fi ${SUDO_PREFIX}${E2E_BIN} \ -minikube-start-args="--driver=${VM_DRIVER} ${EXTRA_START_ARGS}" \ - -test.timeout=120m -test.v \ + -test.timeout=${TIMEOUT} -test.v \ ${EXTRA_TEST_ARGS} \ -binary="${MINIKUBE_BIN}" 2>&1 | tee "${TEST_OUT}" diff --git a/test/integration/net_test.go b/test/integration/net_test.go index fa78212b03..9fff54dc56 100644 --- a/test/integration/net_test.go +++ b/test/integration/net_test.go @@ -70,10 +70,10 @@ func TestNetworkPlugins(t *testing.T) { MaybeParallel(t) profile := UniqueProfileName(tc.name) - ctx, cancel := context.WithTimeout(context.Background(), Minutes(30)) + ctx, cancel := context.WithTimeout(context.Background(), Minutes(40)) defer CleanupWithLogs(t, profile, cancel) - startArgs := append([]string{"start", "-p", profile, "--memory=1800", "--alsologtostderr", "--wait=true", "--wait-timeout=3m"}, tc.args...) + startArgs := append([]string{"start", "-p", profile, "--memory=1800", "--alsologtostderr", "--wait=true", "--wait-timeout=5m"}, tc.args...) startArgs = append(startArgs, StartArgs()...) t.Run("Start", func(t *testing.T) { @@ -158,7 +158,7 @@ func TestNetworkPlugins(t *testing.T) { } // If the coredns process was stable, this retry wouldn't be necessary. - if err := retry.Expo(nslookup, 1*time.Second, Minutes(1)); err != nil { + if err := retry.Expo(nslookup, 1*time.Second, Minutes(6)); err != nil { t.Errorf("failed to do nslookup on kubernetes.default: %v", err) } From 590376760e6bbbc52c8c36acc00cb6d99e9a6e28 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Fri, 29 Jan 2021 15:10:40 -0800 Subject: [PATCH 14/16] Add timeouts to cleaning up --- hack/jenkins/common.sh | 4 ++-- test/integration/pause_test.go | 4 ++-- test/integration/start_stop_delete_test.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh index 0182427673..0e4f4e4e58 100755 --- a/hack/jenkins/common.sh +++ b/hack/jenkins/common.sh @@ -393,8 +393,8 @@ if grep -q html "$HTML_OUT"; then fi echo ">> Cleaning up after ourselves ..." -${SUDO_PREFIX}${MINIKUBE_BIN} tunnel --cleanup || true -${SUDO_PREFIX}${MINIKUBE_BIN} delete --all --purge >/dev/null 2>/dev/null || true +timeout 3m ${SUDO_PREFIX}${MINIKUBE_BIN} tunnel --cleanup || true +timeout 5m ${SUDO_PREFIX}${MINIKUBE_BIN} delete --all --purge >/dev/null 2>/dev/null || true cleanup_stale_routes || true ${SUDO_PREFIX} rm -Rf "${MINIKUBE_HOME}" || true diff --git a/test/integration/pause_test.go b/test/integration/pause_test.go index 81043d4cd8..d9214bec4e 100644 --- a/test/integration/pause_test.go +++ b/test/integration/pause_test.go @@ -71,7 +71,7 @@ func TestPause(t *testing.T) { func validateFreshStart(ctx context.Context, t *testing.T, profile string) { defer PostMortemLogs(t, profile) - args := append([]string{"start", "-p", profile, "--memory=1800", "--install-addons=false", "--wait=all", "--wait-timeout=2m"}, StartArgs()...) + args := append([]string{"start", "-p", profile, "--memory=1800", "--install-addons=false", "--wait=all"}, StartArgs()...) rr, err := Run(t, exec.CommandContext(ctx, Target(), args...)) if err != nil { t.Fatalf("failed to start minikube with args: %q : %v", rr.Command(), err) @@ -82,7 +82,7 @@ func validateFreshStart(ctx context.Context, t *testing.T, profile string) { func validateStartNoReconfigure(ctx context.Context, t *testing.T, profile string) { defer PostMortemLogs(t, profile) - args := []string{"start", "-p", profile, "--alsologtostderr", "-v=1", "--wait-timeout=2m"} + args := []string{"start", "-p", profile, "--alsologtostderr", "-v=1"} args = append(args, StartArgs()...) rr, err := Run(t, exec.CommandContext(ctx, Target(), args...)) if err != nil { diff --git a/test/integration/start_stop_delete_test.go b/test/integration/start_stop_delete_test.go index c7666566c0..8154b53449 100644 --- a/test/integration/start_stop_delete_test.go +++ b/test/integration/start_stop_delete_test.go @@ -94,7 +94,7 @@ func TestStartStop(t *testing.T) { waitFlag = "--wait=apiserver,system_pods,default_sa" } - startArgs := append([]string{"start", "-p", profile, "--memory=2200", "--alsologtostderr", waitFlag, "--wait-timeout=2m"}, tc.args...) + startArgs := append([]string{"start", "-p", profile, "--memory=2200", "--alsologtostderr", waitFlag}, tc.args...) startArgs = append(startArgs, StartArgs()...) startArgs = append(startArgs, fmt.Sprintf("--kubernetes-version=%s", tc.version)) From fa9b3dc9b6156e8b31493c01a6b0a43432be0de8 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Fri, 29 Jan 2021 16:16:27 -0800 Subject: [PATCH 15/16] Switch back to 30m --- test/integration/pause_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/pause_test.go b/test/integration/pause_test.go index d9214bec4e..c412d62942 100644 --- a/test/integration/pause_test.go +++ b/test/integration/pause_test.go @@ -33,7 +33,7 @@ func TestPause(t *testing.T) { type validateFunc func(context.Context, *testing.T, string) profile := UniqueProfileName("pause") - ctx, cancel := context.WithTimeout(context.Background(), Minutes(15)) + ctx, cancel := context.WithTimeout(context.Background(), Minutes(30)) defer Cleanup(t, profile, cancel) // Serial tests From 1d6a1d44b2cec3579fc58599f5ba07aa2bfe2818 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Sun, 31 Jan 2021 22:18:18 -0800 Subject: [PATCH 16/16] Half pause test time --- test/integration/pause_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/pause_test.go b/test/integration/pause_test.go index c412d62942..d9214bec4e 100644 --- a/test/integration/pause_test.go +++ b/test/integration/pause_test.go @@ -33,7 +33,7 @@ func TestPause(t *testing.T) { type validateFunc func(context.Context, *testing.T, string) profile := UniqueProfileName("pause") - ctx, cancel := context.WithTimeout(context.Background(), Minutes(30)) + ctx, cancel := context.WithTimeout(context.Background(), Minutes(15)) defer Cleanup(t, profile, cancel) // Serial tests