Improve start offsets in MaybeSlowParallel by using a schedule

test/integration/addons_test.go

@@ -37,7 +37,6 @@ import (
 // TestAddons tests addons that require no special environment -- in parallel
 func TestAddons(t *testing.T) {
 	MaybeSlowParallel(t)
-
 	profile := UniqueProfileName("addons")
 	ctx, cancel := context.WithTimeout(context.Background(), 40*time.Minute)
 	defer CleanupWithLogs(t, profile, cancel)

test/integration/docker_test.go

@@ -31,7 +31,6 @@ func TestDockerFlags(t *testing.T) {
 		t.Skip("skipping: none driver does not support ssh or bundle docker")
 	}
 	MaybeSlowParallel(t)
-
 	profile := UniqueProfileName("docker-flags")
 	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute)
 	defer CleanupWithLogs(t, profile, cancel)

test/integration/gvisor_addon_test.go

@@ -35,7 +35,6 @@ func TestGvisorAddon(t *testing.T) {
 	}
 
 	MaybeSlowParallel(t)
-
 	profile := UniqueProfileName("gvisor")
 	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute)
 	defer func() {

test/integration/helpers.go

@@ -42,8 +42,8 @@ import (
 )
 
 var (
-	antiRaceCounter = 0
-	antiRaceMutex   = &sync.Mutex{}
+	startSchedule = []time.Time{}
+	antiRaceMutex = &sync.Mutex{}
 )
 
 // RunResult stores the result of an cmd.Run call
@@ -330,24 +330,29 @@ func MaybeParallel(t *testing.T) {
 	t.Parallel()
 }
 
-// MaybeSlowParallel is a terrible workaround for tests which start clusters in a race-filled world
-// TODO: Try removing this hack once certificates are deployed per-profile
+// MaybeSlowParallel offsets cluster starts by the value of --start-offset
+// TODO: Remove when possible
 func MaybeSlowParallel(t *testing.T) {
 	// NoneDriver shouldn't parallelize "minikube start"
 	if NoneDriver() {
 		return
 	}
 
+	wakeup := time.Now()
 	antiRaceMutex.Lock()
-	antiRaceCounter++
+	if len(startSchedule) == 0 {
+		startSchedule = append(startSchedule, wakeup)
+	} else {
+		wakeup = startSchedule[len(startSchedule)-1].Add(*startOffset)
+		startSchedule = append(startSchedule, wakeup)
+	}
 	antiRaceMutex.Unlock()
 
-	if antiRaceCounter > 0 {
-		// Slow enough to offset start, but not slow to be a major source of delay
-		// TODO: Remove or minimize once #5353 is resolved
-		penalty := time.Duration(30*antiRaceCounter) * time.Second
-		t.Logf("MaybeSlowParallel: Sleeping %s to avoid start race ...", penalty)
-		time.Sleep(penalty)
+	if time.Now().Before(wakeup) {
+		d := time.Until(wakeup)
+		t.Logf("MaybeSlowParallel: Sleeping %s (until %s) to avoid start race ...", d, wakeup)
+		time.Sleep(d)
+		// Leave our entry in startSchedule, otherwise we can't guarantee a 30 second offset for the next caller
 	}
 	t.Parallel()
 }

test/integration/main.go

@@ -33,6 +33,7 @@ var defaultDriver = flag.String("expected-default-driver", "", "Expected default
 var forceProfile = flag.String("profile", "", "force tests to run against a particular profile")
 var cleanup = flag.Bool("cleanup", true, "cleanup failed test run")
 var enableGvisor = flag.Bool("gvisor", false, "run gvisor integration test (slow)")
+var startOffset = flag.Duration("start-offset", 30*time.Second, "how much time to offset between cluster starts")
 var postMortemLogs = flag.Bool("postmortem-logs", true, "show logs after a failed test run")
 
 // Paths to files - normally set for CI

test/integration/version_upgrade_test.go

@@ -39,9 +39,9 @@ import (
 // the odlest supported k8s version and then runs the current head minikube
 // and it tries to upgrade from the older supported k8s to news supported k8s
 func TestVersionUpgrade(t *testing.T) {
+	MaybeSlowParallel(t)
 	profile := UniqueProfileName("vupgrade")
 	ctx, cancel := context.WithTimeout(context.Background(), 55*time.Minute)
-	MaybeSlowParallel(t)
 
 	defer CleanupWithLogs(t, profile, cancel)