diff --git a/test/integration/start_stop_delete_test.go b/test/integration/start_stop_delete_test.go
index 7391ad6905..75e2595479 100644
--- a/test/integration/start_stop_delete_test.go
+++ b/test/integration/start_stop_delete_test.go
@@ -29,11 +29,13 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/blang/semver/v4"
 	"github.com/docker/machine/libmachine/state"
 	"github.com/google/go-cmp/cmp"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
 	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/detect"
+	"k8s.io/minikube/pkg/util"
 )
 
 // TestStartStop tests starting, stopping and restarting a minikube clusters with various Kubernetes versions and configurations
@@ -112,6 +114,21 @@ func TestStartStop(t *testing.T) {
 				startArgs = append(startArgs, StartArgs()...)
 				startArgs = append(startArgs, fmt.Sprintf("--kubernetes-version=%s", tc.version))
 
+				version, err := util.ParseKubernetesVersion(tc.version)
+				if err != nil {
+					t.Errorf("failed to parse %s: %v", tc.version, err)
+				}
+				if version.GTE(semver.MustParse("1.24.0-alpha.2")) {
+					args := []string{}
+					for _, arg := range args {
+						if arg == "--extra-config=kubelet.network-plugin=cni" {
+							continue
+						}
+						args = append(args, arg)
+					}
+					startArgs = args
+				}
+
 				t.Run("serial", func(t *testing.T) {
 					serialTests := []struct {
 						name      string