From 46df921da7e2434d2abb45f4f172abcb49a8d1ef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= <anders.f.bjorklund@gmail.com>
Date: Mon, 17 Feb 2020 19:37:07 +0100
Subject: [PATCH 01/26] Add default CNI network for running wth podman

---
 deploy/iso/minikube-iso/package/podman/podman.mk | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/deploy/iso/minikube-iso/package/podman/podman.mk b/deploy/iso/minikube-iso/package/podman/podman.mk
index 911dc30cf3..955266001e 100644
--- a/deploy/iso/minikube-iso/package/podman/podman.mk
+++ b/deploy/iso/minikube-iso/package/podman/podman.mk
@@ -29,6 +29,8 @@ endef
 
 define PODMAN_INSTALL_TARGET_CMDS
 	$(INSTALL) -Dm755 $(@D)/bin/podman $(TARGET_DIR)/usr/bin/podman
+	$(INSTALL) -d -m 755 $(TARGET_DIR)/etc/cni/net.d/
+	$(INSTALL) -m 644 cni/87-podman-bridge.conflist $(TARGET_DIR)/etc/cni/net.d/87-podman-bridge.conflist
 endef
 
 $(eval $(generic-package))

From 270756e93da0778762e19e033707a15ddac9ed9f Mon Sep 17 00:00:00 2001
From: Anshul Sirur <vixus0@gmail.com>
Date: Fri, 17 Apr 2020 09:53:04 +0200
Subject: [PATCH 02/26] Bind mount /var/tmp for Podman

When the CRI-O container storage backend is used, loading images using
`podman load` creates temporary files in /var/tmp. As /var/tmp is
mounted as tmpfs, these temporary files rapidly consume the available memory
and cause processes on the minikube VM to be OOM-killed.

Instead, bind mount /var/tmp like we do with /var/log, which will write
temporary files to disk instead of memory.
---
 deploy/iso/minikube-iso/package/automount/minikube-automount | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/deploy/iso/minikube-iso/package/automount/minikube-automount b/deploy/iso/minikube-iso/package/automount/minikube-automount
index 17fc8b0279..77e5fda1fe 100755
--- a/deploy/iso/minikube-iso/package/automount/minikube-automount
+++ b/deploy/iso/minikube-iso/package/automount/minikube-automount
@@ -109,6 +109,10 @@ if [ -n "$BOOT2DOCKER_DATA" ]; then
     mkdir /var/log
     mount --bind /mnt/$PARTNAME/var/log /var/log
 
+    mkdir -p /mnt/$PARTNAME/var/tmp
+    mkdir /var/tmp
+    mount --bind /mnt/$PARTNAME/var/tmp /var/tmp
+
     mkdir -p /mnt/$PARTNAME/var/lib/kubelet
     mkdir /var/lib/kubelet
     mount --bind /mnt/$PARTNAME/var/lib/kubelet /var/lib/kubelet

From 811030095b29fae27bec7c6cc1076cbe4a933df7 Mon Sep 17 00:00:00 2001
From: Brad Walker <brad@bradmwalker.com>
Date: Sat, 18 Apr 2020 00:44:27 -0600
Subject: [PATCH 03/26] kernel: Enable VRF

---
 deploy/iso/minikube-iso/board/coreos/minikube/linux_defconfig | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/deploy/iso/minikube-iso/board/coreos/minikube/linux_defconfig b/deploy/iso/minikube-iso/board/coreos/minikube/linux_defconfig
index 9611b6a9ac..21f310fb0c 100644
--- a/deploy/iso/minikube-iso/board/coreos/minikube/linux_defconfig
+++ b/deploy/iso/minikube-iso/board/coreos/minikube/linux_defconfig
@@ -114,6 +114,7 @@ CONFIG_TCP_CONG_ADVANCED=y
 CONFIG_TCP_MD5SIG=y
 CONFIG_INET6_AH=y
 CONFIG_INET6_ESP=y
+CONFIG_IPV6_MULTIPLE_TABLES=y
 CONFIG_NETLABEL=y
 CONFIG_NETFILTER=y
 CONFIG_NETFILTER_NETLINK_ACCT=y
@@ -351,6 +352,7 @@ CONFIG_NETCONSOLE=y
 CONFIG_TUN=y
 CONFIG_VETH=y
 CONFIG_VIRTIO_NET=y
+CONFIG_NET_VRF=m
 CONFIG_AMD8111_ETH=m
 CONFIG_PCNET32=m
 CONFIG_PCMCIA_NMCLAN=m

From 169855b29607ed86dc7782db28051334008db119 Mon Sep 17 00:00:00 2001
From: Kenta Iso <type.mafty@gmail.com>
Date: Sat, 18 Apr 2020 14:37:09 +0900
Subject: [PATCH 04/26] Add log if TestFunctional/parallel/MySQL fails: e.g.
 CrashLoopBackOff

---
 test/integration/functional_test.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go
index 5f1ef0124b..58bc6edb3b 100644
--- a/test/integration/functional_test.go
+++ b/test/integration/functional_test.go
@@ -774,6 +774,8 @@ func validateMySQL(ctx context.Context, t *testing.T, profile string) {
 		return err
 	}
 	if err = retry.Expo(mysql, 1*time.Second, Minutes(5)); err != nil {
+		// show mysql pod logs to isolate what is happening: https://github.com/kubernetes/minikube/issues/7401
+		showPodLogs(ctx, t, profile, "default", names)
 		t.Errorf("failed to exec 'mysql -ppassword -e show databases;': %v", err)
 	}
 }

From 90cb3e3331e4120f5949cd693073fb75f24ab359 Mon Sep 17 00:00:00 2001
From: Kenta Iso <type.mafty@gmail.com>
Date: Sat, 18 Apr 2020 17:57:41 +0900
Subject: [PATCH 05/26] disable apparmor for MySQL on Ubuntu

---
 .github/workflows/main.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 38a5b5cd62..4d6f4c3e28 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -116,6 +116,8 @@ jobs:
         mkdir -p testhome
         chmod a+x e2e-*
         chmod a+x minikube-*
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
         START_TIME=$(date -u +%s)
         KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--vm-driver=docker  -test.run TestFunctional -test.timeout=30m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
         END_TIME=$(date -u +%s)

From a471fb385757f3ba5c0793ab1677fb3ec7e7a956 Mon Sep 17 00:00:00 2001
From: Kenta Iso <type.mafty@gmail.com>
Date: Sat, 18 Apr 2020 18:05:50 +0900
Subject: [PATCH 06/26] add disable apparmor for docker_ubuntu_18_04

---
 .github/workflows/main.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 4d6f4c3e28..2d4c33004f 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -207,6 +207,8 @@ jobs:
         mkdir -p testhome
         chmod a+x e2e-*
         chmod a+x minikube-*
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
         START_TIME=$(date -u +%s)
         KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--driver=docker  -test.run TestPause -test.timeout=30m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
         END_TIME=$(date -u +%s)

From ff2c07c9640a85fa4a86c378fcbed9ccda4019b3 Mon Sep 17 00:00:00 2001
From: Kenta Iso <type.mafty@gmail.com>
Date: Sat, 18 Apr 2020 19:01:09 +0900
Subject: [PATCH 07/26] delete showPodLog() from TestFunctional/parallel/MySQL
 because the root cause is found

---
 test/integration/functional_test.go | 2 --
 1 file changed, 2 deletions(-)

diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go
index 58bc6edb3b..5f1ef0124b 100644
--- a/test/integration/functional_test.go
+++ b/test/integration/functional_test.go
@@ -774,8 +774,6 @@ func validateMySQL(ctx context.Context, t *testing.T, profile string) {
 		return err
 	}
 	if err = retry.Expo(mysql, 1*time.Second, Minutes(5)); err != nil {
-		// show mysql pod logs to isolate what is happening: https://github.com/kubernetes/minikube/issues/7401
-		showPodLogs(ctx, t, profile, "default", names)
 		t.Errorf("failed to exec 'mysql -ppassword -e show databases;': %v", err)
 	}
 }

From a6b61a86a5fe3ccaee9f2d996a1d3717218d323f Mon Sep 17 00:00:00 2001
From: Kenta Iso <type.mafty@gmail.com>
Date: Sat, 18 Apr 2020 19:31:22 +0900
Subject: [PATCH 08/26] add how to disbale apparmor for mysql profile for linux
 user

---
 site/content/en/docs/drivers/docker.md | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/site/content/en/docs/drivers/docker.md b/site/content/en/docs/drivers/docker.md
index b26f6a879f..a1a185d75a 100644
--- a/site/content/en/docs/drivers/docker.md
+++ b/site/content/en/docs/drivers/docker.md
@@ -33,3 +33,13 @@ The Docker driver allows you to install Kubernetes into an existing Docker insta
 - On macOS or Windows, you may need to restart Docker for Desktop if a command gets hung
 
 - Run `--alsologtostderr -v=1` for extra debugging information
+
+- On Linux, if you want to run MySQL pod, you need to disable AppArmor for mysql profile
+
+   AppArmor is access control security system for Linux. This is enabled on Ubuntu by default.
+   If you machine enables AppArmor, you need to disable AppArmor for the mysql profile.
+   To disable apparmor for mysql, run the following command on host machine.
+
+    `sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+    sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+    `

From b5b73e43c352ff5de83aea85f0bbd17d28aef310 Mon Sep 17 00:00:00 2001
From: Kenta Iso <type.mafty@gmail.com>
Date: Sat, 18 Apr 2020 19:36:17 +0900
Subject: [PATCH 09/26] fix command layout

---
 site/content/en/docs/drivers/docker.md | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/site/content/en/docs/drivers/docker.md b/site/content/en/docs/drivers/docker.md
index a1a185d75a..0eccba2263 100644
--- a/site/content/en/docs/drivers/docker.md
+++ b/site/content/en/docs/drivers/docker.md
@@ -40,6 +40,4 @@ The Docker driver allows you to install Kubernetes into an existing Docker insta
    If you machine enables AppArmor, you need to disable AppArmor for the mysql profile.
    To disable apparmor for mysql, run the following command on host machine.
 
-    `sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
-    sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
-    `
+    `sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ && sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld`

From eb0e113e6ed08b95970adbef425d17d45eea2e62 Mon Sep 17 00:00:00 2001
From: Kenta Iso <type.mafty@gmail.com>
Date: Sat, 18 Apr 2020 21:34:51 +0900
Subject: [PATCH 10/26] Add log for ServiceCmd to make this flake result
 obvious

---
 test/integration/functional_test.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go
index 5f1ef0124b..c2de8b11d3 100644
--- a/test/integration/functional_test.go
+++ b/test/integration/functional_test.go
@@ -714,6 +714,8 @@ func validateServiceCmd(ctx context.Context, t *testing.T, profile string) {
 	if resp.StatusCode != http.StatusOK {
 		t.Fatalf("expected status code for %q to be -%q- but got *%q*", endpoint, http.StatusOK, resp.StatusCode)
 	}
+	// this log is added to make this flake result obvious: https://github.com/kubernetes/minikube/issues/7765
+	t.Logf("access to %s is success: %d", endpoint, resp.StatusCode)
 }
 
 // validateAddonsCmd asserts basic "addon" command functionality

From 2c81c21e57c3b01a58e4dc2960dda8752462cc6e Mon Sep 17 00:00:00 2001
From: Kenta Iso <type.mafty@gmail.com>
Date: Sat, 18 Apr 2020 21:45:03 +0900
Subject: [PATCH 11/26] fix apparmor trobleshooting to add original issue track

---
 site/content/en/docs/drivers/docker.md | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/site/content/en/docs/drivers/docker.md b/site/content/en/docs/drivers/docker.md
index 0eccba2263..ad5949d0ce 100644
--- a/site/content/en/docs/drivers/docker.md
+++ b/site/content/en/docs/drivers/docker.md
@@ -36,8 +36,5 @@ The Docker driver allows you to install Kubernetes into an existing Docker insta
 
 - On Linux, if you want to run MySQL pod, you need to disable AppArmor for mysql profile
 
-   AppArmor is access control security system for Linux. This is enabled on Ubuntu by default.
-   If you machine enables AppArmor, you need to disable AppArmor for the mysql profile.
-   To disable apparmor for mysql, run the following command on host machine.
-
-    `sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ && sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld`
+   If your docker has [AppArmor](https://wiki.ubuntu.com/AppArmor) enabled, running mysql in privileged mode with docker driver will have the issue [#7401](https://github.com/kubernetes/minikube/issues/7401).
+   There is a workaround - see [moby/moby#7512](https://github.com/moby/moby/issues/7512#issuecomment-61787845).

From d54c6cb45cfb14f2030841fb658e6900a1ce9302 Mon Sep 17 00:00:00 2001
From: Kenta Iso <type.mafty@gmail.com>
Date: Sun, 19 Apr 2020 02:30:38 +0900
Subject: [PATCH 12/26] change t.Errof to t.Logf to avoid docker_ubuntu_16_04
 github action fail

---
 test/integration/functional_test.go | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go
index c2de8b11d3..2da8dadbe3 100644
--- a/test/integration/functional_test.go
+++ b/test/integration/functional_test.go
@@ -670,7 +670,7 @@ func validateServiceCmd(ctx context.Context, t *testing.T, profile string) {
 		t.Fatalf("failed to get service url. args %q : %v", rr.Command(), err)
 	}
 	if rr.Stderr.String() != "" {
-		t.Errorf("expected stderr to be empty but got *%q*", rr.Stderr)
+		t.Logf("expected stderr to be empty but got *%q*", rr.Stderr)
 	}
 
 	endpoint := strings.TrimSpace(rr.Stdout.String())
@@ -714,8 +714,6 @@ func validateServiceCmd(ctx context.Context, t *testing.T, profile string) {
 	if resp.StatusCode != http.StatusOK {
 		t.Fatalf("expected status code for %q to be -%q- but got *%q*", endpoint, http.StatusOK, resp.StatusCode)
 	}
-	// this log is added to make this flake result obvious: https://github.com/kubernetes/minikube/issues/7765
-	t.Logf("access to %s is success: %d", endpoint, resp.StatusCode)
 }
 
 // validateAddonsCmd asserts basic "addon" command functionality

From 24dd9861e034c94c2d95240b7ae19d72b6e206fc Mon Sep 17 00:00:00 2001
From: Medya Gh <medya.gh@gmail.com>
Date: Sat, 18 Apr 2020 14:37:58 -0700
Subject: [PATCH 13/26] change loglevel for freezer

---
 pkg/minikube/bootstrapper/bsutil/kverify/api_server.go | 10 +++++++++-
 test/integration/functional_test.go                    |  2 +-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/api_server.go b/pkg/minikube/bootstrapper/bsutil/kverify/api_server.go
index 6aa6763a26..0fb4a54758 100644
--- a/pkg/minikube/bootstrapper/bsutil/kverify/api_server.go
+++ b/pkg/minikube/bootstrapper/bsutil/kverify/api_server.go
@@ -166,7 +166,15 @@ func APIServerStatus(cr command.Runner, hostname string, port int) (state.State,
 
 	rr, err = cr.RunCmd(exec.Command("sudo", "cat", path.Join("/sys/fs/cgroup/freezer", fparts[2], "freezer.state")))
 	if err != nil {
-		glog.Errorf("unable to get freezer state: %s", rr.Stderr.String())
+		// example error from github action:
+		// cat: /sys/fs/cgroup/freezer/actions_job/e62ef4349cc5a70f4b49f8a150ace391da6ad6df27073c83ecc03dbf81fde1ce/kubepods/burstable/poda1de58db0ce81d19df7999f6808def1b/5df53230fe3483fd65f341923f18a477fda92ae9cd71061168130ef164fe479c/freezer.state: No such file or directory\n"*
+		// TODO: #7770 investigate how to handle this error better.
+		if strings.Contains(rr.Stderr.String(), "freezer.state: No such file or directory\n") {
+			glog.Infof("unable to get freezer state (might be okay and be related to #770): \n ", rr.Stderr.String())
+		} else {
+			glog.Warningf("unable to get freezer state : %s", rr.Stderr.String())
+		}
+
 		return apiServerHealthz(hostname, port)
 	}
 
diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go
index 2da8dadbe3..2f41ea89ef 100644
--- a/test/integration/functional_test.go
+++ b/test/integration/functional_test.go
@@ -670,7 +670,7 @@ func validateServiceCmd(ctx context.Context, t *testing.T, profile string) {
 		t.Fatalf("failed to get service url. args %q : %v", rr.Command(), err)
 	}
 	if rr.Stderr.String() != "" {
-		t.Logf("expected stderr to be empty but got *%q*", rr.Stderr)
+		t.Errorf("expected stderr to be empty but got *%q* . args %q", rr.Stderr, rr.Command())
 	}
 
 	endpoint := strings.TrimSpace(rr.Stdout.String())

From 23f7fb0b9b25686c8f2ef631bf7044a3a50e29fc Mon Sep 17 00:00:00 2001
From: Medya Gh <medya.gh@gmail.com>
Date: Sat, 18 Apr 2020 14:52:45 -0700
Subject: [PATCH 14/26] lint

---
 pkg/minikube/bootstrapper/bsutil/kverify/api_server.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/api_server.go b/pkg/minikube/bootstrapper/bsutil/kverify/api_server.go
index 0fb4a54758..146eabc7c6 100644
--- a/pkg/minikube/bootstrapper/bsutil/kverify/api_server.go
+++ b/pkg/minikube/bootstrapper/bsutil/kverify/api_server.go
@@ -170,7 +170,7 @@ func APIServerStatus(cr command.Runner, hostname string, port int) (state.State,
 		// cat: /sys/fs/cgroup/freezer/actions_job/e62ef4349cc5a70f4b49f8a150ace391da6ad6df27073c83ecc03dbf81fde1ce/kubepods/burstable/poda1de58db0ce81d19df7999f6808def1b/5df53230fe3483fd65f341923f18a477fda92ae9cd71061168130ef164fe479c/freezer.state: No such file or directory\n"*
 		// TODO: #7770 investigate how to handle this error better.
 		if strings.Contains(rr.Stderr.String(), "freezer.state: No such file or directory\n") {
-			glog.Infof("unable to get freezer state (might be okay and be related to #770): \n ", rr.Stderr.String())
+			glog.Infof("unable to get freezer state (might be okay and be related to #770): %s", rr.Stderr.String())
 		} else {
 			glog.Warningf("unable to get freezer state : %s", rr.Stderr.String())
 		}

From d7adbe20f4586d564e0120094ef62260c65a9b36 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya.gh@gmail.com>
Date: Sat, 18 Apr 2020 17:35:56 -0700
Subject: [PATCH 15/26] add more docker tests to github actions

---
 .github/workflows/main.yml         | 175 +++++++++++++++++++++++------
 test/integration/multinode_test.go |   5 -
 2 files changed, 138 insertions(+), 42 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 2d4c33004f..c5e9962411 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -63,14 +63,14 @@ jobs:
       continue-on-error: false
   # Run the following integration tests after the build_minikube
   # They will run in parallel and use the binaries in previous step
-  functional_test_docker_ubuntu_16_04:
+  functional_preload_test_docker_ubuntu:
     needs: [build_minikube]
     env:
       TIME_ELAPSED: time
-      JOB_NAME: "Docker_Ubuntu_16_04"
+      JOB_NAME: "functional_test_docker_ubuntu"
       GOPOGH_RESULT: ""
       SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643
-    runs-on: ubuntu-16.04
+    runs-on: ubuntu-18.04
     steps:    
     - name: Install kubectl
       shell: bash
@@ -141,9 +141,9 @@ jobs:
         echo ::set-env name=STAT::${STAT}
     - uses: actions/upload-artifact@v1
       with:
-        name: docker_ubuntu_16_04
+        name: functional_test_docker_ubuntu
         path: minikube_binaries/report
-    - name: The End Result Docker on ubuntu 16:04
+    - name: The End Result functional_test_docker_ubuntu
       shell: bash
       run: |
         echo ${GOPOGH_RESULT}
@@ -154,11 +154,11 @@ jobs:
         numPass=$(echo $STAT | jq '.NumberOfPass')
         echo "*** $numPass Passed ***"
         if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
-  pause_test_docker_ubuntu_18_04:
+  addons_certs_tests_docker_ubuntu:
     runs-on: ubuntu-18.04
     env:
       TIME_ELAPSED: time
-      JOB_NAME: "Docker_Ubuntu_18_04"
+      JOB_NAME: "addons_certs_tests_docker_ubuntu"
       GOPOGH_RESULT: ""
       SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643
     needs: [build_minikube]
@@ -210,7 +210,7 @@ jobs:
         sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
         sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
         START_TIME=$(date -u +%s)
-        KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--driver=docker  -test.run TestPause -test.timeout=30m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
+        KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--driver=docker  -test.run "(TestAddons|TestCertOptions)" -test.timeout=30m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
         END_TIME=$(date -u +%s)
         TIME_ELAPSED=$(($END_TIME-$START_TIME))
         min=$((${TIME_ELAPSED}/60))
@@ -232,9 +232,9 @@ jobs:
         echo ::set-env name=STAT::${STAT}
     - uses: actions/upload-artifact@v1
       with:
-        name: docker_ubuntu_18_04
+        name: addons_certs_tests_docker_ubuntu
         path: minikube_binaries/report
-    - name: The End Result - Docker On Ubuntu 18:04
+    - name: The End Result - addons_certs_tests_docker_ubuntu
       shell: bash
       run: |
         echo ${GOPOGH_RESULT}
@@ -245,14 +245,14 @@ jobs:
         numPass=$(echo $STAT | jq '.NumberOfPass')
         echo "*** $numPass Passed ***"
         if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
-  baremetal_ubuntu16_04:
-    needs: [build_minikube]
+  multinode_pause_tests_docker_ubuntu:
+    runs-on: ubuntu-18.04
     env:
       TIME_ELAPSED: time
-      JOB_NAME: "None_Ubuntu_16_04"
+      JOB_NAME: "multinode_pause_tests_docker_ubuntu"
       GOPOGH_RESULT: ""
       SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643
-    runs-on: ubuntu-16.04
+    needs: [build_minikube]
     steps:
     - name: Install kubectl
       shell: bash
@@ -260,17 +260,25 @@ jobs:
         curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/amd64/kubectl
         sudo install kubectl /usr/local/bin/kubectl
         kubectl version --client=true
-    # conntrack is required for kubernetes 1.18 and higher
-    # socat is required for kubectl port forward which is used in some tests such as validateHelmTillerAddon
-    - name: Install tools for none
+    - name: Install lz4
       shell: bash
       run: |
         sudo apt-get update -qq
-        sudo apt-get -qq -y install conntrack
-        sudo apt-get -qq -y install socat
-        VERSION="v1.17.0"
-        curl -L https://github.com/kubernetes-sigs/cri-tools/releases/download/$VERSION/crictl-${VERSION}-linux-amd64.tar.gz --output crictl-${VERSION}-linux-amd64.tar.gz
-        sudo tar zxvf crictl-$VERSION-linux-amd64.tar.gz -C /usr/local/bin
+        sudo apt-get -qq -y install liblz4-tool
+    - name: Docker Info
+      shell: bash
+      run: |
+        echo "--------------------------"
+        docker version || true
+        echo "--------------------------"
+        docker info || true
+        echo "--------------------------"
+        docker system df || true
+        echo "--------------------------"
+        docker system info || true
+        echo "--------------------------"
+        docker ps || true
+        echo "--------------------------"
     - name: Install gopogh
       shell: bash
       run: |
@@ -290,8 +298,10 @@ jobs:
         mkdir -p testhome
         chmod a+x e2e-*
         chmod a+x minikube-*
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
         START_TIME=$(date -u +%s)
-        KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome sudo -E ./e2e-linux-amd64 -minikube-start-args=--driver=none -test.timeout=35m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
+        KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--driver=docker  -test.run "(TestPause|TestMultiNode)" -test.timeout=30m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
         END_TIME=$(date -u +%s)
         TIME_ELAPSED=$(($END_TIME-$START_TIME))
         min=$((${TIME_ELAPSED}/60))
@@ -313,9 +323,100 @@ jobs:
         echo ::set-env name=STAT::${STAT}
     - uses: actions/upload-artifact@v1
       with:
-        name: none_ubuntu16_04
+        name: multinode_pause_tests_docker_ubuntu
         path: minikube_binaries/report
-    - name: The End Result - None On Ubuntu 16:04
+    - name: The End Result - multinode_pause_tests_docker_ubuntu
+      shell: bash
+      run: |
+        echo ${GOPOGH_RESULT}
+        numFail=$(echo $STAT | jq '.NumberOfFail')
+        echo "----------------${numFail} Failures----------------------------"
+        echo $STAT | jq '.FailedTests' || true
+        echo "-------------------------------------------------------"
+        numPass=$(echo $STAT | jq '.NumberOfPass')
+        echo "*** $numPass Passed ***"
+        if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
+  preload_docker_flags_tests_docker_ubuntu:
+    runs-on: ubuntu-18.04
+    env:
+      TIME_ELAPSED: time
+      JOB_NAME: "preload_docker_flags_tests_docker_ubuntu"
+      GOPOGH_RESULT: ""
+      SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643
+    needs: [build_minikube]
+    steps:
+    - name: Install kubectl
+      shell: bash
+      run: |
+        curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/amd64/kubectl
+        sudo install kubectl /usr/local/bin/kubectl
+        kubectl version --client=true
+    - name: Install lz4
+      shell: bash
+      run: |
+        sudo apt-get update -qq
+        sudo apt-get -qq -y install liblz4-tool
+    - name: Docker Info
+      shell: bash
+      run: |
+        echo "--------------------------"
+        docker version || true
+        echo "--------------------------"
+        docker info || true
+        echo "--------------------------"
+        docker system df || true
+        echo "--------------------------"
+        docker system info || true
+        echo "--------------------------"
+        docker ps || true
+        echo "--------------------------"
+    - name: Install gopogh
+      shell: bash
+      run: |
+        curl -LO https://github.com/medyagh/gopogh/releases/download/v0.1.19/gopogh-linux-amd64
+        sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
+    - name: Download Binaries
+      uses: actions/download-artifact@v1
+      with:
+        name: minikube_binaries
+    - name: Run Integration Test
+      continue-on-error: true
+      # bash {0} to allow test to continue to next step. in case of
+      shell: bash {0}
+      run: |
+        cd minikube_binaries
+        mkdir -p report
+        mkdir -p testhome
+        chmod a+x e2e-*
+        chmod a+x minikube-*
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        START_TIME=$(date -u +%s)
+        KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--driver=docker  -test.run "(TestPreload|TestDockerFlags)" -test.timeout=30m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
+        END_TIME=$(date -u +%s)
+        TIME_ELAPSED=$(($END_TIME-$START_TIME))
+        min=$((${TIME_ELAPSED}/60))
+        sec=$((${TIME_ELAPSED}%60))
+        TIME_ELAPSED="${min} min $sec seconds "
+        echo ::set-env name=TIME_ELAPSED::${TIME_ELAPSED}
+    - name: Generate HTML Report
+      shell: bash
+      run: |
+        cd minikube_binaries
+        export PATH=${PATH}:`go env GOPATH`/bin
+        go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
+        STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
+        echo status: ${STAT}
+        FailNum=$(echo $STAT | jq '.NumberOfFail')
+        TestsNum=$(echo $STAT | jq '.NumberOfTests')
+        GOPOGH_RESULT="${JOB_NAME} : completed with ${FailNum} / ${TestsNum} failures in ${TIME_ELAPSED}"
+        echo ::set-env name=GOPOGH_RESULT::${GOPOGH_RESULT}
+        echo ::set-env name=STAT::${STAT}
+    - uses: actions/upload-artifact@v1
+      with:
+        name: preload_docker_flags_tests_docker_ubuntu
+        path: minikube_binaries/report
+    - name: The End Result - preload_docker_flags_tests_docker_ubuntu
       shell: bash
       run: |
         echo ${GOPOGH_RESULT}
@@ -411,34 +512,34 @@ jobs:
   # collect all the reports and upload
   upload_all_reports:
     if: always()
-    needs: [functional_test_docker_ubuntu_16_04, pause_test_docker_ubuntu_18_04,baremetal_ubuntu16_04, baremetal_ubuntu18_04]
+    needs: [functional_preload_test_docker_ubuntu, addons_certs_tests_docker_ubuntu, multinode_pause_tests_docker_ubuntu, preload_docker_flags_tests_docker_ubuntu, baremetal_ubuntu18_04]
     runs-on: ubuntu-18.04
     steps:
-    - name: Download Results docker_ubuntu_16_04
+    - name: Download Results functional_test_docker_ubuntu
       uses: actions/download-artifact@v1
       with:
-        name: docker_ubuntu_16_04
-    - name: cp docker_ubuntu_16_04 to all_report
+        name: functional_test_docker_ubuntu
+    - name:  functional_test_docker_ubuntu to all_report
       continue-on-error: true
       shell: bash {0}
       run: |
         mkdir -p all_reports
-        cp -r docker_ubuntu_16_04 ./all_reports/
-    - name: Download Results docker_ubuntu_18_04
+        cp -r functional_test_docker_ubuntu ./all_reports/
+    - name: Download Results multinode_pause_tests_docker_ubuntu
       uses: actions/download-artifact@v1
       with:
-        name: docker_ubuntu_18_04
-    - name: cp docker_ubuntu_18_04 to all_report
+        name: multinode_pause_tests_docker_ubuntu
+    - name: Copy multinode_pause_tests_docker_ubuntu to all_report
       continue-on-error: true
       shell: bash {0}
       run: |
         mkdir -p all_reports
-        cp -r docker_ubuntu_18_04 ./all_reports/
-    - name: Download Results none_ubuntu16_04
+        cp -r multinode_pause_tests_docker_ubuntu ./all_reports/
+    - name: Download Results preload_docker_flags_tests_docker_ubuntu
       uses: actions/download-artifact@v1
       with:
-        name: none_ubuntu16_04
-    - name: cp none_ubuntu16_04 to all_report
+        name: preload_docker_flags_tests_docker_ubuntu
+    - name: Copy preload_docker_flags_tests_docker_ubuntu to all_report
       continue-on-error: true
       shell: bash {0}
       run: |
diff --git a/test/integration/multinode_test.go b/test/integration/multinode_test.go
index 0502cd08e3..efa662964a 100644
--- a/test/integration/multinode_test.go
+++ b/test/integration/multinode_test.go
@@ -30,11 +30,6 @@ func TestMultiNode(t *testing.T) {
 		t.Skip("none driver does not support multinode")
 	}
 
-	// TODO: remove this skip once docker multinode is fixed
-	if KicDriver() {
-		t.Skip("docker driver multinode is currently broken :(")
-	}
-
 	profile := UniqueProfileName("multinode")
 	ctx, cancel := context.WithTimeout(context.Background(), Minutes(30))
 	defer CleanupWithLogs(t, profile, cancel)

From 0d4111972da9e8558d482d617ec9eacd8f587144 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya.gh@gmail.com>
Date: Sat, 18 Apr 2020 17:41:33 -0700
Subject: [PATCH 16/26] remove windows

---
 .github/workflows/main.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index c5e9962411..72dcd6eb26 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -15,8 +15,6 @@ jobs:
       run : |
         make minikube-linux-amd64
         make e2e-linux-amd64
-        make minikube-windows-amd64.exe
-        make e2e-windows-amd64.exe
         cp -r test/integration/testdata ./out
         whoami
         echo github ref $GITHUB_REF

From 8647c395f386ac0f5b7d3e74aa14487197947c30 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya.gh@gmail.com>
Date: Sat, 18 Apr 2020 18:15:56 -0700
Subject: [PATCH 17/26] upgrade our download action to v2-preview

---
 .github/workflows/main.yml | 53 ++++++++++----------------------------
 1 file changed, 14 insertions(+), 39 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 72dcd6eb26..49c3eb2b30 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -61,7 +61,7 @@ jobs:
       continue-on-error: false
   # Run the following integration tests after the build_minikube
   # They will run in parallel and use the binaries in previous step
-  functional_preload_test_docker_ubuntu:
+  functional_test_docker_ubuntu:
     needs: [build_minikube]
     env:
       TIME_ELAPSED: time
@@ -506,53 +506,28 @@ jobs:
         numPass=$(echo $STAT | jq '.NumberOfPass')
         echo "*** $numPass Passed ***"
         if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
-  # After all 4 integration tests finished
+  # After all integration tests finished
   # collect all the reports and upload
   upload_all_reports:
     if: always()
-    needs: [functional_preload_test_docker_ubuntu, addons_certs_tests_docker_ubuntu, multinode_pause_tests_docker_ubuntu, preload_docker_flags_tests_docker_ubuntu, baremetal_ubuntu18_04]
+    needs: [functional_test_docker_ubuntu, addons_certs_tests_docker_ubuntu, multinode_pause_tests_docker_ubuntu, preload_docker_flags_tests_docker_ubuntu, baremetal_ubuntu18_04]
     runs-on: ubuntu-18.04
     steps:
-    - name: Download Results functional_test_docker_ubuntu
-      uses: actions/download-artifact@v1
-      with:
-        name: functional_test_docker_ubuntu
-    - name:  functional_test_docker_ubuntu to all_report
+    - name: upload all results
       continue-on-error: true
+      # using preview version to allow download all https://github.com/actions/download-artifact/tree/v2-preview#download-all-artifacts
+      # skiping uploading baremetal_ubuntu18_04 since that finishes too late
+      uses: actions/download-artifact@v2-preview
       shell: bash {0}
       run: |
         mkdir -p all_reports
-        cp -r functional_test_docker_ubuntu ./all_reports/
-    - name: Download Results multinode_pause_tests_docker_ubuntu
-      uses: actions/download-artifact@v1
-      with:
-        name: multinode_pause_tests_docker_ubuntu
-    - name: Copy multinode_pause_tests_docker_ubuntu to all_report
-      continue-on-error: true
-      shell: bash {0}
-      run: |
-        mkdir -p all_reports
-        cp -r multinode_pause_tests_docker_ubuntu ./all_reports/
-    - name: Download Results preload_docker_flags_tests_docker_ubuntu
-      uses: actions/download-artifact@v1
-      with:
-        name: preload_docker_flags_tests_docker_ubuntu
-    - name: Copy preload_docker_flags_tests_docker_ubuntu to all_report
-      continue-on-error: true
-      shell: bash {0}
-      run: |
-        mkdir -p all_reports
-        cp -r none_ubuntu16_04 ./all_reports/
-    - name: Download Results none_ubuntu18_04
-      uses: actions/download-artifact@v1
-      with:
-        name: none_ubuntu18_04
-    - name: Copy none_ubuntu18_04 to all_report
-      continue-on-error: true
-      shell: bash {0}
-      run: |
-        mkdir -p all_reports
-        cp -r none_ubuntu18_04 ./all_reports/
+        ls -lah
+        ls -lah minikube_binaries
+        ls -lah minikube_binaries/report
+        cp -r ./minikube_binaries/report/functional_test_docker_ubuntu ./all_reports/
+        cp -r ./minikube_binaries/report/addons_certs_tests_docker_ubuntu ./all_reports/
+        cp -r ./minikube_binaries/report/multinode_pause_tests_docker_ubuntu ./all_reports/
+        cp -r ./minikube_binaries/report/preload_docker_flags_tests_docker_ubuntu        
     - uses: actions/upload-artifact@v1
       with:
         name: all_reports

From e9959e3511dea0c71c4ec6d4b83df10df9be70cc Mon Sep 17 00:00:00 2001
From: Medya Gh <medya.gh@gmail.com>
Date: Sat, 18 Apr 2020 18:20:55 -0700
Subject: [PATCH 18/26] trigger

---
 .github/workflows/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 49c3eb2b30..5a21465ac7 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -507,7 +507,7 @@ jobs:
         echo "*** $numPass Passed ***"
         if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
   # After all integration tests finished
-  # collect all the reports and upload
+  # collect all the reports and upload them
   upload_all_reports:
     if: always()
     needs: [functional_test_docker_ubuntu, addons_certs_tests_docker_ubuntu, multinode_pause_tests_docker_ubuntu, preload_docker_flags_tests_docker_ubuntu, baremetal_ubuntu18_04]

From b9d7c6db158a829e185b292ec49f9bde8b569ae3 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya.gh@gmail.com>
Date: Sat, 18 Apr 2020 18:31:05 -0700
Subject: [PATCH 19/26] add push

---
 .github/workflows/main.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 5a21465ac7..071c5fe533 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -1,5 +1,7 @@
 name: CI
-on: [pull_request]
+on: 
+  - pull_request
+  - push
 env:
   GOPROXY: https://proxy.golang.org
 jobs:
@@ -60,7 +62,7 @@ jobs:
         make test
       continue-on-error: false
   # Run the following integration tests after the build_minikube
-  # They will run in parallel and use the binaries in previous step
+  # They will run in parallel and use the binaries in previous step 
   functional_test_docker_ubuntu:
     needs: [build_minikube]
     env:

From 8c6fda6b61535fb33f556df1fb6039fd376b9635 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya.gh@gmail.com>
Date: Sat, 18 Apr 2020 18:42:09 -0700
Subject: [PATCH 20/26] minor

---
 .github/workflows/main.yml | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 071c5fe533..879306034d 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -515,12 +515,11 @@ jobs:
     needs: [functional_test_docker_ubuntu, addons_certs_tests_docker_ubuntu, multinode_pause_tests_docker_ubuntu, preload_docker_flags_tests_docker_ubuntu, baremetal_ubuntu18_04]
     runs-on: ubuntu-18.04
     steps:
-    - name: upload all results
-      continue-on-error: true
-      # using preview version to allow download all https://github.com/actions/download-artifact/tree/v2-preview#download-all-artifacts
-      # skiping uploading baremetal_ubuntu18_04 since that finishes too late
+    - name: download all reports
       uses: actions/download-artifact@v2-preview
+    - name: upload all reports
       shell: bash {0}
+      continue-on-error: true
       run: |
         mkdir -p all_reports
         ls -lah

From 2936722d6d1954fbb681dfcae04c1e9418e8e49b Mon Sep 17 00:00:00 2001
From: Medya Gh <medya.gh@gmail.com>
Date: Sat, 18 Apr 2020 18:57:10 -0700
Subject: [PATCH 21/26] add master workflow

---
 .github/workflows/master.yml           | 456 +++++++++++++++++++++++++
 .github/workflows/{main.yml => pr.yml} |   5 +-
 2 files changed, 459 insertions(+), 2 deletions(-)
 create mode 100644 .github/workflows/master.yml
 rename .github/workflows/{main.yml => pr.yml} (99%)

diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
new file mode 100644
index 0000000000..4c82da0c55
--- /dev/null
+++ b/.github/workflows/master.yml
@@ -0,0 +1,456 @@
+name: master
+on: 
+  push:
+    path:
+      - '**.go'
+    branches:    
+      - master         # Push events on master branch  paths:
+env:
+  GOPROXY: https://proxy.golang.org
+jobs:
+  # Runs before all other jobs
+  # builds the minikube binaries
+  build_minikube:
+    runs-on: ubuntu-18.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: Download Dependencies
+      run : go mod download
+    - name: Build Binaries
+      run : |
+        make minikube-linux-amd64
+        make e2e-linux-amd64
+        cp -r test/integration/testdata ./out
+        whoami
+        echo github ref $GITHUB_REF
+        echo workflow $GITHUB_WORKFLOW
+        echo home $HOME
+        echo event name $GITHUB_EVENT_NAME
+        echo workspace $GITHUB_WORKSPACE
+        echo "end of debug stuff"
+        echo $(which jq)
+    - uses: actions/upload-artifact@v1
+      with:
+        name: minikube_binaries
+        path: out
+  lint:
+    runs-on: ubuntu-18.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install libvirt
+      run : |
+        sudo apt-get update
+        sudo apt-get install -y libvirt-dev
+    - name: Download Dependencies
+      run : go mod download
+    - name: Lint
+      env:
+        TESTSUITE: lintall
+      run : make test
+      continue-on-error: false
+  unit_test:
+    runs-on: ubuntu-18.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install libvirt
+      run : |
+        sudo apt-get update
+        sudo apt-get install -y libvirt-dev
+    - name: Download Dependencies
+      run : go mod download
+    - name: Unit Test
+      env:
+        TESTSUITE: unittest
+      run :
+        make test
+      continue-on-error: false
+  # Run the following integration tests after the build_minikube
+  # They will run in parallel and use the binaries in previous step 
+  functional_test_docker_ubuntu:
+    needs: [build_minikube]
+    env:
+      TIME_ELAPSED: time
+      JOB_NAME: "functional_test_docker_ubuntu"
+      GOPOGH_RESULT: ""
+      SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643
+    runs-on: ubuntu-18.04
+    steps:    
+    - name: Install kubectl
+      shell: bash
+      run: |
+        curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/amd64/kubectl
+        sudo install kubectl /usr/local/bin/kubectl
+        kubectl version --client=true
+    - name: Docker Info
+      shell: bash
+      run: |
+        echo "--------------------------"
+        docker version || true
+        echo "--------------------------"
+        docker info || true
+        echo "--------------------------"
+        docker system df || true
+        echo "--------------------------"
+        docker system info || true
+        echo "--------------------------"
+        docker ps || true
+        echo "--------------------------"
+    - name: Install lz4
+      shell: bash
+      run: |
+        sudo apt-get update -qq
+        sudo apt-get -qq -y install liblz4-tool
+    - name: Install gopogh
+      shell: bash
+      run: |
+        curl -LO https://github.com/medyagh/gopogh/releases/download/v0.1.19/gopogh-linux-amd64
+        sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
+    - name: Download Binaries
+      uses: actions/download-artifact@v1
+      with:
+        name: minikube_binaries
+    - name: Run Integration Test
+      continue-on-error: false
+      # bash {0} to allow test to continue to next step. in case of
+      shell: bash {0}
+      run: |
+        cd minikube_binaries
+        mkdir -p report
+        mkdir -p testhome
+        chmod a+x e2e-*
+        chmod a+x minikube-*
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        START_TIME=$(date -u +%s)
+        KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--vm-driver=docker  -test.run TestFunctional -test.timeout=30m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
+        END_TIME=$(date -u +%s)
+        TIME_ELAPSED=$(($END_TIME-$START_TIME))
+        min=$((${TIME_ELAPSED}/60))
+        sec=$((${TIME_ELAPSED}%60))
+        TIME_ELAPSED="${min} min $sec seconds "
+        echo ::set-env name=TIME_ELAPSED::${TIME_ELAPSED}
+    - name: Generate HTML Report
+      shell: bash
+      run: |
+        cd minikube_binaries
+        export PATH=${PATH}:`go env GOPATH`/bin
+        go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
+        STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
+        echo status: ${STAT}
+        FailNum=$(echo $STAT | jq '.NumberOfFail')
+        TestsNum=$(echo $STAT | jq '.NumberOfTests')
+        GOPOGH_RESULT="${JOB_NAME} : completed with ${FailNum} / ${TestsNum} failures in ${TIME_ELAPSED}"
+        echo ::set-env name=GOPOGH_RESULT::${GOPOGH_RESULT}
+        echo ::set-env name=STAT::${STAT}
+    - uses: actions/upload-artifact@v1
+      with:
+        name: functional_test_docker_ubuntu
+        path: minikube_binaries/report
+    - name: The End Result functional_test_docker_ubuntu
+      shell: bash
+      run: |
+        echo ${GOPOGH_RESULT}
+        numFail=$(echo $STAT | jq '.NumberOfFail')
+        echo "----------------${numFail} Failures----------------------------"
+        echo $STAT | jq '.FailedTests' || true
+        echo "-------------------------------------------------------"
+        numPass=$(echo $STAT | jq '.NumberOfPass')
+        echo "*** $numPass Passed ***"
+        if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
+  addons_certs_tests_docker_ubuntu:
+    runs-on: ubuntu-18.04
+    env:
+      TIME_ELAPSED: time
+      JOB_NAME: "addons_certs_tests_docker_ubuntu"
+      GOPOGH_RESULT: ""
+      SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643
+    needs: [build_minikube]
+    steps:
+    - name: Install kubectl
+      shell: bash
+      run: |
+        curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/amd64/kubectl
+        sudo install kubectl /usr/local/bin/kubectl
+        kubectl version --client=true
+    - name: Install lz4
+      shell: bash
+      run: |
+        sudo apt-get update -qq
+        sudo apt-get -qq -y install liblz4-tool
+    - name: Docker Info
+      shell: bash
+      run: |
+        echo "--------------------------"
+        docker version || true
+        echo "--------------------------"
+        docker info || true
+        echo "--------------------------"
+        docker system df || true
+        echo "--------------------------"
+        docker system info || true
+        echo "--------------------------"
+        docker ps || true
+        echo "--------------------------"
+    - name: Install gopogh
+      shell: bash
+      run: |
+        curl -LO https://github.com/medyagh/gopogh/releases/download/v0.1.19/gopogh-linux-amd64
+        sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
+    - name: Download Binaries
+      uses: actions/download-artifact@v1
+      with:
+        name: minikube_binaries
+    - name: Run Integration Test
+      continue-on-error: true
+      # bash {0} to allow test to continue to next step. in case of
+      shell: bash {0}
+      run: |
+        cd minikube_binaries
+        mkdir -p report
+        mkdir -p testhome
+        chmod a+x e2e-*
+        chmod a+x minikube-*
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        START_TIME=$(date -u +%s)
+        KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--driver=docker  -test.run "(TestAddons|TestCertOptions)" -test.timeout=30m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
+        END_TIME=$(date -u +%s)
+        TIME_ELAPSED=$(($END_TIME-$START_TIME))
+        min=$((${TIME_ELAPSED}/60))
+        sec=$((${TIME_ELAPSED}%60))
+        TIME_ELAPSED="${min} min $sec seconds "
+        echo ::set-env name=TIME_ELAPSED::${TIME_ELAPSED}
+    - name: Generate HTML Report
+      shell: bash
+      run: |
+        cd minikube_binaries
+        export PATH=${PATH}:`go env GOPATH`/bin
+        go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
+        STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
+        echo status: ${STAT}
+        FailNum=$(echo $STAT | jq '.NumberOfFail')
+        TestsNum=$(echo $STAT | jq '.NumberOfTests')
+        GOPOGH_RESULT="${JOB_NAME} : completed with ${FailNum} / ${TestsNum} failures in ${TIME_ELAPSED}"
+        echo ::set-env name=GOPOGH_RESULT::${GOPOGH_RESULT}
+        echo ::set-env name=STAT::${STAT}
+    - uses: actions/upload-artifact@v1
+      with:
+        name: addons_certs_tests_docker_ubuntu
+        path: minikube_binaries/report
+    - name: The End Result - addons_certs_tests_docker_ubuntu
+      shell: bash
+      run: |
+        echo ${GOPOGH_RESULT}
+        numFail=$(echo $STAT | jq '.NumberOfFail')
+        echo "----------------${numFail} Failures----------------------------"
+        echo $STAT | jq '.FailedTests' || true
+        echo "-------------------------------------------------------"
+        numPass=$(echo $STAT | jq '.NumberOfPass')
+        echo "*** $numPass Passed ***"
+        if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
+  multinode_pause_tests_docker_ubuntu:
+    runs-on: ubuntu-18.04
+    env:
+      TIME_ELAPSED: time
+      JOB_NAME: "multinode_pause_tests_docker_ubuntu"
+      GOPOGH_RESULT: ""
+      SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643
+    needs: [build_minikube]
+    steps:
+    - name: Install kubectl
+      shell: bash
+      run: |
+        curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/amd64/kubectl
+        sudo install kubectl /usr/local/bin/kubectl
+        kubectl version --client=true
+    - name: Install lz4
+      shell: bash
+      run: |
+        sudo apt-get update -qq
+        sudo apt-get -qq -y install liblz4-tool
+    - name: Docker Info
+      shell: bash
+      run: |
+        echo "--------------------------"
+        docker version || true
+        echo "--------------------------"
+        docker info || true
+        echo "--------------------------"
+        docker system df || true
+        echo "--------------------------"
+        docker system info || true
+        echo "--------------------------"
+        docker ps || true
+        echo "--------------------------"
+    - name: Install gopogh
+      shell: bash
+      run: |
+        curl -LO https://github.com/medyagh/gopogh/releases/download/v0.1.19/gopogh-linux-amd64
+        sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
+    - name: Download Binaries
+      uses: actions/download-artifact@v1
+      with:
+        name: minikube_binaries
+    - name: Run Integration Test
+      continue-on-error: true
+      # bash {0} to allow test to continue to next step. in case of
+      shell: bash {0}
+      run: |
+        cd minikube_binaries
+        mkdir -p report
+        mkdir -p testhome
+        chmod a+x e2e-*
+        chmod a+x minikube-*
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        START_TIME=$(date -u +%s)
+        KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--driver=docker  -test.run "(TestPause|TestMultiNode)" -test.timeout=30m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
+        END_TIME=$(date -u +%s)
+        TIME_ELAPSED=$(($END_TIME-$START_TIME))
+        min=$((${TIME_ELAPSED}/60))
+        sec=$((${TIME_ELAPSED}%60))
+        TIME_ELAPSED="${min} min $sec seconds "
+        echo ::set-env name=TIME_ELAPSED::${TIME_ELAPSED}
+    - name: Generate HTML Report
+      shell: bash
+      run: |
+        cd minikube_binaries
+        export PATH=${PATH}:`go env GOPATH`/bin
+        go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
+        STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
+        echo status: ${STAT}
+        FailNum=$(echo $STAT | jq '.NumberOfFail')
+        TestsNum=$(echo $STAT | jq '.NumberOfTests')
+        GOPOGH_RESULT="${JOB_NAME} : completed with ${FailNum} / ${TestsNum} failures in ${TIME_ELAPSED}"
+        echo ::set-env name=GOPOGH_RESULT::${GOPOGH_RESULT}
+        echo ::set-env name=STAT::${STAT}
+    - uses: actions/upload-artifact@v1
+      with:
+        name: multinode_pause_tests_docker_ubuntu
+        path: minikube_binaries/report
+    - name: The End Result - multinode_pause_tests_docker_ubuntu
+      shell: bash
+      run: |
+        echo ${GOPOGH_RESULT}
+        numFail=$(echo $STAT | jq '.NumberOfFail')
+        echo "----------------${numFail} Failures----------------------------"
+        echo $STAT | jq '.FailedTests' || true
+        echo "-------------------------------------------------------"
+        numPass=$(echo $STAT | jq '.NumberOfPass')
+        echo "*** $numPass Passed ***"
+        if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
+  preload_docker_flags_tests_docker_ubuntu:
+    runs-on: ubuntu-18.04
+    env:
+      TIME_ELAPSED: time
+      JOB_NAME: "preload_docker_flags_tests_docker_ubuntu"
+      GOPOGH_RESULT: ""
+      SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643
+    needs: [build_minikube]
+    steps:
+    - name: Install kubectl
+      shell: bash
+      run: |
+        curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/amd64/kubectl
+        sudo install kubectl /usr/local/bin/kubectl
+        kubectl version --client=true
+    - name: Install lz4
+      shell: bash
+      run: |
+        sudo apt-get update -qq
+        sudo apt-get -qq -y install liblz4-tool
+    - name: Docker Info
+      shell: bash
+      run: |
+        echo "--------------------------"
+        docker version || true
+        echo "--------------------------"
+        docker info || true
+        echo "--------------------------"
+        docker system df || true
+        echo "--------------------------"
+        docker system info || true
+        echo "--------------------------"
+        docker ps || true
+        echo "--------------------------"
+    - name: Install gopogh
+      shell: bash
+      run: |
+        curl -LO https://github.com/medyagh/gopogh/releases/download/v0.1.19/gopogh-linux-amd64
+        sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
+    - name: Download Binaries
+      uses: actions/download-artifact@v1
+      with:
+        name: minikube_binaries
+    - name: Run Integration Test
+      continue-on-error: true
+      # bash {0} to allow test to continue to next step. in case of
+      shell: bash {0}
+      run: |
+        cd minikube_binaries
+        mkdir -p report
+        mkdir -p testhome
+        chmod a+x e2e-*
+        chmod a+x minikube-*
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        START_TIME=$(date -u +%s)
+        KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--driver=docker  -test.run "(TestPreload|TestDockerFlags)" -test.timeout=30m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
+        END_TIME=$(date -u +%s)
+        TIME_ELAPSED=$(($END_TIME-$START_TIME))
+        min=$((${TIME_ELAPSED}/60))
+        sec=$((${TIME_ELAPSED}%60))
+        TIME_ELAPSED="${min} min $sec seconds "
+        echo ::set-env name=TIME_ELAPSED::${TIME_ELAPSED}
+    - name: Generate HTML Report
+      shell: bash
+      run: |
+        cd minikube_binaries
+        export PATH=${PATH}:`go env GOPATH`/bin
+        go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
+        STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
+        echo status: ${STAT}
+        FailNum=$(echo $STAT | jq '.NumberOfFail')
+        TestsNum=$(echo $STAT | jq '.NumberOfTests')
+        GOPOGH_RESULT="${JOB_NAME} : completed with ${FailNum} / ${TestsNum} failures in ${TIME_ELAPSED}"
+        echo ::set-env name=GOPOGH_RESULT::${GOPOGH_RESULT}
+        echo ::set-env name=STAT::${STAT}
+    - uses: actions/upload-artifact@v1
+      with:
+        name: preload_docker_flags_tests_docker_ubuntu
+        path: minikube_binaries/report
+    - name: The End Result - preload_docker_flags_tests_docker_ubuntu
+      shell: bash
+      run: |
+        echo ${GOPOGH_RESULT}
+        numFail=$(echo $STAT | jq '.NumberOfFail')
+        echo "----------------${numFail} Failures----------------------------"
+        echo $STAT | jq '.FailedTests' || true
+        echo "-------------------------------------------------------"
+        numPass=$(echo $STAT | jq '.NumberOfPass')
+        echo "*** $numPass Passed ***"
+        if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
+  # collect all the reports and upload them
+  upload_all_reports:
+    if: always()
+    needs: [functional_test_docker_ubuntu, addons_certs_tests_docker_ubuntu, multinode_pause_tests_docker_ubuntu, preload_docker_flags_tests_docker_ubuntu]
+    runs-on: ubuntu-18.04
+    steps:
+    - name: download all reports
+      uses: actions/download-artifact@v2-preview
+    - name: upload all reports
+      shell: bash {0}
+      continue-on-error: true
+      run: |
+        mkdir -p all_reports
+        ls -lah
+        ls -lah minikube_binaries
+        ls -lah minikube_binaries/report
+        cp -r ./minikube_binaries/report/functional_test_docker_ubuntu ./all_reports/
+        cp -r ./minikube_binaries/report/addons_certs_tests_docker_ubuntu ./all_reports/
+        cp -r ./minikube_binaries/report/multinode_pause_tests_docker_ubuntu ./all_reports/
+        cp -r ./minikube_binaries/report/preload_docker_flags_tests_docker_ubuntu        
+    - uses: actions/upload-artifact@v1
+      with:
+        name: all_reports
+        path: all_reports
diff --git a/.github/workflows/main.yml b/.github/workflows/pr.yml
similarity index 99%
rename from .github/workflows/main.yml
rename to .github/workflows/pr.yml
index 879306034d..e32bb239ce 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/pr.yml
@@ -1,7 +1,8 @@
 name: CI
 on: 
   - pull_request
-  - push
+  paths:
+  - '**.go'
 env:
   GOPROXY: https://proxy.golang.org
 jobs:
@@ -512,7 +513,7 @@ jobs:
   # collect all the reports and upload them
   upload_all_reports:
     if: always()
-    needs: [functional_test_docker_ubuntu, addons_certs_tests_docker_ubuntu, multinode_pause_tests_docker_ubuntu, preload_docker_flags_tests_docker_ubuntu, baremetal_ubuntu18_04]
+    needs: [functional_test_docker_ubuntu, addons_certs_tests_docker_ubuntu, multinode_pause_tests_docker_ubuntu, preload_docker_flags_tests_docker_ubuntu]
     runs-on: ubuntu-18.04
     steps:
     - name: download all reports

From 37eb445602499ac1d547bd3061c47408654fff7e Mon Sep 17 00:00:00 2001
From: Medya Gh <medya.gh@gmail.com>
Date: Sat, 18 Apr 2020 19:15:54 -0700
Subject: [PATCH 22/26] fix all reports

---
 .github/workflows/master.yml | 13 ++++++-------
 .github/workflows/pr.yml     | 11 +++++------
 2 files changed, 11 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index 4c82da0c55..4fbcb5132a 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -444,13 +444,12 @@ jobs:
       run: |
         mkdir -p all_reports
         ls -lah
-        ls -lah minikube_binaries
-        ls -lah minikube_binaries/report
-        cp -r ./minikube_binaries/report/functional_test_docker_ubuntu ./all_reports/
-        cp -r ./minikube_binaries/report/addons_certs_tests_docker_ubuntu ./all_reports/
-        cp -r ./minikube_binaries/report/multinode_pause_tests_docker_ubuntu ./all_reports/
-        cp -r ./minikube_binaries/report/preload_docker_flags_tests_docker_ubuntu        
+        cp -r ./functional_test_docker_ubuntu ./all_reports/
+        cp -r ./addons_certs_tests_docker_ubuntu ./all_reports/
+        cp -r ./multinode_pause_tests_docker_ubuntu ./all_reports/
+        cp -r ./preload_docker_flags_tests_docker_ubuntu ./all_reports/
+        cp -r ./minikube_binaries ./all_reports/
     - uses: actions/upload-artifact@v1
       with:
         name: all_reports
-        path: all_reports
+        path: all_reports
\ No newline at end of file
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index e32bb239ce..0d64da27e1 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -524,12 +524,11 @@ jobs:
       run: |
         mkdir -p all_reports
         ls -lah
-        ls -lah minikube_binaries
-        ls -lah minikube_binaries/report
-        cp -r ./minikube_binaries/report/functional_test_docker_ubuntu ./all_reports/
-        cp -r ./minikube_binaries/report/addons_certs_tests_docker_ubuntu ./all_reports/
-        cp -r ./minikube_binaries/report/multinode_pause_tests_docker_ubuntu ./all_reports/
-        cp -r ./minikube_binaries/report/preload_docker_flags_tests_docker_ubuntu        
+        cp -r ./functional_test_docker_ubuntu ./all_reports/
+        cp -r ./addons_certs_tests_docker_ubuntu ./all_reports/
+        cp -r ./multinode_pause_tests_docker_ubuntu ./all_reports/
+        cp -r ./preload_docker_flags_tests_docker_ubuntu ./all_reports/
+        cp -r ./minikube_binaries ./all_reports/
     - uses: actions/upload-artifact@v1
       with:
         name: all_reports

From c20e420a03f9a42cd3ab8fc7cc1b5dfd48dbb65f Mon Sep 17 00:00:00 2001
From: Medya Gh <medya.gh@gmail.com>
Date: Sat, 18 Apr 2020 19:22:57 -0700
Subject: [PATCH 23/26] yaml lint

---
 .github/workflows/master.yml | 12 ++++++------
 .github/workflows/pr.yml     |  8 ++++----
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index 4fbcb5132a..10d812c6e2 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -1,10 +1,10 @@
-name: master
-on: 
+name: MasterCI
+on:
   push:
-    path:
-      - '**.go'
-    branches:    
-      - master         # Push events on master branch  paths:
+    branches:
+    - master
+    paths:
+    - '**.go'
 env:
   GOPROXY: https://proxy.golang.org
 jobs:
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 0d64da27e1..219d4c83c7 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -1,8 +1,8 @@
 name: CI
-on: 
-  - pull_request
-  paths:
-  - '**.go'
+on:
+  pull_request:
+    paths:
+    - '**.go'
 env:
   GOPROXY: https://proxy.golang.org
 jobs:

From 059482a34c49deab6ec22d647b4318c3fb0eb184 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya.gh@gmail.com>
Date: Sat, 18 Apr 2020 19:42:47 -0700
Subject: [PATCH 24/26] tune

---
 .github/workflows/master.yml | 86 +++++++++++++++++++++++++++++++++++-
 .github/workflows/pr.yml     |  8 ++--
 2 files changed, 88 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index 10d812c6e2..eb9fd98770 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -430,10 +430,92 @@ jobs:
         numPass=$(echo $STAT | jq '.NumberOfPass')
         echo "*** $numPass Passed ***"
         if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
+  functional_baremetal_ubuntu18_04:
+    needs: [build_minikube]
+    env:
+      TIME_ELAPSED: time
+      JOB_NAME: "functional_baremetal_ubuntu18_04"
+      GOPOGH_RESULT: ""
+      SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643
+    runs-on: ubuntu-18.04
+    steps:
+    - name: Install kubectl
+      shell: bash
+      run: |
+        curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/amd64/kubectl
+        sudo install kubectl /usr/local/bin/kubectl
+        kubectl version --client=true
+    # conntrack is required for kubernetes 1.18 and higher
+    # socat is required for kubectl port forward which is used in some tests such as validateHelmTillerAddon
+    - name: Install tools for none
+      shell: bash
+      run: |
+        sudo apt-get update -qq
+        sudo apt-get -qq -y install conntrack
+        sudo apt-get -qq -y install socat
+        VERSION="v1.17.0"
+        curl -L https://github.com/kubernetes-sigs/cri-tools/releases/download/$VERSION/crictl-${VERSION}-linux-amd64.tar.gz --output crictl-${VERSION}-linux-amd64.tar.gz
+        sudo tar zxvf crictl-$VERSION-linux-amd64.tar.gz -C /usr/local/bin
+    - name: Install gopogh
+      shell: bash
+      run: |
+        curl -LO https://github.com/medyagh/gopogh/releases/download/v0.1.19/gopogh-linux-amd64
+        sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
+    - name: Download Binaries
+      uses: actions/download-artifact@v1
+      with:
+        name: minikube_binaries
+    - name: Run Integration Test
+      continue-on-error: true
+      # bash {0} to allow test to continue to next step. in case of
+      shell: bash {0}
+      run: |
+        cd minikube_binaries
+        mkdir -p report
+        mkdir -p testhome
+        chmod a+x e2e-*
+        chmod a+x minikube-*
+        START_TIME=$(date -u +%s)
+        KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome sudo -E ./e2e-linux-amd64 -minikube-start-args=--driver=none -test.timeout=35m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
+        END_TIME=$(date -u +%s)
+        TIME_ELAPSED=$(($END_TIME-$START_TIME))
+        min=$((${TIME_ELAPSED}/60))
+        sec=$((${TIME_ELAPSED}%60))
+        TIME_ELAPSED="${min} min $sec seconds "
+        echo ::set-env name=TIME_ELAPSED::${TIME_ELAPSED}
+    - name: Generate HTML Report
+      shell: bash
+      run: |
+        cd minikube_binaries
+        export PATH=${PATH}:`go env GOPATH`/bin
+        go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
+        STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
+        echo status: ${STAT}
+        FailNum=$(echo $STAT | jq '.NumberOfFail')
+        TestsNum=$(echo $STAT | jq '.NumberOfTests')
+        GOPOGH_RESULT="${JOB_NAME} : completed with ${FailNum} / ${TestsNum} failures in ${TIME_ELAPSED}"
+        echo ::set-env name=GOPOGH_RESULT::${GOPOGH_RESULT}
+        echo ::set-env name=STAT::${STAT}
+    - uses: actions/upload-artifact@v1
+      with:
+        name: none_ubuntu18_04
+        path: minikube_binaries/report
+    - name: The End Result - None on Ubuntu 18:04
+      shell: bash
+      run: |
+        echo ${GOPOGH_RESULT}
+        numFail=$(echo $STAT | jq '.NumberOfFail')
+        echo "----------------${numFail} Failures----------------------------"
+        echo $STAT | jq '.FailedTests' || true
+        echo "-------------------------------------------------------"
+        numPass=$(echo $STAT | jq '.NumberOfPass')
+        echo "*** $numPass Passed ***"
+        if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
+  # After all integration tests finished
   # collect all the reports and upload them
   upload_all_reports:
     if: always()
-    needs: [functional_test_docker_ubuntu, addons_certs_tests_docker_ubuntu, multinode_pause_tests_docker_ubuntu, preload_docker_flags_tests_docker_ubuntu]
+    needs: [functional_test_docker_ubuntu, addons_certs_tests_docker_ubuntu, multinode_pause_tests_docker_ubuntu, preload_docker_flags_tests_docker_ubuntu, functional_baremetal_ubuntu18_04]
     runs-on: ubuntu-18.04
     steps:
     - name: download all reports
@@ -448,7 +530,7 @@ jobs:
         cp -r ./addons_certs_tests_docker_ubuntu ./all_reports/
         cp -r ./multinode_pause_tests_docker_ubuntu ./all_reports/
         cp -r ./preload_docker_flags_tests_docker_ubuntu ./all_reports/
-        cp -r ./minikube_binaries ./all_reports/
+        cp -r ./functional_baremetal_ubuntu18_04 ./all_reports/
     - uses: actions/upload-artifact@v1
       with:
         name: all_reports
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 219d4c83c7..e2e285cf84 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -428,11 +428,11 @@ jobs:
         numPass=$(echo $STAT | jq '.NumberOfPass')
         echo "*** $numPass Passed ***"
         if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
-  baremetal_ubuntu18_04:
+  functional_baremetal_ubuntu18_04:
     needs: [build_minikube]
     env:
       TIME_ELAPSED: time
-      JOB_NAME: "None_Ubuntu_18_04"
+      JOB_NAME: "functional_baremetal_ubuntu18_04"
       GOPOGH_RESULT: ""
       SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643
     runs-on: ubuntu-18.04
@@ -513,7 +513,7 @@ jobs:
   # collect all the reports and upload them
   upload_all_reports:
     if: always()
-    needs: [functional_test_docker_ubuntu, addons_certs_tests_docker_ubuntu, multinode_pause_tests_docker_ubuntu, preload_docker_flags_tests_docker_ubuntu]
+    needs: [functional_test_docker_ubuntu, addons_certs_tests_docker_ubuntu, multinode_pause_tests_docker_ubuntu, preload_docker_flags_tests_docker_ubuntu, functional_baremetal_ubuntu18_04]
     runs-on: ubuntu-18.04
     steps:
     - name: download all reports
@@ -528,7 +528,7 @@ jobs:
         cp -r ./addons_certs_tests_docker_ubuntu ./all_reports/
         cp -r ./multinode_pause_tests_docker_ubuntu ./all_reports/
         cp -r ./preload_docker_flags_tests_docker_ubuntu ./all_reports/
-        cp -r ./minikube_binaries ./all_reports/
+        cp -r ./functional_baremetal_ubuntu18_04 ./all_reports/
     - uses: actions/upload-artifact@v1
       with:
         name: all_reports

From 77fee467ea0baf69cdcb5f10a173334e92336d3e Mon Sep 17 00:00:00 2001
From: Medya Gh <medya.gh@gmail.com>
Date: Sat, 18 Apr 2020 20:01:20 -0700
Subject: [PATCH 25/26] only functional

---
 .github/workflows/master.yml | 2 +-
 .github/workflows/pr.yml     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index eb9fd98770..a8cc54413e 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -476,7 +476,7 @@ jobs:
         chmod a+x e2e-*
         chmod a+x minikube-*
         START_TIME=$(date -u +%s)
-        KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome sudo -E ./e2e-linux-amd64 -minikube-start-args=--driver=none -test.timeout=35m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
+        KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome sudo -E ./e2e-linux-amd64 -minikube-start-args=--driver=none -test.timeout=35m -test.run TestFunctional -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
         END_TIME=$(date -u +%s)
         TIME_ELAPSED=$(($END_TIME-$START_TIME))
         min=$((${TIME_ELAPSED}/60))
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index e2e285cf84..de141079a8 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -474,7 +474,7 @@ jobs:
         chmod a+x e2e-*
         chmod a+x minikube-*
         START_TIME=$(date -u +%s)
-        KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome sudo -E ./e2e-linux-amd64 -minikube-start-args=--driver=none -test.timeout=35m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
+        KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome sudo -E ./e2e-linux-amd64 -minikube-start-args=--driver=none -test.timeout=35m -test.v -timeout-multiplier=1.5 -test.run TestFunctional -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
         END_TIME=$(date -u +%s)
         TIME_ELAPSED=$(($END_TIME-$START_TIME))
         min=$((${TIME_ELAPSED}/60))

From dc526455f04724d501a09aa3f45aeb88da95a118 Mon Sep 17 00:00:00 2001
From: Prasad Katti <prasadmkatti@gmail.com>
Date: Sat, 18 Apr 2020 20:29:40 -0700
Subject: [PATCH 26/26] Move integration testing doc from the readme.md to the
 website

---
 site/content/en/docs/contrib/testing.en.md | 33 ++++++++++++++++++++--
 test/integration/README.md                 | 30 +-------------------
 2 files changed, 32 insertions(+), 31 deletions(-)

diff --git a/site/content/en/docs/contrib/testing.en.md b/site/content/en/docs/contrib/testing.en.md
index bc8e02eae5..2f88cf99d9 100644
--- a/site/content/en/docs/contrib/testing.en.md
+++ b/site/content/en/docs/contrib/testing.en.md
@@ -16,8 +16,9 @@ make test
 
 ### Integration Tests
 
-Integration tests are currently run manually.
-To run them, build the binary and run the tests:
+#### The basics
+
+From the minikube root directory, build the binary and run the tests:
 
 ```shell
 make integration
@@ -29,6 +30,34 @@ You may find it useful to set various options to test only a particular test aga
  env TEST_ARGS="-minikube-start-args=--driver=hyperkit -test.run TestStartStop" make integration
  ```
 
+#### Quickly iterating on a single test
+
+Run a single test on an active cluster:
+
+```shell
+make integration -e TEST_ARGS="-test.run TestFunctional/parallel/MountCmd --profile=minikube --cleanup=false"
+```
+
+WARNING: For this to work repeatedly, the test must be written so that it cleans up after itself.
+
+The `--cleanup=false` test arg ensures that the cluster will not be deleted after the test is run.
+
+See [main.go](https://github.com/kubernetes/minikube/blob/master/test/integration/main.go) for details.
+
+#### Disabling parallelism
+
+```shell
+make integration -e TEST_ARGS="-test.parallel=1"
+```
+
+#### Testing philosophy
+
+- Tests should be so simple as to be correct by inspection
+- Readers should need to read only the test body to understand the test
+- Top-to-bottom readability is more important than code de-duplication
+
+Tests are typically read with a great air of skepticism, because chances are they are being read only when things are broken. 
+
 ### Conformance Tests
 
 These are Kubernetes tests that run against an arbitrary cluster and exercise a wide range of Kubernetes features.
diff --git a/test/integration/README.md b/test/integration/README.md
index 7af031f6ea..8a62883dde 100644
--- a/test/integration/README.md
+++ b/test/integration/README.md
@@ -1,29 +1 @@
-# Integration tests
-
-## The basics
-
-To run all tests from the minikube root directory:
-
-`make integration`
-
-## Quickly iterating on a single test
-
-Run a single test on an active cluster:
-
-`make integration -e TEST_ARGS="-test.run TestFunctional/parallel/MountCmd --profile=minikube --cleanup=false"`
-
-WARNING: For this to work repeatedly, the test must be written so that it cleans up after itself.
-
-See `main.go` for details.
-
-## Disabling parallelism
-
-`make integration -e TEST_ARGS="-test.parallel=1"`
-
-## Testing philosophy
-
-- Tests should be so simple as to be correct by inspection
-- Readers should need to read only the test body to understand the test
-- Top-to-bottom readability is more important than code de-duplication
-
-Tests are typically read with a great air of skepticism, because chances are they are being read only when things are broken. 
+This document has moved to https://minikube.sigs.k8s.io/docs/contrib/testing/#integration-tests