From fd948c43155ef9ae1ae168e5608ed8a77b4ff111 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Tue, 23 Jun 2020 10:42:43 -0700
Subject: [PATCH 01/11] add more macos tests to github actions

---
 .github/workflows/master.yml | 224 +++++++++++++++++++++++++++++++++++
 .github/workflows/pr.yml     | 224 +++++++++++++++++++++++++++++++++++
 2 files changed, 448 insertions(+)

diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index 52e324709f..616044dfbb 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -632,6 +632,84 @@ jobs:
           if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
           if [ "$numPass" -lt 5 ];then echo "*** Failed to pass at least 5 ! ***";exit 2;fi
           if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
+  addons_certs_virtualbox_macos:
+    runs-on: macos-10.15
+    env:
+      TIME_ELAPSED: time
+      JOB_NAME: "addons_certs_virtualbox_macos"
+      GOPOGH_RESULT: ""
+      SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643
+    needs: [build_minikube]
+    steps:
+      - name: Install kubectl
+        shell: bash
+        run: |
+          curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/darwin/amd64/kubectl
+          sudo install kubectl /usr/local/bin/kubectl
+          kubectl version --client=true
+      - name: Install gopogh
+        shell: bash
+        run: |
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.1.23/gopogh-darwin-amd64
+          sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh
+      - name: Install docker
+        shell: bash
+        run: |
+          brew install docker-machine docker
+          sudo docker --version
+      - name: Download Binaries
+        uses: actions/download-artifact@v1
+        with:
+          name: minikube_binaries
+      - name: Run Integration Test
+        continue-on-error: true
+        # bash {0} to allow test to continue to next step. in case of
+        shell: bash {0}
+        run: |
+          cd minikube_binaries
+          mkdir -p report
+          mkdir -p testhome
+          chmod a+x e2e-*
+          chmod a+x minikube-*
+          START_TIME=$(date -u +%s)
+          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-darwin-amd64 -minikube-start-args=--vm-driver=virtualbox  -test.run "(TestAddons|TestCertOptions)" -test.timeout=15m -test.v -timeout-multiplier=1.5 -binary=./minikube-darwin-amd64 2>&1 | tee ./report/testout.txt
+          END_TIME=$(date -u +%s)
+          TIME_ELAPSED=$(($END_TIME-$START_TIME))
+          min=$((${TIME_ELAPSED}/60))
+          sec=$((${TIME_ELAPSED}%60))
+          TIME_ELAPSED="${min} min $sec seconds "
+          echo ::set-env name=TIME_ELAPSED::${TIME_ELAPSED}
+      - name: Generate HTML Report
+        shell: bash
+        run: |
+          cd minikube_binaries
+          export PATH=${PATH}:`go env GOPATH`/bin
+          go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
+          STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
+          echo status: ${STAT}
+          FailNum=$(echo $STAT | jq '.NumberOfFail')
+          TestsNum=$(echo $STAT | jq '.NumberOfTests')
+          GOPOGH_RESULT="${JOB_NAME} : completed with ${FailNum} / ${TestsNum} failures in ${TIME_ELAPSED}"
+          echo ::set-env name=GOPOGH_RESULT::${GOPOGH_RESULT}
+          echo ::set-env name=STAT::${STAT}
+      - uses: actions/upload-artifact@v1
+        with:
+          name: addons_certs_virtualbox_macos
+          path: minikube_binaries/report
+      - name: The End Result - addons_certs_virtualbox_macos
+        shell: bash
+        run: |
+          echo ${GOPOGH_RESULT}
+          numFail=$(echo $STAT | jq '.NumberOfFail')
+          echo "----------------${numFail} Failures----------------------------"
+          echo $STAT | jq '.FailedTests' || true
+          echo "-------------------------------------------------------"
+          numPass=$(echo $STAT | jq '.NumberOfPass')
+          echo "*** $numPass Passed ***"
+          if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
+          if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
+          if [ "$numPass" -lt 5 ];then echo "*** Failed to pass at least 5 ! ***";exit 2;fi
+          if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
   multinode_pause_docker_ubuntu:
     runs-on: ubuntu-18.04
     env:
@@ -721,6 +799,79 @@ jobs:
           if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
           if [ "$numPass" -lt 12 ];then echo "*** Failed to pass at least 6 ! ***";exit 2;fi
           if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
+  multinode_pause_virtualbox_macos:
+    runs-on: macos-10.15
+    env:
+      TIME_ELAPSED: time
+      JOB_NAME: "multinode_pause_virtualbox_macos"
+      GOPOGH_RESULT: ""
+      SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643
+    needs: [build_minikube]
+    steps:
+      - name: Install kubectl
+        shell: bash
+        run: |
+          curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/darwin/amd64/kubectl
+          sudo install kubectl /usr/local/bin/kubectl
+          kubectl version --client=true
+      - name: Install gopogh
+        shell: bash
+        run: |
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.1.23/gopogh-darwin-amd64
+          sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh
+      - name: Download Binaries
+        uses: actions/download-artifact@v1
+        with:
+          name: minikube_binaries
+      - name: Run Integration Test
+        continue-on-error: true
+        # bash {0} to allow test to continue to next step. in case of
+        shell: bash {0}
+        run: |
+          cd minikube_binaries
+          mkdir -p report
+          mkdir -p testhome
+          chmod a+x e2e-*
+          chmod a+x minikube-*
+          START_TIME=$(date -u +%s)
+          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--driver=docker  -test.run "(TestPause|TestMultiNode)" -test.timeout=10m -test.v -timeout-multiplier=1.5 -binary=./minikube-darwin-amd64 2>&1 | tee ./report/testout.txt
+          END_TIME=$(date -u +%s)
+          TIME_ELAPSED=$(($END_TIME-$START_TIME))
+          min=$((${TIME_ELAPSED}/60))
+          sec=$((${TIME_ELAPSED}%60))
+          TIME_ELAPSED="${min} min $sec seconds "
+          echo ::set-env name=TIME_ELAPSED::${TIME_ELAPSED}
+      - name: Generate HTML Report
+        shell: bash
+        run: |
+          cd minikube_binaries
+          export PATH=${PATH}:`go env GOPATH`/bin
+          go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
+          STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
+          echo status: ${STAT}
+          FailNum=$(echo $STAT | jq '.NumberOfFail')
+          TestsNum=$(echo $STAT | jq '.NumberOfTests')
+          GOPOGH_RESULT="${JOB_NAME} : completed with ${FailNum} / ${TestsNum} failures in ${TIME_ELAPSED}"
+          echo ::set-env name=GOPOGH_RESULT::${GOPOGH_RESULT}
+          echo ::set-env name=STAT::${STAT}
+      - uses: actions/upload-artifact@v1
+        with:
+          name: multinode_pause_virtualbox_macos
+          path: minikube_binaries/report
+      - name: The End Result - multinode_pause_virtualbox_macos
+        shell: bash
+        run: |
+          echo ${GOPOGH_RESULT}
+          numFail=$(echo $STAT | jq '.NumberOfFail')
+          echo "----------------${numFail} Failures----------------------------"
+          echo $STAT | jq '.FailedTests' || true
+          echo "-------------------------------------------------------"
+          numPass=$(echo $STAT | jq '.NumberOfPass')
+          echo "*** $numPass Passed ***"
+          if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
+          if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
+          if [ "$numPass" -lt 12 ];then echo "*** Failed to pass at least 6 ! ***";exit 2;fi
+          if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
   preload_docker_flags_docker_ubuntu:
     runs-on: ubuntu-18.04
     env:
@@ -810,6 +961,79 @@ jobs:
           if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
           if [ "$numPass" -lt 2 ];then echo "*** Failed to pass at least 2 ! ***";exit 2;fi
           if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
+  preload_docker_flags_virtualbox_macos:
+    runs-on: macos-10.15
+    env:
+      TIME_ELAPSED: time
+      JOB_NAME: "preload_docker_flags_virtualbox_macos"
+      GOPOGH_RESULT: ""
+      SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643
+    needs: [build_minikube]
+    steps:
+      - name: Install kubectl
+        shell: bash
+        run: |
+          curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/darwin/amd64/kubectl
+          sudo install kubectl /usr/local/bin/kubectl
+          kubectl version --client=true
+      - name: Install gopogh
+        shell: bash
+        run: |
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.1.23/gopogh-darwin-amd64
+          sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh
+      - name: Download Binaries
+        uses: actions/download-artifact@v1
+        with:
+          name: minikube_binaries
+      - name: Run Integration Test
+        continue-on-error: false
+        # bash {0} to allow test to continue to next step. in case of
+        shell: bash {0}
+        run: |
+          cd minikube_binaries
+          mkdir -p report
+          mkdir -p testhome
+          chmod a+x e2e-*
+          chmod a+x minikube-*
+          START_TIME=$(date -u +%s)
+          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-darwin-amd64 -minikube-start-args=--vm-driver=virtualbox  -test.run "(TestPreload|TestDockerFlags)" -test.timeout=15m -test.v -timeout-multiplier=1.5 -binary=./minikube-darwin-amd64 2>&1 | tee ./report/testout.txt
+          END_TIME=$(date -u +%s)
+          TIME_ELAPSED=$(($END_TIME-$START_TIME))
+          min=$((${TIME_ELAPSED}/60))
+          sec=$((${TIME_ELAPSED}%60))
+          TIME_ELAPSED="${min} min $sec seconds "
+          echo ::set-env name=TIME_ELAPSED::${TIME_ELAPSED}
+      - name: Generate HTML Report
+        shell: bash
+        run: |
+          cd minikube_binaries
+          export PATH=${PATH}:`go env GOPATH`/bin
+          go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
+          STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
+          echo status: ${STAT}
+          FailNum=$(echo $STAT | jq '.NumberOfFail')
+          TestsNum=$(echo $STAT | jq '.NumberOfTests')
+          GOPOGH_RESULT="${JOB_NAME} : completed with ${FailNum} / ${TestsNum} failures in ${TIME_ELAPSED}"
+          echo ::set-env name=GOPOGH_RESULT::${GOPOGH_RESULT}
+          echo ::set-env name=STAT::${STAT}
+      - uses: actions/upload-artifact@v1
+        with:
+          name: preload_docker_flags_virtualbox_macos
+          path: minikube_binaries/report
+      - name: The End Result - preload_docker_flags_virtualbox_macos
+        shell: bash
+        run: |
+          echo ${GOPOGH_RESULT}
+          numFail=$(echo $STAT | jq '.NumberOfFail')
+          echo "----------------${numFail} Failures----------------------------"
+          echo $STAT | jq '.FailedTests' || true
+          echo "-------------------------------------------------------"
+          numPass=$(echo $STAT | jq '.NumberOfPass')
+          echo "*** $numPass Passed ***"
+          if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
+          if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
+          if [ "$numPass" -lt 2 ];then echo "*** Failed to pass at least 2 ! ***";exit 2;fi
+          if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
   # After all integration tests finished
   # collect all the reports and upload them
   upload_all_reports:
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 9d353d9952..d8aa50c254 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -630,6 +630,84 @@ jobs:
           if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
           if [ "$numPass" -lt 5 ];then echo "*** Failed to pass at least 5 ! ***";exit 2;fi
           if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
+  addons_certs_virtualbox_macos:
+    runs-on: macos-10.15
+    env:
+      TIME_ELAPSED: time
+      JOB_NAME: "addons_certs_virtualbox_macos"
+      GOPOGH_RESULT: ""
+      SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643
+    needs: [build_minikube]
+    steps:
+      - name: Install kubectl
+        shell: bash
+        run: |
+          curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/darwin/amd64/kubectl
+          sudo install kubectl /usr/local/bin/kubectl
+          kubectl version --client=true
+      - name: Install gopogh
+        shell: bash
+        run: |
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.1.23/gopogh-darwin-amd64
+          sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh
+      - name: Install docker
+        shell: bash
+        run: |
+          brew install docker-machine docker
+          sudo docker --version
+      - name: Download Binaries
+        uses: actions/download-artifact@v1
+        with:
+          name: minikube_binaries
+      - name: Run Integration Test
+        continue-on-error: true
+        # bash {0} to allow test to continue to next step. in case of
+        shell: bash {0}
+        run: |
+          cd minikube_binaries
+          mkdir -p report
+          mkdir -p testhome
+          chmod a+x e2e-*
+          chmod a+x minikube-*
+          START_TIME=$(date -u +%s)
+          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-darwin-amd64 -minikube-start-args=--vm-driver=virtualbox  -test.run "(TestAddons|TestCertOptions)" -test.timeout=15m -test.v -timeout-multiplier=1.5 -binary=./minikube-darwin-amd64 2>&1 | tee ./report/testout.txt
+          END_TIME=$(date -u +%s)
+          TIME_ELAPSED=$(($END_TIME-$START_TIME))
+          min=$((${TIME_ELAPSED}/60))
+          sec=$((${TIME_ELAPSED}%60))
+          TIME_ELAPSED="${min} min $sec seconds "
+          echo ::set-env name=TIME_ELAPSED::${TIME_ELAPSED}
+      - name: Generate HTML Report
+        shell: bash
+        run: |
+          cd minikube_binaries
+          export PATH=${PATH}:`go env GOPATH`/bin
+          go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
+          STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
+          echo status: ${STAT}
+          FailNum=$(echo $STAT | jq '.NumberOfFail')
+          TestsNum=$(echo $STAT | jq '.NumberOfTests')
+          GOPOGH_RESULT="${JOB_NAME} : completed with ${FailNum} / ${TestsNum} failures in ${TIME_ELAPSED}"
+          echo ::set-env name=GOPOGH_RESULT::${GOPOGH_RESULT}
+          echo ::set-env name=STAT::${STAT}
+      - uses: actions/upload-artifact@v1
+        with:
+          name: addons_certs_virtualbox_macos
+          path: minikube_binaries/report
+      - name: The End Result - addons_certs_virtualbox_macos
+        shell: bash
+        run: |
+          echo ${GOPOGH_RESULT}
+          numFail=$(echo $STAT | jq '.NumberOfFail')
+          echo "----------------${numFail} Failures----------------------------"
+          echo $STAT | jq '.FailedTests' || true
+          echo "-------------------------------------------------------"
+          numPass=$(echo $STAT | jq '.NumberOfPass')
+          echo "*** $numPass Passed ***"
+          if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
+          if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
+          if [ "$numPass" -lt 5 ];then echo "*** Failed to pass at least 5 ! ***";exit 2;fi
+          if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
   multinode_pause_docker_ubuntu:
     runs-on: ubuntu-18.04
     env:
@@ -719,6 +797,79 @@ jobs:
           if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
           if [ "$numPass" -lt 12 ];then echo "*** Failed to pass at least 6 ! ***";exit 2;fi
           if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
+  multinode_pause_virtualbox_macos:
+    runs-on: macos-10.15
+    env:
+      TIME_ELAPSED: time
+      JOB_NAME: "multinode_pause_virtualbox_macos"
+      GOPOGH_RESULT: ""
+      SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643
+    needs: [build_minikube]
+    steps:
+      - name: Install kubectl
+        shell: bash
+        run: |
+          curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/darwin/amd64/kubectl
+          sudo install kubectl /usr/local/bin/kubectl
+          kubectl version --client=true
+      - name: Install gopogh
+        shell: bash
+        run: |
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.1.23/gopogh-darwin-amd64
+          sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh
+      - name: Download Binaries
+        uses: actions/download-artifact@v1
+        with:
+          name: minikube_binaries
+      - name: Run Integration Test
+        continue-on-error: true
+        # bash {0} to allow test to continue to next step. in case of
+        shell: bash {0}
+        run: |
+          cd minikube_binaries
+          mkdir -p report
+          mkdir -p testhome
+          chmod a+x e2e-*
+          chmod a+x minikube-*
+          START_TIME=$(date -u +%s)
+          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--driver=docker  -test.run "(TestPause|TestMultiNode)" -test.timeout=10m -test.v -timeout-multiplier=1.5 -binary=./minikube-darwin-amd64 2>&1 | tee ./report/testout.txt
+          END_TIME=$(date -u +%s)
+          TIME_ELAPSED=$(($END_TIME-$START_TIME))
+          min=$((${TIME_ELAPSED}/60))
+          sec=$((${TIME_ELAPSED}%60))
+          TIME_ELAPSED="${min} min $sec seconds "
+          echo ::set-env name=TIME_ELAPSED::${TIME_ELAPSED}
+      - name: Generate HTML Report
+        shell: bash
+        run: |
+          cd minikube_binaries
+          export PATH=${PATH}:`go env GOPATH`/bin
+          go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
+          STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
+          echo status: ${STAT}
+          FailNum=$(echo $STAT | jq '.NumberOfFail')
+          TestsNum=$(echo $STAT | jq '.NumberOfTests')
+          GOPOGH_RESULT="${JOB_NAME} : completed with ${FailNum} / ${TestsNum} failures in ${TIME_ELAPSED}"
+          echo ::set-env name=GOPOGH_RESULT::${GOPOGH_RESULT}
+          echo ::set-env name=STAT::${STAT}
+      - uses: actions/upload-artifact@v1
+        with:
+          name: multinode_pause_virtualbox_macos
+          path: minikube_binaries/report
+      - name: The End Result - multinode_pause_virtualbox_macos
+        shell: bash
+        run: |
+          echo ${GOPOGH_RESULT}
+          numFail=$(echo $STAT | jq '.NumberOfFail')
+          echo "----------------${numFail} Failures----------------------------"
+          echo $STAT | jq '.FailedTests' || true
+          echo "-------------------------------------------------------"
+          numPass=$(echo $STAT | jq '.NumberOfPass')
+          echo "*** $numPass Passed ***"
+          if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
+          if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
+          if [ "$numPass" -lt 12 ];then echo "*** Failed to pass at least 6 ! ***";exit 2;fi
+          if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
   preload_docker_flags_docker_ubuntu:
     runs-on: ubuntu-18.04
     env:
@@ -808,6 +959,79 @@ jobs:
           if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
           if [ "$numPass" -lt 2 ];then echo "*** Failed to pass at least 2 ! ***";exit 2;fi
           if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
+  preload_docker_flags_virtualbox_macos:
+    runs-on: macos-10.15
+    env:
+      TIME_ELAPSED: time
+      JOB_NAME: "preload_docker_flags_virtualbox_macos"
+      GOPOGH_RESULT: ""
+      SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643
+    needs: [build_minikube]
+    steps:
+      - name: Install kubectl
+        shell: bash
+        run: |
+          curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/darwin/amd64/kubectl
+          sudo install kubectl /usr/local/bin/kubectl
+          kubectl version --client=true
+      - name: Install gopogh
+        shell: bash
+        run: |
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.1.23/gopogh-darwin-amd64
+          sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh
+      - name: Download Binaries
+        uses: actions/download-artifact@v1
+        with:
+          name: minikube_binaries
+      - name: Run Integration Test
+        continue-on-error: false
+        # bash {0} to allow test to continue to next step. in case of
+        shell: bash {0}
+        run: |
+          cd minikube_binaries
+          mkdir -p report
+          mkdir -p testhome
+          chmod a+x e2e-*
+          chmod a+x minikube-*
+          START_TIME=$(date -u +%s)
+          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-darwin-amd64 -minikube-start-args=--vm-driver=virtualbox  -test.run "(TestPreload|TestDockerFlags)" -test.timeout=15m -test.v -timeout-multiplier=1.5 -binary=./minikube-darwin-amd64 2>&1 | tee ./report/testout.txt
+          END_TIME=$(date -u +%s)
+          TIME_ELAPSED=$(($END_TIME-$START_TIME))
+          min=$((${TIME_ELAPSED}/60))
+          sec=$((${TIME_ELAPSED}%60))
+          TIME_ELAPSED="${min} min $sec seconds "
+          echo ::set-env name=TIME_ELAPSED::${TIME_ELAPSED}
+      - name: Generate HTML Report
+        shell: bash
+        run: |
+          cd minikube_binaries
+          export PATH=${PATH}:`go env GOPATH`/bin
+          go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
+          STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
+          echo status: ${STAT}
+          FailNum=$(echo $STAT | jq '.NumberOfFail')
+          TestsNum=$(echo $STAT | jq '.NumberOfTests')
+          GOPOGH_RESULT="${JOB_NAME} : completed with ${FailNum} / ${TestsNum} failures in ${TIME_ELAPSED}"
+          echo ::set-env name=GOPOGH_RESULT::${GOPOGH_RESULT}
+          echo ::set-env name=STAT::${STAT}
+      - uses: actions/upload-artifact@v1
+        with:
+          name: preload_docker_flags_virtualbox_macos
+          path: minikube_binaries/report
+      - name: The End Result - preload_docker_flags_virtualbox_macos
+        shell: bash
+        run: |
+          echo ${GOPOGH_RESULT}
+          numFail=$(echo $STAT | jq '.NumberOfFail')
+          echo "----------------${numFail} Failures----------------------------"
+          echo $STAT | jq '.FailedTests' || true
+          echo "-------------------------------------------------------"
+          numPass=$(echo $STAT | jq '.NumberOfPass')
+          echo "*** $numPass Passed ***"
+          if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
+          if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
+          if [ "$numPass" -lt 2 ];then echo "*** Failed to pass at least 2 ! ***";exit 2;fi
+          if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
   # After all integration tests finished
   # collect all the reports and upload them
   upload_all_reports:

From c2e2e55fb7dfeb4f4afe0d26e3ab5d394ebecdba Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Tue, 23 Jun 2020 11:11:37 -0700
Subject: [PATCH 02/11] fix typo

---
 .github/workflows/master.yml | 2 +-
 .github/workflows/pr.yml     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index 616044dfbb..1339f13516 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -834,7 +834,7 @@ jobs:
           chmod a+x e2e-*
           chmod a+x minikube-*
           START_TIME=$(date -u +%s)
-          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--driver=docker  -test.run "(TestPause|TestMultiNode)" -test.timeout=10m -test.v -timeout-multiplier=1.5 -binary=./minikube-darwin-amd64 2>&1 | tee ./report/testout.txt
+          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-darwin-amd64 -minikube-start-args=--driver=docker  -test.run "(TestPause|TestMultiNode)" -test.timeout=10m -test.v -timeout-multiplier=1.5 -binary=./minikube-darwin-amd64 2>&1 | tee ./report/testout.txt
           END_TIME=$(date -u +%s)
           TIME_ELAPSED=$(($END_TIME-$START_TIME))
           min=$((${TIME_ELAPSED}/60))
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index d8aa50c254..708b383000 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -832,7 +832,7 @@ jobs:
           chmod a+x e2e-*
           chmod a+x minikube-*
           START_TIME=$(date -u +%s)
-          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--driver=docker  -test.run "(TestPause|TestMultiNode)" -test.timeout=10m -test.v -timeout-multiplier=1.5 -binary=./minikube-darwin-amd64 2>&1 | tee ./report/testout.txt
+          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-darwin-amd64 -minikube-start-args=--driver=docker  -test.run "(TestPause|TestMultiNode)" -test.timeout=10m -test.v -timeout-multiplier=1.5 -binary=./minikube-darwin-amd64 2>&1 | tee ./report/testout.txt
           END_TIME=$(date -u +%s)
           TIME_ELAPSED=$(($END_TIME-$START_TIME))
           min=$((${TIME_ELAPSED}/60))

From ae6a38616da8732ce07eac09160ed6f005d90cf9 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Tue, 23 Jun 2020 11:47:02 -0700
Subject: [PATCH 03/11] fix typo

---
 .github/workflows/master.yml | 2 +-
 .github/workflows/pr.yml     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index 1339f13516..6a845a6124 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -834,7 +834,7 @@ jobs:
           chmod a+x e2e-*
           chmod a+x minikube-*
           START_TIME=$(date -u +%s)
-          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-darwin-amd64 -minikube-start-args=--driver=docker  -test.run "(TestPause|TestMultiNode)" -test.timeout=10m -test.v -timeout-multiplier=1.5 -binary=./minikube-darwin-amd64 2>&1 | tee ./report/testout.txt
+          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-darwin-amd64 -minikube-start-args=--driver=virtualbox  -test.run "(TestPause|TestMultiNode)" -test.timeout=10m -test.v -timeout-multiplier=1.5 -binary=./minikube-darwin-amd64 2>&1 | tee ./report/testout.txt
           END_TIME=$(date -u +%s)
           TIME_ELAPSED=$(($END_TIME-$START_TIME))
           min=$((${TIME_ELAPSED}/60))
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 708b383000..3c5c734e18 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -832,7 +832,7 @@ jobs:
           chmod a+x e2e-*
           chmod a+x minikube-*
           START_TIME=$(date -u +%s)
-          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-darwin-amd64 -minikube-start-args=--driver=docker  -test.run "(TestPause|TestMultiNode)" -test.timeout=10m -test.v -timeout-multiplier=1.5 -binary=./minikube-darwin-amd64 2>&1 | tee ./report/testout.txt
+          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-darwin-amd64 -minikube-start-args=--driver=virtualbox  -test.run "(TestPause|TestMultiNode)" -test.timeout=10m -test.v -timeout-multiplier=1.5 -binary=./minikube-darwin-amd64 2>&1 | tee ./report/testout.txt
           END_TIME=$(date -u +%s)
           TIME_ELAPSED=$(($END_TIME-$START_TIME))
           min=$((${TIME_ELAPSED}/60))

From 9d10d4df382e788f6a9cf3ccae0b6ff58cd32513 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Tue, 23 Jun 2020 12:20:27 -0700
Subject: [PATCH 04/11] adjust macos timeout

---
 .github/workflows/master.yml | 12 +++++-------
 .github/workflows/pr.yml     |  6 +++---
 2 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index 6a845a6124..b0bd415ea4 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -1,8 +1,6 @@
-name: Master
+name: PR
 on:
-  push:
-    branches:
-      - master
+  pull_request:
     paths:
       - "**.go"
       - "**.yml"
@@ -797,7 +795,7 @@ jobs:
           echo "*** $numPass Passed ***"
           if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
           if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
-          if [ "$numPass" -lt 12 ];then echo "*** Failed to pass at least 6 ! ***";exit 2;fi
+          if [ "$numPass" -lt 12 ];then echo "*** Failed to pass at least 12 ! ***";exit 2;fi
           if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
   multinode_pause_virtualbox_macos:
     runs-on: macos-10.15
@@ -834,7 +832,7 @@ jobs:
           chmod a+x e2e-*
           chmod a+x minikube-*
           START_TIME=$(date -u +%s)
-          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-darwin-amd64 -minikube-start-args=--driver=virtualbox  -test.run "(TestPause|TestMultiNode)" -test.timeout=10m -test.v -timeout-multiplier=1.5 -binary=./minikube-darwin-amd64 2>&1 | tee ./report/testout.txt
+          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-darwin-amd64 -minikube-start-args=--driver=virtualbox  -test.run "(TestPause|TestMultiNode)" -test.timeout=15m -test.v -timeout-multiplier=1.5 -binary=./minikube-darwin-amd64 2>&1 | tee ./report/testout.txt
           END_TIME=$(date -u +%s)
           TIME_ELAPSED=$(($END_TIME-$START_TIME))
           min=$((${TIME_ELAPSED}/60))
@@ -870,7 +868,7 @@ jobs:
           echo "*** $numPass Passed ***"
           if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
           if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
-          if [ "$numPass" -lt 12 ];then echo "*** Failed to pass at least 6 ! ***";exit 2;fi
+          if [ "$numPass" -lt 12 ];then echo "*** Failed to pass at least 12 ! ***";exit 2;fi
           if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
   preload_docker_flags_docker_ubuntu:
     runs-on: ubuntu-18.04
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 3c5c734e18..b0bd415ea4 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -795,7 +795,7 @@ jobs:
           echo "*** $numPass Passed ***"
           if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
           if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
-          if [ "$numPass" -lt 12 ];then echo "*** Failed to pass at least 6 ! ***";exit 2;fi
+          if [ "$numPass" -lt 12 ];then echo "*** Failed to pass at least 12 ! ***";exit 2;fi
           if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
   multinode_pause_virtualbox_macos:
     runs-on: macos-10.15
@@ -832,7 +832,7 @@ jobs:
           chmod a+x e2e-*
           chmod a+x minikube-*
           START_TIME=$(date -u +%s)
-          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-darwin-amd64 -minikube-start-args=--driver=virtualbox  -test.run "(TestPause|TestMultiNode)" -test.timeout=10m -test.v -timeout-multiplier=1.5 -binary=./minikube-darwin-amd64 2>&1 | tee ./report/testout.txt
+          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-darwin-amd64 -minikube-start-args=--driver=virtualbox  -test.run "(TestPause|TestMultiNode)" -test.timeout=15m -test.v -timeout-multiplier=1.5 -binary=./minikube-darwin-amd64 2>&1 | tee ./report/testout.txt
           END_TIME=$(date -u +%s)
           TIME_ELAPSED=$(($END_TIME-$START_TIME))
           min=$((${TIME_ELAPSED}/60))
@@ -868,7 +868,7 @@ jobs:
           echo "*** $numPass Passed ***"
           if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
           if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
-          if [ "$numPass" -lt 12 ];then echo "*** Failed to pass at least 6 ! ***";exit 2;fi
+          if [ "$numPass" -lt 12 ];then echo "*** Failed to pass at least 12 ! ***";exit 2;fi
           if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
   preload_docker_flags_docker_ubuntu:
     runs-on: ubuntu-18.04

From 6ea74253a951e7e8ea5a2cf7e7634b3a2e28e041 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Tue, 23 Jun 2020 13:21:27 -0700
Subject: [PATCH 05/11] increase multiplier

---
 .github/workflows/pr.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index b0bd415ea4..e169e0850d 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -670,7 +670,7 @@ jobs:
           chmod a+x e2e-*
           chmod a+x minikube-*
           START_TIME=$(date -u +%s)
-          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-darwin-amd64 -minikube-start-args=--vm-driver=virtualbox  -test.run "(TestAddons|TestCertOptions)" -test.timeout=15m -test.v -timeout-multiplier=1.5 -binary=./minikube-darwin-amd64 2>&1 | tee ./report/testout.txt
+          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-darwin-amd64 -minikube-start-args=--vm-driver=virtualbox  -test.run "(TestAddons|TestCertOptions)" -test.timeout=15m -test.v -timeout-multiplier=3 -binary=./minikube-darwin-amd64 2>&1 | tee ./report/testout.txt
           END_TIME=$(date -u +%s)
           TIME_ELAPSED=$(($END_TIME-$START_TIME))
           min=$((${TIME_ELAPSED}/60))

From 35a9bd32f06f96fb0651f7e385bb5f59def6c2fe Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Tue, 23 Jun 2020 14:37:56 -0700
Subject: [PATCH 06/11] add timeout

---
 .github/workflows/pr.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index e169e0850d..bc0bf8b132 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -832,7 +832,7 @@ jobs:
           chmod a+x e2e-*
           chmod a+x minikube-*
           START_TIME=$(date -u +%s)
-          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-darwin-amd64 -minikube-start-args=--driver=virtualbox  -test.run "(TestPause|TestMultiNode)" -test.timeout=15m -test.v -timeout-multiplier=1.5 -binary=./minikube-darwin-amd64 2>&1 | tee ./report/testout.txt
+          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-darwin-amd64 -minikube-start-args=--driver=virtualbox  -test.run "(TestPause|TestMultiNode)" -test.timeout=20m -test.v -timeout-multiplier=1.5 -binary=./minikube-darwin-amd64 2>&1 | tee ./report/testout.txt
           END_TIME=$(date -u +%s)
           TIME_ELAPSED=$(($END_TIME-$START_TIME))
           min=$((${TIME_ELAPSED}/60))

From e3f33d576a30263c18833aa87c9640c98bc64e16 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Tue, 23 Jun 2020 15:00:58 -0700
Subject: [PATCH 07/11] fix mtypo

---
 .github/workflows/master.yml | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index b0bd415ea4..b4027166d4 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -1,6 +1,8 @@
-name: PR
+name: Master
 on:
-  pull_request:
+  push:
+    branches:    
+      - master
     paths:
       - "**.go"
       - "**.yml"
@@ -670,7 +672,7 @@ jobs:
           chmod a+x e2e-*
           chmod a+x minikube-*
           START_TIME=$(date -u +%s)
-          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-darwin-amd64 -minikube-start-args=--vm-driver=virtualbox  -test.run "(TestAddons|TestCertOptions)" -test.timeout=15m -test.v -timeout-multiplier=1.5 -binary=./minikube-darwin-amd64 2>&1 | tee ./report/testout.txt
+          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-darwin-amd64 -minikube-start-args=--vm-driver=virtualbox  -test.run "(TestAddons|TestCertOptions)" -test.timeout=15m -test.v -timeout-multiplier=3 -binary=./minikube-darwin-amd64 2>&1 | tee ./report/testout.txt
           END_TIME=$(date -u +%s)
           TIME_ELAPSED=$(($END_TIME-$START_TIME))
           min=$((${TIME_ELAPSED}/60))
@@ -832,7 +834,7 @@ jobs:
           chmod a+x e2e-*
           chmod a+x minikube-*
           START_TIME=$(date -u +%s)
-          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-darwin-amd64 -minikube-start-args=--driver=virtualbox  -test.run "(TestPause|TestMultiNode)" -test.timeout=15m -test.v -timeout-multiplier=1.5 -binary=./minikube-darwin-amd64 2>&1 | tee ./report/testout.txt
+          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-darwin-amd64 -minikube-start-args=--driver=virtualbox  -test.run "(TestPause|TestMultiNode)" -test.timeout=20m -test.v -timeout-multiplier=1.5 -binary=./minikube-darwin-amd64 2>&1 | tee ./report/testout.txt
           END_TIME=$(date -u +%s)
           TIME_ELAPSED=$(($END_TIME-$START_TIME))
           min=$((${TIME_ELAPSED}/60))

From 6f2bbb987f98d9221a5851789e5429ed0388cb86 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Tue, 23 Jun 2020 17:03:09 -0700
Subject: [PATCH 08/11] balance test per vm

---
 .github/workflows/master.yml | 65 +++++++++++++++++-------------------
 .github/workflows/pr.yml     | 63 +++++++++++++++++-----------------
 2 files changed, 61 insertions(+), 67 deletions(-)

diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index b4027166d4..a1ff1ee560 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -1,7 +1,7 @@
 name: Master
 on:
   push:
-    branches:    
+    branches:
       - master
     paths:
       - "**.go"
@@ -591,8 +591,6 @@ jobs:
           mkdir -p testhome
           chmod a+x e2e-*
           chmod a+x minikube-*
-          sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
-          sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
           START_TIME=$(date -u +%s)
           KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--driver=docker  -test.run "(TestAddons|TestCertOptions)" -test.timeout=10m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
           END_TIME=$(date -u +%s)
@@ -710,11 +708,11 @@ jobs:
           if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
           if [ "$numPass" -lt 5 ];then echo "*** Failed to pass at least 5 ! ***";exit 2;fi
           if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
-  multinode_pause_docker_ubuntu:
+  multinode_docker_ubuntu:
     runs-on: ubuntu-18.04
     env:
       TIME_ELAPSED: time
-      JOB_NAME: "multinode_pause_docker_ubuntu"
+      JOB_NAME: "multinode_docker_ubuntu"
       GOPOGH_RESULT: ""
       SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643
     needs: [build_minikube]
@@ -761,7 +759,7 @@ jobs:
           sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
           sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
           START_TIME=$(date -u +%s)
-          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--driver=docker  -test.run "(TestPause|TestMultiNode)" -test.timeout=10m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
+          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--driver=docker  -test.run "TestMultiNode" -test.timeout=10m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
           END_TIME=$(date -u +%s)
           TIME_ELAPSED=$(($END_TIME-$START_TIME))
           min=$((${TIME_ELAPSED}/60))
@@ -783,9 +781,9 @@ jobs:
           echo ::set-env name=STAT::${STAT}
       - uses: actions/upload-artifact@v1
         with:
-          name: multinode_pause_docker_ubuntu
+          name: multinode_docker_ubuntu
           path: minikube_binaries/report
-      - name: The End Result - multinode_pause_docker_ubuntu
+      - name: The End Result - multinode_docker_ubuntu
         shell: bash
         run: |
           echo ${GOPOGH_RESULT}
@@ -797,13 +795,12 @@ jobs:
           echo "*** $numPass Passed ***"
           if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
           if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
-          if [ "$numPass" -lt 12 ];then echo "*** Failed to pass at least 12 ! ***";exit 2;fi
-          if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
-  multinode_pause_virtualbox_macos:
+          if [ "$numPass" -lt 5 ];then echo "*** Failed to pass at least 5 ! ***";exit 2;fi
+  multinode_virtualbox_macos:
     runs-on: macos-10.15
     env:
       TIME_ELAPSED: time
-      JOB_NAME: "multinode_pause_virtualbox_macos"
+      JOB_NAME: "multinode_virtualbox_macos"
       GOPOGH_RESULT: ""
       SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643
     needs: [build_minikube]
@@ -834,7 +831,7 @@ jobs:
           chmod a+x e2e-*
           chmod a+x minikube-*
           START_TIME=$(date -u +%s)
-          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-darwin-amd64 -minikube-start-args=--driver=virtualbox  -test.run "(TestPause|TestMultiNode)" -test.timeout=20m -test.v -timeout-multiplier=1.5 -binary=./minikube-darwin-amd64 2>&1 | tee ./report/testout.txt
+          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-darwin-amd64 -minikube-start-args=--driver=virtualbox  -test.run "TestMultiNode" -test.timeout=15m -test.v -timeout-multiplier=1.5 -binary=./minikube-darwin-amd64 2>&1 | tee ./report/testout.txt
           END_TIME=$(date -u +%s)
           TIME_ELAPSED=$(($END_TIME-$START_TIME))
           min=$((${TIME_ELAPSED}/60))
@@ -856,9 +853,9 @@ jobs:
           echo ::set-env name=STAT::${STAT}
       - uses: actions/upload-artifact@v1
         with:
-          name: multinode_pause_virtualbox_macos
+          name: multinode_virtualbox_macos
           path: minikube_binaries/report
-      - name: The End Result - multinode_pause_virtualbox_macos
+      - name: The End Result - multinode_virtualbox_macos
         shell: bash
         run: |
           echo ${GOPOGH_RESULT}
@@ -870,13 +867,13 @@ jobs:
           echo "*** $numPass Passed ***"
           if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
           if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
-          if [ "$numPass" -lt 12 ];then echo "*** Failed to pass at least 12 ! ***";exit 2;fi
+          if [ "$numPass" -lt 5 ];then echo "*** Failed to pass at least 5 ! ***";exit 2;fi
           if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
-  preload_docker_flags_docker_ubuntu:
+  pause_preload_dockerflags_docker_ubuntu:
     runs-on: ubuntu-18.04
     env:
       TIME_ELAPSED: time
-      JOB_NAME: "preload_docker_flags_docker_ubuntu"
+      JOB_NAME: "pause_preload_dockerflags_docker_ubuntu"
       GOPOGH_RESULT: ""
       SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643
     needs: [build_minikube]
@@ -920,10 +917,8 @@ jobs:
           mkdir -p testhome
           chmod a+x e2e-*
           chmod a+x minikube-*
-          sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
-          sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
           START_TIME=$(date -u +%s)
-          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--driver=docker  -test.run "(TestPreload|TestDockerFlags)" -test.timeout=10m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
+          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--driver=docker  -test.run "(TestPause|TestPreload|TestDockerFlags)" -test.timeout=15m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
           END_TIME=$(date -u +%s)
           TIME_ELAPSED=$(($END_TIME-$START_TIME))
           min=$((${TIME_ELAPSED}/60))
@@ -945,9 +940,9 @@ jobs:
           echo ::set-env name=STAT::${STAT}
       - uses: actions/upload-artifact@v1
         with:
-          name: preload_docker_flags_docker_ubuntu
+          name: pause_preload_dockerflags_docker_ubuntu
           path: minikube_binaries/report
-      - name: The End Result - preload_docker_flags_docker_ubuntu
+      - name: The End Result - pause_preload_dockerflags_docker_ubuntu
         shell: bash
         run: |
           echo ${GOPOGH_RESULT}
@@ -960,12 +955,11 @@ jobs:
           if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
           if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
           if [ "$numPass" -lt 2 ];then echo "*** Failed to pass at least 2 ! ***";exit 2;fi
-          if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
-  preload_docker_flags_virtualbox_macos:
+  pause_preload_dockerflags_virtualbox_macos:
     runs-on: macos-10.15
     env:
       TIME_ELAPSED: time
-      JOB_NAME: "preload_docker_flags_virtualbox_macos"
+      JOB_NAME: "pause_preload_dockerflags_virtualbox_macos"
       GOPOGH_RESULT: ""
       SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643
     needs: [build_minikube]
@@ -996,7 +990,7 @@ jobs:
           chmod a+x e2e-*
           chmod a+x minikube-*
           START_TIME=$(date -u +%s)
-          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-darwin-amd64 -minikube-start-args=--vm-driver=virtualbox  -test.run "(TestPreload|TestDockerFlags)" -test.timeout=15m -test.v -timeout-multiplier=1.5 -binary=./minikube-darwin-amd64 2>&1 | tee ./report/testout.txt
+          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-darwin-amd64 -minikube-start-args=--vm-driver=virtualbox  -test.run "(TestPause|TestPreload|TestDockerFlags)" -test.timeout=15m -test.v -timeout-multiplier=1.5 -binary=./minikube-darwin-amd64 2>&1 | tee ./report/testout.txt
           END_TIME=$(date -u +%s)
           TIME_ELAPSED=$(($END_TIME-$START_TIME))
           min=$((${TIME_ELAPSED}/60))
@@ -1018,9 +1012,9 @@ jobs:
           echo ::set-env name=STAT::${STAT}
       - uses: actions/upload-artifact@v1
         with:
-          name: preload_docker_flags_virtualbox_macos
+          name: pause_preload_dockerflags_virtualbox_macos
           path: minikube_binaries/report
-      - name: The End Result - preload_docker_flags_virtualbox_macos
+      - name: The End Result - pause_preload_dockerflags_virtualbox_macos
         shell: bash
         run: |
           echo ${GOPOGH_RESULT}
@@ -1060,13 +1054,16 @@ jobs:
           mkdir -p all_reports
           ls -lah
           cp -r ./functional_docker_ubuntu ./all_reports/
-          cp -r ./addons_certs_docker_ubuntu ./all_reports/
-          cp -r ./multinode_pause_docker_ubuntu ./all_reports/
-          cp -r ./preload_docker_flags_docker_ubuntu ./all_reports/
-          cp -r ./functional_baremetal_ubuntu18_04 ./all_reports/
+          cp -r ./functional_virtualbox_macos ./all_reports/
           cp -r ./functional_docker_windows ./all_reports/
           cp -r ./functional_hyperv_windows ./all_reports/
-          cp -r ./functional_virtualbox_macos ./all_reports/          
+          cp -r ./functional_baremetal_ubuntu18_04 ./all_reports/
+          cp -r ./addons_certs_docker_ubuntu ./all_reports/
+          cp -r ./addons_certs_virtualbox_macos ./all_reports/
+          cp -r ./multinode_docker_ubuntu ./all_reports/
+          cp -r ./multinode_virtualbox_macos ./all_reports/
+          cp -r ./pause_preload_dockerflags_docker_ubuntu ./all_reports/
+          cp -r ./pause_preload_dockerflags_virtualbox_macos ./all_reports/
       - uses: actions/upload-artifact@v1
         with:
           name: all_reports
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index bc0bf8b132..83feb93872 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -589,8 +589,6 @@ jobs:
           mkdir -p testhome
           chmod a+x e2e-*
           chmod a+x minikube-*
-          sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
-          sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
           START_TIME=$(date -u +%s)
           KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--driver=docker  -test.run "(TestAddons|TestCertOptions)" -test.timeout=10m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
           END_TIME=$(date -u +%s)
@@ -708,11 +706,11 @@ jobs:
           if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
           if [ "$numPass" -lt 5 ];then echo "*** Failed to pass at least 5 ! ***";exit 2;fi
           if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
-  multinode_pause_docker_ubuntu:
+  multinode_docker_ubuntu:
     runs-on: ubuntu-18.04
     env:
       TIME_ELAPSED: time
-      JOB_NAME: "multinode_pause_docker_ubuntu"
+      JOB_NAME: "multinode_docker_ubuntu"
       GOPOGH_RESULT: ""
       SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643
     needs: [build_minikube]
@@ -759,7 +757,7 @@ jobs:
           sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
           sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
           START_TIME=$(date -u +%s)
-          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--driver=docker  -test.run "(TestPause|TestMultiNode)" -test.timeout=10m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
+          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--driver=docker  -test.run "TestMultiNode" -test.timeout=10m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
           END_TIME=$(date -u +%s)
           TIME_ELAPSED=$(($END_TIME-$START_TIME))
           min=$((${TIME_ELAPSED}/60))
@@ -781,9 +779,9 @@ jobs:
           echo ::set-env name=STAT::${STAT}
       - uses: actions/upload-artifact@v1
         with:
-          name: multinode_pause_docker_ubuntu
+          name: multinode_docker_ubuntu
           path: minikube_binaries/report
-      - name: The End Result - multinode_pause_docker_ubuntu
+      - name: The End Result - multinode_docker_ubuntu
         shell: bash
         run: |
           echo ${GOPOGH_RESULT}
@@ -795,13 +793,12 @@ jobs:
           echo "*** $numPass Passed ***"
           if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
           if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
-          if [ "$numPass" -lt 12 ];then echo "*** Failed to pass at least 12 ! ***";exit 2;fi
-          if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
-  multinode_pause_virtualbox_macos:
+          if [ "$numPass" -lt 5 ];then echo "*** Failed to pass at least 5 ! ***";exit 2;fi
+  multinode_virtualbox_macos:
     runs-on: macos-10.15
     env:
       TIME_ELAPSED: time
-      JOB_NAME: "multinode_pause_virtualbox_macos"
+      JOB_NAME: "multinode_virtualbox_macos"
       GOPOGH_RESULT: ""
       SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643
     needs: [build_minikube]
@@ -832,7 +829,7 @@ jobs:
           chmod a+x e2e-*
           chmod a+x minikube-*
           START_TIME=$(date -u +%s)
-          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-darwin-amd64 -minikube-start-args=--driver=virtualbox  -test.run "(TestPause|TestMultiNode)" -test.timeout=20m -test.v -timeout-multiplier=1.5 -binary=./minikube-darwin-amd64 2>&1 | tee ./report/testout.txt
+          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-darwin-amd64 -minikube-start-args=--driver=virtualbox  -test.run "TestMultiNode" -test.timeout=15m -test.v -timeout-multiplier=1.5 -binary=./minikube-darwin-amd64 2>&1 | tee ./report/testout.txt
           END_TIME=$(date -u +%s)
           TIME_ELAPSED=$(($END_TIME-$START_TIME))
           min=$((${TIME_ELAPSED}/60))
@@ -854,9 +851,9 @@ jobs:
           echo ::set-env name=STAT::${STAT}
       - uses: actions/upload-artifact@v1
         with:
-          name: multinode_pause_virtualbox_macos
+          name: multinode_virtualbox_macos
           path: minikube_binaries/report
-      - name: The End Result - multinode_pause_virtualbox_macos
+      - name: The End Result - multinode_virtualbox_macos
         shell: bash
         run: |
           echo ${GOPOGH_RESULT}
@@ -868,13 +865,13 @@ jobs:
           echo "*** $numPass Passed ***"
           if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
           if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
-          if [ "$numPass" -lt 12 ];then echo "*** Failed to pass at least 12 ! ***";exit 2;fi
+          if [ "$numPass" -lt 5 ];then echo "*** Failed to pass at least 5 ! ***";exit 2;fi
           if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
-  preload_docker_flags_docker_ubuntu:
+  pause_preload_dockerflags_docker_ubuntu:
     runs-on: ubuntu-18.04
     env:
       TIME_ELAPSED: time
-      JOB_NAME: "preload_docker_flags_docker_ubuntu"
+      JOB_NAME: "pause_preload_dockerflags_docker_ubuntu"
       GOPOGH_RESULT: ""
       SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643
     needs: [build_minikube]
@@ -918,10 +915,8 @@ jobs:
           mkdir -p testhome
           chmod a+x e2e-*
           chmod a+x minikube-*
-          sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
-          sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
           START_TIME=$(date -u +%s)
-          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--driver=docker  -test.run "(TestPreload|TestDockerFlags)" -test.timeout=10m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
+          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--driver=docker  -test.run "(TestPause|TestPreload|TestDockerFlags)" -test.timeout=15m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
           END_TIME=$(date -u +%s)
           TIME_ELAPSED=$(($END_TIME-$START_TIME))
           min=$((${TIME_ELAPSED}/60))
@@ -943,9 +938,9 @@ jobs:
           echo ::set-env name=STAT::${STAT}
       - uses: actions/upload-artifact@v1
         with:
-          name: preload_docker_flags_docker_ubuntu
+          name: pause_preload_dockerflags_docker_ubuntu
           path: minikube_binaries/report
-      - name: The End Result - preload_docker_flags_docker_ubuntu
+      - name: The End Result - pause_preload_dockerflags_docker_ubuntu
         shell: bash
         run: |
           echo ${GOPOGH_RESULT}
@@ -958,12 +953,11 @@ jobs:
           if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
           if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
           if [ "$numPass" -lt 2 ];then echo "*** Failed to pass at least 2 ! ***";exit 2;fi
-          if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
-  preload_docker_flags_virtualbox_macos:
+  pause_preload_dockerflags_virtualbox_macos:
     runs-on: macos-10.15
     env:
       TIME_ELAPSED: time
-      JOB_NAME: "preload_docker_flags_virtualbox_macos"
+      JOB_NAME: "pause_preload_dockerflags_virtualbox_macos"
       GOPOGH_RESULT: ""
       SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643
     needs: [build_minikube]
@@ -994,7 +988,7 @@ jobs:
           chmod a+x e2e-*
           chmod a+x minikube-*
           START_TIME=$(date -u +%s)
-          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-darwin-amd64 -minikube-start-args=--vm-driver=virtualbox  -test.run "(TestPreload|TestDockerFlags)" -test.timeout=15m -test.v -timeout-multiplier=1.5 -binary=./minikube-darwin-amd64 2>&1 | tee ./report/testout.txt
+          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-darwin-amd64 -minikube-start-args=--vm-driver=virtualbox  -test.run "(TestPause|TestPreload|TestDockerFlags)" -test.timeout=15m -test.v -timeout-multiplier=1.5 -binary=./minikube-darwin-amd64 2>&1 | tee ./report/testout.txt
           END_TIME=$(date -u +%s)
           TIME_ELAPSED=$(($END_TIME-$START_TIME))
           min=$((${TIME_ELAPSED}/60))
@@ -1016,9 +1010,9 @@ jobs:
           echo ::set-env name=STAT::${STAT}
       - uses: actions/upload-artifact@v1
         with:
-          name: preload_docker_flags_virtualbox_macos
+          name: pause_preload_dockerflags_virtualbox_macos
           path: minikube_binaries/report
-      - name: The End Result - preload_docker_flags_virtualbox_macos
+      - name: The End Result - pause_preload_dockerflags_virtualbox_macos
         shell: bash
         run: |
           echo ${GOPOGH_RESULT}
@@ -1058,13 +1052,16 @@ jobs:
           mkdir -p all_reports
           ls -lah
           cp -r ./functional_docker_ubuntu ./all_reports/
-          cp -r ./addons_certs_docker_ubuntu ./all_reports/
-          cp -r ./multinode_pause_docker_ubuntu ./all_reports/
-          cp -r ./preload_docker_flags_docker_ubuntu ./all_reports/
-          cp -r ./functional_baremetal_ubuntu18_04 ./all_reports/
+          cp -r ./functional_virtualbox_macos ./all_reports/
           cp -r ./functional_docker_windows ./all_reports/
           cp -r ./functional_hyperv_windows ./all_reports/
-          cp -r ./functional_virtualbox_macos ./all_reports/          
+          cp -r ./functional_baremetal_ubuntu18_04 ./all_reports/
+          cp -r ./addons_certs_docker_ubuntu ./all_reports/
+          cp -r ./addons_certs_virtualbox_macos ./all_reports/
+          cp -r ./multinode_docker_ubuntu ./all_reports/
+          cp -r ./multinode_virtualbox_macos ./all_reports/
+          cp -r ./pause_preload_dockerflags_docker_ubuntu ./all_reports/
+          cp -r ./pause_preload_dockerflags_virtualbox_macos ./all_reports/
       - uses: actions/upload-artifact@v1
         with:
           name: all_reports

From 76434fc6a7d8a6165e8688a86d9d0ef74a95ba84 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Tue, 23 Jun 2020 17:09:57 -0700
Subject: [PATCH 09/11] balance jobs

---
 .github/workflows/master.yml | 19 +++++++++++--------
 .github/workflows/pr.yml     | 19 +++++++++++--------
 2 files changed, 22 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index a1ff1ee560..4c72aa7b6b 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -1034,14 +1034,17 @@ jobs:
     if: always()
     needs:
       [
-        functional_docker_ubuntu,
-        addons_certs_docker_ubuntu,
-        multinode_pause_docker_ubuntu,
-        preload_docker_flags_docker_ubuntu,
-        functional_baremetal_ubuntu18_04,
-        functional_docker_windows,
-        functional_hyperv_windows,
-        functional_virtualbox_macos,
+          functional_docker_ubuntu
+          functional_virtualbox_macos
+          functional_docker_windows
+          functional_hyperv_windows
+          functional_baremetal_ubuntu18_04
+          addons_certs_docker_ubuntu
+          addons_certs_virtualbox_macos
+          multinode_docker_ubuntu
+          multinode_virtualbox_macos
+          pause_preload_dockerflags_docker_ubuntu
+          pause_preload_dockerflags_virtualbox_macos
       ]
     runs-on: ubuntu-18.04
     steps:
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 83feb93872..5bce398e53 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -1032,14 +1032,17 @@ jobs:
     if: always()
     needs:
       [
-        functional_docker_ubuntu,
-        addons_certs_docker_ubuntu,
-        multinode_pause_docker_ubuntu,
-        preload_docker_flags_docker_ubuntu,
-        functional_baremetal_ubuntu18_04,
-        functional_docker_windows,
-        functional_hyperv_windows,
-        functional_virtualbox_macos,
+          functional_docker_ubuntu
+          functional_virtualbox_macos
+          functional_docker_windows
+          functional_hyperv_windows
+          functional_baremetal_ubuntu18_04
+          addons_certs_docker_ubuntu
+          addons_certs_virtualbox_macos
+          multinode_docker_ubuntu
+          multinode_virtualbox_macos
+          pause_preload_dockerflags_docker_ubuntu
+          pause_preload_dockerflags_virtualbox_macos
       ]
     runs-on: ubuntu-18.04
     steps:

From 63605e1e774bd28688fc2707eb1ce432adc49e09 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Tue, 23 Jun 2020 17:17:14 -0700
Subject: [PATCH 10/11] missing commas

---
 .github/workflows/master.yml | 22 +++++++++++-----------
 .github/workflows/pr.yml     | 22 +++++++++++-----------
 2 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index 4c72aa7b6b..daad679041 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -1034,17 +1034,17 @@ jobs:
     if: always()
     needs:
       [
-          functional_docker_ubuntu
-          functional_virtualbox_macos
-          functional_docker_windows
-          functional_hyperv_windows
-          functional_baremetal_ubuntu18_04
-          addons_certs_docker_ubuntu
-          addons_certs_virtualbox_macos
-          multinode_docker_ubuntu
-          multinode_virtualbox_macos
-          pause_preload_dockerflags_docker_ubuntu
-          pause_preload_dockerflags_virtualbox_macos
+          functional_docker_ubuntu,
+          functional_virtualbox_macos,
+          functional_docker_windows,
+          functional_hyperv_windows,
+          functional_baremetal_ubuntu18_04,
+          addons_certs_docker_ubuntu,
+          addons_certs_virtualbox_macos,
+          multinode_docker_ubuntu,
+          multinode_virtualbox_macos,
+          pause_preload_dockerflags_docker_ubuntu,
+          pause_preload_dockerflags_virtualbox_macos,
       ]
     runs-on: ubuntu-18.04
     steps:
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 5bce398e53..f77f0ba7ec 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -1032,17 +1032,17 @@ jobs:
     if: always()
     needs:
       [
-          functional_docker_ubuntu
-          functional_virtualbox_macos
-          functional_docker_windows
-          functional_hyperv_windows
-          functional_baremetal_ubuntu18_04
-          addons_certs_docker_ubuntu
-          addons_certs_virtualbox_macos
-          multinode_docker_ubuntu
-          multinode_virtualbox_macos
-          pause_preload_dockerflags_docker_ubuntu
-          pause_preload_dockerflags_virtualbox_macos
+          functional_docker_ubuntu,
+          functional_virtualbox_macos,
+          functional_docker_windows,
+          functional_hyperv_windows,
+          functional_baremetal_ubuntu18_04,
+          addons_certs_docker_ubuntu,
+          addons_certs_virtualbox_macos,
+          multinode_docker_ubuntu,
+          multinode_virtualbox_macos,
+          pause_preload_dockerflags_docker_ubuntu,
+          pause_preload_dockerflags_virtualbox_macos,
       ]
     runs-on: ubuntu-18.04
     steps:

From f4d6e152668d9781f317c0dbb6a9f2b66e9326db Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Wed, 24 Jun 2020 11:01:12 -0700
Subject: [PATCH 11/11] adjust

---
 .github/workflows/master.yml | 10 +++-------
 .github/workflows/pr.yml     | 10 +++-------
 2 files changed, 6 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index daad679041..e7900142a4 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -629,7 +629,6 @@ jobs:
           if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
           if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
           if [ "$numPass" -lt 5 ];then echo "*** Failed to pass at least 5 ! ***";exit 2;fi
-          if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
   addons_certs_virtualbox_macos:
     runs-on: macos-10.15
     env:
@@ -707,7 +706,6 @@ jobs:
           if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
           if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
           if [ "$numPass" -lt 5 ];then echo "*** Failed to pass at least 5 ! ***";exit 2;fi
-          if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
   multinode_docker_ubuntu:
     runs-on: ubuntu-18.04
     env:
@@ -867,8 +865,7 @@ jobs:
           echo "*** $numPass Passed ***"
           if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
           if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
-          if [ "$numPass" -lt 5 ];then echo "*** Failed to pass at least 5 ! ***";exit 2;fi
-          if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
+          if [ "$numPass" -lt 7 ];then echo "*** Failed to pass at least 7 ! ***";exit 2;fi
   pause_preload_dockerflags_docker_ubuntu:
     runs-on: ubuntu-18.04
     env:
@@ -954,7 +951,7 @@ jobs:
           echo "*** $numPass Passed ***"
           if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
           if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
-          if [ "$numPass" -lt 2 ];then echo "*** Failed to pass at least 2 ! ***";exit 2;fi
+          if [ "$numPass" -lt 9 ];then echo "*** Failed to pass at least 9 ! ***";exit 2;fi
   pause_preload_dockerflags_virtualbox_macos:
     runs-on: macos-10.15
     env:
@@ -1026,8 +1023,7 @@ jobs:
           echo "*** $numPass Passed ***"
           if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
           if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
-          if [ "$numPass" -lt 2 ];then echo "*** Failed to pass at least 2 ! ***";exit 2;fi
-          if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
+          if [ "$numPass" -lt 9 ];then echo "*** Failed to pass at least 9 ! ***";exit 2;fi
   # After all integration tests finished
   # collect all the reports and upload them
   upload_all_reports:
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index f77f0ba7ec..89e7d60c5f 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -627,7 +627,6 @@ jobs:
           if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
           if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
           if [ "$numPass" -lt 5 ];then echo "*** Failed to pass at least 5 ! ***";exit 2;fi
-          if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
   addons_certs_virtualbox_macos:
     runs-on: macos-10.15
     env:
@@ -705,7 +704,6 @@ jobs:
           if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
           if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
           if [ "$numPass" -lt 5 ];then echo "*** Failed to pass at least 5 ! ***";exit 2;fi
-          if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
   multinode_docker_ubuntu:
     runs-on: ubuntu-18.04
     env:
@@ -865,8 +863,7 @@ jobs:
           echo "*** $numPass Passed ***"
           if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
           if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
-          if [ "$numPass" -lt 5 ];then echo "*** Failed to pass at least 5 ! ***";exit 2;fi
-          if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
+          if [ "$numPass" -lt 7 ];then echo "*** Failed to pass at least 7 ! ***";exit 2;fi
   pause_preload_dockerflags_docker_ubuntu:
     runs-on: ubuntu-18.04
     env:
@@ -952,7 +949,7 @@ jobs:
           echo "*** $numPass Passed ***"
           if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
           if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
-          if [ "$numPass" -lt 2 ];then echo "*** Failed to pass at least 2 ! ***";exit 2;fi
+          if [ "$numPass" -lt 9 ];then echo "*** Failed to pass at least 9 ! ***";exit 2;fi
   pause_preload_dockerflags_virtualbox_macos:
     runs-on: macos-10.15
     env:
@@ -1024,8 +1021,7 @@ jobs:
           echo "*** $numPass Passed ***"
           if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
           if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
-          if [ "$numPass" -lt 2 ];then echo "*** Failed to pass at least 2 ! ***";exit 2;fi
-          if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
+          if [ "$numPass" -lt 9 ];then echo "*** Failed to pass at least 9 ! ***";exit 2;fi
   # After all integration tests finished
   # collect all the reports and upload them
   upload_all_reports: