From b6e55e6c979d1610624e43ab02ac015cfe354bea Mon Sep 17 00:00:00 2001
From: Andriy Dzikh <andriydzikh@gmail.com>
Date: Fri, 9 Jul 2021 10:28:56 -0700
Subject: [PATCH 1/3] Create common.ps1 to simplify Windows integration tests.

---
 hack/jenkins/common.ps1                       | 78 +++++++++++++++++++
 hack/jenkins/minikube_set_pending.sh          |  2 +-
 hack/jenkins/windows_integration_setup.ps1    |  6 ++
 hack/jenkins/windows_integration_teardown.ps1 |  8 ++
 .../windows_integration_test_docker.ps1       | 72 ++---------------
 .../windows_integration_test_hyperv.ps1       | 31 ++------
 .../windows_integration_test_virtualbox.ps1   | 32 ++------
 7 files changed, 108 insertions(+), 121 deletions(-)
 create mode 100644 hack/jenkins/common.ps1

diff --git a/hack/jenkins/common.ps1 b/hack/jenkins/common.ps1
new file mode 100644
index 0000000000..32f242efce
--- /dev/null
+++ b/hack/jenkins/common.ps1
@@ -0,0 +1,78 @@
+# Copyright 2019 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+mkdir -p out
+
+(New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.8.0/gopogh.exe", "C:\Go\bin\gopogh.exe")
+
+gsutil.cmd -m cp gs://minikube-builds/$env:MINIKUBE_LOCATION/minikube-windows-amd64.exe out/
+gsutil.cmd -m cp gs://minikube-builds/$env:MINIKUBE_LOCATION/e2e-windows-amd64.exe out/
+gsutil.cmd -m cp -r gs://minikube-builds/$env:MINIKUBE_LOCATION/testdata .
+gsutil.cmd -m cp -r gs://minikube-builds/$env:MINIKUBE_LOCATION/windows_integration_setup.ps1 out/
+gsutil.cmd -m cp -r gs://minikube-builds/$env:MINIKUBE_LOCATION/windows_integration_teardown.ps1 out/
+
+./out/minikube-windows-amd64.exe delete --all
+
+./out/windows_integration_setup.ps1
+
+$started=Get-Date -UFormat %s
+
+out/e2e-windows-amd64.exe --minikube-start-args="--driver=$driver" --binary=out/minikube-windows-amd64.exe --test.v --test.timeout=$timeout | Tee-Object -FilePath testout.txt
+
+$env:result=$lastexitcode
+# If the last exit code was 0->success, x>0->error
+If($env:result -eq 0){
+	$env:status="success"
+	echo "minikube: SUCCESS"
+} Else {
+	$env:status="failure"
+	echo "minikube: FAIL"
+}
+
+$ended=Get-Date -UFormat %s
+$elapsed=$ended-$started
+$elapsed=$elapsed/60
+$elapsed=[math]::Round($elapsed, 2)
+
+Get-Content testout.txt -Encoding ASCII | go tool test2json -t | Out-File -FilePath testout.json -Encoding ASCII
+
+$gopogh_status=gopogh --in testout.json --out_html testout.html --out_summary testout_summary.json --name "$env:JOB_NAME" -pr $env:MINIKUBE_LOCATION --repo github.com/kubernetes/minikube/ --details $env:COMMIT
+
+$failures=echo $gopogh_status | jq '.NumberOfFail'
+$tests=echo $gopogh_status | jq '.NumberOfTests'
+$bad_status="$failures / $tests failures"
+
+$description="$status in $elapsed minute(s)."
+If($env:status -eq "failure") {
+	$description="completed with $bad_status in $elapsed minute(s)."
+}
+echo $description
+
+$env:SHORT_COMMIT=$env:COMMIT.substring(0, 7)
+$gcs_bucket="minikube-builds/logs/$env:MINIKUBE_LOCATION/$env:SHORT_COMMIT"
+
+#Upload logs to gcs
+gsutil -qm cp testout.txt gs://$gcs_bucket/${env:JOB_NAME}out.txt
+gsutil -qm cp testout.json gs://$gcs_bucket/${env:JOB_NAME}.json
+gsutil -qm cp testout.html gs://$gcs_bucket/${env:JOB_NAME}.html
+gsutil -qm cp testout_summary.json gs://$gcs_bucket/${env:JOB_NAME}_summary.json
+
+$env:target_url="https://storage.googleapis.com/$gcs_bucket/$env:JOB_NAME.html"
+# Update the PR with the new info
+$json = "{`"state`": `"$env:status`", `"description`": `"Jenkins: $description`", `"target_url`": `"$env:target_url`", `"context`": `"${env:JOB_NAME}`"}"
+Invoke-WebRequest -Uri "https://api.github.com/repos/kubernetes/minikube/statuses/$env:COMMIT`?access_token=$env:access_token" -Body $json -ContentType "application/json" -Method Post -usebasicparsing
+
+./out/windows_integration_teardown.ps1
+
+Exit $env:result
diff --git a/hack/jenkins/minikube_set_pending.sh b/hack/jenkins/minikube_set_pending.sh
index 3c998f8398..371ba645d3 100755
--- a/hack/jenkins/minikube_set_pending.sh
+++ b/hack/jenkins/minikube_set_pending.sh
@@ -31,7 +31,7 @@ jobs=(
      # 'Hyper-V_Windows'
      # 'VirtualBox_Linux'
      # 'VirtualBox_macOS'
-     'VirtualBox_Windows'
+     # 'VirtualBox_Windows'
      # 'KVM-GPU_Linux' - Disabled
      'KVM_Linux'
      'KVM_Linux_containerd'
diff --git a/hack/jenkins/windows_integration_setup.ps1 b/hack/jenkins/windows_integration_setup.ps1
index 510cccbfc3..1c20b66c2b 100644
--- a/hack/jenkins/windows_integration_setup.ps1
+++ b/hack/jenkins/windows_integration_setup.ps1
@@ -16,6 +16,12 @@ $test_home="$env:HOMEDRIVE$env:HOMEPATH\minikube-integration"
 $env:KUBECONFIG="$test_home\kubeconfig"
 $env:MINIKUBE_HOME="$test_home\.minikube"
 
+if ($driver == "docker") {
+  # Remove unused images and containers
+  docker system prune --all --force --volumes
+  docker ps -aq | ForEach -Process {docker rm -fv $_}
+}
+
 # delete in case previous test was unexpectedly ended and teardown wasn't run
 rm -r -Force $test_home
 mkdir -p $test_home
diff --git a/hack/jenkins/windows_integration_teardown.ps1 b/hack/jenkins/windows_integration_teardown.ps1
index 2dc1248f7e..6a66241338 100644
--- a/hack/jenkins/windows_integration_teardown.ps1
+++ b/hack/jenkins/windows_integration_teardown.ps1
@@ -14,4 +14,12 @@
 
 $test_home="$env:HOMEDRIVE$env:HOMEPATH\minikube-integration"
 
+if ($driver == "docker") {
+  # Remove unused images and containers
+  docker system prune --all --force
+
+  # Just shutdown Docker, it's safer than anything else
+  Get-Process "*Docker Desktop*" | Stop-Process
+}
+
 rm -r -Force $test_home
diff --git a/hack/jenkins/windows_integration_test_docker.ps1 b/hack/jenkins/windows_integration_test_docker.ps1
index cd6764067a..186fff9f37 100644
--- a/hack/jenkins/windows_integration_test_docker.ps1
+++ b/hack/jenkins/windows_integration_test_docker.ps1
@@ -14,15 +14,10 @@
 
 mkdir -p out
 
-(New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.8.0/gopogh.exe", "C:\Go\bin\gopogh.exe")
 (New-Object Net.WebClient).DownloadFile("https://dl.k8s.io/release/v1.20.0/bin/windows/amd64/kubectl.exe", "C:\Program Files\Docker\Docker\resources\bin\kubectl.exe")
 
-gsutil.cmd -m cp gs://minikube-builds/$env:MINIKUBE_LOCATION/minikube-windows-amd64.exe out/
-gsutil.cmd -m cp gs://minikube-builds/$env:MINIKUBE_LOCATION/e2e-windows-amd64.exe out/
-gsutil.cmd -m cp -r gs://minikube-builds/$env:MINIKUBE_LOCATION/testdata .
 gsutil.cmd -m cp -r gs://minikube-builds/$env:MINIKUBE_LOCATION/setup_docker_desktop_windows.ps1 out/
-gsutil.cmd -m cp -r gs://minikube-builds/$env:MINIKUBE_LOCATION/windows_integration_setup.ps1 out/
-gsutil.cmd -m cp -r gs://minikube-builds/$env:MINIKUBE_LOCATION/windows_integration_teardown.ps1 out/
+gsutil.cmd -m cp -r gs://minikube-builds/$env:MINIKUBE_LOCATION/common.ps1 out/
 
 $env:SHORT_COMMIT=$env:COMMIT.substring(0, 7)
 $gcs_bucket="minikube-builds/logs/$env:MINIKUBE_LOCATION/$env:SHORT_COMMIT"
@@ -39,65 +34,8 @@ If ($lastexitcode -gt 0) {
 	Exit $lastexitcode
 }
 
-./out/minikube-windows-amd64.exe delete --all
+$driver="docker"
+$timeout="180m"
+$env:JOB_NAME="Docker_Windows"
 
-# Remove unused images and containers
-docker system prune --all --force --volumes
-
-./out/windows_integration_setup.ps1
-
-docker ps -aq | ForEach -Process {docker rm -fv $_}
-
-$started=Get-Date -UFormat %s
-
-out/e2e-windows-amd64.exe --minikube-start-args="--driver=docker" --binary=out/minikube-windows-amd64.exe --test.v --test.timeout=180m | Tee-Object -FilePath testout.txt
-$env:result=$lastexitcode
-# If the last exit code was 0->success, x>0->error
-If($env:result -eq 0){
-	$env:status="success"
-	echo "minikube: SUCCESS"
-} Else {
-	$env:status="failure"
-	echo "minikube: FAIL"
-}
-
-$ended=Get-Date -UFormat %s
-$elapsed=$ended-$started
-$elapsed=$elapsed/60
-$elapsed=[math]::Round($elapsed, 2)
-
-Get-Content testout.txt -Encoding ASCII | go tool test2json -t | Out-File -FilePath testout.json -Encoding ASCII
-
-$gopogh_status=gopogh --in testout.json --out_html testout.html --out_summary testout_summary.json --name "Docker_Windows" -pr $env:MINIKUBE_LOCATION --repo github.com/kubernetes/minikube/ --details $env:COMMIT
-
-$failures=echo $gopogh_status | jq '.NumberOfFail'
-$tests=echo $gopogh_status | jq '.NumberOfTests'
-$bad_status="$failures / $tests failures"
-
-$description="$status in $elapsed minute(s)."
-If($env:status -eq "failure") {
-	$description="completed with $bad_status in $elapsed minute(s)."
-}
-echo $description
-
-$env:target_url="https://storage.googleapis.com/$gcs_bucket/Docker_Windows.html"
-
-#Upload logs to gcs
-gsutil -qm cp testout.txt gs://$gcs_bucket/Docker_Windowsout.txt
-gsutil -qm cp testout.json gs://$gcs_bucket/Docker_Windows.json
-gsutil -qm cp testout.html gs://$gcs_bucket/Docker_Windows.html
-gsutil -qm cp testout_summary.json gs://$gcs_bucket/Docker_Windows_summary.json
-
-# Update the PR with the new info
-$json = "{`"state`": `"$env:status`", `"description`": `"Jenkins: $description`", `"target_url`": `"$env:target_url`", `"context`": `"Docker_Windows`"}"
-Invoke-WebRequest -Uri "https://api.github.com/repos/kubernetes/minikube/statuses/$env:COMMIT`?access_token=$env:access_token" -Body $json -ContentType "application/json" -Method Post -usebasicparsing
-
-# Remove unused images and containers
-docker system prune --all --force
-
-# Just shutdown Docker, it's safer than anything else
-Get-Process "*Docker Desktop*" | Stop-Process
-
-./out/windows_integration_teardown.ps1
-
-Exit $env:result
+. ./out/common.ps1
diff --git a/hack/jenkins/windows_integration_test_hyperv.ps1 b/hack/jenkins/windows_integration_test_hyperv.ps1
index 067e64b695..41520a9f8f 100644
--- a/hack/jenkins/windows_integration_test_hyperv.ps1
+++ b/hack/jenkins/windows_integration_test_hyperv.ps1
@@ -12,31 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-(New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.8.0/gopogh.exe", "C:\Go\bin\gopogh.exe")
+gsutil.cmd -m cp -r gs://minikube-builds/$env:MINIKUBE_LOCATION/common.ps1 out/
 
-mkdir -p out
-gsutil.cmd -m cp gs://minikube-builds/$env:MINIKUBE_LOCATION/minikube-windows-amd64.exe out/
-gsutil.cmd -m cp gs://minikube-builds/$env:MINIKUBE_LOCATION/e2e-windows-amd64.exe out/
-gsutil.cmd -m cp -r gs://minikube-builds/$env:MINIKUBE_LOCATION/testdata .
-gsutil.cmd -m cp -r gs://minikube-builds/$env:MINIKUBE_LOCATION/windows_integration_setup.ps1 out/
-gsutil.cmd -m cp -r gs://minikube-builds/$env:MINIKUBE_LOCATION/windows_integration_teardown.ps1 out/
+$driver="hyperv"
+$timeout="65m"
+$env:JOB_NAME="Hyper-V_Windows"
 
-./out/minikube-windows-amd64.exe delete --all
-
-./out/windows_integration_setup.ps1
-
-out/e2e-windows-amd64.exe -minikube-start-args="--driver=hyperv" -binary=out/minikube-windows-amd64.exe -test.v -test.timeout=65m
-$env:result=$lastexitcode
-# If the last exit code was 0->success, x>0->error
-If($env:result -eq 0){$env:status="success"}
-Else {$env:status="failure"}
-
-# $env:SHORT_COMMIT=$env:COMMIT.substring(0, 7)
-# to be used later to implement https://github.com/kubernetes/minikube/issues/6593
-$env:target_url="https://storage.googleapis.com/minikube-builds/logs/$env:MINIKUBE_LOCATION/Hyper-V_Windows.txt"
-$json = "{`"state`": `"$env:status`", `"description`": `"Jenkins`", `"target_url`": `"$env:target_url`", `"context`": `"Hyper-V_Windows`"}"
-Invoke-WebRequest -Uri "https://api.github.com/repos/kubernetes/minikube/statuses/$env:COMMIT`?access_token=$env:access_token" -Body $json -ContentType "application/json" -Method Post -usebasicparsing
-
-./out/windows_integration_teardown.ps1
-
-Exit $env:result
+. ./out/common.ps1
diff --git a/hack/jenkins/windows_integration_test_virtualbox.ps1 b/hack/jenkins/windows_integration_test_virtualbox.ps1
index e4100ddf83..fd3d58d541 100644
--- a/hack/jenkins/windows_integration_test_virtualbox.ps1
+++ b/hack/jenkins/windows_integration_test_virtualbox.ps1
@@ -12,32 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-(New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.8.0/gopogh.exe", "C:\Go\bin\gopogh.exe")
+gsutil.cmd -m cp -r gs://minikube-builds/$env:MINIKUBE_LOCATION/common.ps1 out/
 
+$driver="virtualbox"
+$timeout="30m"
+$env:JOB_NAME="VirtualBox_Windows"
 
-mkdir -p out
-gsutil.cmd -m cp gs://minikube-builds/$env:MINIKUBE_LOCATION/minikube-windows-amd64.exe out/
-gsutil.cmd -m cp gs://minikube-builds/$env:MINIKUBE_LOCATION/e2e-windows-amd64.exe out/
-gsutil.cmd -m cp -r gs://minikube-builds/$env:MINIKUBE_LOCATION/testdata .
-gsutil.cmd -m cp -r gs://minikube-builds/$env:MINIKUBE_LOCATION/windows_integration_setup.ps1 out/
-gsutil.cmd -m cp -r gs://minikube-builds/$env:MINIKUBE_LOCATION/windows_integration_teardown.ps1 out/
-
-./out/minikube-windows-amd64.exe delete
-
-./out/windows_integration_setup.ps1
-
-out/e2e-windows-amd64.exe -minikube-start-args="--driver=virtualbox" -binary=out/minikube-windows-amd64.exe -test.v -test.timeout=30m
-$env:result=$lastexitcode
-# If the last exit code was 0->success, x>0->error
-If($env:result -eq 0){$env:status="success"}
-Else {$env:status="failure"}
-
-# $env:SHORT_COMMIT=$env:COMMIT.substring(0, 7)
-# to be used later 
-$env:target_url="https://storage.googleapis.com/minikube-builds/logs/$env:MINIKUBE_LOCATION/VirtualBox_Windows.txt"
-$json = "{`"state`": `"$env:status`", `"description`": `"Jenkins`", `"target_url`": `"$env:target_url`", `"context`": `"VirtualBox_Windows`"}"
-Invoke-WebRequest -Uri "https://api.github.com/repos/kubernetes/minikube/statuses/$env:COMMIT`?access_token=$env:access_token" -Body $json -ContentType "application/json" -Method Post -usebasicparsing
-
-./out/windows_integration_teardown.ps1
-
-Exit $env:result
+. ./out/common.ps1

From c64ba2c0808d0be0c2af47315b8774e03acc81ab Mon Sep 17 00:00:00 2001
From: Andriy Dzikh <andriydzikh@gmail.com>
Date: Mon, 12 Jul 2021 10:10:54 -0700
Subject: [PATCH 2/3] Fix report scripts to print out only started
 environments, and use quotes around script names.

---
 hack/jenkins/test-flake-chart/report_flakes.sh | 4 ++--
 hack/jenkins/test-flake-chart/sync_tests.sh    | 7 +++++--
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/hack/jenkins/test-flake-chart/report_flakes.sh b/hack/jenkins/test-flake-chart/report_flakes.sh
index cffc628e9c..9040a71577 100755
--- a/hack/jenkins/test-flake-chart/report_flakes.sh
+++ b/hack/jenkins/test-flake-chart/report_flakes.sh
@@ -40,7 +40,7 @@ TMP_DATA=$(mktemp)
 # 3) Sort by environment, then test name.
 # 4) Store in file $TMP_DATA.
 gsutil cat $(< "${ENVIRONMENT_LIST}" sed -r "s/^/gs:\\/\\/minikube-builds\\/logs\\/${PR_NUMBER}\\/${SHORT_COMMIT}\\/; s/$/_summary.json/") \
-  | $DIR/process_data.sh \
+  | "$DIR/process_data.sh" \
   | sed -n -r -e "s/[0-9a-f]*,[0-9-]*,([a-zA-Z\/_0-9-]*),([a-zA-Z\/_0-9-]*),Failed,[.0-9]*/\1:\2/p" \
   | sort -t, -k\
   > "$TMP_DATA"
@@ -85,6 +85,6 @@ fi
 printf "\n\nTo see the flake rates of all tests on $ENVIRONMENT, click [here](https:\/\/storage.googleapis.com\/minikube-flake-rate\/flake_chart.html?env=$ENVIRONMENT)." >> "$TMP_COMMENT"
 
 # install gh if not present
-$DIR/../installers/check_install_gh.sh
+"$DIR/../installers/check_install_gh.sh"
 
 gh pr comment "https://github.com/kubernetes/minikube/pull/$PR_NUMBER" --body "$(cat $TMP_COMMENT)"
diff --git a/hack/jenkins/test-flake-chart/sync_tests.sh b/hack/jenkins/test-flake-chart/sync_tests.sh
index b44002cd31..03b4cf364a 100644
--- a/hack/jenkins/test-flake-chart/sync_tests.sh
+++ b/hack/jenkins/test-flake-chart/sync_tests.sh
@@ -54,9 +54,11 @@ gsutil cat "${FINISHED_LIST_REMOTE}"\
   | uniq > "${FINISHED_LIST}"
 
 STARTED_COUNT=$(echo "${STARTED_LIST}" | wc -l)
-FINISHED_COUNT=$(\
+FINISHED_LIST_JOINED=$(\
   echo "${STARTED_LIST}"\
-  | join - "${FINISHED_LIST}"\
+  | join - "${FINISHED_LIST}")
+FINISHED_COUNT=$(\
+  echo "${FINISHED_LIST_JOINED}"\
   | wc -l)
 
 if [ ${STARTED_COUNT} -ne ${FINISHED_COUNT} ]; then
@@ -68,6 +70,7 @@ fi
 gsutil rm "${BUCKET_PATH}/started_environments_${ROOT_JOB_ID}.txt"
 
 # At this point, we know all integration tests are done and we can process all summaries safely.
+echo "${FINISHED_LIST_JOINED}" > ${FINISHED_LIST}
 
 # Get directory of this script.
 DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )

From 0589a42fdf7b751a8059d3ec2a315399c41d2ac9 Mon Sep 17 00:00:00 2001
From: Andriy Dzikh <andriydzikh@gmail.com>
Date: Mon, 12 Jul 2021 14:10:50 -0700
Subject: [PATCH 3/3] Use -eq instead of == because Powershell bad.

---
 hack/jenkins/windows_integration_setup.ps1    | 2 +-
 hack/jenkins/windows_integration_teardown.ps1 | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/hack/jenkins/windows_integration_setup.ps1 b/hack/jenkins/windows_integration_setup.ps1
index 1c20b66c2b..e1e2912de4 100644
--- a/hack/jenkins/windows_integration_setup.ps1
+++ b/hack/jenkins/windows_integration_setup.ps1
@@ -16,7 +16,7 @@ $test_home="$env:HOMEDRIVE$env:HOMEPATH\minikube-integration"
 $env:KUBECONFIG="$test_home\kubeconfig"
 $env:MINIKUBE_HOME="$test_home\.minikube"
 
-if ($driver == "docker") {
+if ($driver -eq "docker") {
   # Remove unused images and containers
   docker system prune --all --force --volumes
   docker ps -aq | ForEach -Process {docker rm -fv $_}
diff --git a/hack/jenkins/windows_integration_teardown.ps1 b/hack/jenkins/windows_integration_teardown.ps1
index 6a66241338..eb1c0d953f 100644
--- a/hack/jenkins/windows_integration_teardown.ps1
+++ b/hack/jenkins/windows_integration_teardown.ps1
@@ -14,7 +14,7 @@
 
 $test_home="$env:HOMEDRIVE$env:HOMEPATH\minikube-integration"
 
-if ($driver == "docker") {
+if ($driver -eq "docker") {
   # Remove unused images and containers
   docker system prune --all --force