From 2c62a991ce303b8a892af85e82c62034b03bdd04 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Thu, 14 Jan 2021 10:44:03 -0800 Subject: [PATCH 01/11] adding proper integration testing script for docker on windows --- go.sum | 4 +- hack/jenkins/minikube_set_pending.sh | 2 +- .../windows_integration_test_docker.ps1 | 56 ++++++++++++++++--- 3 files changed, 50 insertions(+), 12 deletions(-) diff --git a/go.sum b/go.sum index e8cf127822..3f6096a3a3 100644 --- a/go.sum +++ b/go.sum @@ -986,8 +986,8 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/ulikunitz/xz v0.5.5 h1:pFrO0lVpTBXLpYw+pnLj6TbvHuyjXMfjGeCwSqCVwok= -github.com/ulikunitz/xz v0.5.5/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= +github.com/ulikunitz/xz v0.5.8 h1:ERv8V6GKqVi23rgu5cj9pVfVzJbOqAY2Ntl88O6c2nQ= +github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ultraware/funlen v0.0.1/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA= github.com/ultraware/funlen v0.0.2/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA= github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= diff --git a/hack/jenkins/minikube_set_pending.sh b/hack/jenkins/minikube_set_pending.sh index 1f2f5e4075..9955708d51 100755 --- a/hack/jenkins/minikube_set_pending.sh +++ b/hack/jenkins/minikube_set_pending.sh @@ -42,7 +42,7 @@ jobs=( 'none_Linux' 'Docker_Linux' # 'Docker_macOS' - # 'Docker_Windows' + 'Docker_Windows' # 'Podman_Linux' ) diff --git a/hack/jenkins/windows_integration_test_docker.ps1 b/hack/jenkins/windows_integration_test_docker.ps1 index b30125c27d..d2ec7bd36d 100644 --- a/hack/jenkins/windows_integration_test_docker.ps1 +++ b/hack/jenkins/windows_integration_test_docker.ps1 @@ -6,12 +6,13 @@ # # http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, +# Unless required by applicable law or agreed to in writing, software# distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +docker ps -aq | ForEach -Process {docker rm -fv $_} + mkdir -p out gsutil.cmd -m cp gs://minikube-builds/$env:MINIKUBE_LOCATION/minikube-windows-amd64.exe out/ gsutil.cmd -m cp gs://minikube-builds/$env:MINIKUBE_LOCATION/e2e-windows-amd64.exe out/ @@ -19,16 +20,53 @@ gsutil.cmd -m cp -r gs://minikube-builds/$env:MINIKUBE_LOCATION/testdata . ./out/minikube-windows-amd64.exe delete --all -out/e2e-windows-amd64.exe -minikube-start-args="--driver=docker" -binary=out/minikube-windows-amd64.exe -test.v -test.timeout=65m +$started=Get-Date -UFormat %s + +out/e2e-windows-amd64.exe --minikube-start-args="--driver=docker" --binary=out/minikube-windows-amd64.exe --test.v --test.timeout=180m | tee testout.txt $env:result=$lastexitcode # If the last exit code was 0->success, x>0->error -If($env:result -eq 0){$env:status="success"} -Else {$env:status="failure"} +If($env:result -eq 0){ + $env:status="success" + echo "minikube: SUCCESS" +} Else { + $env:status="failure" + echo "minikube: FAIL" +} -# $env:SHORT_COMMIT=$env:COMMIT.substring(0, 7) -# to be used later to implement https://github.com/kubernetes/minikube/issues/6593 -$env:target_url="https://storage.googleapis.com/minikube-builds/logs/$env:MINIKUBE_LOCATION/Docker_Windows.txt" +$ended=Get-Date -UFormat %s +$elapsed=$ended-$started +$elapsed=$elapsed/60 +$elapsed=[math]::Round($elapsed) + +Get-Content testout.txt | go tool test2json -t > testout.json + +$gopogh_status=gopogh --in testout.json --out testout.html --name "Docker_Windows" -pr $env:MINIKUBE_LOCATION --repo github.com/kubernetes/minikube/ --details $env:COMMIT + +$failures=echo $gopogh_status | jq '.NumberOfFail' +$tests=echo $gopogh_status | jq '.NumberOfTests' +$bad_status="$failures / $tests failures" + +$description="$status in $elapsed minutes" +If($env:status -eq "failure") { + $description="completed with $bad_status in $elapsed minutes" +} +echo $description + +$env:SHORT_COMMIT=$env:COMMIT.substring(0, 7) +$gcs_bucket="minikube-builds/logs/$env:MINIKUBE_LOCATION/$env:SHORT_COMMIT" +$env:target_url="https://storage.googleapis.com/$gcs_bucket/Docker_Windows.txt" + +#Upload logs to gcs +gsutil -qm cp testout.txt gs://$gcs_bucket/Docker_Windowsout.txt +gsutil -qm cp testout.json gs://$gcs_bucket/Docker_Windows.json +gsutil -qm cp testout.html gs://$gcs_bucket/Docker_Windows.html + +# Update the PR with the new info $json = "{`"state`": `"$env:status`", `"description`": `"Jenkins`", `"target_url`": `"$env:target_url`", `"context`": `"Docker_Windows`"}" Invoke-WebRequest -Uri "https://api.github.com/repos/kubernetes/minikube/statuses/$env:COMMIT`?access_token=$env:access_token" -Body $json -ContentType "application/json" -Method Post -usebasicparsing -Exit $env:result \ No newline at end of file +# Uncomment once tunnel is fixed on Windows: https://github.com/kubernetes/minikube/issues/8304 +#./out/minikube-windows-amd64.exe tunnel --cleanup +./out/minikube-windows-amd64.exe delete --all --purge + +Exit $env:result From efc240deab0b77746455e7beb127a1adc1e7912c Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Fri, 15 Jan 2021 10:06:45 -0800 Subject: [PATCH 02/11] add script to explicitly start docker desktop on windows because we can't have nice things --- hack/jenkins/setup_docker_desktop_windows.ps1 | 18 ++++++++++++++++++ .../windows_integration_test_docker.ps1 | 8 +++++++- 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 hack/jenkins/setup_docker_desktop_windows.ps1 diff --git a/hack/jenkins/setup_docker_desktop_windows.ps1 b/hack/jenkins/setup_docker_desktop_windows.ps1 new file mode 100644 index 0000000000..6d4593960a --- /dev/null +++ b/hack/jenkins/setup_docker_desktop_windows.ps1 @@ -0,0 +1,18 @@ +$attempt = 10 +while($attempt -ne 0) { + Write-Host "Attempt ", $attempt + Write-Host "Wait for 3 minutes" + & "C:\Program Files\Docker\Docker\Docker Desktop.exe" + Start-Sleep 180 + $dockerInfo = docker info + Write-Host "Docker Info ", $dockerInfo + $serverVersion = $dockerInfo | Where-Object {$_ -Match "Server Version"} + Write-Host "Server Version ", $serverVersion + if (![System.String]::IsNullOrEmpty($serverVersion)) { + Write-Host "Docker successfully started!" + exit 0 + } + Write-Host "Restarting Docker Desktop" + Get-Process "*Docker Desktop*" | Stop-Process + $attempt -= 1 +} diff --git a/hack/jenkins/windows_integration_test_docker.ps1 b/hack/jenkins/windows_integration_test_docker.ps1 index d2ec7bd36d..3f475123ee 100644 --- a/hack/jenkins/windows_integration_test_docker.ps1 +++ b/hack/jenkins/windows_integration_test_docker.ps1 @@ -17,6 +17,9 @@ mkdir -p out gsutil.cmd -m cp gs://minikube-builds/$env:MINIKUBE_LOCATION/minikube-windows-amd64.exe out/ gsutil.cmd -m cp gs://minikube-builds/$env:MINIKUBE_LOCATION/e2e-windows-amd64.exe out/ gsutil.cmd -m cp -r gs://minikube-builds/$env:MINIKUBE_LOCATION/testdata . +gsutil.cmd -m cp -r gs://minikube-builds/$env:MINIKUBE_LOCATION/setup_docker_desktop_windows.ps1 out/ + +./setup_docker_desktop_windows.ps1 ./out/minikube-windows-amd64.exe delete --all @@ -65,8 +68,11 @@ gsutil -qm cp testout.html gs://$gcs_bucket/Docker_Windows.html $json = "{`"state`": `"$env:status`", `"description`": `"Jenkins`", `"target_url`": `"$env:target_url`", `"context`": `"Docker_Windows`"}" Invoke-WebRequest -Uri "https://api.github.com/repos/kubernetes/minikube/statuses/$env:COMMIT`?access_token=$env:access_token" -Body $json -ContentType "application/json" -Method Post -usebasicparsing +# Just shutdown Docker, it's safer than anything else +Get-Process "*Docker Desktop*" | Stop-Process + # Uncomment once tunnel is fixed on Windows: https://github.com/kubernetes/minikube/issues/8304 #./out/minikube-windows-amd64.exe tunnel --cleanup -./out/minikube-windows-amd64.exe delete --all --purge +#./out/minikube-windows-amd64.exe delete --all --purge Exit $env:result From 2dd37713a403c26b597fbc8ffe96162a175939d3 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Fri, 15 Jan 2021 10:11:59 -0800 Subject: [PATCH 03/11] fix script --- hack/jenkins/windows_integration_test_docker.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/jenkins/windows_integration_test_docker.ps1 b/hack/jenkins/windows_integration_test_docker.ps1 index 3f475123ee..3849edf414 100644 --- a/hack/jenkins/windows_integration_test_docker.ps1 +++ b/hack/jenkins/windows_integration_test_docker.ps1 @@ -19,7 +19,7 @@ gsutil.cmd -m cp gs://minikube-builds/$env:MINIKUBE_LOCATION/e2e-windows-amd64.e gsutil.cmd -m cp -r gs://minikube-builds/$env:MINIKUBE_LOCATION/testdata . gsutil.cmd -m cp -r gs://minikube-builds/$env:MINIKUBE_LOCATION/setup_docker_desktop_windows.ps1 out/ -./setup_docker_desktop_windows.ps1 +./out/setup_docker_desktop_windows.ps1 ./out/minikube-windows-amd64.exe delete --all From 46a07ffc20376283de92772f7b7191b0789af60f Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Tue, 19 Jan 2021 13:17:58 -0800 Subject: [PATCH 04/11] add detailed description to jenkins status --- hack/jenkins/windows_integration_test_docker.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/jenkins/windows_integration_test_docker.ps1 b/hack/jenkins/windows_integration_test_docker.ps1 index 3849edf414..e7b369b89d 100644 --- a/hack/jenkins/windows_integration_test_docker.ps1 +++ b/hack/jenkins/windows_integration_test_docker.ps1 @@ -65,7 +65,7 @@ gsutil -qm cp testout.json gs://$gcs_bucket/Docker_Windows.json gsutil -qm cp testout.html gs://$gcs_bucket/Docker_Windows.html # Update the PR with the new info -$json = "{`"state`": `"$env:status`", `"description`": `"Jenkins`", `"target_url`": `"$env:target_url`", `"context`": `"Docker_Windows`"}" +$json = "{`"state`": `"$env:status`", `"description`": `"Jenkins: $description`", `"target_url`": `"$env:target_url`", `"context`": `"Docker_Windows`"}" Invoke-WebRequest -Uri "https://api.github.com/repos/kubernetes/minikube/statuses/$env:COMMIT`?access_token=$env:access_token" -Body $json -ContentType "application/json" -Method Post -usebasicparsing # Just shutdown Docker, it's safer than anything else From 80c1a4d8b5c2bdc88fec0b69e4108d81c13b5b3e Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Tue, 19 Jan 2021 17:06:13 -0800 Subject: [PATCH 05/11] conform with other tests --- hack/jenkins/windows_integration_test_docker.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hack/jenkins/windows_integration_test_docker.ps1 b/hack/jenkins/windows_integration_test_docker.ps1 index e7b369b89d..52177a0b87 100644 --- a/hack/jenkins/windows_integration_test_docker.ps1 +++ b/hack/jenkins/windows_integration_test_docker.ps1 @@ -49,7 +49,7 @@ $failures=echo $gopogh_status | jq '.NumberOfFail' $tests=echo $gopogh_status | jq '.NumberOfTests' $bad_status="$failures / $tests failures" -$description="$status in $elapsed minutes" +$description="$status in $elapsed minute(s)." If($env:status -eq "failure") { $description="completed with $bad_status in $elapsed minutes" } @@ -57,7 +57,7 @@ echo $description $env:SHORT_COMMIT=$env:COMMIT.substring(0, 7) $gcs_bucket="minikube-builds/logs/$env:MINIKUBE_LOCATION/$env:SHORT_COMMIT" -$env:target_url="https://storage.googleapis.com/$gcs_bucket/Docker_Windows.txt" +$env:target_url="https://storage.googleapis.com/$gcs_bucket/Docker_Windows.html" #Upload logs to gcs gsutil -qm cp testout.txt gs://$gcs_bucket/Docker_Windowsout.txt From 68e6ee36b2c7cb762fad1e56c4f16432720a4dff Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Tue, 19 Jan 2021 18:18:53 -0800 Subject: [PATCH 06/11] run docker rm after starting it --- hack/jenkins/windows_integration_test_docker.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hack/jenkins/windows_integration_test_docker.ps1 b/hack/jenkins/windows_integration_test_docker.ps1 index 52177a0b87..122939fb16 100644 --- a/hack/jenkins/windows_integration_test_docker.ps1 +++ b/hack/jenkins/windows_integration_test_docker.ps1 @@ -11,8 +11,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -docker ps -aq | ForEach -Process {docker rm -fv $_} - mkdir -p out gsutil.cmd -m cp gs://minikube-builds/$env:MINIKUBE_LOCATION/minikube-windows-amd64.exe out/ gsutil.cmd -m cp gs://minikube-builds/$env:MINIKUBE_LOCATION/e2e-windows-amd64.exe out/ @@ -21,6 +19,8 @@ gsutil.cmd -m cp -r gs://minikube-builds/$env:MINIKUBE_LOCATION/setup_docker_des ./out/setup_docker_desktop_windows.ps1 +docker ps -aq | ForEach -Process {docker rm -fv $_} + ./out/minikube-windows-amd64.exe delete --all $started=Get-Date -UFormat %s From 4f4c2923100eebe455f949ccc1d84e458522bae4 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Wed, 20 Jan 2021 09:43:20 -0800 Subject: [PATCH 07/11] move docker rm after minikube delete --- hack/jenkins/windows_integration_test_docker.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hack/jenkins/windows_integration_test_docker.ps1 b/hack/jenkins/windows_integration_test_docker.ps1 index 122939fb16..630700746c 100644 --- a/hack/jenkins/windows_integration_test_docker.ps1 +++ b/hack/jenkins/windows_integration_test_docker.ps1 @@ -19,10 +19,10 @@ gsutil.cmd -m cp -r gs://minikube-builds/$env:MINIKUBE_LOCATION/setup_docker_des ./out/setup_docker_desktop_windows.ps1 -docker ps -aq | ForEach -Process {docker rm -fv $_} - ./out/minikube-windows-amd64.exe delete --all +docker ps -aq | ForEach -Process {docker rm -fv $_} + $started=Get-Date -UFormat %s out/e2e-windows-amd64.exe --minikube-start-args="--driver=docker" --binary=out/minikube-windows-amd64.exe --test.v --test.timeout=180m | tee testout.txt From f5a7aa2201620a99c9c32f79e865888f1bbcc227 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Wed, 20 Jan 2021 13:19:04 -0800 Subject: [PATCH 08/11] fix description for failure --- hack/jenkins/windows_integration_test_docker.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/jenkins/windows_integration_test_docker.ps1 b/hack/jenkins/windows_integration_test_docker.ps1 index 630700746c..a51a00820b 100644 --- a/hack/jenkins/windows_integration_test_docker.ps1 +++ b/hack/jenkins/windows_integration_test_docker.ps1 @@ -51,7 +51,7 @@ $bad_status="$failures / $tests failures" $description="$status in $elapsed minute(s)." If($env:status -eq "failure") { - $description="completed with $bad_status in $elapsed minutes" + $description="completed with $bad_status in $elapsed minute(s)." } echo $description From 44699da49ffa22c3665ad298dce44218fce3f589 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Wed, 20 Jan 2021 13:20:38 -0800 Subject: [PATCH 09/11] round elapsed minutes properly --- hack/jenkins/windows_integration_test_docker.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/jenkins/windows_integration_test_docker.ps1 b/hack/jenkins/windows_integration_test_docker.ps1 index a51a00820b..0442500f3f 100644 --- a/hack/jenkins/windows_integration_test_docker.ps1 +++ b/hack/jenkins/windows_integration_test_docker.ps1 @@ -39,7 +39,7 @@ If($env:result -eq 0){ $ended=Get-Date -UFormat %s $elapsed=$ended-$started $elapsed=$elapsed/60 -$elapsed=[math]::Round($elapsed) +$elapsed=[math]::Round($elapsed, 2) Get-Content testout.txt | go tool test2json -t > testout.json From 244b9768d2db631e024854b44322b9d5e8474e19 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Wed, 20 Jan 2021 14:30:40 -0800 Subject: [PATCH 10/11] fix encoding for gopogh --- hack/jenkins/windows_integration_test_docker.ps1 | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/hack/jenkins/windows_integration_test_docker.ps1 b/hack/jenkins/windows_integration_test_docker.ps1 index 0442500f3f..e222b5fc8c 100644 --- a/hack/jenkins/windows_integration_test_docker.ps1 +++ b/hack/jenkins/windows_integration_test_docker.ps1 @@ -25,7 +25,7 @@ docker ps -aq | ForEach -Process {docker rm -fv $_} $started=Get-Date -UFormat %s -out/e2e-windows-amd64.exe --minikube-start-args="--driver=docker" --binary=out/minikube-windows-amd64.exe --test.v --test.timeout=180m | tee testout.txt +out/e2e-windows-amd64.exe --minikube-start-args="--driver=docker" --binary=out/minikube-windows-amd64.exe --test.v --test.timeout=180m | Tee-Object -FilePath testout.txt $env:result=$lastexitcode # If the last exit code was 0->success, x>0->error If($env:result -eq 0){ @@ -41,7 +41,7 @@ $elapsed=$ended-$started $elapsed=$elapsed/60 $elapsed=[math]::Round($elapsed, 2) -Get-Content testout.txt | go tool test2json -t > testout.json +Get-Content testout.txt -Encoding ASCII | go tool test2json -t | Out-File -FilePath testout.json -Encoding ASCII $gopogh_status=gopogh --in testout.json --out testout.html --name "Docker_Windows" -pr $env:MINIKUBE_LOCATION --repo github.com/kubernetes/minikube/ --details $env:COMMIT @@ -71,8 +71,4 @@ Invoke-WebRequest -Uri "https://api.github.com/repos/kubernetes/minikube/statuse # Just shutdown Docker, it's safer than anything else Get-Process "*Docker Desktop*" | Stop-Process -# Uncomment once tunnel is fixed on Windows: https://github.com/kubernetes/minikube/issues/8304 -#./out/minikube-windows-amd64.exe tunnel --cleanup -#./out/minikube-windows-amd64.exe delete --all --purge - Exit $env:result From 52ab6a503ec6d8a2b153aa91ab5b35d037af4e0b Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Tue, 26 Jan 2021 12:46:26 -0800 Subject: [PATCH 11/11] boilerplate --- hack/jenkins/setup_docker_desktop_windows.ps1 | 15 +++++++++++++++ hack/jenkins/windows_integration_test_docker.ps1 | 3 ++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/hack/jenkins/setup_docker_desktop_windows.ps1 b/hack/jenkins/setup_docker_desktop_windows.ps1 index 6d4593960a..14aba3d940 100644 --- a/hack/jenkins/setup_docker_desktop_windows.ps1 +++ b/hack/jenkins/setup_docker_desktop_windows.ps1 @@ -1,3 +1,18 @@ +# Copyright 2021 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + $attempt = 10 while($attempt -ne 0) { Write-Host "Attempt ", $attempt diff --git a/hack/jenkins/windows_integration_test_docker.ps1 b/hack/jenkins/windows_integration_test_docker.ps1 index e222b5fc8c..8791fbee9e 100644 --- a/hack/jenkins/windows_integration_test_docker.ps1 +++ b/hack/jenkins/windows_integration_test_docker.ps1 @@ -6,7 +6,8 @@ # # http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software# distributed under the License is distributed on an "AS IS" BASIS, +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License.