From 1c4c158327bef4f98f57ebdaf7a528845c5c29fa Mon Sep 17 00:00:00 2001 From: Alonyb Date: Mon, 10 Feb 2020 22:57:30 -0500 Subject: [PATCH 01/67] add new validations to windows workflow --- Makefile | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/Makefile b/Makefile index bfc0a8656e..0d88f76fc9 100755 --- a/Makefile +++ b/Makefile @@ -277,6 +277,18 @@ ifeq ($(MINIKUBE_BUILD_IN_DOCKER),y) $(call DOCKER,$(BUILD_IMAGE),/usr/bin/make $@) endif which go-bindata || GO111MODULE=off GOBIN="$(GOPATH)$(DIRSEP)bin" go get github.com/jteeuwen/go-bindata/... +ifeq ($(OS),Windows_NT) + echo "%cd%" + echo "$(GOPATH)" + dir + PATH="$(PATH)$(PATHSEP)$(GOPATH)$(DIRSEP)bin" + "$(GOPATH)\bin\go-bindata.exe" -nomemcopy -o $@ -pkg assets deploy/addons/... + -gofmt -s -w $@ + @#golint: Dns should be DNS (compat sed) + @sed -i -e 's/Dns/DNS/g' $@ && rm -f ./-e + @#golint: Html should be HTML (compat sed) + @sed -i -e 's/Html/HTML/g' $@ && rm -f ./-e +else PATH="$(PATH)$(PATHSEP)$(GOPATH)$(DIRSEP)bin" go-bindata -nomemcopy -o $@ -pkg assets deploy/addons/... -gofmt -s -w $@ @#golint: Dns should be DNS (compat sed) @@ -284,15 +296,26 @@ endif @#golint: Html should be HTML (compat sed) @sed -i -e 's/Html/HTML/g' $@ && rm -f ./-e +endif + pkg/minikube/translate/translations.go: $(shell find "translations/" -type f) ifeq ($(MINIKUBE_BUILD_IN_DOCKER),y) $(call DOCKER,$(BUILD_IMAGE),/usr/bin/make $@) endif +ifeq ($(OS),Windows_NT) + which go-bindata || GO111MODULE=off GOBIN="$(GOPATH)$(DIRSEP)bin" go get github.com/jteeuwen/go-bindata/... + PATH="$(PATH)$(PATHSEP)$(GOPATH)$(DIRSEP)bin" + "$(GOPATH)\bin\go-bindata.exe" -nomemcopy -o $@ -pkg translate translations/... + -gofmt -s -w $@ + @#golint: Json should be JSON (compat sed) + @sed -i -e 's/Json/JSON/' $@ && rm -f ./-e +else which go-bindata || GO111MODULE=off GOBIN="$(GOPATH)$(DIRSEP)bin" go get github.com/jteeuwen/go-bindata/... PATH="$(PATH)$(PATHSEP)$(GOPATH)$(DIRSEP)bin" go-bindata -nomemcopy -o $@ -pkg translate translations/... -gofmt -s -w $@ @#golint: Json should be JSON (compat sed) @sed -i -e 's/Json/JSON/' $@ && rm -f ./-e +endif .PHONY: cross cross: minikube-linux-amd64 minikube-linux-arm64 minikube-darwin-amd64 minikube-windows-amd64.exe ## Build minikube for all platform From bcb71106b1ec553f82dfe0500b9b68de72ddadad Mon Sep 17 00:00:00 2001 From: Alonyb Date: Mon, 10 Feb 2020 22:57:43 -0500 Subject: [PATCH 02/67] add windows base yml --- .github/workflows/main.yml | 271 ++++++++++++++++++++----------------- 1 file changed, 150 insertions(+), 121 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2ec7b9b32c..33e54c3319 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,139 +6,168 @@ jobs: docker_ubuntu_16_04: runs-on: ubuntu-16.04 steps: - - uses: actions/checkout@v2 - - name: build binaries - run : | - make minikube-linux-amd64 - make e2e-linux-amd64 - mkdir -p report - - name: install gopogh - run: | - cd /tmp - GO111MODULE="on" go get github.com/medyagh/gopogh@v0.0.17 || true - cd - - - name: run integration test - run: | - mkdir -p /tmp/testhome - MINIKUBE_HOME=/tmp/testhome ./out/e2e-linux-amd64 -minikube-start-args=--vm-driver=docker -expected-default-driver= -test.timeout=70m -test.v -binary=out/minikube-linux-amd64 2>&1 | tee ./report/testout.txt - - name: generate gopogh report - run: | - export PATH=${PATH}:`go env GOPATH`/bin - go tool test2json -t < ./report/testout.txt > ./report/testout.json || true - gopogh -in ./report/testout.json -out ./report/testout.html -name "docker ubuntu" -repo github.com/kubernetes/minikube/ || true - - uses: actions/upload-artifact@v1 - with: - name: docker_on_ubuntu_16_04_report - path: report + - uses: actions/checkout@v2 + - name: build binaries + run: | + make minikube-linux-amd64 + make e2e-linux-amd64 + mkdir -p report + - name: install gopogh + run: | + cd /tmp + GO111MODULE="on" go get github.com/medyagh/gopogh@v0.0.17 || true + cd - + - name: run integration test + run: | + mkdir -p /tmp/testhome + MINIKUBE_HOME=/tmp/testhome ./out/e2e-linux-amd64 -minikube-start-args=--vm-driver=docker -expected-default-driver= -test.timeout=70m -test.v -binary=out/minikube-linux-amd64 2>&1 | tee ./report/testout.txt + - name: generate gopogh report + run: | + export PATH=${PATH}:`go env GOPATH`/bin + go tool test2json -t < ./report/testout.txt > ./report/testout.json || true + gopogh -in ./report/testout.json -out ./report/testout.html -name "docker ubuntu" -repo github.com/kubernetes/minikube/ || true + - uses: actions/upload-artifact@v1 + with: + name: docker_on_ubuntu_16_04_report + path: report docker_ubuntu_18_04: runs-on: ubuntu-18.04 steps: - - uses: actions/checkout@v2 - - name: build binaries - run : | - make minikube-linux-amd64 - make e2e-linux-amd64 - mkdir -p report - - name: install gopogh - run: | - cd /tmp - GO111MODULE="on" go get github.com/medyagh/gopogh@v0.0.17 || true - cd - - - name: run integration test - run: | - mkdir -p /tmp/testhome - MINIKUBE_HOME=/tmp/testhome ./out/e2e-linux-amd64 -minikube-start-args=--vm-driver=docker -expected-default-driver= -test.timeout=70m -test.v -binary=out/minikube-linux-amd64 2>&1 | tee ./report/testout.txt - - name: generate gopogh report - run: | - export PATH=${PATH}:`go env GOPATH`/bin - go tool test2json -t < ./report/testout.txt > ./report/testout.json || true - gopogh -in ./report/testout.json -out ./report/testout.html -name "docker ubuntu" -repo github.com/kubernetes/minikube/ || true - - uses: actions/upload-artifact@v1 - with: - name: docker_on_ubuntu_18_04_report - path: report + - uses: actions/checkout@v2 + - name: build binaries + run: | + make minikube-linux-amd64 + make e2e-linux-amd64 + mkdir -p report + - name: install gopogh + run: | + cd /tmp + GO111MODULE="on" go get github.com/medyagh/gopogh@v0.0.17 || true + cd - + - name: run integration test + run: | + mkdir -p /tmp/testhome + MINIKUBE_HOME=/tmp/testhome ./out/e2e-linux-amd64 -minikube-start-args=--vm-driver=docker -expected-default-driver= -test.timeout=70m -test.v -binary=out/minikube-linux-amd64 2>&1 | tee ./report/testout.txt + - name: generate gopogh report + run: | + export PATH=${PATH}:`go env GOPATH`/bin + go tool test2json -t < ./report/testout.txt > ./report/testout.json || true + gopogh -in ./report/testout.json -out ./report/testout.html -name "docker ubuntu" -repo github.com/kubernetes/minikube/ || true + - uses: actions/upload-artifact@v1 + with: + name: docker_on_ubuntu_18_04_report + path: report docker_macos: runs-on: macos-latest steps: - - uses: actions/checkout@v2 - - name: build binaries - run : | - make minikube-darwin-amd64 - make e2e-darwin-amd64 - mkdir -p report - - name: install docker - run: | + - uses: actions/checkout@v2 + - name: build binaries + run: | + make minikube-darwin-amd64 + make e2e-darwin-amd64 + mkdir -p report + - name: install docker + run: | brew install docker-machine docker || true brew services start docker-machine || true docker version || true - - name: install gopogh - run: | - cd /tmp - GO111MODULE="on" go get github.com/medyagh/gopogh@v0.0.17 || true - cd - - - name: run integration test - run: | - mkdir -p /tmp/testhome - MINIKUBE_HOME=/tmp/testhome ./out/e2e-darwin-amd64 -minikube-start-args=--vm-driver=docker -expected-default-driver= -test.timeout=70m -test.v -binary=./out/minikube-darwin-amd64 2>&1 | tee ./report/testout.txt - - name: generate gopogh report - run: | - export PATH=${PATH}:`go env GOPATH`/bin - go tool test2json -t < ./report/testout.txt > ./report/testout.json || true - gopogh -in ./report/testout.json -out ./report/testout.html -name "docker macos" -repo github.com/kubernetes/minikube/ || true - - uses: actions/upload-artifact@v1 - with: - name: docker_on_macos_report - path: ./report + - name: install gopogh + run: | + cd /tmp + GO111MODULE="on" go get github.com/medyagh/gopogh@v0.0.17 || true + cd - + - name: run integration test + run: | + mkdir -p /tmp/testhome + MINIKUBE_HOME=/tmp/testhome ./out/e2e-darwin-amd64 -minikube-start-args=--vm-driver=docker -expected-default-driver= -test.timeout=70m -test.v -binary=./out/minikube-darwin-amd64 2>&1 | tee ./report/testout.txt + - name: generate gopogh report + run: | + export PATH=${PATH}:`go env GOPATH`/bin + go tool test2json -t < ./report/testout.txt > ./report/testout.json || true + gopogh -in ./report/testout.json -out ./report/testout.html -name "docker macos" -repo github.com/kubernetes/minikube/ || true + - uses: actions/upload-artifact@v1 + with: + name: docker_on_macos_report + path: ./report none_ubuntu16_04: runs-on: ubuntu-16.04 steps: - - uses: actions/checkout@v2 - - name: build binaries - run : | - make minikube-linux-amd64 - make e2e-linux-amd64 - mkdir -p report - - name: install gopogh - run: | - cd /tmp - GO111MODULE="on" go get github.com/medyagh/gopogh@v0.0.17 || true - cd - - - name: run integration test - run: | - mkdir -p /tmp/testhome - MINIKUBE_HOME=/tmp/testhome sudo -E ./out/e2e-linux-amd64 -minikube-start-args=--vm-driver=none -expected-default-driver= -test.timeout=70m -test.v -binary=out/minikube-linux-amd64 2>&1 | tee ./report/testout.txt - - name: generate gopogh report - run: | - export PATH=${PATH}:`go env GOPATH`/bin - go tool test2json -t < ./report/testout.txt > ./report/testout.json || true - gopogh -in ./report/testout.json -out ./report/testout.html -name "docker ubuntu" -repo github.com/kubernetes/minikube/ || true - - uses: actions/upload-artifact@v1 - with: - name: none_on_ubuntu_16_04 - path: report + - uses: actions/checkout@v2 + - name: build binaries + run: | + make minikube-linux-amd64 + make e2e-linux-amd64 + mkdir -p report + - name: install gopogh + run: | + cd /tmp + GO111MODULE="on" go get github.com/medyagh/gopogh@v0.0.17 || true + cd - + - name: run integration test + run: | + mkdir -p /tmp/testhome + MINIKUBE_HOME=/tmp/testhome sudo -E ./out/e2e-linux-amd64 -minikube-start-args=--vm-driver=none -expected-default-driver= -test.timeout=70m -test.v -binary=out/minikube-linux-amd64 2>&1 | tee ./report/testout.txt + - name: generate gopogh report + run: | + export PATH=${PATH}:`go env GOPATH`/bin + go tool test2json -t < ./report/testout.txt > ./report/testout.json || true + gopogh -in ./report/testout.json -out ./report/testout.html -name "docker ubuntu" -repo github.com/kubernetes/minikube/ || true + - uses: actions/upload-artifact@v1 + with: + name: none_on_ubuntu_16_04 + path: report none_ubuntu_18_04: runs-on: ubuntu-18.04 steps: - - uses: actions/checkout@v2 - - name: build binaries - run : | - make minikube-linux-amd64 - make e2e-linux-amd64 - - name: install gopogh - run: | - cd /tmp - GO111MODULE="on" go get github.com/medyagh/gopogh@v0.0.17 || true - cd - - - name: run integration test - run: | - mkdir -p /tmp/testhome - MINIKUBE_HOME=/tmp/testhome sudo -E ./out/e2e-linux-amd64 -minikube-start-args=--vm-driver=none -expected-default-driver= -test.timeout=70m -test.v -binary=out/minikube-linux-amd64 2>&1 | tee ./report/testout.txt - - name: generate gopogh report - run: | - export PATH=${PATH}:`go env GOPATH`/bin - go tool test2json -t < ./report/testout.txt > ./report/testout.json || true - gopogh -in ./report/testout.json -out ./report/testout.html -name "docker ubuntu" -repo github.com/kubernetes/minikube/ || true - - uses: actions/upload-artifact@v1 - with: - name: none_on_ubuntu_latest_report - path: report + - uses: actions/checkout@v2 + - name: build binaries + run: | + make minikube-linux-amd64 + make e2e-linux-amd64 + - name: install gopogh + run: | + cd /tmp + GO111MODULE="on" go get github.com/medyagh/gopogh@v0.0.17 || true + cd - + - name: run integration test + run: | + mkdir -p /tmp/testhome + MINIKUBE_HOME=/tmp/testhome sudo -E ./out/e2e-linux-amd64 -minikube-start-args=--vm-driver=none -expected-default-driver= -test.timeout=70m -test.v -binary=out/minikube-linux-amd64 2>&1 | tee ./report/testout.txt + - name: generate gopogh report + run: | + export PATH=${PATH}:`go env GOPATH`/bin + go tool test2json -t < ./report/testout.txt > ./report/testout.json || true + gopogh -in ./report/testout.json -out ./report/testout.html -name "docker ubuntu" -repo github.com/kubernetes/minikube/ || true + - uses: actions/upload-artifact@v1 + with: + name: none_on_ubuntu_latest_report + path: report + + windows_amd_64: + runs-on: windows-latest + steps: + - uses: actions/checkout@v2 + - name: build binaries + run: | + echo "::add-path::$GOPATH\bin" + make minikube-windows-amd64.exe + make e2e-windows-amd64.exe + - name: install gopogh + run: | + # go get github.com/medyagh/gopogh@v0.0.17 + # cd - + env: + GO111MODULE: on + - name: run integration test + run: | + out/e2e-windows-amd64.exe --expected-default-driver=hyperv -minikube-start-args="--vm-driver=hyperv --hyperv-virtual-switch=primary-virtual-switch" -binary=out/minikube-windows-amd64.exe -test.v -test.timeout=65m + - name: generate gopogh report + run: | + echo "{}" > ./report/testout.json + go tool test2json -t > ./report/testout.json + Get-Content ./report/testout.txt | ./report/testout.json + gopogh -in ./report/testout.json -out ./report/testout.html -name "docker ubuntu" -repo github.com/kubernetes/minikube/ + - uses: actions/upload-artifact@v1 + with: + name: none_on_ubuntu_latest_report + path: report From 5a72ca3ae7446687e2918327894e33d28f2ea396 Mon Sep 17 00:00:00 2001 From: Alonyb Date: Tue, 11 Feb 2020 23:44:32 -0500 Subject: [PATCH 03/67] add new test line --- pkg/minikube/registry/drvs/hyperv/hyperv.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/minikube/registry/drvs/hyperv/hyperv.go b/pkg/minikube/registry/drvs/hyperv/hyperv.go index 9f15d0c470..72c96f47d3 100644 --- a/pkg/minikube/registry/drvs/hyperv/hyperv.go +++ b/pkg/minikube/registry/drvs/hyperv/hyperv.go @@ -88,7 +88,8 @@ func status() registry.State { // Allow no more than 2 seconds for querying state ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() - + + exec.CommandContext(ctx, path, "Enable-WindowsOptionalFeature", "-Online", "-FeatureName", "Microsoft-Hyper-V -All") cmd := exec.CommandContext(ctx, path, "Get-WindowsOptionalFeature", "-FeatureName", "Microsoft-Hyper-V-All", "-Online") out, err := cmd.CombinedOutput() if err != nil { From 82bb6b1aa7c278561975f43e724ef28e45ff449c Mon Sep 17 00:00:00 2001 From: Ruben Baez Date: Thu, 13 Feb 2020 22:01:10 -0500 Subject: [PATCH 04/67] Update hyperv.go --- pkg/minikube/registry/drvs/hyperv/hyperv.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/minikube/registry/drvs/hyperv/hyperv.go b/pkg/minikube/registry/drvs/hyperv/hyperv.go index 72c96f47d3..f17b9517fe 100644 --- a/pkg/minikube/registry/drvs/hyperv/hyperv.go +++ b/pkg/minikube/registry/drvs/hyperv/hyperv.go @@ -89,7 +89,6 @@ func status() registry.State { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() - exec.CommandContext(ctx, path, "Enable-WindowsOptionalFeature", "-Online", "-FeatureName", "Microsoft-Hyper-V -All") cmd := exec.CommandContext(ctx, path, "Get-WindowsOptionalFeature", "-FeatureName", "Microsoft-Hyper-V-All", "-Online") out, err := cmd.CombinedOutput() if err != nil { From dbaa6a33251465d77df7dc55c6541ba05ba7aafa Mon Sep 17 00:00:00 2001 From: Ruben Baez Date: Sun, 16 Feb 2020 22:52:11 -0500 Subject: [PATCH 05/67] add binaries, build stage --- .github/workflows/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c45087c0cd..5826b943d2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -15,6 +15,8 @@ jobs: run : | make minikube-linux-amd64 make e2e-linux-amd64 + make minikube-windows-amd64.exe + make e2e-windows-amd64.exe cp -r test/integration/testdata ./out whoami echo github ref $GITHUB_REF From 85cf5154417056575687f8bcd90a670315769c4b Mon Sep 17 00:00:00 2001 From: Ruben Baez Date: Sun, 16 Feb 2020 22:55:19 -0500 Subject: [PATCH 06/67] add windows stage --- .github/workflows/main.yml | 67 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5826b943d2..2e3cbd6783 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -177,6 +177,73 @@ jobs: echo "--------------------------------------------" numberOfFailures=$(echo $STAT | jq '.NumberOfFail') if [ "$numberOfFailures" -gt 0 ];then echo "*** $numberOfFailures Failed ***";exit 2;fi + docker_windows_NT: + needs: [build_minikube] + env: + TIME_ELAPSED: time + JOB_NAME: "Docker_windows_NT" + COMMIT_STATUS: "" + runs-on: windows-latest + steps: + - uses: actions/checkout@v2 + - name: Install gopogh + run: | + find ~/ -iname gopogh.exe + echo $GOPATH + echo $GITHUB_WORKSPACE + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.1.16/gopogh.exe + ls + find ~/ -iname gopogh.exe + pwd + shell: bash + - name: Download binaries + uses: actions/download-artifact@v1 + with: + name: minikube_binaries + - name: run integration test + continue-on-error: true + run: | + set +euo pipefail + mkdir -p report + mkdir -p testhome + chmod a+x e2e-* + chmod a+x minikube-* + START_TIME=$(date -u +%s) + KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome minikube_binaries/e2e-windows-amd64.exe --expected-default-driver=hyperv -minikube-start-args=--vm-driver=hyperv -binary=minikube_binaries/minikube-windows-amd64.exe -test.v -test.timeout=65m 2>&1 | tee ./report/testout.txt + END_TIME=$(date -u +%s) + TIME_ELAPSED=$(($END_TIME-$START_TIME)) + min=$((${TIME_ELAPSED}/60)) + sec=$((${TIME_ELAPSED}%60)) + TIME_ELAPSED="${min} min $sec seconds" + echo ::set-env name=TIME_ELAPSED::${TIME_ELAPSED} + shell: bash + - name: Generate html report + run: | + pwd + ls + echo $GITHUB_WORKSPACE + go tool test2json -t < ./report/testout.txt > ./report/testout.json || true + STAT=$(${GITHUB_WORKSPACE}$(DIRSEP)gopogh.exe -in ./report/testout.json -out ./report/testout.html -name " $GITHUB_REF" -repo "${JOB_NAME} ${GITHUB_REF} ${GITHUB_REPOSITORY}" -details "${GITHUB_SHA}") || true + echo status: ${STAT} + FailNum=$(echo $STAT | jq '.NumberOfFail') + TestsNum=$(echo $STAT | jq '.NumberOfTests') + GOPOGH_RESULT="${JOB_NAME} : completed with ${FailNum} / ${TestsNum} failures in ${TIME_ELAPSED}" + echo ::set-env name=GOPOGH_RESULT::${GOPOGH_RESULT} + echo ::set-env name=STAT::${STAT} + shell: bash + - name: The End Result + run: | + echo ${GOPOGH_RESULT} + echo "--------------------------------------------" + echo $STAT | jq '.FailedTests' || true + echo "--------------------------------------------" + numberOfFailures=$(echo $STAT | jq '.NumberOfFail') + if [ "$numberOfFailures" -gt 0 ];then echo "*** $numberOfFailures Failed ***";exit 2;fi + shell: bash + - uses: actions/upload-artifact@v1 + with: + name: docker_windows_NT + path: minikube_binaries/report none_ubuntu16_04: needs: [build_minikube] env: From 566eb23ccd1ca5549bf0e055b5c2ee362dcb4786 Mon Sep 17 00:00:00 2001 From: Ruben Baez Date: Sun, 16 Feb 2020 22:56:39 -0500 Subject: [PATCH 07/67] add windows report --- .github/workflows/main.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2e3cbd6783..335deb5aa5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -463,6 +463,15 @@ jobs: run: | mkdir -p all_reports cp -r docker_ubuntu_18_04 ./all_reports/ + - name: download results docker_windows_NT + uses: actions/download-artifact@v1 + with: + name: docker_windows_NT + - name: cp to all_report + shell: bash + run: | + mkdir -p all_reports + cp -r docker_Windows_NT ./all_reports/ - name: download results none_ubuntu16_04 uses: actions/download-artifact@v1 with: From f784001468b3f8e8887dd5aa9b7fe3fb30b0ca26 Mon Sep 17 00:00:00 2001 From: Ruben Baez Date: Sun, 16 Feb 2020 22:57:59 -0500 Subject: [PATCH 08/67] giving some love to windows workflow --- .github/workflows/main.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 335deb5aa5..312058d4ef 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -186,15 +186,9 @@ jobs: runs-on: windows-latest steps: - uses: actions/checkout@v2 - - name: Install gopogh + - name: Download gopogh run: | - find ~/ -iname gopogh.exe - echo $GOPATH - echo $GITHUB_WORKSPACE curl -LO https://github.com/medyagh/gopogh/releases/download/v0.1.16/gopogh.exe - ls - find ~/ -iname gopogh.exe - pwd shell: bash - name: Download binaries uses: actions/download-artifact@v1 From bc619df5bf15c08d405d32358b1819ac7379247c Mon Sep 17 00:00:00 2001 From: Ruben Baez Date: Sun, 16 Feb 2020 22:59:57 -0500 Subject: [PATCH 09/67] Delete linux stuff --- .github/workflows/main.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 312058d4ef..fd85349459 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -200,8 +200,6 @@ jobs: set +euo pipefail mkdir -p report mkdir -p testhome - chmod a+x e2e-* - chmod a+x minikube-* START_TIME=$(date -u +%s) KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome minikube_binaries/e2e-windows-amd64.exe --expected-default-driver=hyperv -minikube-start-args=--vm-driver=hyperv -binary=minikube_binaries/minikube-windows-amd64.exe -test.v -test.timeout=65m 2>&1 | tee ./report/testout.txt END_TIME=$(date -u +%s) From f3ee969dd1343cf7960cf6d9a66f56b8807c5226 Mon Sep 17 00:00:00 2001 From: Ruben Baez Date: Sun, 16 Feb 2020 23:01:40 -0500 Subject: [PATCH 10/67] it's almost done --- .github/workflows/main.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fd85349459..373f6e0f13 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -211,11 +211,8 @@ jobs: shell: bash - name: Generate html report run: | - pwd - ls - echo $GITHUB_WORKSPACE go tool test2json -t < ./report/testout.txt > ./report/testout.json || true - STAT=$(${GITHUB_WORKSPACE}$(DIRSEP)gopogh.exe -in ./report/testout.json -out ./report/testout.html -name " $GITHUB_REF" -repo "${JOB_NAME} ${GITHUB_REF} ${GITHUB_REPOSITORY}" -details "${GITHUB_SHA}") || true + STAT=$(${GITHUB_WORKSPACE}/gopogh.exe -in ./report/testout.json -out ./report/testout.html -name " $GITHUB_REF" -repo "${JOB_NAME} ${GITHUB_REF} ${GITHUB_REPOSITORY}" -details "${GITHUB_SHA}") || true echo status: ${STAT} FailNum=$(echo $STAT | jq '.NumberOfFail') TestsNum=$(echo $STAT | jq '.NumberOfTests') From bceb07159d4b04d1d5f2bb6ec6a268efeaa80ef6 Mon Sep 17 00:00:00 2001 From: Ruben Baez Date: Sun, 16 Feb 2020 23:03:24 -0500 Subject: [PATCH 11/67] perfect --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 373f6e0f13..9110594ed1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -460,7 +460,7 @@ jobs: shell: bash run: | mkdir -p all_reports - cp -r docker_Windows_NT ./all_reports/ + cp -r docker_windows_NT ./all_reports/ - name: download results none_ubuntu16_04 uses: actions/download-artifact@v1 with: From 24fcd1521997020b323793ff4275f78a0db86ae7 Mon Sep 17 00:00:00 2001 From: Ruben Baez Date: Sun, 16 Feb 2020 23:07:16 -0500 Subject: [PATCH 12/67] giving some love to makefile --- Makefile | 3 --- 1 file changed, 3 deletions(-) diff --git a/Makefile b/Makefile index c56aa28f09..2f831e01d0 100755 --- a/Makefile +++ b/Makefile @@ -281,9 +281,6 @@ ifeq ($(MINIKUBE_BUILD_IN_DOCKER),y) endif which go-bindata || GO111MODULE=off GOBIN="$(GOPATH)$(DIRSEP)bin" go get github.com/jteeuwen/go-bindata/... ifeq ($(OS),Windows_NT) - echo "%cd%" - echo "$(GOPATH)" - dir PATH="$(PATH)$(PATHSEP)$(GOPATH)$(DIRSEP)bin" "$(GOPATH)\bin\go-bindata.exe" -nomemcopy -o $@ -pkg assets deploy/addons/... -gofmt -s -w $@ From 9834408c68e23bb1f2a136b8b9dcdce7d6602179 Mon Sep 17 00:00:00 2001 From: Ruben Baez Date: Mon, 17 Feb 2020 07:43:23 -0500 Subject: [PATCH 13/67] change the order of steps --- .github/workflows/main.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9110594ed1..86af8e60f5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -220,6 +220,10 @@ jobs: echo ::set-env name=GOPOGH_RESULT::${GOPOGH_RESULT} echo ::set-env name=STAT::${STAT} shell: bash + - uses: actions/upload-artifact@v1 + with: + name: docker_windows_NT + path: minikube_binaries/report - name: The End Result run: | echo ${GOPOGH_RESULT} @@ -229,10 +233,6 @@ jobs: numberOfFailures=$(echo $STAT | jq '.NumberOfFail') if [ "$numberOfFailures" -gt 0 ];then echo "*** $numberOfFailures Failed ***";exit 2;fi shell: bash - - uses: actions/upload-artifact@v1 - with: - name: docker_windows_NT - path: minikube_binaries/report none_ubuntu16_04: needs: [build_minikube] env: From 788402e4db4a82fb3a6b003bd7880464fecc00b7 Mon Sep 17 00:00:00 2001 From: Ruben Baez Date: Mon, 17 Feb 2020 08:23:54 -0500 Subject: [PATCH 14/67] Update main.yml --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 86af8e60f5..fb26bd2596 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -223,7 +223,7 @@ jobs: - uses: actions/upload-artifact@v1 with: name: docker_windows_NT - path: minikube_binaries/report + path: report - name: The End Result run: | echo ${GOPOGH_RESULT} From a04239566948347360e3ac8373f87d0c2a0deb53 Mon Sep 17 00:00:00 2001 From: Ruben Baez Date: Wed, 19 Feb 2020 07:22:57 -0500 Subject: [PATCH 15/67] Update Makefile --- Makefile | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/Makefile b/Makefile index 2f831e01d0..28d5bc1147 100755 --- a/Makefile +++ b/Makefile @@ -280,15 +280,6 @@ ifeq ($(MINIKUBE_BUILD_IN_DOCKER),y) $(call DOCKER,$(BUILD_IMAGE),/usr/bin/make $@) endif which go-bindata || GO111MODULE=off GOBIN="$(GOPATH)$(DIRSEP)bin" go get github.com/jteeuwen/go-bindata/... -ifeq ($(OS),Windows_NT) - PATH="$(PATH)$(PATHSEP)$(GOPATH)$(DIRSEP)bin" - "$(GOPATH)\bin\go-bindata.exe" -nomemcopy -o $@ -pkg assets deploy/addons/... - -gofmt -s -w $@ - @#golint: Dns should be DNS (compat sed) - @sed -i -e 's/Dns/DNS/g' $@ && rm -f ./-e - @#golint: Html should be HTML (compat sed) - @sed -i -e 's/Html/HTML/g' $@ && rm -f ./-e -else PATH="$(PATH)$(PATHSEP)$(GOPATH)$(DIRSEP)bin" go-bindata -nomemcopy -o $@ -pkg assets deploy/addons/... -gofmt -s -w $@ @#golint: Dns should be DNS (compat sed) @@ -296,26 +287,15 @@ else @#golint: Html should be HTML (compat sed) @sed -i -e 's/Html/HTML/g' $@ && rm -f ./-e -endif - pkg/minikube/translate/translations.go: $(shell find "translations/" -type f) ifeq ($(MINIKUBE_BUILD_IN_DOCKER),y) $(call DOCKER,$(BUILD_IMAGE),/usr/bin/make $@) endif -ifeq ($(OS),Windows_NT) - which go-bindata || GO111MODULE=off GOBIN="$(GOPATH)$(DIRSEP)bin" go get github.com/jteeuwen/go-bindata/... - PATH="$(PATH)$(PATHSEP)$(GOPATH)$(DIRSEP)bin" - "$(GOPATH)\bin\go-bindata.exe" -nomemcopy -o $@ -pkg translate translations/... - -gofmt -s -w $@ - @#golint: Json should be JSON (compat sed) - @sed -i -e 's/Json/JSON/' $@ && rm -f ./-e -else which go-bindata || GO111MODULE=off GOBIN="$(GOPATH)$(DIRSEP)bin" go get github.com/jteeuwen/go-bindata/... PATH="$(PATH)$(PATHSEP)$(GOPATH)$(DIRSEP)bin" go-bindata -nomemcopy -o $@ -pkg translate translations/... -gofmt -s -w $@ @#golint: Json should be JSON (compat sed) @sed -i -e 's/Json/JSON/' $@ && rm -f ./-e -endif .PHONY: cross cross: minikube-linux-amd64 minikube-linux-arm64 minikube-darwin-amd64 minikube-windows-amd64.exe ## Build minikube for all platform From 3bccafb66bb644e2a77aaa0da93f20df1cc557d3 Mon Sep 17 00:00:00 2001 From: Ruben Baez Date: Wed, 19 Feb 2020 20:23:06 -0500 Subject: [PATCH 16/67] Update hyperv.go --- pkg/minikube/registry/drvs/hyperv/hyperv.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/minikube/registry/drvs/hyperv/hyperv.go b/pkg/minikube/registry/drvs/hyperv/hyperv.go index f17b9517fe..9f15d0c470 100644 --- a/pkg/minikube/registry/drvs/hyperv/hyperv.go +++ b/pkg/minikube/registry/drvs/hyperv/hyperv.go @@ -88,7 +88,7 @@ func status() registry.State { // Allow no more than 2 seconds for querying state ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() - + cmd := exec.CommandContext(ctx, path, "Get-WindowsOptionalFeature", "-FeatureName", "Microsoft-Hyper-V-All", "-Online") out, err := cmd.CombinedOutput() if err != nil { From 2e9054f495ddc99cecf2c80daef1dbe7a99ac641 Mon Sep 17 00:00:00 2001 From: ZouYu Date: Wed, 25 Mar 2020 11:08:27 +0800 Subject: [PATCH 17/67] add unit test for pkg/drivers/hyperkit/iso.go Signed-off-by: ZouYu --- pkg/drivers/hyperkit/iso_test.go | 91 ++++++++++++++++++++++++++++++ pkg/drivers/hyperkit/iso_test.iso | Bin 0 -> 362496 bytes 2 files changed, 91 insertions(+) create mode 100644 pkg/drivers/hyperkit/iso_test.go create mode 100644 pkg/drivers/hyperkit/iso_test.iso diff --git a/pkg/drivers/hyperkit/iso_test.go b/pkg/drivers/hyperkit/iso_test.go new file mode 100644 index 0000000000..8dda1e4a6e --- /dev/null +++ b/pkg/drivers/hyperkit/iso_test.go @@ -0,0 +1,91 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package hyperkit + +import ( + "io/ioutil" + "os" + "testing" +) + +func TestExtractFile(t *testing.T) { + testDir, err := ioutil.TempDir(os.TempDir(), "") + if nil != err { + return + } + defer os.Remove(testDir) + + tests := []struct { + name string + isoPath string + srcPath string + destPath string + expectedError bool + }{ + { + name: "all is right", + isoPath: "iso_test.iso", + srcPath: "/test1.txt", + destPath: testDir + "/test1.txt", + expectedError: false, + }, + { + name: "isoPath is error", + isoPath: "tests.iso", + srcPath: "/test1.txt", + destPath: testDir + "/test1.txt", + expectedError: true, + }, + { + name: "srcPath is empty", + isoPath: "iso_tests.iso", + srcPath: "", + destPath: testDir + "/test1.txt", + expectedError: true, + }, + { + name: "srcPath is error", + isoPath: "iso_tests.iso", + srcPath: "/t1.txt", + destPath: testDir + "/test1.txt", + expectedError: true, + }, + { + name: "destPath is empty", + isoPath: "iso_test.iso", + srcPath: "/test1.txt", + destPath: "", + expectedError: true, + }, + { + name: "find files in a folder", + isoPath: "./iso_test.iso", + srcPath: "/test2/test2.txt", + destPath: testDir + "/test2.txt", + expectedError: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ExtractFile(tt.isoPath, tt.srcPath, tt.destPath) + if (nil != err) != tt.expectedError { + t.Errorf("expectedError = %v, get = %v", tt.expectedError, err) + return + } + }) + } +} diff --git a/pkg/drivers/hyperkit/iso_test.iso b/pkg/drivers/hyperkit/iso_test.iso new file mode 100644 index 0000000000000000000000000000000000000000..dbed69dd413fd8b02f0f019e33ac700e6f75d699 GIT binary patch literal 362496 zcmeI(+fEZf7y#f|4Wu5pQp4fGOt>(bVA-|=(sot@pze!J2-Y%)CV4T@MbVt02p-#$E;#e?Cf zn4Qha;yCuFMX#JpEwAHZ*)|h*GjlCA`A1G zo3s8e0`-5IErCZM@arP}KZ2>40s#U92oNAZfB*pk1PBlyFeicRe2u+YO%F@w>+o49 zi&@#8)2}%#0RjXF5FkK+009C72oNCf5CZAVzghyQs-@Ig3z_%-1fdWhK!5-N0t5&U zAV7csfkhIyJIC&p5pS=u@UpG2UX!dZ*Y$GB(|frbt8Xb!??qZtXf?~z@>A>GRkq*X zbNZGnKh%b{1PBlyKwypnn|H+H?Cvc5D*kWOGu*fq|6j+d_bUFk>-ay%OL1xf1PBly zK;WJN-@ktQ-g Date: Thu, 26 Mar 2020 08:18:08 -0500 Subject: [PATCH 18/67] give some love to actions --- .github/workflows/main.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a36dfb1a19..8d6a7f8b82 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -211,15 +211,21 @@ jobs: echo $STAT | jq '.FailedTests' || true echo "-------------------------------------------------------" if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi - docker_windows_NT: + docker_windows_shell: needs: [build_minikube] env: TIME_ELAPSED: time - JOB_NAME: "Docker_windows_NT" + JOB_NAME: "Docker_windows_shell" COMMIT_STATUS: "" runs-on: windows-latest steps: - uses: actions/checkout@v2 + - name: Docker Info + shell: bash + run: | + docker info || true + docker version || true + docker ps || true - name: Download gopogh run: | curl -LO https://github.com/medyagh/gopogh/releases/download/v0.1.16/gopogh.exe @@ -235,7 +241,7 @@ jobs: mkdir -p report mkdir -p testhome START_TIME=$(date -u +%s) - KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome minikube_binaries/e2e-windows-amd64.exe --expected-default-driver=hyperv -minikube-start-args=--vm-driver=hyperv -binary=minikube_binaries/minikube-windows-amd64.exe -test.v -test.timeout=65m 2>&1 | tee ./report/testout.txt + KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome minikube_binaries/e2e-windows-amd64.exe --expected-default-driver=hyperv -minikube-start-args=--vm-driver=docker -binary=minikube_binaries/minikube-windows-amd64.exe -test.v -test.timeout=65m 2>&1 | tee ./report/testout.txt END_TIME=$(date -u +%s) TIME_ELAPSED=$(($END_TIME-$START_TIME)) min=$((${TIME_ELAPSED}/60)) @@ -256,7 +262,7 @@ jobs: shell: bash - uses: actions/upload-artifact@v1 with: - name: docker_windows_NT + name: docker_windows_shell path: report - name: The End Result run: | @@ -265,6 +271,8 @@ jobs: echo "----------------${numFail} Failures----------------------------" echo $STAT | jq '.FailedTests' || true echo "--------------------------------------------" + numPass=$(echo $STAT | jq '.NumberOfPass') + echo "*** $numPass Passed ***" if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi shell: bash none_ubuntu16_04: From 1520b592768096fb620e20fce7ba8c97275a8e93 Mon Sep 17 00:00:00 2001 From: Ruben Baez Date: Thu, 26 Mar 2020 08:19:48 -0500 Subject: [PATCH 19/67] update stage name --- .github/workflows/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8d6a7f8b82..31be5e710c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -499,15 +499,15 @@ jobs: run: | mkdir -p all_reports cp -r docker_ubuntu_18_04 ./all_reports/ - - name: download results docker_windows_NT + - name: download results docker_windows_shell uses: actions/download-artifact@v1 with: - name: docker_windows_NT + name: docker_windows_shell - name: cp to all_report shell: bash run: | mkdir -p all_reports - cp -r docker_windows_NT ./all_reports/ + cp -r docker_windows_shell ./all_reports/ - name: Download Results none_ubuntu16_04 uses: actions/download-artifact@v1 with: From 42a28d014e5167b9ce453a08b4038ecdb4986517 Mon Sep 17 00:00:00 2001 From: Ruben Baez Date: Thu, 26 Mar 2020 08:32:59 -0500 Subject: [PATCH 20/67] Update main.yml --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 31be5e710c..c6bf2a4574 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -241,7 +241,7 @@ jobs: mkdir -p report mkdir -p testhome START_TIME=$(date -u +%s) - KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome minikube_binaries/e2e-windows-amd64.exe --expected-default-driver=hyperv -minikube-start-args=--vm-driver=docker -binary=minikube_binaries/minikube-windows-amd64.exe -test.v -test.timeout=65m 2>&1 | tee ./report/testout.txt + KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome minikube_binaries/e2e-windows-amd64.exe --expected-default-driver=docker -minikube-start-args=--vm-driver=docker -binary=minikube_binaries/minikube-windows-amd64.exe -test.v -test.timeout=65m 2>&1 | tee ./report/testout.txt END_TIME=$(date -u +%s) TIME_ELAPSED=$(($END_TIME-$START_TIME)) min=$((${TIME_ELAPSED}/60)) From 4f1ae0cd4fe50e5ffb924ece8b1b1947966c3c8f Mon Sep 17 00:00:00 2001 From: Ruben Baez Date: Thu, 26 Mar 2020 08:47:00 -0500 Subject: [PATCH 21/67] Update main.yml --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c6bf2a4574..4eb9b9706a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -241,7 +241,7 @@ jobs: mkdir -p report mkdir -p testhome START_TIME=$(date -u +%s) - KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome minikube_binaries/e2e-windows-amd64.exe --expected-default-driver=docker -minikube-start-args=--vm-driver=docker -binary=minikube_binaries/minikube-windows-amd64.exe -test.v -test.timeout=65m 2>&1 | tee ./report/testout.txt + KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome minikube_binaries/e2e-windows-amd64.exe --expected-default-driver=hyperv -minikube-start-args=--vm-driver=hyperv -binary=minikube_binaries/minikube-windows-amd64.exe -test.v -test.timeout=65m 2>&1 | tee ./report/testout.txt END_TIME=$(date -u +%s) TIME_ELAPSED=$(($END_TIME-$START_TIME)) min=$((${TIME_ELAPSED}/60)) From da578b8727ef257052e3b77ec19a182083d0fb91 Mon Sep 17 00:00:00 2001 From: Ruben Baez Date: Thu, 26 Mar 2020 21:00:45 -0500 Subject: [PATCH 22/67] change continue on error to false --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4eb9b9706a..f94d59a89b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -100,7 +100,7 @@ jobs: with: name: minikube_binaries - name: Run Integration Test - continue-on-error: true + continue-on-error: false # bash {0} to allow test to continue to next step. in case of shell: bash {0} run: | From d8cf3339abd9e71f0ce6e2b389327fcc97a85253 Mon Sep 17 00:00:00 2001 From: Ruben Baez Date: Thu, 26 Mar 2020 21:38:07 -0500 Subject: [PATCH 23/67] test old file --- .github/workflows/main.yml | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f94d59a89b..f19cf8b641 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -211,21 +211,15 @@ jobs: echo $STAT | jq '.FailedTests' || true echo "-------------------------------------------------------" if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi - docker_windows_shell: + docker_windows_NT: needs: [build_minikube] env: TIME_ELAPSED: time - JOB_NAME: "Docker_windows_shell" + JOB_NAME: "Docker_windows_NT" COMMIT_STATUS: "" runs-on: windows-latest steps: - uses: actions/checkout@v2 - - name: Docker Info - shell: bash - run: | - docker info || true - docker version || true - docker ps || true - name: Download gopogh run: | curl -LO https://github.com/medyagh/gopogh/releases/download/v0.1.16/gopogh.exe @@ -262,7 +256,7 @@ jobs: shell: bash - uses: actions/upload-artifact@v1 with: - name: docker_windows_shell + name: docker_windows_NT path: report - name: The End Result run: | @@ -271,8 +265,6 @@ jobs: echo "----------------${numFail} Failures----------------------------" echo $STAT | jq '.FailedTests' || true echo "--------------------------------------------" - numPass=$(echo $STAT | jq '.NumberOfPass') - echo "*** $numPass Passed ***" if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi shell: bash none_ubuntu16_04: From 0bc5adee14bd5b45ef04b76ac78764719611b79b Mon Sep 17 00:00:00 2001 From: Ruben Baez Date: Thu, 26 Mar 2020 21:45:22 -0500 Subject: [PATCH 24/67] change to new file --- .github/workflows/main.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f19cf8b641..dcdc6d15a3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -211,11 +211,11 @@ jobs: echo $STAT | jq '.FailedTests' || true echo "-------------------------------------------------------" if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi - docker_windows_NT: + docker_windows_shell: needs: [build_minikube] env: TIME_ELAPSED: time - JOB_NAME: "Docker_windows_NT" + JOB_NAME: "Docker_windows_shell" COMMIT_STATUS: "" runs-on: windows-latest steps: @@ -256,7 +256,7 @@ jobs: shell: bash - uses: actions/upload-artifact@v1 with: - name: docker_windows_NT + name: docker_windows_shell path: report - name: The End Result run: | @@ -265,6 +265,8 @@ jobs: echo "----------------${numFail} Failures----------------------------" echo $STAT | jq '.FailedTests' || true echo "--------------------------------------------" + numPass=$(echo $STAT | jq '.NumberOfPass') + echo "*** $numPass Passed ***" if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi shell: bash none_ubuntu16_04: From b7bcda88180f632eb69050e5863cafa4e573ea69 Mon Sep 17 00:00:00 2001 From: Ruben Baez Date: Thu, 26 Mar 2020 21:58:39 -0500 Subject: [PATCH 25/67] delete expected-default-driver --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index dcdc6d15a3..6854bcb1f2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -235,7 +235,7 @@ jobs: mkdir -p report mkdir -p testhome START_TIME=$(date -u +%s) - KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome minikube_binaries/e2e-windows-amd64.exe --expected-default-driver=hyperv -minikube-start-args=--vm-driver=hyperv -binary=minikube_binaries/minikube-windows-amd64.exe -test.v -test.timeout=65m 2>&1 | tee ./report/testout.txt + KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome minikube_binaries/e2e-windows-amd64.exe -minikube-start-args=--vm-driver=hyperv -binary=minikube_binaries/minikube-windows-amd64.exe -test.v -test.timeout=65m 2>&1 | tee ./report/testout.txt END_TIME=$(date -u +%s) TIME_ELAPSED=$(($END_TIME-$START_TIME)) min=$((${TIME_ELAPSED}/60)) From 5466cbf4ac1b1ad09713e21d64f5a0340b67a51a Mon Sep 17 00:00:00 2001 From: Ruben Baez Date: Thu, 26 Mar 2020 22:10:17 -0500 Subject: [PATCH 26/67] rename the stage --- .github/workflows/main.yml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6854bcb1f2..cd3d6844fc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -211,15 +211,21 @@ jobs: echo $STAT | jq '.FailedTests' || true echo "-------------------------------------------------------" if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi - docker_windows_shell: + docker_on_windows: needs: [build_minikube] env: TIME_ELAPSED: time - JOB_NAME: "Docker_windows_shell" + JOB_NAME: "Docker_on_windows" COMMIT_STATUS: "" runs-on: windows-latest steps: - uses: actions/checkout@v2 + - name: Docker Info + shell: bash + run: | + docker info || true + docker version || true + docker ps || true - name: Download gopogh run: | curl -LO https://github.com/medyagh/gopogh/releases/download/v0.1.16/gopogh.exe @@ -256,7 +262,7 @@ jobs: shell: bash - uses: actions/upload-artifact@v1 with: - name: docker_windows_shell + name: docker_on_windows path: report - name: The End Result run: | @@ -493,15 +499,15 @@ jobs: run: | mkdir -p all_reports cp -r docker_ubuntu_18_04 ./all_reports/ - - name: download results docker_windows_shell + - name: download results docker_on_windows uses: actions/download-artifact@v1 with: - name: docker_windows_shell + name: docker_on_windows - name: cp to all_report shell: bash run: | mkdir -p all_reports - cp -r docker_windows_shell ./all_reports/ + cp -r docker_on_windows ./all_reports/ - name: Download Results none_ubuntu16_04 uses: actions/download-artifact@v1 with: From 7c9068c991fcb5b80fb22a59984c219817f9a56d Mon Sep 17 00:00:00 2001 From: Ruben Baez Date: Fri, 27 Mar 2020 08:53:49 -0500 Subject: [PATCH 27/67] change driver --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cd3d6844fc..168b01261e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -241,7 +241,7 @@ jobs: mkdir -p report mkdir -p testhome START_TIME=$(date -u +%s) - KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome minikube_binaries/e2e-windows-amd64.exe -minikube-start-args=--vm-driver=hyperv -binary=minikube_binaries/minikube-windows-amd64.exe -test.v -test.timeout=65m 2>&1 | tee ./report/testout.txt + KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome minikube_binaries/e2e-windows-amd64.exe -minikube-start-args=--vm-driver=docker -binary=minikube_binaries/minikube-windows-amd64.exe -test.v -test.timeout=65m 2>&1 | tee ./report/testout.txt END_TIME=$(date -u +%s) TIME_ELAPSED=$(($END_TIME-$START_TIME)) min=$((${TIME_ELAPSED}/60)) From 23ab8d59060ac3850bc3d0eff2b54c55a9a81cef Mon Sep 17 00:00:00 2001 From: gorbondiga Date: Tue, 7 Apr 2020 19:04:23 +0200 Subject: [PATCH 28/67] Fix broken community links Signed-off-by: Gorka B --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c9f85e00a8..0dce7cc77c 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,6 @@ minikube is a Kubernetes [#sig-cluster-lifecycle](https://github.com/kubernetes/ * [minikube-dev mailing list](https://groups.google.com/forum/#!forum/minikube-dev) * [Bi-weekly office hours, Mondays @ 11am PST](https://tinyurl.com/minikube-oh) -* [Contributing](https://minikube.sigs.k8s.io/docs/contributing/) -* [Development Roadmap](https://minikube.sigs.k8s.io/docs/contributing/roadmap/) +* [Contributing](https://minikube.sigs.k8s.io/docs/contrib/) +* [Development Roadmap](https://minikube.sigs.k8s.io/docs/contrib/roadmap/) From b509d69182fc227c50fa6905feb96e08046a9673 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Tue, 7 Apr 2020 14:19:20 -0700 Subject: [PATCH 29/67] Extract preloaded tarball as soon as minikube volume is created Extracting preload currently takes 6 seconds. This PR begins preload extraction as soon as the minikube volume is created, and runs it in parallell with creating the container node and setting up SSH. This shaves 2-3 secnods off of `node.StartHost` --- pkg/drivers/kic/kic.go | 37 ++++++++++++++++++++++++------------- pkg/drivers/kic/oci/oci.go | 16 ++++++++++++---- 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go index 692bcdf485..081850a124 100644 --- a/pkg/drivers/kic/kic.go +++ b/pkg/drivers/kic/kic.go @@ -22,6 +22,7 @@ import ( "os/exec" "strconv" "strings" + "sync" "time" "github.com/docker/machine/libmachine/drivers" @@ -112,6 +113,28 @@ func (d *Driver) Create() error { } } + if err := oci.SetupContainerNode(params); err != nil { + return errors.Wrap(err, "setting up container node") + } + + var waitForPreload sync.WaitGroup + waitForPreload.Add(1) + go func() { + defer waitForPreload.Done() + // If preload doesn't exist, don't bother extracting tarball to volume + if !download.PreloadExists(d.NodeConfig.KubernetesVersion, d.NodeConfig.ContainerRuntime) { + return + } + t := time.Now() + glog.Infof("Starting extracting preloaded images to volume") + // Extract preloaded images to container + if err := oci.ExtractTarballToVolume(download.TarballPath(d.NodeConfig.KubernetesVersion, d.NodeConfig.ContainerRuntime), params.Name, BaseImage); err != nil { + glog.Infof("Unable to extract preloaded tarball to volume: %v", err) + } else { + glog.Infof("Took %f seconds to extract preloaded images to volume", time.Since(t).Seconds()) + } + }() + if err := oci.CreateContainerNode(params); err != nil { return errors.Wrap(err, "create kic node") } @@ -120,19 +143,7 @@ func (d *Driver) Create() error { return errors.Wrap(err, "prepare kic ssh") } - // If preload doesn't exist, don't bother extracting tarball to volume - if !download.PreloadExists(d.NodeConfig.KubernetesVersion, d.NodeConfig.ContainerRuntime) { - return nil - } - t := time.Now() - glog.Infof("Starting extracting preloaded images to volume") - // Extract preloaded images to container - if err := oci.ExtractTarballToVolume(download.TarballPath(d.NodeConfig.KubernetesVersion, d.NodeConfig.ContainerRuntime), params.Name, BaseImage); err != nil { - glog.Infof("Unable to extract preloaded tarball to volume: %v", err) - } else { - glog.Infof("Took %f seconds to extract preloaded images to volume", time.Since(t).Seconds()) - } - + waitForPreload.Wait() return nil } diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go index 29b6ddbd9b..6605c3e332 100644 --- a/pkg/drivers/kic/oci/oci.go +++ b/pkg/drivers/kic/oci/oci.go @@ -84,6 +84,18 @@ func DeleteContainer(ociBin string, name string) error { return nil } +// SetupContainerNode sets up the container node +func SetupContainerNode(p CreateParams) error { + if p.OCIBinary != Docker { + return nil + } + if err := createDockerVolume(p.Name, p.Name); err != nil { + return errors.Wrapf(err, "creating volume for %s container", p.Name) + } + glog.Infof("Successfully created a docker volume %s", p.Name) + return nil +} + // CreateContainerNode creates a new container node func CreateContainerNode(p CreateParams) error { runArgs := []string{ @@ -122,10 +134,6 @@ func CreateContainerNode(p CreateParams) error { runArgs = append(runArgs, "--volume", fmt.Sprintf("%s:/var:exec", hostVarVolPath)) } if p.OCIBinary == Docker { - if err := createDockerVolume(p.Name, p.Name); err != nil { - return errors.Wrapf(err, "creating volume for %s container", p.Name) - } - glog.Infof("Successfully created a docker volume %s", p.Name) runArgs = append(runArgs, "--volume", fmt.Sprintf("%s:/var", p.Name)) // setting resource limit in privileged mode is only supported by docker // podman error: "Error: invalid configuration, cannot set resources with rootless containers not using cgroups v2 unified mode" From accba6aa7fa673ae3c4276f77f9f0f44df85f4e0 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Tue, 7 Apr 2020 15:00:28 -0700 Subject: [PATCH 30/67] code review comments --- pkg/drivers/kic/kic.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go index 081850a124..a1468b53e5 100644 --- a/pkg/drivers/kic/kic.go +++ b/pkg/drivers/kic/kic.go @@ -131,7 +131,7 @@ func (d *Driver) Create() error { if err := oci.ExtractTarballToVolume(download.TarballPath(d.NodeConfig.KubernetesVersion, d.NodeConfig.ContainerRuntime), params.Name, BaseImage); err != nil { glog.Infof("Unable to extract preloaded tarball to volume: %v", err) } else { - glog.Infof("Took %f seconds to extract preloaded images to volume", time.Since(t).Seconds()) + glog.Infof("duration metric: took %f seconds to extract preloaded images to volume", time.Since(t).Seconds()) } }() From df5da4d45332821a6f3481df7d4f404a5967fb5e Mon Sep 17 00:00:00 2001 From: nestoralonso Date: Tue, 7 Apr 2020 22:01:38 -0500 Subject: [PATCH 31/67] Update start.go It's necessary to add the start argument in this scenario because the suggested command doesn't work as printed: 'sudo minikube --driver=none'->'sudo minikube start --driver=none' --- cmd/minikube/cmd/start.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index 7e3dba8f7f..9e296e195d 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -584,7 +584,7 @@ func validateUser(drvName string) { useForce := viper.GetBool(force) if driver.NeedsRoot(drvName) && u.Uid != "0" && !useForce { - exit.WithCodeT(exit.Permissions, `The "{{.driver_name}}" driver requires root privileges. Please run minikube using 'sudo minikube --driver={{.driver_name}}'.`, out.V{"driver_name": drvName}) + exit.WithCodeT(exit.Permissions, `The "{{.driver_name}}" driver requires root privileges. Please run minikube using 'sudo minikube start --driver={{.driver_name}}'.`, out.V{"driver_name": drvName}) } if driver.NeedsRoot(drvName) || u.Uid != "0" { From 3a9bdd5328ba262949ef32f0bc4c207a32880fcb Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Wed, 8 Apr 2020 11:35:36 -0700 Subject: [PATCH 32/67] Add df and desc po to post-mortem out --- test/integration/helpers.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test/integration/helpers.go b/test/integration/helpers.go index 89eef10dc2..2007d0b6e3 100644 --- a/test/integration/helpers.go +++ b/test/integration/helpers.go @@ -209,6 +209,13 @@ func clusterLogs(t *testing.T, profile string) { } t.Logf("%s logs: %s", t.Name(), rr.Stdout) + rr, err := Run(t, exec.Command(Target(), "-p", profile, "ssh", "df -h /var/lib/docker/overlay2 /var /; du -hs /var/lib/docker/overlay2")) + if err != nil { + t.Logf("failed logs error: %v", err) + return + } + t.Logf("%s logs: %s", t.Name(), rr.Stdout) + st = Status(context.Background(), t, Target(), profile, "APIServer") if st != state.Running.String() { t.Logf("%q apiserver is not running, skipping kubectl commands (state=%q)", profile, st) @@ -228,6 +235,14 @@ func clusterLogs(t *testing.T, profile string) { } else { t.Logf("(dbg) %s:\n%s", rr.Command(), rr.Stdout) } + + rr, err = Run(t, exec.Command("kubectl", "--context", profile, "describe", "po", "-A")) + if err != nil { + t.Logf("%s: %v", rr.Command(), err) + } else { + t.Logf("(dbg) %s:\n%s", rr.Command(), rr.Stdout) + } + t.Logf("<<< %s FAILED: end of post-mortem logs <<<", t.Name()) } From 70329bf51d859f98977c4e6b847cf8b9b61b993a Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Wed, 8 Apr 2020 13:44:04 -0700 Subject: [PATCH 33/67] Code review comments --- pkg/drivers/kic/kic.go | 2 +- pkg/drivers/kic/oci/oci.go | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go index a1468b53e5..f8607c0c4d 100644 --- a/pkg/drivers/kic/kic.go +++ b/pkg/drivers/kic/kic.go @@ -113,7 +113,7 @@ func (d *Driver) Create() error { } } - if err := oci.SetupContainerNode(params); err != nil { + if err := oci.PrepareContainerNode(params); err != nil { return errors.Wrap(err, "setting up container node") } diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go index 6605c3e332..372dad8620 100644 --- a/pkg/drivers/kic/oci/oci.go +++ b/pkg/drivers/kic/oci/oci.go @@ -84,8 +84,9 @@ func DeleteContainer(ociBin string, name string) error { return nil } -// SetupContainerNode sets up the container node -func SetupContainerNode(p CreateParams) error { +// PrepareContainerNode sets up the container node befpre CreateContainerNode is caleld +// for the docker runtime, it creates a docker volume which will be mounted into kic +func PrepareContainerNode(p CreateParams) error { if p.OCIBinary != Docker { return nil } From e0271c53d8e0ad49fc0d73f21020bf0510b22729 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Wed, 8 Apr 2020 14:39:47 -0700 Subject: [PATCH 34/67] update docsy theme --- site/themes/docsy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/themes/docsy b/site/themes/docsy index e13afeb5c0..dd303fd19c 160000 --- a/site/themes/docsy +++ b/site/themes/docsy @@ -1 +1 @@ -Subproject commit e13afeb5c0ffa5a2dff237fffa77bea80319f578 +Subproject commit dd303fd19cc13ffc01bcbe86ff54f21e423d04de From ae74bb0bce0bf26075f744a1938a9cb8971b2f87 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Wed, 8 Apr 2020 14:41:41 -0700 Subject: [PATCH 35/67] make site search wider --- site/assets/scss/_variables_project.scss | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/site/assets/scss/_variables_project.scss b/site/assets/scss/_variables_project.scss index 01a924faa8..5d5d9059c4 100644 --- a/site/assets/scss/_variables_project.scss +++ b/site/assets/scss/_variables_project.scss @@ -163,3 +163,8 @@ div.td-content { font-size: 1.3rem !important; } } + +// make search bar wider https://www.docsy.dev/docs/adding-content/navigation/#configure-local-search-with-lunr +.popover.offline-search-result { + max-width: 460px; +} \ No newline at end of file From 6af9002d9c3b2bbbb120054fc78cfe475cd074fb Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Wed, 8 Apr 2020 14:45:17 -0700 Subject: [PATCH 36/67] change bg color --- site/assets/scss/_variables_project.scss | 1 + 1 file changed, 1 insertion(+) diff --git a/site/assets/scss/_variables_project.scss b/site/assets/scss/_variables_project.scss index 5d5d9059c4..857958fb5d 100644 --- a/site/assets/scss/_variables_project.scss +++ b/site/assets/scss/_variables_project.scss @@ -166,5 +166,6 @@ div.td-content { // make search bar wider https://www.docsy.dev/docs/adding-content/navigation/#configure-local-search-with-lunr .popover.offline-search-result { + background-color: $mk-dark !important; max-width: 460px; } \ No newline at end of file From e03f079eb385f6a53f49a58e28a4182eac65f615 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Wed, 8 Apr 2020 14:53:52 -0700 Subject: [PATCH 37/67] change bg color --- site/assets/scss/_variables_project.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/assets/scss/_variables_project.scss b/site/assets/scss/_variables_project.scss index 857958fb5d..f9c97d406a 100644 --- a/site/assets/scss/_variables_project.scss +++ b/site/assets/scss/_variables_project.scss @@ -166,6 +166,6 @@ div.td-content { // make search bar wider https://www.docsy.dev/docs/adding-content/navigation/#configure-local-search-with-lunr .popover.offline-search-result { - background-color: $mk-dark !important; + background-color: $gray-200 !important; max-width: 460px; } \ No newline at end of file From 546c8daa13e316f1990b93d1f5448adef26f61bf Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Wed, 8 Apr 2020 15:04:12 -0700 Subject: [PATCH 38/67] cert sync: ignore 0-byte certs, test for them explicitly --- pkg/minikube/bootstrapper/certs.go | 68 ++++++++++++++++++----------- test/integration/functional_test.go | 37 ++++++++++++++-- 2 files changed, 76 insertions(+), 29 deletions(-) diff --git a/pkg/minikube/bootstrapper/certs.go b/pkg/minikube/bootstrapper/certs.go index 28ea6e5295..35833705e0 100644 --- a/pkg/minikube/bootstrapper/certs.go +++ b/pkg/minikube/bootstrapper/certs.go @@ -323,20 +323,33 @@ func collectCACerts() (map[string]string, error) { if err != nil { return err } + if info == nil { + return nil + } + if info.IsDir() { + return nil + } - if info != nil && !info.IsDir() { - ext := strings.ToLower(filepath.Ext(hostpath)) - if ext == ".crt" || ext == ".pem" { - glog.Infof("found cert: %s (%d bytes)", info.Name(), info.Size()) - validPem, err := isValidPEMCertificate(hostpath) - if err != nil { - return err - } - if validPem { - filename := filepath.Base(hostpath) - dst := fmt.Sprintf("%s.%s", strings.TrimSuffix(filename, ext), "pem") - certFiles[hostpath] = path.Join(vmpath.GuestCertAuthDir, dst) - } + fullPath := filepath.Join(certsDir, hostpath) + ext := strings.ToLower(filepath.Ext(hostpath)) + + if ext == ".crt" || ext == ".pem" { + if info.Size() < 32 { + glog.Warningf("ignoring %s, impossibly tiny %d bytes", fullPath, info.Size()) + return nil + } + + glog.Infof("found cert: %s (%d bytes)", fullPath, info.Size()) + + validPem, err := isValidPEMCertificate(hostpath) + if err != nil { + return err + } + + if validPem { + filename := filepath.Base(hostpath) + dst := fmt.Sprintf("%s.%s", strings.TrimSuffix(filename, ext), "pem") + certFiles[hostpath] = path.Join(vmpath.GuestCertAuthDir, dst) } } return nil @@ -394,23 +407,26 @@ func installCertSymlinks(cr command.Runner, caCerts map[string]string) error { for _, caCertFile := range caCerts { dstFilename := path.Base(caCertFile) certStorePath := path.Join(vmpath.GuestCertStoreDir, dstFilename) - // If the cert really exists, add a named symlink - cmd := fmt.Sprintf("test -f %s && ln -fs %s %s", caCertFile, caCertFile, certStorePath) + + cmd := fmt.Sprintf("test -s %s && ln -fs %s %s", caCertFile, caCertFile, certStorePath) if _, err := cr.RunCmd(exec.Command("sudo", "/bin/bash", "-c", cmd)); err != nil { return errors.Wrapf(err, "create symlink for %s", caCertFile) } - if hasSSLBinary { - subjectHash, err := getSubjectHash(cr, caCertFile) - if err != nil { - return errors.Wrapf(err, "calculate hash for cacert %s", caCertFile) - } - subjectHashLink := path.Join(vmpath.GuestCertStoreDir, fmt.Sprintf("%s.0", subjectHash)) - // NOTE: This symlink may exist, but point to a missing file - cmd := fmt.Sprintf("test -L %s || ln -fs %s %s", subjectHashLink, certStorePath, subjectHashLink) - if _, err := cr.RunCmd(exec.Command("sudo", "/bin/bash", "-c", cmd)); err != nil { - return errors.Wrapf(err, "create symlink for %s", caCertFile) - } + if !hasSSLBinary { + continue + } + + subjectHash, err := getSubjectHash(cr, caCertFile) + if err != nil { + return errors.Wrapf(err, "calculate hash for cacert %s", caCertFile) + } + subjectHashLink := path.Join(vmpath.GuestCertStoreDir, fmt.Sprintf("%s.0", subjectHash)) + + // NOTE: This symlink may exist, but point to a missing file + cmd = fmt.Sprintf("test -L %s || ln -fs %s %s", subjectHashLink, certStorePath, subjectHashLink) + if _, err := cr.RunCmd(exec.Command("sudo", "/bin/bash", "-c", cmd)); err != nil { + return errors.Wrapf(err, "create symlink for %s", caCertFile) } } return nil diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go index 9dd63bbc29..8545e811a7 100644 --- a/test/integration/functional_test.go +++ b/test/integration/functional_test.go @@ -73,6 +73,11 @@ func TestFunctional(t *testing.T) { if err := os.Remove(p); err != nil { t.Logf("unable to remove %q: %v", p, err) } + p = localEmptyCertPath() + if err := os.Remove(p); err != nil { + t.Logf("unable to remove %q: %v", p, err) + } + CleanupWithLogs(t, profile, cancel) }() @@ -793,18 +798,44 @@ func localTestCertPath() string { return filepath.Join(localpath.MiniPath(), "/certs", testCert()) } +// localEmptyCertPath is where the test file will be synced into the VM +func localEmptyCertPath() string { + return filepath.Join(localpath.MiniPath(), "/certs", fmt.Sprintf("%d_empty.pem", os.Getpid())) +} + // Copy extra file into minikube home folder for file sync test func setupFileSync(ctx context.Context, t *testing.T, profile string) { p := localSyncTestPath() t.Logf("local sync path: %s", p) err := copy.Copy("./testdata/sync.test", p) if err != nil { - t.Fatalf("failed to copy ./testdata/sync.test : %v", err) + t.Fatalf("failed to copy ./testdata/sync.test: %v", err) } - err = copy.Copy("./testdata/minikube_test.pem", localTestCertPath()) + testPem := "./testdata/minikube_test.pem" + + err = copy.Copy(testPem, localTestCertPath()) if err != nil { - t.Fatalf("failed to copy ./testdata/minikube_test.pem : %v", err) + t.Fatalf("failed to copy %s: %v", testPem, err) + } + + want, err := os.Stat(testPem) + if err != nil { + t.Fatalf("stat failed: %v", err) + } + + got, err := os.Stat(localTestCertPath()) + if err != nil { + t.Fatalf("stat failed: %v", err) + } + + if want.Size() != got.Size() { + t.Errorf("%s size=%d, want %d", localTestCertPath(), got.Size(), want.Size()) + } + + // Create an empty file just to mess with people + if _, err := os.Create(localEmptyCertPath()); err != nil { + t.Fatalf("create failed: %v", err) } } From 1a0f86a7af568a759c8519c79e2efbe53f08dcb3 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Wed, 8 Apr 2020 15:27:54 -0700 Subject: [PATCH 39/67] Update unit tests for new command line --- pkg/minikube/bootstrapper/certs_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/minikube/bootstrapper/certs_test.go b/pkg/minikube/bootstrapper/certs_test.go index fd96b6a838..4f93aad180 100644 --- a/pkg/minikube/bootstrapper/certs_test.go +++ b/pkg/minikube/bootstrapper/certs_test.go @@ -51,8 +51,8 @@ func TestSetupCerts(t *testing.T) { } expected := map[string]string{ - `sudo /bin/bash -c "test -f /usr/share/ca-certificates/mycert.pem && ln -fs /usr/share/ca-certificates/mycert.pem /etc/ssl/certs/mycert.pem"`: "-", - `sudo /bin/bash -c "test -f /usr/share/ca-certificates/minikubeCA.pem && ln -fs /usr/share/ca-certificates/minikubeCA.pem /etc/ssl/certs/minikubeCA.pem"`: "-", + `sudo /bin/bash -c "test -s /usr/share/ca-certificates/mycert.pem && ln -fs /usr/share/ca-certificates/mycert.pem /etc/ssl/certs/mycert.pem"`: "-", + `sudo /bin/bash -c "test -s /usr/share/ca-certificates/minikubeCA.pem && ln -fs /usr/share/ca-certificates/minikubeCA.pem /etc/ssl/certs/minikubeCA.pem"`: "-", } f := command.NewFakeCommandRunner() f.SetCommandToOutput(expected) From 194b8cd7d8261908ec22f925b009ef2c2fc2fbee Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Wed, 8 Apr 2020 15:34:16 -0700 Subject: [PATCH 40/67] Modify post-mortem logs so I don't need to count dashes --- test/integration/helpers.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/test/integration/helpers.go b/test/integration/helpers.go index 8304e879ed..2ec166a4ac 100644 --- a/test/integration/helpers.go +++ b/test/integration/helpers.go @@ -201,9 +201,9 @@ func clusterLogs(t *testing.T, profile string) { return } - t.Logf("-----------------------post-mortem--------------------------------") + t.Logf("------------------------------------------------------------------") t.Logf("<<< %s FAILED: start of post-mortem logs <<<", t.Name()) - t.Logf("-------------------post-mortem minikube logs----------------------") + t.Logf("------++> post-mortem[%s]: minikube logs", t.Name()) rr, err := Run(t, exec.Command(Target(), "-p", profile, "logs", "--problems")) if err != nil { @@ -212,21 +212,20 @@ func clusterLogs(t *testing.T, profile string) { } t.Logf("%s logs: %s", t.Name(), rr.Output()) - t.Logf("------------------post-mortem disk usage-------------------") + t.Logf("------++> post-mortem[%s]: disk usage", t.Name()) rr, err = Run(t, exec.Command(Target(), "-p", profile, "ssh", "df -h /var/lib/docker/overlay2 /var /; du -hs /var/lib/docker/overlay2")) if err != nil { t.Logf("failed df error: %v", err) } t.Logf("%s df: %s", t.Name(), rr.Stdout) - t.Logf("------------------post-mortem api server status-------------------") st = Status(context.Background(), t, Target(), profile, "APIServer") if st != state.Running.String() { t.Logf("%q apiserver is not running, skipping kubectl commands (state=%q)", profile, st) return } - t.Logf("--------------------post-mortem get pods--------------------------") + t.Logf("------++> post-mortem[%s]: get pods", t.Name()) rr, rerr := Run(t, exec.Command("kubectl", "--context", profile, "get", "po", "-A", "--show-labels")) if rerr != nil { t.Logf("%s: %v", rr.Command(), rerr) @@ -234,7 +233,7 @@ func clusterLogs(t *testing.T, profile string) { } t.Logf("(dbg) %s:\n%s", rr.Command(), rr.Output()) - t.Logf("-------------------post-mortem describe node----------------------") + t.Logf("------++> post-mortem[%s]: describe node", t.Name()) rr, err = Run(t, exec.Command("kubectl", "--context", profile, "describe", "node")) if err != nil { t.Logf("%s: %v", rr.Command(), err) @@ -242,7 +241,7 @@ func clusterLogs(t *testing.T, profile string) { t.Logf("(dbg) %s:\n%s", rr.Command(), rr.Output()) } - t.Logf("-------------------post-mortem describe pods----------------------") + t.Logf("------++> post-mortem[%s]: describe pods", t.Name()) rr, err = Run(t, exec.Command("kubectl", "--context", profile, "describe", "po", "-A")) if err != nil { t.Logf("%s: %v", rr.Command(), err) @@ -250,7 +249,6 @@ func clusterLogs(t *testing.T, profile string) { t.Logf("(dbg) %s:\n%s", rr.Command(), rr.Stdout) } - t.Logf("------------------------------------------------------------------") t.Logf("<<< %s FAILED: end of post-mortem logs <<<", t.Name()) t.Logf("---------------------/post-mortem---------------------------------") } From 93060d17d190574a1b99443f8893e376bce8ec24 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Wed, 8 Apr 2020 15:35:09 -0700 Subject: [PATCH 41/67] keep header --- test/integration/helpers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/helpers.go b/test/integration/helpers.go index 2ec166a4ac..7daac4aa6f 100644 --- a/test/integration/helpers.go +++ b/test/integration/helpers.go @@ -201,7 +201,7 @@ func clusterLogs(t *testing.T, profile string) { return } - t.Logf("------------------------------------------------------------------") + t.Logf("-----------------------post-mortem--------------------------------") t.Logf("<<< %s FAILED: start of post-mortem logs <<<", t.Name()) t.Logf("------++> post-mortem[%s]: minikube logs", t.Name()) From 7a2f62dbe79b14d08deded612df7f0ba1e409328 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Wed, 8 Apr 2020 17:18:43 -0700 Subject: [PATCH 42/67] create comparision table for methods to push img to minikube --- site/content/en/docs/handbook/pushing.md | 103 +++++++++++++++-------- 1 file changed, 68 insertions(+), 35 deletions(-) diff --git a/site/content/en/docs/handbook/pushing.md b/site/content/en/docs/handbook/pushing.md index a8ac2d93a5..3ea44e0623 100644 --- a/site/content/en/docs/handbook/pushing.md +++ b/site/content/en/docs/handbook/pushing.md @@ -2,7 +2,7 @@ title: "Pushing images" weight: 5 description: > - There are many ways to push images into minikube. + comparing 3 ways to push your image into minikiube. aliases: - /docs/tasks/building - /docs/tasks/caching @@ -10,9 +10,59 @@ aliases: - /docs/tasks/docker_daemon --- -# Cached Images +## What is best method to push image to minikiube ? +the answer depends on the container-runtime driver you choose. +Here is a comparison table to help you choose: -From the host, you can push a Docker image directly to minikube. It will also be cached for future cluster starts. + + +| Method | Supported Runtimes | Supported Drivers* | Performance | +|--- |--- |--- |--- |--- | +| [docker-env command](http://localhost:1313/docs/handbook/pushing/#pushing-directly-to-the-in-cluster-docker-daemon) | only docker | all | best | +| [podman-env command](http://localhost:1313/docs/handbook/pushing/#pushing-directly-to-in-cluster-crio) | only cri-o | all | best | +| [cache add command](http://localhost:1313/docs/handbook/pushing/#push-images-using-cache-command) | all | all | good | +| [registry addon](http://localhost:1313/docs/handbook/pushing/#pushing-to-an-in-cluster-using-a-registry-addon) | all | all but [docker on mac](https://github.com/kubernetes/minikube/issues/7535) | good | +| [minikube ssh](http://localhost:1313/docs/handbook/pushing/#building-images-inside-of-minikube-using-ssh) | all | all | best | + + +* note1 : the default container-runtime on minikube is 'docker'. +* note2 : 'none' driver (bare metal) does not need pushing image to the cluster, as any image on your system is already available to the kuberentes. + + +# Pushing directly to the in-cluster Docker daemon +When using a container or VM driver (all drivers except none), you can reuse the Docker daemon inside minikube cluster. +this means you don't have to build on your host machine and push the image into a docker registry. You can just build inside the same docker daemon as minikube which speeds up local experiments. + +To point your terminal to use the docker daemon inside minikube run this: + +```shell +eval $(minikube docker-env) +``` + +now any 'docker' command you run in this current terminal will run against the docker inside minikube VM or Container. +Try it: + +```shell +docker ps +``` + +now you 'build' against the docker inside minikube. which is instantly accessible to kubernetes cluster. + +''' +docker build -t myimage . +''' + +Remember to turn off the `imagePullPolicy:Always` (use `imagePullPolicy:IfNotPresent` or `imagePullPolicy:Never`), as otherwise Kubernetes won't use images you built locally. + +#### docker-env will be +Please remember by closing the terminal, you will go back to using your own system's docker daemon. +to verify your terminal is using minikuber's docker-env you can check the value of the environment MINIKUBE_ACTIVE_DOCKERD + +more information on [docker-env](https://minikube.sigs.k8s.io/docs/commands/docker-env/) + +# Push images using Cache Command. + +From the host, you can push a Docker image directly to minikube. It will also be cached for all minikube clusters . ```shell minikube cache add ubuntu:16.04 @@ -26,7 +76,13 @@ To display images you have added to the cache: minikube cache list ``` -This listing will not include the images which are built-in to minikube. +This listing will not include the images minikube's built-in system images. + +to ensure your running cluster has cached the updated images use reload: + +```shell +minikube cache reload +``` ```shell @@ -37,34 +93,6 @@ For more information, see: * [Reference: cache command]({{< ref "/docs/commands/cache.md" >}}) -You must be using minikube with the container runtime set to Docker. This is the default setting. - -# Pushing directly to the in-cluster Docker daemon -When user a container or VM driver, it's really handy to reuse the Docker daemon inside minikube; as this means you don't have to build on your host machine and push the image into a docker registry - you can just build inside the same docker daemon as minikube which speeds up local experiments. - -To point your terminal to use the docker daemon inside minikube run this: - -```shell -eval $(minikube docker-env) -``` - -now any command you run in this current terminal will run against the docker inside minikube VM or Container. -Try it: - -```shell -docker ps -``` - -now you can use same docker build command against the docker inside minikube. which is instantly accessible to kubernetes cluster. - -''' -docker build -t myimage . -''' - - -Remember to turn off the `imagePullPolicy:Always` (use `imagePullPolicy:IfNotPresent` or `imagePullPolicy:Never`), as otherwise Kubernetes won't use images you built locally. - -more information on [docker-env](https://minikube.sigs.k8s.io/docs/commands/docker-env/) # Pushing directly to in-cluster CRIO @@ -82,7 +110,7 @@ podman-remote help Remember to turn off the `imagePullPolicy:Always` (use `imagePullPolicy:IfNotPresent` or `imagePullPolicy:Never`), as otherwise Kubernetes won't use images you built locally. -# Pushing to an in-cluster Registry +# Pushing to an in-cluster using a Registry addon For illustration purpose, we will assume that minikube VM has one of the ip from `192.168.39.0/24` subnet. If you have not overridden these subnets as per [networking guide](https://minikube.sigs.k8s.io/reference/networking/), you can find out default subnet being used by minikube for a specific OS and driver combination [here](https://github.com/kubernetes/minikube/blob/dfd9b6b83d0ca2eeab55588a16032688bc26c348/pkg/minikube/cluster/cluster.go#L408) which is subject to change. Replace `192.168.39.0/24` with appropriate values for your environment wherever applicable. @@ -108,9 +136,10 @@ Push docker image to minikube registry: docker push $(minikube ip):5000/test-img ``` -# Building images inside of minikube +# Building images inside of minikube using SSH -Use `minikube ssh` to connect to the virtual machine, and run the `docker build` there: +Use `minikube ssh` to go inside a minikube node, and run the `docker build` directly there. +any command you run there will run against the same daemon that kubernetes is using. ```shell docker build @@ -126,3 +155,7 @@ sudo -E podman build For more information on the `podman build` command, read the [Podman documentation](https://github.com/containers/libpod/blob/master/docs/source/markdown/podman-build.1.md) (podman.io). +to exit minikube ssh and come back to your terminal type: +```shell +exit +``` From 5ac676e7bb7ef92aaabc92a45eae72a5c1183e07 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Wed, 8 Apr 2020 17:23:18 -0700 Subject: [PATCH 43/67] change perforamnce words --- site/content/en/docs/handbook/pushing.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/site/content/en/docs/handbook/pushing.md b/site/content/en/docs/handbook/pushing.md index 3ea44e0623..47a42524a9 100644 --- a/site/content/en/docs/handbook/pushing.md +++ b/site/content/en/docs/handbook/pushing.md @@ -18,10 +18,10 @@ Here is a comparison table to help you choose: | Method | Supported Runtimes | Supported Drivers* | Performance | |--- |--- |--- |--- |--- | -| [docker-env command](http://localhost:1313/docs/handbook/pushing/#pushing-directly-to-the-in-cluster-docker-daemon) | only docker | all | best | -| [podman-env command](http://localhost:1313/docs/handbook/pushing/#pushing-directly-to-in-cluster-crio) | only cri-o | all | best | -| [cache add command](http://localhost:1313/docs/handbook/pushing/#push-images-using-cache-command) | all | all | good | -| [registry addon](http://localhost:1313/docs/handbook/pushing/#pushing-to-an-in-cluster-using-a-registry-addon) | all | all but [docker on mac](https://github.com/kubernetes/minikube/issues/7535) | good | +| [docker-env command](http://localhost:1313/docs/handbook/pushing/#pushing-directly-to-the-in-cluster-docker-daemon) | only docker | all | good | +| [podman-env command](http://localhost:1313/docs/handbook/pushing/#pushing-directly-to-in-cluster-crio) | only cri-o | all | good | +| [cache add command](http://localhost:1313/docs/handbook/pushing/#push-images-using-cache-command) | all | all | ok | +| [registry addon](http://localhost:1313/docs/handbook/pushing/#pushing-to-an-in-cluster-using-a-registry-addon) | all | all but [docker on mac](https://github.com/kubernetes/minikube/issues/7535) | ok | | [minikube ssh](http://localhost:1313/docs/handbook/pushing/#building-images-inside-of-minikube-using-ssh) | all | all | best | From 3575367b110132950d67d93fe2f936c42a333803 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Wed, 8 Apr 2020 17:25:00 -0700 Subject: [PATCH 44/67] fix number words --- site/content/en/docs/handbook/pushing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/content/en/docs/handbook/pushing.md b/site/content/en/docs/handbook/pushing.md index 47a42524a9..576626004f 100644 --- a/site/content/en/docs/handbook/pushing.md +++ b/site/content/en/docs/handbook/pushing.md @@ -2,7 +2,7 @@ title: "Pushing images" weight: 5 description: > - comparing 3 ways to push your image into minikiube. + comparing 5 ways to push your image into a minikiube cluster. aliases: - /docs/tasks/building - /docs/tasks/caching From 880f3a7ce9df302c5d6368408fe1ec0b9304ca68 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Wed, 8 Apr 2020 17:37:29 -0700 Subject: [PATCH 45/67] issues --- site/content/en/docs/handbook/pushing.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/site/content/en/docs/handbook/pushing.md b/site/content/en/docs/handbook/pushing.md index 576626004f..79b6b388dd 100644 --- a/site/content/en/docs/handbook/pushing.md +++ b/site/content/en/docs/handbook/pushing.md @@ -16,13 +16,13 @@ Here is a comparison table to help you choose: -| Method | Supported Runtimes | Supported Drivers* | Performance | +| Method | Supported Runtimes | Issues | Performance | |--- |--- |--- |--- |--- | -| [docker-env command](http://localhost:1313/docs/handbook/pushing/#pushing-directly-to-the-in-cluster-docker-daemon) | only docker | all | good | -| [podman-env command](http://localhost:1313/docs/handbook/pushing/#pushing-directly-to-in-cluster-crio) | only cri-o | all | good | -| [cache add command](http://localhost:1313/docs/handbook/pushing/#push-images-using-cache-command) | all | all | ok | -| [registry addon](http://localhost:1313/docs/handbook/pushing/#pushing-to-an-in-cluster-using-a-registry-addon) | all | all but [docker on mac](https://github.com/kubernetes/minikube/issues/7535) | ok | -| [minikube ssh](http://localhost:1313/docs/handbook/pushing/#building-images-inside-of-minikube-using-ssh) | all | all | best | +| [docker-env command](http://localhost:1313/docs/handbook/pushing/#pushing-directly-to-the-in-cluster-docker-daemon) | only docker | | good | +| [podman-env command](http://localhost:1313/docs/handbook/pushing/#pushing-directly-to-in-cluster-crio) | only cri-o | | good | +| [cache add command](http://localhost:1313/docs/handbook/pushing/#push-images-using-cache-command) | all | | ok | +| [registry addon](http://localhost:1313/docs/handbook/pushing/#pushing-to-an-in-cluster-using-a-registry-addon) | all | [docker on mac](https://github.com/kubernetes/minikube/issues/7535) | ok | +| [minikube ssh](http://localhost:1313/docs/handbook/pushing/#building-images-inside-of-minikube-using-ssh) | all | | best | * note1 : the default container-runtime on minikube is 'docker'. From ce3754465aff02e7b0cf45e1d5bfe5ec2b88c5cb Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Wed, 8 Apr 2020 17:38:13 -0700 Subject: [PATCH 46/67] add wip --- site/content/en/docs/handbook/pushing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/content/en/docs/handbook/pushing.md b/site/content/en/docs/handbook/pushing.md index 79b6b388dd..5a53b5f259 100644 --- a/site/content/en/docs/handbook/pushing.md +++ b/site/content/en/docs/handbook/pushing.md @@ -21,7 +21,7 @@ Here is a comparison table to help you choose: | [docker-env command](http://localhost:1313/docs/handbook/pushing/#pushing-directly-to-the-in-cluster-docker-daemon) | only docker | | good | | [podman-env command](http://localhost:1313/docs/handbook/pushing/#pushing-directly-to-in-cluster-crio) | only cri-o | | good | | [cache add command](http://localhost:1313/docs/handbook/pushing/#push-images-using-cache-command) | all | | ok | -| [registry addon](http://localhost:1313/docs/handbook/pushing/#pushing-to-an-in-cluster-using-a-registry-addon) | all | [docker on mac](https://github.com/kubernetes/minikube/issues/7535) | ok | +| [registry addon](http://localhost:1313/docs/handbook/pushing/#pushing-to-an-in-cluster-using-a-registry-addon) | all | work in progress for [docker on mac](https://github.com/kubernetes/minikube/issues/7535) | ok | | [minikube ssh](http://localhost:1313/docs/handbook/pushing/#building-images-inside-of-minikube-using-ssh) | all | | best | From ed67b770d410d868b9d8df21f37d36b8d5229c44 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Wed, 8 Apr 2020 17:53:48 -0700 Subject: [PATCH 47/67] bullet number the items --- site/content/en/docs/handbook/pushing.md | 26 ++++++++++++++++-------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/site/content/en/docs/handbook/pushing.md b/site/content/en/docs/handbook/pushing.md index 5a53b5f259..0f43ce99c2 100644 --- a/site/content/en/docs/handbook/pushing.md +++ b/site/content/en/docs/handbook/pushing.md @@ -10,12 +10,11 @@ aliases: - /docs/tasks/docker_daemon --- -## What is best method to push image to minikiube ? +## Comparison table for different methods the answer depends on the container-runtime driver you choose. Here is a comparison table to help you choose: - | Method | Supported Runtimes | Issues | Performance | |--- |--- |--- |--- |--- | | [docker-env command](http://localhost:1313/docs/handbook/pushing/#pushing-directly-to-the-in-cluster-docker-daemon) | only docker | | good | @@ -28,8 +27,9 @@ Here is a comparison table to help you choose: * note1 : the default container-runtime on minikube is 'docker'. * note2 : 'none' driver (bare metal) does not need pushing image to the cluster, as any image on your system is already available to the kuberentes. +--- -# Pushing directly to the in-cluster Docker daemon +## 1.Pushing directly to the in-cluster Docker daemon (docker-env) When using a container or VM driver (all drivers except none), you can reuse the Docker daemon inside minikube cluster. this means you don't have to build on your host machine and push the image into a docker registry. You can just build inside the same docker daemon as minikube which speeds up local experiments. @@ -54,13 +54,16 @@ docker build -t myimage . Remember to turn off the `imagePullPolicy:Always` (use `imagePullPolicy:IfNotPresent` or `imagePullPolicy:Never`), as otherwise Kubernetes won't use images you built locally. -#### docker-env will be -Please remember by closing the terminal, you will go back to using your own system's docker daemon. + +Please remember evalulating the docker-env is only valid for the current terminal. +and by closing the terminal, you will go back to using your own system's docker daemon. to verify your terminal is using minikuber's docker-env you can check the value of the environment MINIKUBE_ACTIVE_DOCKERD more information on [docker-env](https://minikube.sigs.k8s.io/docs/commands/docker-env/) -# Push images using Cache Command. +--- + +## 2.Push images using 'cache' command. From the host, you can push a Docker image directly to minikube. It will also be cached for all minikube clusters . @@ -93,8 +96,9 @@ For more information, see: * [Reference: cache command]({{< ref "/docs/commands/cache.md" >}}) +--- -# Pushing directly to in-cluster CRIO +## 3. Pushing directly to in-cluster CRIO. (podman-env) To push directly to CRIO, configure podman client on your mac/linux host using the podman-env command in your shell: @@ -110,7 +114,9 @@ podman-remote help Remember to turn off the `imagePullPolicy:Always` (use `imagePullPolicy:IfNotPresent` or `imagePullPolicy:Never`), as otherwise Kubernetes won't use images you built locally. -# Pushing to an in-cluster using a Registry addon +--- + +## 4. Pushing to an in-cluster using Registry addon For illustration purpose, we will assume that minikube VM has one of the ip from `192.168.39.0/24` subnet. If you have not overridden these subnets as per [networking guide](https://minikube.sigs.k8s.io/reference/networking/), you can find out default subnet being used by minikube for a specific OS and driver combination [here](https://github.com/kubernetes/minikube/blob/dfd9b6b83d0ca2eeab55588a16032688bc26c348/pkg/minikube/cluster/cluster.go#L408) which is subject to change. Replace `192.168.39.0/24` with appropriate values for your environment wherever applicable. @@ -136,7 +142,9 @@ Push docker image to minikube registry: docker push $(minikube ip):5000/test-img ``` -# Building images inside of minikube using SSH +--- + +## 5. Building images inside of minikube using SSH Use `minikube ssh` to go inside a minikube node, and run the `docker build` directly there. any command you run there will run against the same daemon that kubernetes is using. From 16a8c386821dbba82296b9437f0071a19f882554 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Wed, 8 Apr 2020 18:20:30 -0700 Subject: [PATCH 48/67] none: Add support for OpenRC init (Google CloudShell) --- cmd/minikube/cmd/docker-env.go | 6 +- cmd/minikube/cmd/status.go | 12 +- hack/preload-images/generate.go | 5 +- pkg/drivers/kic/kic.go | 8 +- pkg/drivers/none/none.go | 10 +- pkg/minikube/bootstrapper/bootstrapper.go | 1 - pkg/minikube/bootstrapper/bsutil/binaries.go | 14 ++- pkg/minikube/bootstrapper/bsutil/files.go | 22 +--- .../bsutil/kverify/system_pods.go | 23 +--- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 75 ++++++----- pkg/minikube/cluster/pause.go | 19 ++- pkg/minikube/cruntime/containerd.go | 19 +-- pkg/minikube/cruntime/crio.go | 17 +-- pkg/minikube/cruntime/cruntime.go | 21 +++- pkg/minikube/cruntime/cruntime_test.go | 26 ++-- pkg/minikube/cruntime/docker.go | 25 ++-- pkg/minikube/kubelet/kubelet.go | 117 ------------------ pkg/minikube/node/start.go | 1 + pkg/minikube/registry/drvs/none/none.go | 9 +- 19 files changed, 154 insertions(+), 276 deletions(-) delete mode 100644 pkg/minikube/kubelet/kubelet.go diff --git a/cmd/minikube/cmd/docker-env.go b/cmd/minikube/cmd/docker-env.go index 76ae9d4f5a..c4432c6b58 100644 --- a/cmd/minikube/cmd/docker-env.go +++ b/cmd/minikube/cmd/docker-env.go @@ -24,7 +24,6 @@ import ( "io" "net" "os" - "os/exec" "strconv" "strings" @@ -38,6 +37,7 @@ import ( "k8s.io/minikube/pkg/minikube/mustload" "k8s.io/minikube/pkg/minikube/out" "k8s.io/minikube/pkg/minikube/shell" + "k8s.io/minikube/pkg/minikube/sysinit" ) var dockerEnvTmpl = fmt.Sprintf("{{ .Prefix }}%s{{ .Delimiter }}{{ .DockerTLSVerify }}{{ .Suffix }}{{ .Prefix }}%s{{ .Delimiter }}{{ .DockerHost }}{{ .Suffix }}{{ .Prefix }}%s{{ .Delimiter }}{{ .DockerCertPath }}{{ .Suffix }}{{ .Prefix }}%s{{ .Delimiter }}{{ .MinikubeDockerdProfile }}{{ .Suffix }}{{ if .NoProxyVar }}{{ .Prefix }}{{ .NoProxyVar }}{{ .Delimiter }}{{ .NoProxyValue }}{{ .Suffix }}{{end}}{{ .UsageHint }}", constants.DockerTLSVerifyEnv, constants.DockerHostEnv, constants.DockerCertPathEnv, constants.MinikubeActiveDockerdEnv) @@ -116,9 +116,7 @@ func (EnvNoProxyGetter) GetNoProxyVar() (string, string) { // isDockerActive checks if Docker is active func isDockerActive(r command.Runner) bool { - c := exec.Command("sudo", "systemctl", "is-active", "--quiet", "service", "docker") - _, err := r.RunCmd(c) - return err == nil + return sysinit.New(r).Active("docker") } // dockerEnvCmd represents the docker-env command diff --git a/cmd/minikube/cmd/status.go b/cmd/minikube/cmd/status.go index 95d21e3079..a95c7978cb 100644 --- a/cmd/minikube/cmd/status.go +++ b/cmd/minikube/cmd/status.go @@ -208,15 +208,9 @@ func status(api libmachine.API, cc config.ClusterConfig, n config.Node) (*Status return st, err } - stk, err := kverify.KubeletStatus(cr) - glog.Infof("%s kubelet status = %s (err=%v)", name, stk, err) - - if err != nil { - glog.Warningf("kubelet err: %v", err) - st.Kubelet = state.Error.String() - } else { - st.Kubelet = stk.String() - } + stk := kverify.KubeletStatus(cr) + glog.Infof("%s kubelet status = %s", name, stk) + st.Kubelet = stk.String() // Early exit for regular nodes if !controlPlane { diff --git a/hack/preload-images/generate.go b/hack/preload-images/generate.go index f178a6592b..1a22e7404d 100644 --- a/hack/preload-images/generate.go +++ b/hack/preload-images/generate.go @@ -31,6 +31,7 @@ import ( "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/driver" "k8s.io/minikube/pkg/minikube/localpath" + "k8s.io/minikube/pkg/minikube/sysinit" ) func generateTarball(kubernetesVersion, containerRuntime, tarballFilename string) error { @@ -86,7 +87,9 @@ func generateTarball(kubernetesVersion, containerRuntime, tarballFilename string KubernetesVersion: kubernetesVersion, } runner := command.NewKICRunner(profile, driver.OCIBinary) - if err := bsutil.TransferBinaries(kcfg, runner); err != nil { + sm := sysinit.New(runner) + + if err := bsutil.TransferBinaries(kcfg, runner, sm); err != nil { return errors.Wrap(err, "transferring k8s binaries") } // Create image tarball diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go index 692bcdf485..7a342afcbc 100644 --- a/pkg/drivers/kic/kic.go +++ b/pkg/drivers/kic/kic.go @@ -37,7 +37,7 @@ import ( "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/cruntime" "k8s.io/minikube/pkg/minikube/download" - "k8s.io/minikube/pkg/minikube/kubelet" + "k8s.io/minikube/pkg/minikube/sysinit" ) // Driver represents a kic driver https://minikube.sigs.k8s.io/docs/reference/drivers/docker @@ -245,7 +245,7 @@ func (d *Driver) GetState() (state.State, error) { func (d *Driver) Kill() error { // on init this doesn't get filled when called from cmd d.exec = command.NewKICRunner(d.MachineName, d.OCIBinary) - if err := kubelet.ForceStop(d.exec); err != nil { + if err := sysinit.New(d.exec).ForceStop("kubelet"); err != nil { glog.Warningf("couldn't force stop kubelet. will continue with kill anyways: %v", err) } cmd := exec.Command(d.NodeConfig.OCIBinary, "kill", d.MachineName) @@ -318,9 +318,9 @@ func (d *Driver) Stop() error { d.exec = command.NewKICRunner(d.MachineName, d.OCIBinary) // docker does not send right SIG for systemd to know to stop the systemd. // to avoid bind address be taken on an upgrade. more info https://github.com/kubernetes/minikube/issues/7171 - if err := kubelet.Stop(d.exec); err != nil { + if err := sysinit.New(d.exec).Stop("kubelet"); err != nil { glog.Warningf("couldn't stop kubelet. will continue with stop anyways: %v", err) - if err := kubelet.ForceStop(d.exec); err != nil { + if err := sysinit.New(d.exec).ForceStop("kubelet"); err != nil { glog.Warningf("couldn't force stop kubelet. will continue with stop anyways: %v", err) } } diff --git a/pkg/drivers/none/none.go b/pkg/drivers/none/none.go index 91ab305464..1049f713fe 100644 --- a/pkg/drivers/none/none.go +++ b/pkg/drivers/none/none.go @@ -31,7 +31,7 @@ import ( "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/cruntime" "k8s.io/minikube/pkg/minikube/kubeconfig" - "k8s.io/minikube/pkg/minikube/kubelet" + "k8s.io/minikube/pkg/minikube/sysinit" "k8s.io/minikube/pkg/minikube/vmpath" ) @@ -142,12 +142,12 @@ func (d *Driver) GetState() (state.State, error) { return state.Running, nil } - return kverify.KubeletStatus(d.exec) + return kverify.KubeletStatus(d.exec), nil } // Kill stops a host forcefully, including any containers that we are managing. func (d *Driver) Kill() error { - if err := kubelet.ForceStop(d.exec); err != nil { + if err := sysinit.New(d.exec).ForceStop("kubelet"); err != nil { glog.Warningf("couldn't force stop kubelet. will continue with kill anyways: %v", err) } @@ -211,9 +211,9 @@ func (d *Driver) Start() error { // Stop a host gracefully, including any containers that we are managing. func (d *Driver) Stop() error { - if err := kubelet.Stop(d.exec); err != nil { + if err := sysinit.New(d.exec).Stop("kubelet"); err != nil { glog.Warningf("couldn't stop kubelet. will continue with stop anyways: %v", err) - if err := kubelet.ForceStop(d.exec); err != nil { + if err := sysinit.New(d.exec).ForceStop("kubelet"); err != nil { glog.Warningf("couldn't force stop kubelet. will continue with stop anyways: %v", err) } } diff --git a/pkg/minikube/bootstrapper/bootstrapper.go b/pkg/minikube/bootstrapper/bootstrapper.go index c724b2607f..0019b1750d 100644 --- a/pkg/minikube/bootstrapper/bootstrapper.go +++ b/pkg/minikube/bootstrapper/bootstrapper.go @@ -45,7 +45,6 @@ type Bootstrapper interface { // LogCommands returns a map of log type to a command which will display that log. LogCommands(config.ClusterConfig, LogOptions) map[string]string SetupCerts(config.KubernetesConfig, config.Node) error - GetKubeletStatus() (string, error) GetAPIServerStatus(string, int) (string, error) } diff --git a/pkg/minikube/bootstrapper/bsutil/binaries.go b/pkg/minikube/bootstrapper/bsutil/binaries.go index 0ffcaa05ad..341a5b295b 100644 --- a/pkg/minikube/bootstrapper/bsutil/binaries.go +++ b/pkg/minikube/bootstrapper/bsutil/binaries.go @@ -32,11 +32,12 @@ import ( "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/download" "k8s.io/minikube/pkg/minikube/machine" + "k8s.io/minikube/pkg/minikube/sysinit" "k8s.io/minikube/pkg/minikube/vmpath" ) // TransferBinaries transfers all required Kubernetes binaries -func TransferBinaries(cfg config.KubernetesConfig, c command.Runner) error { +func TransferBinaries(cfg config.KubernetesConfig, c command.Runner, sm sysinit.Manager) error { ok, err := binariesExist(cfg, c) if err == nil && ok { glog.Info("Found k8s binaries, skipping transfer") @@ -50,11 +51,6 @@ func TransferBinaries(cfg config.KubernetesConfig, c command.Runner) error { return err } - // stop kubelet to avoid "Text File Busy" error - if _, err := c.RunCmd(exec.Command("/bin/bash", "-c", "pgrep kubelet && sudo systemctl stop kubelet")); err != nil { - glog.Warningf("unable to stop kubelet: %s", err) - } - var g errgroup.Group for _, name := range constants.KubernetesReleaseBinaries { name := name @@ -64,6 +60,12 @@ func TransferBinaries(cfg config.KubernetesConfig, c command.Runner) error { return errors.Wrapf(err, "downloading %s", name) } + if name == "kubelet" { + if err := sm.ForceStop("kubelet"); err != nil { + glog.Errorf("unable to stop kubelet: %v", err) + } + } + dst := path.Join(dir, name) if err := machine.CopyBinary(c, src, dst); err != nil { return errors.Wrapf(err, "copybinary %s -> %s", src, dst) diff --git a/pkg/minikube/bootstrapper/bsutil/files.go b/pkg/minikube/bootstrapper/bsutil/files.go index cec6d69085..8d6bc05bc3 100644 --- a/pkg/minikube/bootstrapper/bsutil/files.go +++ b/pkg/minikube/bootstrapper/bsutil/files.go @@ -20,8 +20,6 @@ package bsutil import ( "path" - "k8s.io/minikube/pkg/minikube/assets" - "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/vmpath" ) @@ -35,20 +33,8 @@ const ( KubeletServiceFile = "/lib/systemd/system/kubelet.service" // KubeletSystemdConfFile is config for the systemd kubelet.service KubeletSystemdConfFile = "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf" + // InitRestartWrapper is ... + InitRestartWrapper = "/etc/init.d/.restart_wrapper.sh" + // KubeletInitPath is where Sys-V style init script is installed + KubeletInitPath = "/etc/init.d/kubelet" ) - -// ConfigFileAssets returns configuration file assets -func ConfigFileAssets(cfg config.KubernetesConfig, kubeadm []byte, kubelet []byte, kubeletSvc []byte, defaultCNIConfig []byte) []assets.CopyableFile { - fs := []assets.CopyableFile{ - assets.NewMemoryAssetTarget(kubeadm, KubeadmYamlPath+".new", "0640"), - assets.NewMemoryAssetTarget(kubelet, KubeletSystemdConfFile+".new", "0644"), - assets.NewMemoryAssetTarget(kubeletSvc, KubeletServiceFile+".new", "0644"), - } - // Copy the default CNI config (k8s.conf), so that kubelet can successfully - // start a Pod in the case a user hasn't manually installed any CNI plugin - // and minikube was started with "--extra-config=kubelet.network-plugin=cni". - if defaultCNIConfig != nil { - fs = append(fs, assets.NewMemoryAssetTarget(defaultCNIConfig, DefaultCNIConfigPath, "0644")) - } - return fs -} diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/system_pods.go b/pkg/minikube/bootstrapper/bsutil/kverify/system_pods.go index d13f3b3ea3..35dd40a858 100644 --- a/pkg/minikube/bootstrapper/bsutil/kverify/system_pods.go +++ b/pkg/minikube/bootstrapper/bsutil/kverify/system_pods.go @@ -19,7 +19,6 @@ package kverify import ( "fmt" - "os/exec" "strings" "time" @@ -36,6 +35,7 @@ import ( "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/cruntime" "k8s.io/minikube/pkg/minikube/logs" + "k8s.io/minikube/pkg/minikube/sysinit" ) // WaitForSystemPods verifies essential pods for running kurnetes is running @@ -156,22 +156,11 @@ func announceProblems(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg conf } // KubeletStatus checks the kubelet status -func KubeletStatus(cr command.Runner) (state.State, error) { +func KubeletStatus(cr command.Runner) state.State { glog.Infof("Checking kubelet status ...") - rr, err := cr.RunCmd(exec.Command("sudo", "systemctl", "is-active", "kubelet")) - if err != nil { - // Do not return now, as we still have parsing to do! - glog.Warningf("%s returned error: %v", rr.Command(), err) + active := sysinit.New(cr).Active("kubelet") + if active { + return state.Running } - s := strings.TrimSpace(rr.Stdout.String()) - glog.Infof("kubelet is-active: %s", s) - switch s { - case "active": - return state.Running, nil - case "inactive": - return state.Stopped, nil - case "activating": - return state.Starting, nil - } - return state.Error, nil + return state.Stopped } diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 10bac2d726..b97707a5cf 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -51,9 +51,9 @@ import ( "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/cruntime" "k8s.io/minikube/pkg/minikube/driver" - "k8s.io/minikube/pkg/minikube/kubelet" "k8s.io/minikube/pkg/minikube/machine" "k8s.io/minikube/pkg/minikube/out" + "k8s.io/minikube/pkg/minikube/sysinit" "k8s.io/minikube/pkg/minikube/vmpath" "k8s.io/minikube/pkg/util" "k8s.io/minikube/pkg/util/retry" @@ -82,26 +82,6 @@ func NewBootstrapper(api libmachine.API, cc config.ClusterConfig, n config.Node) return &Bootstrapper{c: runner, contextName: cc.Name, k8sClient: nil}, nil } -// GetKubeletStatus returns the kubelet status -func (k *Bootstrapper) GetKubeletStatus() (string, error) { - rr, err := k.c.RunCmd(exec.Command("sudo", "systemctl", "is-active", "kubelet")) - if err != nil { - // Do not return now, as we still have parsing to do! - glog.Warningf("%s returned error: %v", rr.Command(), err) - } - s := strings.TrimSpace(rr.Stdout.String()) - glog.Infof("kubelet is-active: %s", s) - switch s { - case "active": - return state.Running.String(), nil - case "inactive": - return state.Stopped.String(), nil - case "activating": - return state.Starting.String(), nil - } - return state.Error.String(), nil -} - // GetAPIServerStatus returns the api-server status func (k *Bootstrapper) GetAPIServerStatus(hostname string, port int) (string, error) { s, err := kverify.APIServerStatus(k.c, hostname, port) @@ -190,7 +170,7 @@ func (k *Bootstrapper) init(cfg config.ClusterConfig) error { } extraFlags := bsutil.CreateFlagsFromExtraArgs(cfg.KubernetesConfig.ExtraOptions) - r, err := cruntime.New(cruntime.Config{Type: cfg.KubernetesConfig.ContainerRuntime}) + r, err := cruntime.New(cruntime.Config{Type: cfg.KubernetesConfig.ContainerRuntime, Runner: k.c}) if err != nil { return err } @@ -615,7 +595,7 @@ func (k *Bootstrapper) DeleteCluster(k8s config.KubernetesConfig) error { glog.Warningf("%s: %v", rr.Command(), err) } - if err := kubelet.ForceStop(k.c); err != nil { + if err := sysinit.New(k.c).ForceStop("kubelet"); err != nil { glog.Warningf("stop kubelet: %v", err) } @@ -679,6 +659,11 @@ func (k *Bootstrapper) UpdateCluster(cfg config.ClusterConfig) error { // UpdateNode updates a node. func (k *Bootstrapper) UpdateNode(cfg config.ClusterConfig, n config.Node, r cruntime.Manager) error { + now := time.Now() + defer func() { + glog.Infof("reloadKubelet took %s", time.Since(now)) + }() + kubeadmCfg, err := bsutil.GenerateKubeadmYAML(cfg, n, r) if err != nil { return errors.Wrap(err, "generating kubeadm cfg") @@ -696,24 +681,40 @@ func (k *Bootstrapper) UpdateNode(cfg config.ClusterConfig, n config.Node, r cru glog.Infof("kubelet %s config:\n%+v", kubeletCfg, cfg.KubernetesConfig) - if err := bsutil.TransferBinaries(cfg.KubernetesConfig, k.c); err != nil { + sm := sysinit.New(k.c) + + if err := bsutil.TransferBinaries(cfg.KubernetesConfig, k.c, sm); err != nil { return errors.Wrap(err, "downloading binaries") } - var cniFile []byte + files := []assets.CopyableFile{ + assets.NewMemoryAssetTarget(kubeadmCfg, bsutil.KubeadmYamlPath+".new", "0640"), + assets.NewMemoryAssetTarget(kubeletCfg, bsutil.KubeletSystemdConfFile+".new", "0644"), + assets.NewMemoryAssetTarget(kubeletService, bsutil.KubeletServiceFile+".new", "0644"), + } + // Copy the default CNI config (k8s.conf), so that kubelet can successfully + // start a Pod in the case a user hasn't manually installed any CNI plugin + // and minikube was started with "--extra-config=kubelet.network-plugin=cni". if cfg.KubernetesConfig.EnableDefaultCNI { - cniFile = []byte(defaultCNIConfig) + files = append(files, assets.NewMemoryAssetTarget([]byte(defaultCNIConfig), bsutil.DefaultCNIConfigPath, "0644")) } - // Install assets into temporary files - files := bsutil.ConfigFileAssets(cfg.KubernetesConfig, kubeadmCfg, kubeletCfg, kubeletService, cniFile) + // Installs compatibility shims for non-systemd environments + kubeletPath := path.Join(vmpath.GuestPersistentDir, "binaries", cfg.KubernetesConfig.KubernetesVersion, "kubectl") + shims, err := sm.GenerateInitShim("kubelet", kubeletPath, bsutil.KubeletSystemdConfFile) + if err != nil { + return errors.Wrap(err, "shim") + } + files = append(files, shims...) + if err := copyFiles(k.c, files); err != nil { - return err + return errors.Wrap(err, "copy") } - if err := reloadKubelet(k.c); err != nil { - return err + if err := startKubeletIfRequired(k.c, sm); err != nil { + return errors.Wrap(err, "reload") } + return nil } @@ -736,7 +737,12 @@ func copyFiles(runner command.Runner, files []assets.CopyableFile) error { return nil } -func reloadKubelet(runner command.Runner) error { +func startKubeletIfRequired(runner command.Runner, sm sysinit.Manager) error { + now := time.Now() + defer func() { + glog.Infof("reloadKubelet took %s", time.Since(now)) + }() + svc := bsutil.KubeletServiceFile conf := bsutil.KubeletSystemdConfFile @@ -746,11 +752,12 @@ func reloadKubelet(runner command.Runner) error { return nil } - startCmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo cp %s.new %s && sudo cp %s.new %s && sudo systemctl daemon-reload && sudo systemctl restart kubelet", svc, svc, conf, conf)) + startCmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo cp %s.new %s && sudo cp %s.new %s", svc, svc, conf, conf)) if _, err := runner.RunCmd(startCmd); err != nil { return errors.Wrap(err, "starting kubelet") } - return nil + + return sm.Start("kubelet") } // applyKicOverlay applies the CNI plugin needed to make kic work diff --git a/pkg/minikube/cluster/pause.go b/pkg/minikube/cluster/pause.go index d7661f1274..09b1a10c24 100644 --- a/pkg/minikube/cluster/pause.go +++ b/pkg/minikube/cluster/pause.go @@ -21,27 +21,33 @@ import ( "github.com/pkg/errors" "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/cruntime" - "k8s.io/minikube/pkg/minikube/kubelet" + "k8s.io/minikube/pkg/minikube/sysinit" ) // Pause pauses a Kubernetes cluster func Pause(cr cruntime.Manager, r command.Runner, namespaces []string) ([]string, error) { ids := []string{} + // Disable the kubelet so it does not attempt to restart paused pods - if err := kubelet.Disable(r); err != nil { + sm := sysinit.New(r) + if err := sm.Disable("kubelet"); err != nil { return ids, errors.Wrap(err, "kubelet disable") } - if err := kubelet.Stop(r); err != nil { + + if err := sm.Stop("kubelet"); err != nil { return ids, errors.Wrap(err, "kubelet stop") } + ids, err := cr.ListContainers(cruntime.ListOptions{State: cruntime.Running, Namespaces: namespaces}) if err != nil { return ids, errors.Wrap(err, "list running") } + if len(ids) == 0 { glog.Warningf("no running containers to pause") return ids, nil } + return ids, cr.PauseContainers(ids) } @@ -59,11 +65,14 @@ func Unpause(cr cruntime.Manager, r command.Runner, namespaces []string) ([]stri return ids, errors.Wrap(err, "unpause") } - if err := kubelet.Enable(r); err != nil { + sm := sysinit.New(r) + if err := sm.Enable("kubelet"); err != nil { return ids, errors.Wrap(err, "kubelet enable") } - if err := kubelet.Start(r); err != nil { + + if err := sm.Start("kubelet"); err != nil { return ids, errors.Wrap(err, "kubelet start") } + return ids, nil } diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go index 89a2e3912a..8eadb57392 100644 --- a/pkg/minikube/cruntime/containerd.go +++ b/pkg/minikube/cruntime/containerd.go @@ -32,6 +32,7 @@ import ( "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/download" "k8s.io/minikube/pkg/minikube/out" + "k8s.io/minikube/pkg/minikube/sysinit" ) const ( @@ -116,6 +117,7 @@ type Containerd struct { Runner CommandRunner ImageRepository string KubernetesVersion semver.Version + Init sysinit.Manager } // Name is a human readable name for containerd @@ -158,9 +160,7 @@ func (r *Containerd) DefaultCNI() bool { // Active returns if containerd is active on the host func (r *Containerd) Active() bool { - c := exec.Command("sudo", "systemctl", "is-active", "--quiet", "service", "containerd") - _, err := r.Runner.RunCmd(c) - return err == nil + return r.Init.Active("containerd") } // Available returns an error if it is not possible to use this runtime on a host @@ -208,21 +208,14 @@ func (r *Containerd) Enable(disOthers bool) error { if err := enableIPForwarding(r.Runner); err != nil { return err } + // Otherwise, containerd will fail API requests with 'Unimplemented' - c := exec.Command("sudo", "systemctl", "restart", "containerd") - if _, err := r.Runner.RunCmd(c); err != nil { - return errors.Wrap(err, "restart containerd") - } - return nil + return r.Init.Restart("containerd") } // Disable idempotently disables containerd on a host func (r *Containerd) Disable() error { - c := exec.Command("sudo", "systemctl", "stop", "-f", "containerd") - if _, err := r.Runner.RunCmd(c); err != nil { - return errors.Wrapf(err, "stop containerd") - } - return nil + return r.Init.ForceStop("containerd") } // ImageExists checks if an image exists, expected input format diff --git a/pkg/minikube/cruntime/crio.go b/pkg/minikube/cruntime/crio.go index 5678a9d6a4..250d765df4 100644 --- a/pkg/minikube/cruntime/crio.go +++ b/pkg/minikube/cruntime/crio.go @@ -28,6 +28,7 @@ import ( "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/download" "k8s.io/minikube/pkg/minikube/out" + "k8s.io/minikube/pkg/minikube/sysinit" ) const ( @@ -41,6 +42,7 @@ type CRIO struct { Runner CommandRunner ImageRepository string KubernetesVersion semver.Version + Init sysinit.Manager } // generateCRIOConfig sets up /etc/crio/crio.conf @@ -104,9 +106,7 @@ func (r *CRIO) Available() error { // Active returns if CRIO is active on the host func (r *CRIO) Active() bool { - c := exec.Command("sudo", "systemctl", "is-active", "--quiet", "service", "crio") - _, err := r.Runner.RunCmd(c) - return err == nil + return r.Init.Active("crio") } // Enable idempotently enables CRIO on a host @@ -125,19 +125,12 @@ func (r *CRIO) Enable(disOthers bool) error { if err := enableIPForwarding(r.Runner); err != nil { return err } - - if _, err := r.Runner.RunCmd(exec.Command("sudo", "systemctl", "restart", "crio")); err != nil { - return errors.Wrapf(err, "enable crio.") - } - return nil + return r.Init.Start("crio") } // Disable idempotently disables CRIO on a host func (r *CRIO) Disable() error { - if _, err := r.Runner.RunCmd(exec.Command("sudo", "systemctl", "stop", "-f", "crio")); err != nil { - return errors.Wrapf(err, "disable crio.") - } - return nil + return r.Init.ForceStop("crio") } // ImageExists checks if an image exists diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go index d7153d0830..9bde921993 100644 --- a/pkg/minikube/cruntime/cruntime.go +++ b/pkg/minikube/cruntime/cruntime.go @@ -28,6 +28,7 @@ import ( "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/out" + "k8s.io/minikube/pkg/minikube/sysinit" ) // ContainerState is the run state of a container @@ -131,13 +132,27 @@ type ListOptions struct { // New returns an appropriately configured runtime func New(c Config) (Manager, error) { + sm := sysinit.New(c.Runner) + switch c.Type { case "", "docker": - return &Docker{Socket: c.Socket, Runner: c.Runner}, nil + return &Docker{Socket: c.Socket, Runner: c.Runner, Init: sm}, nil case "crio", "cri-o": - return &CRIO{Socket: c.Socket, Runner: c.Runner, ImageRepository: c.ImageRepository, KubernetesVersion: c.KubernetesVersion}, nil + return &CRIO{ + Socket: c.Socket, + Runner: c.Runner, + ImageRepository: c.ImageRepository, + KubernetesVersion: c.KubernetesVersion, + Init: sm, + }, nil case "containerd": - return &Containerd{Socket: c.Socket, Runner: c.Runner, ImageRepository: c.ImageRepository, KubernetesVersion: c.KubernetesVersion}, nil + return &Containerd{ + Socket: c.Socket, + Runner: c.Runner, + ImageRepository: c.ImageRepository, + KubernetesVersion: c.KubernetesVersion, + Init: sm, + }, nil default: return nil, fmt.Errorf("unknown runtime type: %q", c.Type) } diff --git a/pkg/minikube/cruntime/cruntime_test.go b/pkg/minikube/cruntime/cruntime_test.go index 9523f1126a..aef420cf84 100644 --- a/pkg/minikube/cruntime/cruntime_test.go +++ b/pkg/minikube/cruntime/cruntime_test.go @@ -23,6 +23,7 @@ import ( "strings" "testing" + "github.com/golang/glog" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/pkg/errors" @@ -406,12 +407,27 @@ func (f *FakeRunner) crictl(args []string, _ bool) (string, error) { // systemctl is a fake implementation of systemctl func (f *FakeRunner) systemctl(args []string, root bool) (string, error) { // nolint result 0 (string) is always "" + glog.Infof("fake systemctl: %v", args) action := args[0] - svcs := args[1:] + + if action == "--version" { + return "systemd 123 (321.2-1)", nil + } + + if action == "daemon-reload" { + return "ok", nil + } + + var svcs []string + if len(args) > 0 { + svcs = args[1:] + } + // force if svcs[0] == "-f" { svcs = svcs[1:] } + out := "" for i, arg := range args { @@ -496,7 +512,6 @@ func TestVersion(t *testing.T) { // defaultServices reflects the default boot state for the minikube VM var defaultServices = map[string]serviceState{ "docker": SvcRunning, - "docker.socket": SvcRunning, "crio": SvcExited, "crio-shutdown": SvcExited, "containerd": SvcExited, @@ -507,7 +522,7 @@ func TestDisable(t *testing.T) { runtime string want []string }{ - {"docker", []string{"sudo", "systemctl", "stop", "-f", "docker", "docker.socket"}}, + {"docker", []string{"sudo", "systemctl", "stop", "-f", "docker"}}, {"crio", []string{"sudo", "systemctl", "stop", "-f", "crio"}}, {"containerd", []string{"sudo", "systemctl", "stop", "-f", "containerd"}}, } @@ -539,23 +554,20 @@ func TestEnable(t *testing.T) { }{ {"docker", map[string]serviceState{ "docker": SvcRunning, - "docker.socket": SvcRunning, "containerd": SvcExited, "crio": SvcExited, "crio-shutdown": SvcExited, }}, {"containerd", map[string]serviceState{ "docker": SvcExited, - "docker.socket": SvcExited, "containerd": SvcRestarted, "crio": SvcExited, "crio-shutdown": SvcExited, }}, {"crio", map[string]serviceState{ "docker": SvcExited, - "docker.socket": SvcExited, "containerd": SvcExited, - "crio": SvcRestarted, + "crio": SvcRunning, "crio-shutdown": SvcExited, }}, } diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go index df9577f47c..236ae3510a 100644 --- a/pkg/minikube/cruntime/docker.go +++ b/pkg/minikube/cruntime/docker.go @@ -32,6 +32,7 @@ import ( "k8s.io/minikube/pkg/minikube/docker" "k8s.io/minikube/pkg/minikube/download" "k8s.io/minikube/pkg/minikube/out" + "k8s.io/minikube/pkg/minikube/sysinit" ) // KubernetesContainerPrefix is the prefix of each kubernetes container @@ -56,6 +57,7 @@ func (e *ErrISOFeature) Error() string { type Docker struct { Socket string Runner CommandRunner + Init sysinit.Manager } // Name is a human readable name for Docker @@ -97,9 +99,7 @@ func (r *Docker) Available() error { // Active returns if docker is active on the host func (r *Docker) Active() bool { - c := exec.Command("sudo", "systemctl", "is-active", "--quiet", "service", "docker") - _, err := r.Runner.RunCmd(c) - return err == nil + return r.Init.Active("docker") } // Enable idempotently enables Docker on a host @@ -109,29 +109,18 @@ func (r *Docker) Enable(disOthers bool) error { glog.Warningf("disableOthers: %v", err) } } - c := exec.Command("sudo", "systemctl", "start", "docker") - if _, err := r.Runner.RunCmd(c); err != nil { - return errors.Wrap(err, "enable docker.") - } - return nil + + return r.Init.Start("docker") } // Restart restarts Docker on a host func (r *Docker) Restart() error { - c := exec.Command("sudo", "systemctl", "restart", "docker") - if _, err := r.Runner.RunCmd(c); err != nil { - return errors.Wrap(err, "restarting docker.") - } - return nil + return r.Init.Restart("docker") } // Disable idempotently disables Docker on a host func (r *Docker) Disable() error { - c := exec.Command("sudo", "systemctl", "stop", "-f", "docker", "docker.socket") - if _, err := r.Runner.RunCmd(c); err != nil { - return errors.Wrap(err, "disable docker") - } - return nil + return r.Init.ForceStop("docker") } // ImageExists checks if an image exists diff --git a/pkg/minikube/kubelet/kubelet.go b/pkg/minikube/kubelet/kubelet.go deleted file mode 100644 index 8f15e6aca0..0000000000 --- a/pkg/minikube/kubelet/kubelet.go +++ /dev/null @@ -1,117 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubelet - -import ( - "fmt" - "os/exec" - "strings" - "time" - - "github.com/golang/glog" - "github.com/pkg/errors" - "k8s.io/minikube/pkg/minikube/command" - "k8s.io/minikube/pkg/util/retry" -) - -// Stop idempotently stops the kubelet -func Stop(cr command.Runner) error { - return stop(cr, false) -} - -// ForceStop idempotently force stops the kubelet -func ForceStop(cr command.Runner) error { - return stop(cr, true) -} - -// stop dempotently stops the kubelet -func stop(cr command.Runner, force bool) error { - glog.Infof("stopping kubelet ...") - stp := func() error { - cmd := exec.Command("sudo", "systemctl", "stop", "kubelet.service") - if force { - cmd = exec.Command("sudo", "systemctl", "stop", "-f", "kubelet.service") - } - if rr, err := cr.RunCmd(cmd); err != nil { - return fmt.Errorf("temporary error for %q : %v", rr.Command(), err) - } - cmd = exec.Command("sudo", "systemctl", "show", "-p", "SubState", "kubelet") - rr, err := cr.RunCmd(cmd) - if err != nil { - return fmt.Errorf("temporary error: for %q : %v", rr.Command(), err) - } - if !strings.Contains(rr.Stdout.String(), "dead") && !strings.Contains(rr.Stdout.String(), "failed") { - return fmt.Errorf("unexpected kubelet state: %q", rr.Stdout.String()) - } - return nil - } - - if err := retry.Expo(stp, 1*time.Second, time.Minute, 2); err != nil { - return errors.Wrapf(err, "error stopping kubelet") - } - return nil -} - -// Start starts the kubelet -func Start(cr command.Runner) error { - glog.Infof("restarting kubelet.service ...") - c := exec.Command("sudo", "systemctl", "start", "kubelet") - if _, err := cr.RunCmd(c); err != nil { - return err - } - return nil -} - -// Restart restarts the kubelet -func Restart(cr command.Runner) error { - glog.Infof("restarting kubelet.service ...") - c := exec.Command("sudo", "systemctl", "restart", "kubelet.service") - if _, err := cr.RunCmd(c); err != nil { - return err - } - return nil -} - -// Check checks on the status of the kubelet -func Check(cr command.Runner) error { - glog.Infof("checking for running kubelet ...") - c := exec.Command("sudo", "systemctl", "is-active", "--quiet", "service", "kubelet") - if _, err := cr.RunCmd(c); err != nil { - return errors.Wrap(err, "check kubelet") - } - return nil -} - -// Disable disables the Kubelet -func Disable(cr command.Runner) error { - glog.Infof("disabling kubelet ...") - c := exec.Command("sudo", "systemctl", "disable", "kubelet") - if _, err := cr.RunCmd(c); err != nil { - return errors.Wrap(err, "disable") - } - return nil -} - -// Enable enables the Kubelet -func Enable(cr command.Runner) error { - glog.Infof("enabling kubelet ...") - c := exec.Command("sudo", "systemctl", "enable", "kubelet") - if _, err := cr.RunCmd(c); err != nil { - return errors.Wrap(err, "enable") - } - return nil -} diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index b2129662cd..46af9f8bd8 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -125,6 +125,7 @@ func Start(cc config.ClusterConfig, n config.Node, existingAddons map[string]boo // setup kubeadm (must come after setupKubeconfig) bs = setupKubeAdm(machineAPI, cc, n) + err = bs.StartCluster(cc) if err != nil { exit.WithLogEntries("Error starting cluster", err, logs.FindProblems(cr, bs, cc, mRunner)) diff --git a/pkg/minikube/registry/drvs/none/none.go b/pkg/minikube/registry/drvs/none/none.go index fd7e42a6de..3c7ef6b48c 100644 --- a/pkg/minikube/registry/drvs/none/none.go +++ b/pkg/minikube/registry/drvs/none/none.go @@ -52,10 +52,15 @@ func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) { } func status() registry.State { - if _, err := exec.LookPath("systemctl"); err != nil { - return registry.State{Error: err, Fix: "Use a systemd based Linux distribution", Doc: "https://minikube.sigs.k8s.io/docs/reference/drivers/none/"} + _, err := exec.LookPath("iptables") + if err != nil { + return registry.State{Error: err, Fix: "iptables must be installed", Doc: "https://minikube.sigs.k8s.io/docs/reference/drivers/none/"} } + /* if _, err := exec.LookPath("systemctl"); err != nil { + return registry.State{Error: err, Fix: "Use a systemd based Linux distribution", Doc: "https://minikube.sigs.k8s.io/docs/reference/drivers/none/"} + }*/ + if _, err := exec.LookPath("docker"); err != nil { return registry.State{Error: err, Installed: false, Fix: "Install docker", Doc: "https://minikube.sigs.k8s.io/docs/reference/drivers/none/"} } From 7d2b045c40eb6eabf0607f8c1d32835b6bfb2318 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Wed, 8 Apr 2020 18:23:09 -0700 Subject: [PATCH 49/67] Remove obsolete comment --- pkg/minikube/registry/drvs/none/none.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pkg/minikube/registry/drvs/none/none.go b/pkg/minikube/registry/drvs/none/none.go index 3c7ef6b48c..ec8cea71d8 100644 --- a/pkg/minikube/registry/drvs/none/none.go +++ b/pkg/minikube/registry/drvs/none/none.go @@ -57,10 +57,6 @@ func status() registry.State { return registry.State{Error: err, Fix: "iptables must be installed", Doc: "https://minikube.sigs.k8s.io/docs/reference/drivers/none/"} } - /* if _, err := exec.LookPath("systemctl"); err != nil { - return registry.State{Error: err, Fix: "Use a systemd based Linux distribution", Doc: "https://minikube.sigs.k8s.io/docs/reference/drivers/none/"} - }*/ - if _, err := exec.LookPath("docker"); err != nil { return registry.State{Error: err, Installed: false, Fix: "Install docker", Doc: "https://minikube.sigs.k8s.io/docs/reference/drivers/none/"} } From b742c84ee7686546c9d4e9822620fbe3f3b01259 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Wed, 8 Apr 2020 18:26:17 -0700 Subject: [PATCH 50/67] Add sysinit package --- pkg/minikube/sysinit/openrc.go | 170 ++++++++++++++++++++++++++++++++ pkg/minikube/sysinit/sysinit.go | 85 ++++++++++++++++ pkg/minikube/sysinit/systemd.go | 98 ++++++++++++++++++ 3 files changed, 353 insertions(+) create mode 100644 pkg/minikube/sysinit/openrc.go create mode 100644 pkg/minikube/sysinit/sysinit.go create mode 100644 pkg/minikube/sysinit/systemd.go diff --git a/pkg/minikube/sysinit/openrc.go b/pkg/minikube/sysinit/openrc.go new file mode 100644 index 0000000000..822c4157d0 --- /dev/null +++ b/pkg/minikube/sysinit/openrc.go @@ -0,0 +1,170 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package sysinit provides an abstraction over init systems like systemctl +package sysinit + +import ( + "bytes" + "context" + "html/template" + "os/exec" + "path" + "time" + + "github.com/golang/glog" + "github.com/pkg/errors" + "k8s.io/minikube/pkg/minikube/assets" + "k8s.io/minikube/pkg/minikube/vmpath" +) + +var restartWrapper = `#!/bin/bash +# Wrapper script to emulate systemd restart on non-systemd systems +readonly UNIT_PATH=$1 + +while true; do + if [[ -f "${UNIT_PATH}" ]]; then + eval $(egrep "^ExecStart=" "${UNIT_PATH}" | cut -d"=" -f2-) + fi + sleep 1 +done +` + +var initScriptTmpl = template.Must(template.New("initScript").Parse(`#!/bin/bash +# OpenRC init script shim for systemd units +readonly NAME="{{.Name}}" +readonly RESTART_WRAPPER="{{.Wrapper}}" +readonly UNIT_PATH="{{.Unit}}" +readonly PID_PATH="/var/run/${NAME}.pid" + +function start() { + start-stop-daemon --oknodo --pidfile "${PID_PATH}" --background --start --make-pid --exec "${RESTART_WRAPPER}" "${UNIT_PATH}" +} + +function stop() { + if [[ -f "${PID_PATH}" ]]; then + pkill -P "$(cat ${PID_PATH})" + fi + start-stop-daemon --oknodo --pidfile "${PID_PATH}" --stop +} + +case "$1" in + start) + start + ;; + stop) + stop + ;; + restart) + stop + start + ;; + status) + start-stop-daemon --pidfile "${PID_PATH}" --status + ;; + *) + echo "Usage: {{.Name}} {start|stop|restart|status}" + exit 1 + ;; +esac +`)) + +// OpenRC is a service manager for OpenRC-like init systems +type OpenRC struct { + r Runner +} + +// Name returns the name of the init system +func (s *OpenRC) Name() string { + return "OpenRC" +} + +// Active checks if a service is running +func (s *OpenRC) Active(svc string) bool { + _, err := s.r.RunCmd(exec.Command("sudo", "service", svc, "status")) + return err == nil +} + +// Start starts a service idempotently +func (s *OpenRC) Start(svc string) error { + if s.Active(svc) { + return nil + } + ctx, cb := context.WithTimeout(context.Background(), 5*time.Second) + defer cb() + + rr, err := s.r.RunCmd(exec.CommandContext(ctx, "sudo", "service", svc, "start")) + glog.Infof("start output: %s", rr.Output()) + return err +} + +// Disable does nothing +func (s *OpenRC) Disable(svc string) error { + return nil +} + +// Enable does nothing +func (s *OpenRC) Enable(svc string) error { + return nil +} + +// Restart restarts a service +func (s *OpenRC) Restart(svc string) error { + rr, err := s.r.RunCmd(exec.Command("sudo", "service", svc, "restart")) + glog.Infof("restart output: %s", rr.Output()) + return err +} + +// Stop stops a service +func (s *OpenRC) Stop(svc string) error { + rr, err := s.r.RunCmd(exec.Command("sudo", "service", svc, "stop")) + glog.Infof("stop output: %s", rr.Output()) + return err +} + +// ForceStop stops a service with prejuidice +func (s *OpenRC) ForceStop(svc string) error { + return s.Stop(svc) +} + +// GenerateInitShim generates any additional init files required for this service +func (s *OpenRC) GenerateInitShim(svc string, binary string, unit string) ([]assets.CopyableFile, error) { + restartWrapperPath := path.Join(vmpath.GuestPersistentDir, "openrc-restart-wrapper.sh") + + opts := struct { + Binary string + Wrapper string + Name string + Unit string + }{ + Name: svc, + Binary: binary, + Wrapper: restartWrapperPath, + Unit: unit, + } + + var b bytes.Buffer + if err := initScriptTmpl.Execute(&b, opts); err != nil { + return nil, errors.Wrap(err, "template execute") + } + + files := []assets.CopyableFile{ + assets.NewMemoryAssetTarget([]byte(restartWrapper), restartWrapperPath, "0755"), + assets.NewMemoryAssetTarget(b.Bytes(), path.Join("/etc/init.d/", svc), "0755"), + } + + return files, nil +} diff --git a/pkg/minikube/sysinit/sysinit.go b/pkg/minikube/sysinit/sysinit.go new file mode 100644 index 0000000000..fc20479ded --- /dev/null +++ b/pkg/minikube/sysinit/sysinit.go @@ -0,0 +1,85 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package sysinit + +import ( + "os/exec" + + "k8s.io/minikube/pkg/minikube/assets" + "k8s.io/minikube/pkg/minikube/command" +) + +var cachedSystemdCheck *bool + +// Runner is the subset of command.Runner this package consumes +type Runner interface { + RunCmd(cmd *exec.Cmd) (*command.RunResult, error) +} + +// Manager is a common interface for init systems +type Manager interface { + // Name returns the name of the init manager + Name() string + + // Active returns if a service is active + Active(string) bool + + // Disable disables a service + Disable(string) error + + // Enable enables a service + Enable(string) error + + // Start starts a service idempotently + Start(string) error + + // Restart restarts a service + Restart(string) error + + // Stop stops a service + Stop(string) error + + // ForceStop stops a service with prejudice + ForceStop(string) error + + // GenerateInitShim generates any additional init files required for this service + GenerateInitShim(svc string, binary string, unit string) ([]assets.CopyableFile, error) +} + +// New returns an appropriately configured service manager +func New(r Runner) Manager { + // If we are not provided a runner, we can't do anything anyways + if r == nil { + return nil + } + + var systemd bool + + // Caching the result is important, as this manager may be created in many places, + // and ssh calls are expensive on some drivers, such as Docker. + if cachedSystemdCheck != nil { + systemd = *cachedSystemdCheck + } else { + systemd = usesSystemd(r) + cachedSystemdCheck = &systemd + } + + if systemd { + return &Systemd{r: r} + } + return &OpenRC{r: r} +} diff --git a/pkg/minikube/sysinit/systemd.go b/pkg/minikube/sysinit/systemd.go new file mode 100644 index 0000000000..4bb53a186b --- /dev/null +++ b/pkg/minikube/sysinit/systemd.go @@ -0,0 +1,98 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package sysinit provides an abstraction over init systems like systemctl +package sysinit + +import ( + "os/exec" + + "k8s.io/minikube/pkg/minikube/assets" +) + +// Systemd is a service manager for systemd distributions +type Systemd struct { + r Runner +} + +// Name returns the name of the init system +func (s *Systemd) Name() string { + return "systemd" +} + +// reload reloads systemd configuration +func (s *Systemd) reload() error { + _, err := s.r.RunCmd(exec.Command("sudo", "systemctl", "daemon-reload")) + return err +} + +// Active checks if a service is running +func (s *Systemd) Active(svc string) bool { + _, err := s.r.RunCmd(exec.Command("sudo", "systemctl", "is-active", "--quiet", "service", svc)) + return err == nil +} + +// Disable disables a service +func (s *Systemd) Disable(svc string) error { + _, err := s.r.RunCmd(exec.Command("sudo", "systemctl", "disable", svc)) + return err +} + +// Enable enables a service +func (s *Systemd) Enable(svc string) error { + _, err := s.r.RunCmd(exec.Command("sudo", "systemctl", "enable", svc)) + return err +} + +// Start starts a service +func (s *Systemd) Start(svc string) error { + if err := s.reload(); err != nil { + return err + } + _, err := s.r.RunCmd(exec.Command("sudo", "systemctl", "start", svc)) + return err +} + +// Restart restarts a service +func (s *Systemd) Restart(svc string) error { + if err := s.reload(); err != nil { + return err + } + _, err := s.r.RunCmd(exec.Command("sudo", "systemctl", "restart", svc)) + return err +} + +// Stop stops a service +func (s *Systemd) Stop(svc string) error { + _, err := s.r.RunCmd(exec.Command("sudo", "systemctl", "stop", svc)) + return err +} + +// ForceStop terminates a service with prejudice +func (s *Systemd) ForceStop(svc string) error { + _, err := s.r.RunCmd(exec.Command("sudo", "systemctl", "stop", "-f", svc)) + return err +} + +// GenerateInitShim does nothing for systemd +func (s *Systemd) GenerateInitShim(svc string, binary string, unit string) ([]assets.CopyableFile, error) { + return nil, nil +} + +func usesSystemd(r Runner) bool { + _, err := r.RunCmd(exec.Command("systemctl", "--version")) + return err == nil +} From 93d9e523583129561768893f16af947e3d4b51d0 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Wed, 8 Apr 2020 18:50:18 -0700 Subject: [PATCH 51/67] address review comments --- site/content/en/docs/handbook/pushing.md | 26 ++++++++++++++---------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/site/content/en/docs/handbook/pushing.md b/site/content/en/docs/handbook/pushing.md index 0f43ce99c2..d636640042 100644 --- a/site/content/en/docs/handbook/pushing.md +++ b/site/content/en/docs/handbook/pushing.md @@ -11,17 +11,17 @@ aliases: --- ## Comparison table for different methods -the answer depends on the container-runtime driver you choose. +The best method to push your image to minikube depends on the container-runtime you built your cluster with (default docker). Here is a comparison table to help you choose: | Method | Supported Runtimes | Issues | Performance | |--- |--- |--- |--- |--- | -| [docker-env command](http://localhost:1313/docs/handbook/pushing/#pushing-directly-to-the-in-cluster-docker-daemon) | only docker | | good | -| [podman-env command](http://localhost:1313/docs/handbook/pushing/#pushing-directly-to-in-cluster-crio) | only cri-o | | good | -| [cache add command](http://localhost:1313/docs/handbook/pushing/#push-images-using-cache-command) | all | | ok | -| [registry addon](http://localhost:1313/docs/handbook/pushing/#pushing-to-an-in-cluster-using-a-registry-addon) | all | work in progress for [docker on mac](https://github.com/kubernetes/minikube/issues/7535) | ok | -| [minikube ssh](http://localhost:1313/docs/handbook/pushing/#building-images-inside-of-minikube-using-ssh) | all | | best | +| [docker-env command](/docs/handbook/pushing/#1pushing-directly-to-the-in-cluster-docker-daemon-docker-env) | only docker | | good | +| [podman-env command](/docs/handbook/pushing/#3-pushing-directly-to-in-cluster-crio-podman-env) | only cri-o | | good | +| [cache add command](/pushing/#push-images-using-cache-command) | all | | ok | +| [registry addon](/docs/handbook/pushing/#4-pushing-to-an-in-cluster-using-registry-addon) | all | work in progress for [docker on mac](https://github.com/kubernetes/minikube/issues/7535) | ok | +| [minikube ssh](/docs/handbook/pushing/#5-building-images-inside-of-minikube-using-ssh) | all | | best | * note1 : the default container-runtime on minikube is 'docker'. @@ -54,10 +54,14 @@ docker build -t myimage . Remember to turn off the `imagePullPolicy:Always` (use `imagePullPolicy:IfNotPresent` or `imagePullPolicy:Never`), as otherwise Kubernetes won't use images you built locally. - -Please remember evalulating the docker-env is only valid for the current terminal. +{{% pageinfo %}} +Evaluating the docker-env is only valid for the current terminal. and by closing the terminal, you will go back to using your own system's docker daemon. -to verify your terminal is using minikuber's docker-env you can check the value of the environment MINIKUBE_ACTIVE_DOCKERD + +in some drivers such as Docker or Podman, you will need to re-do docker-env each time you restart your minikube. +{{% /pageinfo %}} + +To verify your terminal is using minikuber's docker-env you can check the value of the environment variable MINIKUBE_ACTIVE_DOCKERD to reflect the profile name. more information on [docker-env](https://minikube.sigs.k8s.io/docs/commands/docker-env/) @@ -146,8 +150,8 @@ docker push $(minikube ip):5000/test-img ## 5. Building images inside of minikube using SSH -Use `minikube ssh` to go inside a minikube node, and run the `docker build` directly there. -any command you run there will run against the same daemon that kubernetes is using. +Use `minikube ssh` to run commands inside the minikube node, and run the `docker build` directly there. +Any command you run there will run against the same daemon that kubernetes cluster is using. ```shell docker build From 794d78a788c988fa926b55cae3a8046ec210b44d Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Wed, 8 Apr 2020 18:55:21 -0700 Subject: [PATCH 52/67] document registery limitation on docker mac --- site/content/en/docs/drivers/docker.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/site/content/en/docs/drivers/docker.md b/site/content/en/docs/drivers/docker.md index bf47ed840b..248428753d 100644 --- a/site/content/en/docs/drivers/docker.md +++ b/site/content/en/docs/drivers/docker.md @@ -21,12 +21,16 @@ The Docker driver allows you to install Kubernetes into an existing Docker insta - On macOS, containers might get hung and require a restart of Docker for Desktop. See [docker/for-mac#1835](https://github.com/docker/for-mac/issues/1835) -- The `ingress` and `ingress-dns` addons are currently only supported on Linux. See [#7332](https://github.com/kubernetes/minikube/issues/7332) +- The `ingress`, `ingress-dns` and `registry` addons are currently only supported on Linux. See [#7332](https://github.com/kubernetes/minikube/issues/7332) and [#7535](https://github.com/kubernetes/minikube/issues/7535) - On WSL2 (experimental - see [#5392](https://github.com/kubernetes/minikube/issues/5392)), you may need to run: `sudo mkdir /sys/fs/cgroup/systemd && sudo mount -t cgroup -o none,name=systemd cgroup /sys/fs/cgroup/systemd`. +- Addon 'registry' for mac and windows is not supported yet and it is [a work in progress](https://github.com/kubernetes/minikube/issues/7535). + + + ## Troubleshooting - On macOS or Windows, you may need to restart Docker for Desktop if a command gets hung From a823b9b80b9646a00c3937ce0708b53cd0aea3a5 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Wed, 8 Apr 2020 19:09:02 -0700 Subject: [PATCH 53/67] address review comments --- site/content/en/docs/handbook/pushing.md | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/site/content/en/docs/handbook/pushing.md b/site/content/en/docs/handbook/pushing.md index d636640042..3647aa4058 100644 --- a/site/content/en/docs/handbook/pushing.md +++ b/site/content/en/docs/handbook/pushing.md @@ -72,10 +72,19 @@ more information on [docker-env](https://minikube.sigs.k8s.io/docs/commands/dock From the host, you can push a Docker image directly to minikube. It will also be cached for all minikube clusters . ```shell -minikube cache add ubuntu:16.04 +minikube cache add alpine:latest ``` -The add command will store the requested image to `$MINIKUBE_HOME/cache/images`, and load it into the VM's container runtime environment next time `minikube start` is called. +The add command will store the requested image to `$MINIKUBE_HOME/cache/images`, and load it into the minikube cluster's container runtime environment automatically. + +{{% pageinfo %}} +if your image changes after your cached it, you could do `cache reload` to ensure minikube gets the last updates. + +``` +shell +minikube cache reload +``` +{{% /pageinfo %}} To display images you have added to the cache: @@ -85,11 +94,6 @@ minikube cache list This listing will not include the images minikube's built-in system images. -to ensure your running cluster has cached the updated images use reload: - -```shell -minikube cache reload -``` ```shell From 620823e77b0483dbacc5eab8592ee6aaf09145c3 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Wed, 8 Apr 2020 19:10:04 -0700 Subject: [PATCH 54/67] address review comments --- site/content/en/docs/handbook/pushing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/content/en/docs/handbook/pushing.md b/site/content/en/docs/handbook/pushing.md index 3647aa4058..ed2cc65506 100644 --- a/site/content/en/docs/handbook/pushing.md +++ b/site/content/en/docs/handbook/pushing.md @@ -69,7 +69,7 @@ more information on [docker-env](https://minikube.sigs.k8s.io/docs/commands/dock ## 2.Push images using 'cache' command. -From the host, you can push a Docker image directly to minikube. It will also be cached for all minikube clusters . +From your host, you can push a Docker image directly to minikube. This image will be cached and automatically pulled into all future minikube clusters created on the machine ```shell minikube cache add alpine:latest From 89b28c482e9e0549125cee0b8b88e1ed09dd803a Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Wed, 8 Apr 2020 20:14:55 -0700 Subject: [PATCH 55/67] Filter describe for non-running pods --- test/integration/helpers.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/helpers.go b/test/integration/helpers.go index 7daac4aa6f..af8f8698d9 100644 --- a/test/integration/helpers.go +++ b/test/integration/helpers.go @@ -241,8 +241,8 @@ func clusterLogs(t *testing.T, profile string) { t.Logf("(dbg) %s:\n%s", rr.Command(), rr.Output()) } - t.Logf("------++> post-mortem[%s]: describe pods", t.Name()) - rr, err = Run(t, exec.Command("kubectl", "--context", profile, "describe", "po", "-A")) + t.Logf("------++> post-mortem[%s]: !running pods", t.Name()) + rr, err = Run(t, exec.Command("kubectl", "--context", profile, "describe", "po", "-A", "--field-selector=status.phase!=Running")) if err != nil { t.Logf("%s: %v", rr.Command(), err) } else { From 9637790fa0f69d0fa6a1a05d97b94903baa1865c Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Tue, 7 Apr 2020 18:23:50 -0700 Subject: [PATCH 56/67] detect docker service not running --- pkg/minikube/registry/drvs/docker/docker.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pkg/minikube/registry/drvs/docker/docker.go b/pkg/minikube/registry/drvs/docker/docker.go index a9191a35d9..e6e0d39e8a 100644 --- a/pkg/minikube/registry/drvs/docker/docker.go +++ b/pkg/minikube/registry/drvs/docker/docker.go @@ -78,9 +78,14 @@ func status() registry.State { defer cancel() err = exec.CommandContext(ctx, oci.Docker, "info").Run() - if err != nil { - return registry.State{Error: err, Installed: true, Healthy: false, Fix: "Docker is not running or is responding too slow. Try: restarting docker desktop."} + + if ctx.Err() == context.DeadlineExceeded { + return registry.State{Error: err, Installed: true, Healthy: false, Fix: "Docker is responding too slow. Try: restarting docker."} + } + if err != nil{ + return registry.State{Error: err, Installed: true, Healthy: false, Fix: "Docker service is not running. Try: starting docker service."} } return registry.State{Installed: true, Healthy: true} } + From 62d271c3fe2e8f3c11e45cc43ea65fe89f2ea08e Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Tue, 7 Apr 2020 19:15:17 -0700 Subject: [PATCH 57/67] more user-friendly exit when docker is not running --- pkg/minikube/node/cache.go | 10 ++++++++-- pkg/minikube/registry/drvs/docker/docker.go | 3 +-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/pkg/minikube/node/cache.go b/pkg/minikube/node/cache.go index ebeb04fdc0..233f0d5ff9 100644 --- a/pkg/minikube/node/cache.go +++ b/pkg/minikube/node/cache.go @@ -19,6 +19,7 @@ package node import ( "os" "runtime" + "strings" "github.com/golang/glog" "github.com/spf13/viper" @@ -111,8 +112,13 @@ func beginDownloadKicArtifacts(g *errgroup.Group) { // WaitDownloadKicArtifacts blocks until the required artifacts for KIC are downloaded. func waitDownloadKicArtifacts(g *errgroup.Group) { if err := g.Wait(); err != nil { - glog.Errorln("Error downloading kic artifacts: ", err) - return + if strings.Contains(err.Error(), "Cannot connect to the Docker daemon") { + out.WarningT("Failed to connect to docker daemon: {{.error}}", out.V{"error": err.Error()}) + exit.UsageT("Please make sure Docker service is running.") + } else { + exit.WithError("Failed to download kic base image", err) + } + } glog.Info("Successfully downloaded all kic artifacts") } diff --git a/pkg/minikube/registry/drvs/docker/docker.go b/pkg/minikube/registry/drvs/docker/docker.go index e6e0d39e8a..113cebb54f 100644 --- a/pkg/minikube/registry/drvs/docker/docker.go +++ b/pkg/minikube/registry/drvs/docker/docker.go @@ -82,10 +82,9 @@ func status() registry.State { if ctx.Err() == context.DeadlineExceeded { return registry.State{Error: err, Installed: true, Healthy: false, Fix: "Docker is responding too slow. Try: restarting docker."} } - if err != nil{ + if err != nil { return registry.State{Error: err, Installed: true, Healthy: false, Fix: "Docker service is not running. Try: starting docker service."} } return registry.State{Installed: true, Healthy: true} } - From 3d8bb44fdbad6c6ff9a38b89de298c842ffec601 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Tue, 7 Apr 2020 19:25:24 -0700 Subject: [PATCH 58/67] tweak error msg --- pkg/minikube/node/cache.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/minikube/node/cache.go b/pkg/minikube/node/cache.go index 233f0d5ff9..f17ee30b66 100644 --- a/pkg/minikube/node/cache.go +++ b/pkg/minikube/node/cache.go @@ -113,7 +113,7 @@ func beginDownloadKicArtifacts(g *errgroup.Group) { func waitDownloadKicArtifacts(g *errgroup.Group) { if err := g.Wait(); err != nil { if strings.Contains(err.Error(), "Cannot connect to the Docker daemon") { - out.WarningT("Failed to connect to docker daemon: {{.error}}", out.V{"error": err.Error()}) + out.WarningT("Failed to connect to docker daemon : {{.error}}", out.V{"error": err.Error()}) exit.UsageT("Please make sure Docker service is running.") } else { exit.WithError("Failed to download kic base image", err) From 0109af5b778c29b33d2107d0127cd0f62a6ed97d Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Wed, 8 Apr 2020 15:37:55 -0700 Subject: [PATCH 59/67] revert handling --- pkg/minikube/node/cache.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/pkg/minikube/node/cache.go b/pkg/minikube/node/cache.go index f17ee30b66..ebeb04fdc0 100644 --- a/pkg/minikube/node/cache.go +++ b/pkg/minikube/node/cache.go @@ -19,7 +19,6 @@ package node import ( "os" "runtime" - "strings" "github.com/golang/glog" "github.com/spf13/viper" @@ -112,13 +111,8 @@ func beginDownloadKicArtifacts(g *errgroup.Group) { // WaitDownloadKicArtifacts blocks until the required artifacts for KIC are downloaded. func waitDownloadKicArtifacts(g *errgroup.Group) { if err := g.Wait(); err != nil { - if strings.Contains(err.Error(), "Cannot connect to the Docker daemon") { - out.WarningT("Failed to connect to docker daemon : {{.error}}", out.V{"error": err.Error()}) - exit.UsageT("Please make sure Docker service is running.") - } else { - exit.WithError("Failed to download kic base image", err) - } - + glog.Errorln("Error downloading kic artifacts: ", err) + return } glog.Info("Successfully downloaded all kic artifacts") } From 030228406cab9ecad062d8e0cc516732a45da26d Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Wed, 8 Apr 2020 15:40:46 -0700 Subject: [PATCH 60/67] add better doc link --- pkg/minikube/registry/drvs/docker/docker.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/minikube/registry/drvs/docker/docker.go b/pkg/minikube/registry/drvs/docker/docker.go index 113cebb54f..8d592db6c7 100644 --- a/pkg/minikube/registry/drvs/docker/docker.go +++ b/pkg/minikube/registry/drvs/docker/docker.go @@ -70,7 +70,7 @@ func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) { func status() registry.State { _, err := exec.LookPath(oci.Docker) if err != nil { - return registry.State{Error: err, Installed: false, Healthy: false, Fix: "Docker is required.", Doc: "https://minikube.sigs.k8s.io/docs/reference/drivers/docker/"} + return registry.State{Error: err, Installed: false, Healthy: false, Fix: "Docker is required.", Doc: "https://minikube.sigs.k8s.io/docs/drivers/docker/#install-docker"} } // Allow no more than 3 seconds for docker info @@ -80,7 +80,7 @@ func status() registry.State { err = exec.CommandContext(ctx, oci.Docker, "info").Run() if ctx.Err() == context.DeadlineExceeded { - return registry.State{Error: err, Installed: true, Healthy: false, Fix: "Docker is responding too slow. Try: restarting docker."} + return registry.State{Error: err, Installed: true, Healthy: false, Fix: "Docker was too slow to respond. Try: restarting docker."} } if err != nil { return registry.State{Error: err, Installed: true, Healthy: false, Fix: "Docker service is not running. Try: starting docker service."} From ef0c1aea8b790080452423e71934133206d02ed8 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Wed, 8 Apr 2020 15:51:24 -0700 Subject: [PATCH 61/67] add driver docs url to failure --- cmd/minikube/cmd/start.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index eba9b9c7e9..fe0cfe2e85 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -521,7 +521,9 @@ func validateDriver(ds registry.DriverState, existing *config.ClusterConfig) { } if !viper.GetBool(force) { - exit.WithCodeT(exit.Unavailable, "Failed to validate '{{.driver}}' driver", out.V{"driver": name}) + docURL := fmt.Sprintf("https://minikube.sigs.k8s.io/docs/drivers/%s/", strings.ToLower(name)) + out.ErrT(out.URL, "Read more : {{.URL}}", out.V{"URL": docURL}) + exit.WithCodeT(exit.Unavailable, "Failed to validate '{{.driver}}' driver.", out.V{"driver": name}) } } } From 34ce512474948b7f0efcd00ebc0a28180363fc6c Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Wed, 8 Apr 2020 16:15:56 -0700 Subject: [PATCH 62/67] address code review --- pkg/minikube/registry/drvs/docker/docker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/minikube/registry/drvs/docker/docker.go b/pkg/minikube/registry/drvs/docker/docker.go index 8d592db6c7..c4a5cbfb59 100644 --- a/pkg/minikube/registry/drvs/docker/docker.go +++ b/pkg/minikube/registry/drvs/docker/docker.go @@ -83,7 +83,7 @@ func status() registry.State { return registry.State{Error: err, Installed: true, Healthy: false, Fix: "Docker was too slow to respond. Try: restarting docker."} } if err != nil { - return registry.State{Error: err, Installed: true, Healthy: false, Fix: "Docker service is not running. Try: starting docker service."} + return registry.State{Error: err, Installed: true, Healthy: false, Fix: "Docker is not running. Please start the docker service."} } return registry.State{Installed: true, Healthy: true} From 3057b02384045d8c85925aa74c030f1f67bacf58 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Wed, 8 Apr 2020 21:48:10 -0700 Subject: [PATCH 63/67] add doc links correctly --- cmd/minikube/cmd/start.go | 4 +--- pkg/minikube/registry/drvs/docker/docker.go | 6 +++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index fe0cfe2e85..eba9b9c7e9 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -521,9 +521,7 @@ func validateDriver(ds registry.DriverState, existing *config.ClusterConfig) { } if !viper.GetBool(force) { - docURL := fmt.Sprintf("https://minikube.sigs.k8s.io/docs/drivers/%s/", strings.ToLower(name)) - out.ErrT(out.URL, "Read more : {{.URL}}", out.V{"URL": docURL}) - exit.WithCodeT(exit.Unavailable, "Failed to validate '{{.driver}}' driver.", out.V{"driver": name}) + exit.WithCodeT(exit.Unavailable, "Failed to validate '{{.driver}}' driver", out.V{"driver": name}) } } } diff --git a/pkg/minikube/registry/drvs/docker/docker.go b/pkg/minikube/registry/drvs/docker/docker.go index c4a5cbfb59..fee546774e 100644 --- a/pkg/minikube/registry/drvs/docker/docker.go +++ b/pkg/minikube/registry/drvs/docker/docker.go @@ -70,7 +70,7 @@ func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) { func status() registry.State { _, err := exec.LookPath(oci.Docker) if err != nil { - return registry.State{Error: err, Installed: false, Healthy: false, Fix: "Docker is required.", Doc: "https://minikube.sigs.k8s.io/docs/drivers/docker/#install-docker"} + return registry.State{Error: err, Installed: false, Healthy: false, Fix: "Install Docker.", Doc: "https://minikube.sigs.k8s.io/docs/drivers/docker/#install-docker"} } // Allow no more than 3 seconds for docker info @@ -80,10 +80,10 @@ func status() registry.State { err = exec.CommandContext(ctx, oci.Docker, "info").Run() if ctx.Err() == context.DeadlineExceeded { - return registry.State{Error: err, Installed: true, Healthy: false, Fix: "Docker was too slow to respond. Try: restarting docker."} + return registry.State{Error: err, Installed: true, Healthy: false, Fix: "Docker responds too slow. Restart the Docker Service.", Doc: "https://minikube.sigs.k8s.io/docs/drivers/docker"} } if err != nil { - return registry.State{Error: err, Installed: true, Healthy: false, Fix: "Docker is not running. Please start the docker service."} + return registry.State{Error: err, Installed: true, Healthy: false, Fix: "Start the Docker Service.", Doc: "https://minikube.sigs.k8s.io/docs/drivers/docker"} } return registry.State{Installed: true, Healthy: true} From aba681ac301c198df7a5143d37c18409b623a241 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Wed, 8 Apr 2020 22:08:10 -0700 Subject: [PATCH 64/67] improve suggestions --- pkg/minikube/registry/drvs/docker/docker.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pkg/minikube/registry/drvs/docker/docker.go b/pkg/minikube/registry/drvs/docker/docker.go index fee546774e..1e512d9f87 100644 --- a/pkg/minikube/registry/drvs/docker/docker.go +++ b/pkg/minikube/registry/drvs/docker/docker.go @@ -21,9 +21,11 @@ import ( "fmt" "os/exec" "runtime" + "strings" "time" "github.com/docker/machine/libmachine/drivers" + "github.com/golang/glog" "k8s.io/minikube/pkg/drivers/kic" "k8s.io/minikube/pkg/drivers/kic/oci" "k8s.io/minikube/pkg/minikube/config" @@ -83,7 +85,14 @@ func status() registry.State { return registry.State{Error: err, Installed: true, Healthy: false, Fix: "Docker responds too slow. Restart the Docker Service.", Doc: "https://minikube.sigs.k8s.io/docs/drivers/docker"} } if err != nil { - return registry.State{Error: err, Installed: true, Healthy: false, Fix: "Start the Docker Service.", Doc: "https://minikube.sigs.k8s.io/docs/drivers/docker"} + glog.Infof("docker info returned error: %v", err) + if strings.Contains(err.Error(), "Cannot connect to the Docker daemon") { + return registry.State{Error: err, Installed: true, Healthy: false, Fix: "Start the Docker Service.", Doc: "https://minikube.sigs.k8s.io/docs/drivers/docker"} + } + // if we get here, something is really wrong on their docker. + // our best suggestion would be re-install latest docker. + return registry.State{Error: err, Installed: true, Healthy: false, Fix: "Re-install the latest version of Docker.", Doc: "https://minikube.sigs.k8s.io/docs/drivers/docker"} + } return registry.State{Installed: true, Healthy: true} From af87f3ddcb5f1b61c06a05d8772a8f28a61c8ed9 Mon Sep 17 00:00:00 2001 From: Kenta Iso Date: Thu, 9 Apr 2020 19:26:18 +0900 Subject: [PATCH 65/67] Doc fix: use LC_ALL which has priority for system locale instead of LANG --- site/content/en/docs/contrib/translations.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/site/content/en/docs/contrib/translations.md b/site/content/en/docs/contrib/translations.md index 0fb40d0b10..3250965698 100644 --- a/site/content/en/docs/contrib/translations.md +++ b/site/content/en/docs/contrib/translations.md @@ -79,11 +79,12 @@ All translations are stored in the top-level `translations` directory. * You now have a fresh minikube binary in the `out` directory. If your system locale is that of the language you added translations for, a simple `out/minikube start` will work as a test, assuming you translated phrases from `minikube start`. You can use whatever command you'd like in that way. -* If you have a different system locale, you can override the printed language using the LANG environment variable: +* If you have a different system locale, you can override the printed language using the LC_ALL environment variable: ``` - ~/minikube$ LANG=fr out/minikube start - 😄 minikube v1.9.0-beta.2 sur Darwin 10.14.6 - ✨ Choix automatique du driver hyperkit + ~/minikube$ LC_ALL=fr out/minikube start + 😄 minikube v1.9.2 sur Darwin 10.14.5 + ✨ Choix automatique du driver hyperkit. Autres choix: + 👍 Starting control plane node minikube in cluster minikube 🔥 Création de VM hyperkit (CPUs=2, Mémoire=4000MB, Disque=20000MB)... 🐳 Préparation de Kubernetes v1.18.0 sur Docker 19.03.8... 🌟 Installation des addons: default-storageclass, storage-provisioner From 4d24e651a4f4e1e800878ccc3069da61e7d1ac14 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Thu, 9 Apr 2020 05:17:28 -0700 Subject: [PATCH 66/67] Remove field selector --- test/integration/helpers.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/helpers.go b/test/integration/helpers.go index af8f8698d9..7daac4aa6f 100644 --- a/test/integration/helpers.go +++ b/test/integration/helpers.go @@ -241,8 +241,8 @@ func clusterLogs(t *testing.T, profile string) { t.Logf("(dbg) %s:\n%s", rr.Command(), rr.Output()) } - t.Logf("------++> post-mortem[%s]: !running pods", t.Name()) - rr, err = Run(t, exec.Command("kubectl", "--context", profile, "describe", "po", "-A", "--field-selector=status.phase!=Running")) + t.Logf("------++> post-mortem[%s]: describe pods", t.Name()) + rr, err = Run(t, exec.Command("kubectl", "--context", profile, "describe", "po", "-A")) if err != nil { t.Logf("%s: %v", rr.Command(), err) } else { From 8733f07e442c26941a158e4c637f7d70bad3a4a6 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Thu, 9 Apr 2020 05:21:05 -0700 Subject: [PATCH 67/67] Higher vis --- test/integration/helpers.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/integration/helpers.go b/test/integration/helpers.go index 7daac4aa6f..e8bd0b1d9f 100644 --- a/test/integration/helpers.go +++ b/test/integration/helpers.go @@ -203,7 +203,7 @@ func clusterLogs(t *testing.T, profile string) { t.Logf("-----------------------post-mortem--------------------------------") t.Logf("<<< %s FAILED: start of post-mortem logs <<<", t.Name()) - t.Logf("------++> post-mortem[%s]: minikube logs", t.Name()) + t.Logf("======> post-mortem[%s]: minikube logs <======", t.Name()) rr, err := Run(t, exec.Command(Target(), "-p", profile, "logs", "--problems")) if err != nil { @@ -212,7 +212,7 @@ func clusterLogs(t *testing.T, profile string) { } t.Logf("%s logs: %s", t.Name(), rr.Output()) - t.Logf("------++> post-mortem[%s]: disk usage", t.Name()) + t.Logf("======> post-mortem[%s]: disk usage <======", t.Name()) rr, err = Run(t, exec.Command(Target(), "-p", profile, "ssh", "df -h /var/lib/docker/overlay2 /var /; du -hs /var/lib/docker/overlay2")) if err != nil { t.Logf("failed df error: %v", err) @@ -225,7 +225,7 @@ func clusterLogs(t *testing.T, profile string) { return } - t.Logf("------++> post-mortem[%s]: get pods", t.Name()) + t.Logf("======> post-mortem[%s]: get pods <======", t.Name()) rr, rerr := Run(t, exec.Command("kubectl", "--context", profile, "get", "po", "-A", "--show-labels")) if rerr != nil { t.Logf("%s: %v", rr.Command(), rerr) @@ -233,7 +233,7 @@ func clusterLogs(t *testing.T, profile string) { } t.Logf("(dbg) %s:\n%s", rr.Command(), rr.Output()) - t.Logf("------++> post-mortem[%s]: describe node", t.Name()) + t.Logf("======> post-mortem[%s]: describe node <======", t.Name()) rr, err = Run(t, exec.Command("kubectl", "--context", profile, "describe", "node")) if err != nil { t.Logf("%s: %v", rr.Command(), err) @@ -241,7 +241,7 @@ func clusterLogs(t *testing.T, profile string) { t.Logf("(dbg) %s:\n%s", rr.Command(), rr.Output()) } - t.Logf("------++> post-mortem[%s]: describe pods", t.Name()) + t.Logf("======> post-mortem[%s]: describe pods <======", t.Name()) rr, err = Run(t, exec.Command("kubectl", "--context", profile, "describe", "po", "-A")) if err != nil { t.Logf("%s: %v", rr.Command(), err)