diff --git a/.github/workflows/iso.yml b/.github/workflows/iso.yml index 372baa8288..2eac6cc640 100644 --- a/.github/workflows/iso.yml +++ b/.github/workflows/iso.yml @@ -82,7 +82,7 @@ jobs: - name: Install gopogh shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-linux-amd64 sudo install gopogh-linux-amd64 /usr/local/bin/gopogh sudo apt-get install -y jq - name: Run Integration Test diff --git a/.github/workflows/kic_image.yml b/.github/workflows/kic_image.yml index 65a826627d..85028af43f 100644 --- a/.github/workflows/kic_image.yml +++ b/.github/workflows/kic_image.yml @@ -60,7 +60,7 @@ jobs: - name: Install gopogh shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-linux-amd64 sudo install gopogh-linux-amd64 /usr/local/bin/gopogh sudo apt-get install -y jq rm -f gopogh-linux-amd64 || true diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index bdac426be1..df2caf3749 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -120,7 +120,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-linux-amd64 sudo install gopogh-linux-amd64 /usr/local/bin/gopogh - name: Download Binaries uses: actions/download-artifact@v1 @@ -205,7 +205,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-darwin-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-darwin-amd64 sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh - name: Install docker shell: bash @@ -350,7 +350,7 @@ jobs: continue-on-error: true shell: powershell run: | - (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe") + (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe") choco install -y kubernetes-cli choco install -y jq choco install -y caffeine @@ -487,7 +487,7 @@ jobs: shell: powershell run: | $ErrorActionPreference = "SilentlyContinue" - (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe") + (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe") choco install -y kubernetes-cli choco install -y jq choco install -y caffeine @@ -592,7 +592,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-linux-amd64 sudo install gopogh-linux-amd64 /usr/local/bin/gopogh - name: Download Binaries uses: actions/download-artifact@v1 @@ -651,6 +651,117 @@ jobs: if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi if [ "$numPass" -lt 26 ];then echo "*** Failed to pass at least 26 ! ***";exit 2;fi if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi + functional_docker_ubuntu_arm64: + needs: [ build_minikube ] + runs-on: [ self-hosted, arm64 ] + env: + TIME_ELAPSED: time + JOB_NAME: "functional_docker_ubuntu_arm64" + GOPOGH_RESULT: "" + SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643 + steps: + - name: Install kubectl + shell: bash + run: | + curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/arm64/kubectl + sudo install kubectl /usr/local/bin/kubectl + kubectl version --client=true + + - name: Install gopogh + shell: bash + run: | + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-linux-arm64 + sudo install gopogh-linux-arm64 /usr/local/bin/gopogh + + - name: Install tools + shell: bash + run: | + sudo apt update + sudo apt install -y jq docker + + - name: Docker Info + shell: bash + run: | + echo "--------------------------" + docker version || true + echo "--------------------------" + docker info || true + echo "--------------------------" + docker system df || true + echo "--------------------------" + docker system info --format='{{json .}}'|| true + echo "--------------------------" + docker ps || true + echo "--------------------------" + # go 1.14.6+ is needed because of this bug https://github.com/golang/go/issues/39308 + - uses: actions/setup-go@v2 + with: + go-version: '1.15.2' + stable: true + + - name: Download Binaries + uses: actions/download-artifact@v1 + with: + name: minikube_binaries + + - name: Run Integration Test + continue-on-error: false + # bash {0} to allow test to continue to next step. in case of + shell: bash {0} + run: | + cd minikube_binaries + mkdir -p report + mkdir -p testhome + chmod a+x e2e-* + chmod a+x minikube-* + START_TIME=$(date -u +%s) + KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-arm64 -minikube-start-args=--vm-driver=docker -test.run TestFunctional -test.timeout=10m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-arm64 2>&1 | tee ./report/testout.txt + END_TIME=$(date -u +%s) + TIME_ELAPSED=$(($END_TIME-$START_TIME)) + min=$((${TIME_ELAPSED}/60)) + sec=$((${TIME_ELAPSED}%60)) + TIME_ELAPSED="${min} min $sec seconds " + echo "TIME_ELAPSED=${TIME_ELAPSED}" >> $GITHUB_ENV + + - name: Generate HTML Report + shell: bash + run: | + cd minikube_binaries + export PATH=${PATH}:`go env GOPATH`/bin + go tool test2json -t < ./report/testout.txt > ./report/testout.json || true + STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}" -details "${GITHUB_SHA}") || true + echo status: ${STAT} + FailNum=$(echo $STAT | jq '.NumberOfFail') + TestsNum=$(echo $STAT | jq '.NumberOfTests') + GOPOGH_RESULT="${JOB_NAME} : completed with ${FailNum} / ${TestsNum} failures in ${TIME_ELAPSED}" + echo "GOPOGH_RESULT=${GOPOGH_RESULT}" >> $GITHUB_ENV + echo 'STAT<> $GITHUB_ENV + echo "${STAT}" >> $GITHUB_ENV + echo 'EOF' >> $GITHUB_ENV + + - uses: actions/upload-artifact@v1 + with: + name: functional_docker_ubuntu_arm64 + path: minikube_binaries/report + + - name: The End Result - functional_docker_ubuntu_arm64 + shell: bash + run: | + echo ${GOPOGH_RESULT} + numFail=$(echo $STAT | jq '.NumberOfFail') + numPass=$(echo $STAT | jq '.NumberOfPass') + echo "*******************${numPass} Passes :) *******************" + echo $STAT | jq '.PassedTests' || true + echo "*******************************************************" + echo "---------------- ${numFail} Failures :( ----------------------------" + echo $STAT | jq '.FailedTests' || true + echo "-------------------------------------------------------" + if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi + if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi + if [ "$numPass" -lt 20 ];then echo "*** Failed to pass at least 20 ! ***";exit 2;fi + if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi + + addons_certs_docker_ubuntu: runs-on: ubuntu-18.04 env: @@ -689,7 +800,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-linux-amd64 sudo install gopogh-linux-amd64 /usr/local/bin/gopogh - name: Download Binaries uses: actions/download-artifact@v1 @@ -771,7 +882,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-darwin-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-darwin-amd64 sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh - name: Install docker shell: bash @@ -883,7 +994,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-linux-amd64 sudo install gopogh-linux-amd64 /usr/local/bin/gopogh - name: Download Binaries uses: actions/download-artifact@v1 @@ -967,7 +1078,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-darwin-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-darwin-amd64 sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh - name: Download Binaries uses: actions/download-artifact@v1 @@ -1074,7 +1185,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-linux-amd64 sudo install gopogh-linux-amd64 /usr/local/bin/gopogh - name: Download Binaries uses: actions/download-artifact@v1 @@ -1156,7 +1267,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-darwin-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-darwin-amd64 sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh - name: Download Binaries uses: actions/download-artifact@v1 @@ -1235,6 +1346,7 @@ jobs: functional_docker_windows, functional_hyperv_windows, functional_baremetal_ubuntu18_04, + functional_docker_ubuntu_arm64, addons_certs_docker_ubuntu, addons_certs_virtualbox_macos, multinode_docker_ubuntu, @@ -1257,12 +1369,14 @@ jobs: cp -r ./functional_docker_windows ./all_reports/ cp -r ./functional_hyperv_windows ./all_reports/ cp -r ./functional_baremetal_ubuntu18_04 ./all_reports/ + cp -r ./functional_docker_ubuntu_arm64 ./all_reports/ cp -r ./addons_certs_docker_ubuntu ./all_reports/ cp -r ./addons_certs_virtualbox_macos ./all_reports/ cp -r ./multinode_docker_ubuntu ./all_reports/ cp -r ./multinode_virtualbox_macos ./all_reports/ cp -r ./preload_dockerflags_docker_ubuntu ./all_reports/ cp -r ./pause_preload_dockerflags_virtualbox_macos ./all_reports/ + - uses: actions/upload-artifact@v1 with: name: all_reports diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index ea8515c994..8c409289de 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -118,7 +118,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-linux-amd64 sudo install gopogh-linux-amd64 /usr/local/bin/gopogh - name: Download Binaries uses: actions/download-artifact@v1 @@ -203,7 +203,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-darwin-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-darwin-amd64 sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh - name: Install docker shell: bash @@ -348,7 +348,7 @@ jobs: continue-on-error: true shell: powershell run: | - (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe") + (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe") choco install -y kubernetes-cli choco install -y jq choco install -y caffeine @@ -485,7 +485,7 @@ jobs: shell: powershell run: | $ErrorActionPreference = "SilentlyContinue" - (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe") + (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe") choco install -y kubernetes-cli choco install -y jq choco install -y caffeine @@ -590,7 +590,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-linux-amd64 sudo install gopogh-linux-amd64 /usr/local/bin/gopogh - name: Download Binaries uses: actions/download-artifact@v1 @@ -649,6 +649,118 @@ jobs: if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi if [ "$numPass" -lt 26 ];then echo "*** Failed to pass at least 26 ! ***";exit 2;fi if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi + + functional_docker_ubuntu_arm64: + needs: [ build_minikube ] + runs-on: [ self-hosted, arm64 ] + env: + TIME_ELAPSED: time + JOB_NAME: "functional_docker_ubuntu_arm64" + GOPOGH_RESULT: "" + SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643 + steps: + - name: Install kubectl + shell: bash + run: | + curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/arm64/kubectl + sudo install kubectl /usr/local/bin/kubectl + kubectl version --client=true + + - name: Install gopogh + shell: bash + run: | + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-linux-arm64 + sudo install gopogh-linux-arm64 /usr/local/bin/gopogh + + - name: Install tools + shell: bash + run: | + sudo apt update + sudo apt install -y jq docker + + - name: Docker Info + shell: bash + run: | + echo "--------------------------" + docker version || true + echo "--------------------------" + docker info || true + echo "--------------------------" + docker system df || true + echo "--------------------------" + docker system info --format='{{json .}}'|| true + echo "--------------------------" + docker ps || true + echo "--------------------------" + + # go 1.14.6+ is needed because of this bug https://github.com/golang/go/issues/39308 + - uses: actions/setup-go@v2 + with: + go-version: '1.15.2' + stable: true + + - name: Download Binaries + uses: actions/download-artifact@v1 + with: + name: minikube_binaries + + - name: Run Integration Test + continue-on-error: false + # bash {0} to allow test to continue to next step. in case of + shell: bash {0} + run: | + cd minikube_binaries + mkdir -p report + mkdir -p testhome + chmod a+x e2e-* + chmod a+x minikube-* + START_TIME=$(date -u +%s) + KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-arm64 -minikube-start-args=--vm-driver=docker -test.run TestFunctional -test.timeout=10m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-arm64 2>&1 | tee ./report/testout.txt + END_TIME=$(date -u +%s) + TIME_ELAPSED=$(($END_TIME-$START_TIME)) + min=$((${TIME_ELAPSED}/60)) + sec=$((${TIME_ELAPSED}%60)) + TIME_ELAPSED="${min} min $sec seconds " + echo "TIME_ELAPSED=${TIME_ELAPSED}" >> $GITHUB_ENV + + - name: Generate HTML Report + shell: bash + run: | + cd minikube_binaries + export PATH=${PATH}:`go env GOPATH`/bin + go tool test2json -t < ./report/testout.txt > ./report/testout.json || true + STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}" -details "${GITHUB_SHA}") || true + echo status: ${STAT} + FailNum=$(echo $STAT | jq '.NumberOfFail') + TestsNum=$(echo $STAT | jq '.NumberOfTests') + GOPOGH_RESULT="${JOB_NAME} : completed with ${FailNum} / ${TestsNum} failures in ${TIME_ELAPSED}" + echo "GOPOGH_RESULT=${GOPOGH_RESULT}" >> $GITHUB_ENV + echo 'STAT<> $GITHUB_ENV + echo "${STAT}" >> $GITHUB_ENV + echo 'EOF' >> $GITHUB_ENV + + - uses: actions/upload-artifact@v1 + with: + name: functional_docker_ubuntu_arm64 + path: minikube_binaries/report + + - name: The End Result - functional_docker_ubuntu_arm64 + shell: bash + run: | + echo ${GOPOGH_RESULT} + numFail=$(echo $STAT | jq '.NumberOfFail') + numPass=$(echo $STAT | jq '.NumberOfPass') + echo "*******************${numPass} Passes :) *******************" + echo $STAT | jq '.PassedTests' || true + echo "*******************************************************" + echo "---------------- ${numFail} Failures :( ----------------------------" + echo $STAT | jq '.FailedTests' || true + echo "-------------------------------------------------------" + if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi + if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi + if [ "$numPass" -lt 0 ];then echo "*** Failed to pass at least 20! ***";exit 2;fi + if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi + addons_certs_docker_ubuntu: runs-on: ubuntu-18.04 env: @@ -684,10 +796,9 @@ jobs: go-version: '1.15.5' stable: true - name: Install gopogh - shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-linux-amd64 sudo install gopogh-linux-amd64 /usr/local/bin/gopogh - name: Download Binaries uses: actions/download-artifact@v1 @@ -769,7 +880,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-darwin-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-darwin-amd64 sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh - name: Install docker shell: bash @@ -877,11 +988,11 @@ jobs: with: go-version: '1.15.5' stable: true - - name: Install gopogh + - name: Install gopogh shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-linux-amd64 sudo install gopogh-linux-amd64 /usr/local/bin/gopogh - name: Download Binaries uses: actions/download-artifact@v1 @@ -965,7 +1076,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-darwin-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-darwin-amd64 sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh - name: Download Binaries uses: actions/download-artifact@v1 @@ -1072,7 +1183,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-linux-amd64 sudo install gopogh-linux-amd64 /usr/local/bin/gopogh - name: Download Binaries uses: actions/download-artifact@v1 @@ -1154,7 +1265,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-darwin-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-darwin-amd64 sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh - name: Download Binaries uses: actions/download-artifact@v1 @@ -1229,6 +1340,7 @@ jobs: needs: [ functional_docker_ubuntu, + functional_docker_ubuntu_arm64, functional_virtualbox_macos, functional_docker_windows, functional_hyperv_windows, @@ -1251,6 +1363,7 @@ jobs: mkdir -p all_reports ls -lah cp -r ./functional_docker_ubuntu ./all_reports/ + cp -r ./functional_docker_ubuntu_arm64 ./all_reports/ cp -r ./functional_virtualbox_macos ./all_reports/ cp -r ./functional_docker_windows ./all_reports/ cp -r ./functional_hyperv_windows ./all_reports/ diff --git a/Makefile b/Makefile index 7e4ca12576..f61a0dbc28 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ KUBERNETES_VERSION ?= $(shell egrep "DefaultKubernetesVersion =" pkg/minikube/co KIC_VERSION ?= $(shell egrep "Version =" pkg/drivers/kic/types.go | cut -d \" -f2) # Default to .0 for higher cache hit rates, as build increments typically don't require new ISO versions -ISO_VERSION ?= v1.16.0 +ISO_VERSION ?= v1.17.0 # Dashes are valid in semver, but not Linux packaging. Use ~ to delimit alpha/beta DEB_VERSION ?= $(subst -,~,$(RAW_VERSION)) RPM_VERSION ?= $(DEB_VERSION) @@ -216,8 +216,9 @@ else go build -tags "$(MINIKUBE_BUILD_TAGS)" -ldflags="$(MINIKUBE_LDFLAGS)" -a -o $@ k8s.io/minikube/cmd/minikube endif -.PHONY: e2e-linux-amd64 e2e-darwin-amd64 e2e-windows-amd64.exe +.PHONY: e2e-linux-amd64 e2e-linux-arm64 e2e-darwin-amd64 e2e-windows-amd64.exe e2e-linux-amd64: out/e2e-linux-amd64 ## Execute end-to-end testing for Linux 64bit +e2e-linux-arm64: out/e2e-linux-arm64 ## Execute end-to-end testing for Linux ARM 64bit e2e-darwin-amd64: out/e2e-darwin-amd64 ## Execute end-to-end testing for Darwin 64bit e2e-windows-amd64.exe: out/e2e-windows-amd64.exe ## Execute end-to-end testing for Windows 64bit @@ -377,7 +378,7 @@ darwin: minikube-darwin-amd64 ## Build minikube for Darwin 64bit linux: minikube-linux-amd64 ## Build minikube for Linux 64bit .PHONY: e2e-cross -e2e-cross: e2e-linux-amd64 e2e-darwin-amd64 e2e-windows-amd64.exe ## End-to-end cross test +e2e-cross: e2e-linux-amd64 e2e-linux-arm64 e2e-darwin-amd64 e2e-windows-amd64.exe ## End-to-end cross test .PHONY: checksum checksum: ## Generate checksums @@ -846,3 +847,7 @@ else go run update_kubernetes_version.go) endif +.PHONY: update-gopogh-version +update-gopogh-version: ## update gopogh version + (cd hack/update/gopogh_version && \ + go run update_gopogh_version.go) diff --git a/cmd/minikube/cmd/delete.go b/cmd/minikube/cmd/delete.go index 6afb85924b..dbc4bd47fd 100644 --- a/cmd/minikube/cmd/delete.go +++ b/cmd/minikube/cmd/delete.go @@ -309,7 +309,7 @@ func deleteProfile(profile *config.Profile) error { return DeletionError{Err: delErr, Errtype: MissingProfile} } - if err == nil && driver.BareMetal(cc.Driver) { + if err == nil && (driver.BareMetal(cc.Driver) || driver.IsSSH(cc.Driver)) { if err := uninstallKubernetes(api, *cc, cc.Nodes[0], viper.GetString(cmdcfg.Bootstrapper)); err != nil { deletionError, ok := err.(DeletionError) if ok { diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index c30b930ae6..cef933ef4b 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -306,7 +306,7 @@ func provisionWithDriver(cmd *cobra.Command, ds registry.DriverState, existing * os.Exit(0) } - if driver.IsVM(driverName) { + if driver.IsVM(driverName) && !driver.IsSSH(driverName) { url, err := download.ISO(viper.GetStringSlice(isoURL), cmd.Flags().Changed(isoURL)) if err != nil { return node.Starter{}, errors.Wrap(err, "Failed to cache ISO") @@ -855,7 +855,7 @@ func validateUser(drvName string) { // memoryLimits returns the amount of memory allocated to the system and hypervisor, the return value is in MiB func memoryLimits(drvName string) (int, int, error) { - info, cpuErr, memErr, diskErr := machine.CachedHostInfo() + info, cpuErr, memErr, diskErr := machine.LocalHostInfo() if cpuErr != nil { klog.Warningf("could not get system cpu info while verifying memory limits, which might be okay: %v", cpuErr) } diff --git a/cmd/minikube/cmd/start_flags.go b/cmd/minikube/cmd/start_flags.go index 8cd3fdc6ac..510e9a685f 100644 --- a/cmd/minikube/cmd/start_flags.go +++ b/cmd/minikube/cmd/start_flags.go @@ -95,7 +95,7 @@ const ( waitTimeout = "wait-timeout" nativeSSH = "native-ssh" minUsableMem = 1800 // Kubernetes (kubeadm) will not start with less - minRecommendedMem = 2000 // Warn at no lower than existing configurations + minRecommendedMem = 1900 // Warn at no lower than existing configurations minimumCPUS = 2 minimumDiskSize = 2000 autoUpdate = "auto-update-drivers" @@ -110,6 +110,12 @@ const ( network = "network" startNamespace = "namespace" trace = "trace" + sshIPAddress = "ssh-ip-address" + sshSSHUser = "ssh-user" + sshSSHKey = "ssh-key" + sshSSHPort = "ssh-port" + defaultSSHUser = "root" + defaultSSHPort = 22 ) var ( @@ -221,6 +227,12 @@ func initNetworkingFlags() { startCmd.Flags().String(serviceCIDR, constants.DefaultServiceCIDR, "The CIDR to be used for service cluster IPs.") startCmd.Flags().StringArrayVar(&config.DockerEnv, "docker-env", nil, "Environment variables to pass to the Docker daemon. (format: key=value)") startCmd.Flags().StringArrayVar(&config.DockerOpt, "docker-opt", nil, "Specify arbitrary flags to pass to the Docker daemon. (format: key=value)") + + // ssh + startCmd.Flags().String(sshIPAddress, "", "IP address (ssh driver only)") + startCmd.Flags().String(sshSSHUser, defaultSSHUser, "SSH user (ssh driver only)") + startCmd.Flags().String(sshSSHKey, "", "SSH key (ssh driver only)") + startCmd.Flags().Int(sshSSHPort, defaultSSHPort, "SSH port (ssh driver only)") } // ClusterFlagValue returns the current cluster name based on flags @@ -335,6 +347,10 @@ func generateClusterConfig(cmd *cobra.Command, existing *config.ClusterConfig, k NatNicType: viper.GetString(natNicType), StartHostTimeout: viper.GetDuration(waitTimeout), ExposedPorts: viper.GetStringSlice(ports), + SSHIPAddress: viper.GetString(sshIPAddress), + SSHUser: viper.GetString(sshSSHUser), + SSHKey: viper.GetString(sshSSHKey), + SSHPort: viper.GetInt(sshSSHPort), KubernetesConfig: config.KubernetesConfig{ KubernetesVersion: k8sVersion, ClusterName: ClusterFlagValue(), diff --git a/cmd/minikube/cmd/update-context.go b/cmd/minikube/cmd/update-context.go index 709a65ad61..3688df3839 100644 --- a/cmd/minikube/cmd/update-context.go +++ b/cmd/minikube/cmd/update-context.go @@ -35,8 +35,9 @@ var updateContextCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { cname := ClusterFlagValue() co := mustload.Running(cname) + // cluster extension metada for kubeconfig - updated, err := kubeconfig.UpdateEndpoint(cname, co.CP.Hostname, co.CP.Port, kubeconfig.PathFromEnv()) + updated, err := kubeconfig.UpdateEndpoint(cname, co.CP.Hostname, co.CP.Port, kubeconfig.PathFromEnv(), kubeconfig.NewExtension()) if err != nil { exit.Error(reason.HostKubeconfigUpdate, "update config", err) } diff --git a/deploy/iso/minikube-iso/package/docker-bin/docker-bin.hash b/deploy/iso/minikube-iso/package/docker-bin/docker-bin.hash index 4364f390f6..7d1aa5a788 100644 --- a/deploy/iso/minikube-iso/package/docker-bin/docker-bin.hash +++ b/deploy/iso/minikube-iso/package/docker-bin/docker-bin.hash @@ -25,3 +25,4 @@ sha256 ddb13aff1fcdcceb710bf71a210169b9c1abfd7420eeaf42cf7975f8fae2fcc8 docker- sha256 9f1ec28e357a8f18e9561129239caf9c0807d74756e21cc63637c7fdeaafe847 docker-19.03.14.tgz sha256 02936a3585f12f13b21b95e02ae722d74eaf1870b536997e914659ee307b2ac4 docker-20.10.0.tgz sha256 8790f3b94ee07ca69a9fdbd1310cbffc729af0a07e5bf9f34a79df1e13d2e50e docker-20.10.1.tgz +sha256 97017e32a8ecbdd1826bb3c7b1424303ee0dea3f900d33591b1df5e394ed4eed docker-20.10.2.tgz diff --git a/deploy/iso/minikube-iso/package/docker-bin/docker-bin.mk b/deploy/iso/minikube-iso/package/docker-bin/docker-bin.mk index 7b2eb7b77f..db38e6a105 100644 --- a/deploy/iso/minikube-iso/package/docker-bin/docker-bin.mk +++ b/deploy/iso/minikube-iso/package/docker-bin/docker-bin.mk @@ -4,7 +4,7 @@ # ################################################################################ -DOCKER_BIN_VERSION = 20.10.1 +DOCKER_BIN_VERSION = 20.10.2 DOCKER_BIN_SITE = https://download.docker.com/linux/static/stable/x86_64 DOCKER_BIN_SOURCE = docker-$(DOCKER_BIN_VERSION).tgz diff --git a/enhancements/proposed/20201230-containerd-default/container-default.md b/enhancements/proposed/20201230-containerd-default/container-default.md new file mode 100644 index 0000000000..667666c194 --- /dev/null +++ b/enhancements/proposed/20201230-containerd-default/container-default.md @@ -0,0 +1,45 @@ +# CRI: Containerd by default + +* First proposed: 2020-11-08 +* Authors: Anders F Björklund (@afbjorklund) + +## Reviewer Priorities + +Please review this proposal with the following priorities: + +* Does this fit with minikube's [principles](https://minikube.sigs.k8s.io/docs/concepts/principles/)? +* Are there other approaches to consider? +* Could the implementation be made simpler? +* Are there usability, reliability, or technical debt concerns? + +Please leave the above text in your proposal as instructions to the reader. + +## Summary + +Change default container runtime, from current "docker" to replacement "containerd". + +## Goals + +* Change from docker to containerd as default +* Still allow fast-building images with minikube + +## Non-Goals + +* Remove the docker support from minikube +* Change anything in the docker driver + +## Design Details + +The containerd container runtime is already included, and is passing certification. + +Unlike [Docker](https://www.docker.com/products/docker-engine), will need to include the CRI (runtime) and CNI (network) by default. + +Use [BuildKit](https://github.com/moby/buildkit) as a complement to [Containerd](https://containerd.io/), for producing an image from Dockerfile. + +Only run buildkitd on-demand (i.e. when building), default to running only containerd. + +## Alternatives Considered + +Keep Docker as the default, and add the new CRI-Docker to replace the old dockershim. + +Use [CRI-O](https://cri-o.io/)/[Podman](https://podman.io/) as default, which is a bigger change (since dockerd uses containerd). diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh index 07673308a0..0182427673 100755 --- a/hack/jenkins/common.sh +++ b/hack/jenkins/common.sh @@ -20,12 +20,13 @@ # The script expects the following env variables: # OS_ARCH: The operating system and the architecture separated by a hyphen '-' (e.g. darwin-amd64, linux-amd64, windows-amd64) # VM_DRIVER: the driver to use for the test +# CONTAINER_RUNTIME: the container runtime to use for the test # EXTRA_START_ARGS: additional flags to pass into minikube start # EXTRA_TEST_ARGS: additional flags to pass into go test # JOB_NAME: the name of the logfile and check name to update on github readonly TEST_ROOT="${HOME}/minikube-integration" -readonly TEST_HOME="${TEST_ROOT}/${OS_ARCH}-${VM_DRIVER}-${MINIKUBE_LOCATION}-$$-${COMMIT}" +readonly TEST_HOME="${TEST_ROOT}/${OS_ARCH}-${VM_DRIVER}-${CONTAINER_RUNTIME}-${MINIKUBE_LOCATION}-$$-${COMMIT}" export GOPATH="$HOME/go" export KUBECONFIG="${TEST_HOME}/kubeconfig" export PATH=$PATH:"/usr/local/bin/:/usr/local/go/bin/:$GOPATH/bin" @@ -52,6 +53,7 @@ echo "" echo "arch: ${OS_ARCH}" echo "build: ${MINIKUBE_LOCATION}" echo "driver: ${VM_DRIVER}" +echo "runtime: ${CONTAINER_RUNTIME}" echo "job: ${JOB_NAME}" echo "test home: ${TEST_HOME}" echo "sudo: ${SUDO_PREFIX}" @@ -298,6 +300,12 @@ if test -f "${TEST_OUT}"; then rm "${TEST_OUT}" || true # clean up previous runs of same build fi touch "${TEST_OUT}" + +if [ ! -z "${CONTAINER_RUNTIME}" ] +then + EXTRA_START_ARGS="${EXTRA_START_ARGS} --container-runtime=${CONTAINER_RUNTIME}" +fi + ${SUDO_PREFIX}${E2E_BIN} \ -minikube-start-args="--driver=${VM_DRIVER} ${EXTRA_START_ARGS}" \ -test.timeout=${TIMEOUT} -test.v \ @@ -352,9 +360,9 @@ fi echo ">> Installing gopogh" if [ "$(uname)" != "Darwin" ]; then - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64 && sudo install gopogh-linux-amd64 /usr/local/bin/gopogh + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-linux-amd64 && sudo install gopogh-linux-amd64 /usr/local/bin/gopogh else - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-darwin-amd64 && sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.4.0/gopogh-darwin-amd64 && sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh fi echo ">> Running gopogh" diff --git a/hack/jenkins/linux_integration_tests_docker.sh b/hack/jenkins/linux_integration_tests_docker.sh index fb912d2391..faba877c8c 100755 --- a/hack/jenkins/linux_integration_tests_docker.sh +++ b/hack/jenkins/linux_integration_tests_docker.sh @@ -28,6 +28,7 @@ set -e OS_ARCH="linux-amd64" VM_DRIVER="docker" JOB_NAME="Docker_Linux" +CONTAINER_RUNTIME="docker" mkdir -p cron && gsutil -qm rsync "gs://minikube-builds/${MINIKUBE_LOCATION}/cron" cron || echo "FAILED TO GET CRON FILES" sudo install cron/cleanup_and_reboot_Linux.sh /etc/cron.hourly/cleanup_and_reboot || echo "FAILED TO INSTALL CLEANUP" diff --git a/hack/jenkins/linux_integration_tests_docker_containerd.sh b/hack/jenkins/linux_integration_tests_docker_containerd.sh new file mode 100644 index 0000000000..1549688b94 --- /dev/null +++ b/hack/jenkins/linux_integration_tests_docker_containerd.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# Copyright 2019 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# This script runs the integration tests on a Linux machine for the KVM Driver + +# The script expects the following env variables: +# MINIKUBE_LOCATION: GIT_COMMIT from upstream build. +# COMMIT: Actual commit ID from upstream build +# EXTRA_BUILD_ARGS (optional): Extra args to be passed into the minikube integrations tests +# access_token: The Github API access token. Injected by the Jenkins credential provider. + +set -e + +OS_ARCH="linux-amd64" +VM_DRIVER="docker" +JOB_NAME="Docker_Linux_containerd" +CONTAINER_RUNTIME="containerd" + + + +mkdir -p cron && gsutil -qm rsync "gs://minikube-builds/${MINIKUBE_LOCATION}/cron" cron || echo "FAILED TO GET CRON FILES" +sudo install cron/cleanup_and_reboot_Linux.sh /etc/cron.hourly/cleanup_and_reboot || echo "FAILED TO INSTALL CLEANUP" + +# removing possible left over docker containers from previous runs +docker rm -f -v $(docker ps -aq) >/dev/null 2>&1 || true + +source ./common.sh diff --git a/hack/jenkins/linux_integration_tests_podman.sh b/hack/jenkins/linux_integration_tests_podman.sh index 186e9e0a43..f712f38658 100755 --- a/hack/jenkins/linux_integration_tests_podman.sh +++ b/hack/jenkins/linux_integration_tests_podman.sh @@ -28,12 +28,11 @@ set -e OS_ARCH="linux-amd64" VM_DRIVER="podman" JOB_NAME="Experimental_Podman_Linux" +CONTAINER_RUNTIME="containerd" mkdir -p cron && gsutil -qm rsync "gs://minikube-builds/${MINIKUBE_LOCATION}/cron" cron || echo "FAILED TO GET CRON FILES" sudo install cron/cleanup_and_reboot_Linux.sh /etc/cron.hourly/cleanup_and_reboot || echo "FAILED TO INSTALL CLEANUP" -EXTRA_START_ARGS="--container-runtime=containerd" - # remove possible left over podman containers sudo podman rm -f -v $(sudo podman ps -aq) || true diff --git a/hack/jenkins/minikube_set_pending.sh b/hack/jenkins/minikube_set_pending.sh index 1f2f5e4075..7bcdd813c6 100755 --- a/hack/jenkins/minikube_set_pending.sh +++ b/hack/jenkins/minikube_set_pending.sh @@ -41,6 +41,7 @@ jobs=( 'KVM_Linux' 'none_Linux' 'Docker_Linux' + 'Docker_Linux_containerd' # 'Docker_macOS' # 'Docker_Windows' # 'Podman_Linux' diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go index f126f6fe8f..419945a92e 100644 --- a/pkg/drivers/kic/types.go +++ b/pkg/drivers/kic/types.go @@ -24,9 +24,9 @@ import ( const ( // Version is the current version of kic - Version = "v0.0.16-snapshot1" + Version = "v0.0.17" // SHA of the kic base image - baseImageSHA = "dff16232547bb3ac3f2a9e09a42246a96ecf8f40d9a1c5bcf5a37953690954b6" + baseImageSHA = "1cd2e039ec9d418e6380b2fa0280503a72e5b282adea674ee67882f59f4f546e" ) var ( diff --git a/pkg/drivers/ssh/ssh.go b/pkg/drivers/ssh/ssh.go new file mode 100644 index 0000000000..d36fd8b073 --- /dev/null +++ b/pkg/drivers/ssh/ssh.go @@ -0,0 +1,241 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ssh + +import ( + "fmt" + "net" + "os" + "os/exec" + "path" + "strconv" + "time" + + "github.com/docker/machine/libmachine/drivers" + "github.com/docker/machine/libmachine/engine" + "github.com/docker/machine/libmachine/log" + "github.com/docker/machine/libmachine/mcnutils" + "github.com/docker/machine/libmachine/state" + "github.com/pkg/errors" + "k8s.io/klog/v2" + pkgdrivers "k8s.io/minikube/pkg/drivers" + "k8s.io/minikube/pkg/minikube/command" + "k8s.io/minikube/pkg/minikube/cruntime" + "k8s.io/minikube/pkg/minikube/sysinit" +) + +// Driver is a driver designed to run kubeadm w/o VM management. +// https://minikube.sigs.k8s.io/docs/reference/drivers/ssh/ +type Driver struct { + *drivers.BaseDriver + *pkgdrivers.CommonDriver + EnginePort int + SSHKey string + runtime cruntime.Manager + exec command.Runner +} + +// Config is configuration for the SSH driver +type Config struct { + MachineName string + StorePath string + ContainerRuntime string +} + +const ( + defaultTimeout = 15 * time.Second +) + +// NewDriver creates and returns a new instance of the driver +func NewDriver(c Config) *Driver { + d := &Driver{ + EnginePort: engine.DefaultPort, + BaseDriver: &drivers.BaseDriver{ + MachineName: c.MachineName, + StorePath: c.StorePath, + }, + } + runner := command.NewSSHRunner(d) + runtime, err := cruntime.New(cruntime.Config{Type: c.ContainerRuntime, Runner: runner}) + // Libraries shouldn't panic, but there is no way for drivers to return error :( + if err != nil { + klog.Fatalf("unable to create container runtime: %v", err) + } + d.runtime = runtime + d.exec = runner + return d +} + +// DriverName returns the name of the driver +func (d *Driver) DriverName() string { + return "ssh" +} + +func (d *Driver) GetSSHHostname() (string, error) { + return d.GetIP() +} + +func (d *Driver) GetSSHUsername() string { + return d.SSHUser +} + +func (d *Driver) GetSSHKeyPath() string { + return d.SSHKeyPath +} + +func (d *Driver) PreCreateCheck() error { + if d.SSHKey != "" { + if _, err := os.Stat(d.SSHKey); os.IsNotExist(err) { + return fmt.Errorf("SSH key does not exist: %q", d.SSHKey) + } + } + + return nil +} + +func (d *Driver) Create() error { + if d.SSHKey == "" { + log.Info("No SSH key specified. Assuming an existing key at the default location.") + } else { + log.Info("Importing SSH key...") + + d.SSHKeyPath = d.ResolveStorePath(path.Base(d.SSHKey)) + if err := copySSHKey(d.SSHKey, d.SSHKeyPath); err != nil { + return err + } + + if err := copySSHKey(d.SSHKey+".pub", d.SSHKeyPath+".pub"); err != nil { + log.Infof("Couldn't copy SSH public key : %s", err) + } + } + + if d.runtime.Name() == "Docker" { + if _, err := d.exec.RunCmd(exec.Command("sudo", "usermod", "-aG", "docker", d.GetSSHUsername())); err != nil { + return errors.Wrap(err, "usermod") + } + } + + log.Debugf("IP: %s", d.IPAddress) + + return nil +} + +func (d *Driver) GetURL() (string, error) { + if err := drivers.MustBeRunning(d); err != nil { + return "", err + } + + ip, err := d.GetIP() + if err != nil { + return "", err + } + + return fmt.Sprintf("tcp://%s", net.JoinHostPort(ip, strconv.Itoa(d.EnginePort))), nil +} + +func (d *Driver) GetState() (state.State, error) { + address := net.JoinHostPort(d.IPAddress, strconv.Itoa(d.SSHPort)) + + _, err := net.DialTimeout("tcp", address, defaultTimeout) + if err != nil { + return state.Stopped, nil + } + + return state.Running, nil +} + +// Start a host +func (d *Driver) Start() error { + return nil +} + +// Stop a host gracefully, including any containers that we are managing. +func (d *Driver) Stop() error { + if err := sysinit.New(d.exec).Stop("kubelet"); err != nil { + klog.Warningf("couldn't stop kubelet. will continue with stop anyways: %v", err) + if err := sysinit.New(d.exec).ForceStop("kubelet"); err != nil { + klog.Warningf("couldn't force stop kubelet. will continue with stop anyways: %v", err) + } + } + containers, err := d.runtime.ListContainers(cruntime.ListOptions{}) + if err != nil { + return errors.Wrap(err, "containers") + } + if len(containers) > 0 { + if err := d.runtime.StopContainers(containers); err != nil { + return errors.Wrap(err, "stop containers") + } + } + klog.Infof("ssh driver is stopped!") + return nil +} + +// Restart a host +func (d *Driver) Restart() error { + if err := sysinit.New(d.exec).Restart("kubelet"); err != nil { + return err + } + return nil +} + +// Kill stops a host forcefully, including any containers that we are managing. +func (d *Driver) Kill() error { + if err := sysinit.New(d.exec).ForceStop("kubelet"); err != nil { + klog.Warningf("couldn't force stop kubelet. will continue with kill anyways: %v", err) + } + + // First try to gracefully stop containers + containers, err := d.runtime.ListContainers(cruntime.ListOptions{}) + if err != nil { + return errors.Wrap(err, "containers") + } + if len(containers) == 0 { + return nil + } + // Try to be graceful before sending SIGKILL everywhere. + if err := d.runtime.StopContainers(containers); err != nil { + return errors.Wrap(err, "stop") + } + + containers, err = d.runtime.ListContainers(cruntime.ListOptions{}) + if err != nil { + return errors.Wrap(err, "containers") + } + if len(containers) == 0 { + return nil + } + if err := d.runtime.KillContainers(containers); err != nil { + return errors.Wrap(err, "kill") + } + return nil +} + +func (d *Driver) Remove() error { + return nil +} + +func copySSHKey(src, dst string) error { + if err := mcnutils.CopyFile(src, dst); err != nil { + return fmt.Errorf("unable to copy ssh key: %s", err) + } + + if err := os.Chmod(dst, 0600); err != nil { + return fmt.Errorf("unable to set permissions on the ssh key: %s", err) + } + + return nil +} diff --git a/pkg/minikube/bootstrapper/certs.go b/pkg/minikube/bootstrapper/certs.go index 90fd9870fa..136e9d54fd 100644 --- a/pkg/minikube/bootstrapper/certs.go +++ b/pkg/minikube/bootstrapper/certs.go @@ -99,6 +99,8 @@ func SetupCerts(cmd command.Runner, k8s config.KubernetesConfig, n config.Node) ClientCertificate: path.Join(vmpath.GuestKubernetesCertsDir, "apiserver.crt"), ClientKey: path.Join(vmpath.GuestKubernetesCertsDir, "apiserver.key"), CertificateAuthority: path.Join(vmpath.GuestKubernetesCertsDir, "ca.crt"), + ExtensionContext: kubeconfig.NewExtension(), + ExtensionCluster: kubeconfig.NewExtension(), KeepContext: false, } diff --git a/pkg/minikube/bootstrapper/images/images.go b/pkg/minikube/bootstrapper/images/images.go index 0a9d4db0d2..7d5226de0d 100644 --- a/pkg/minikube/bootstrapper/images/images.go +++ b/pkg/minikube/bootstrapper/images/images.go @@ -20,9 +20,9 @@ package images import ( "fmt" "path" - "runtime" "github.com/blang/semver" + "k8s.io/minikube/pkg/version" ) @@ -34,7 +34,7 @@ func Pause(v semver.Version, mirror string) string { if semver.MustParseRange("<1.18.0-alpha.0")(v) { pv = "3.1" } - return path.Join(kubernetesRepo(mirror), "pause"+archTag(false)+pv) + return path.Join(kubernetesRepo(mirror), "pause:"+pv) } // essentials returns images needed too bootstrap a Kubernetes @@ -53,13 +53,7 @@ func essentials(mirror string, v semver.Version) []string { // componentImage returns a Kubernetes component image to pull func componentImage(name string, v semver.Version, mirror string) string { - needsArchSuffix := false - ancient := semver.MustParseRange("<1.12.0") - if ancient(v) { - needsArchSuffix = true - } - - return fmt.Sprintf("%sv%s", path.Join(kubernetesRepo(mirror), name+archTag(needsArchSuffix)), v) + return fmt.Sprintf("%s:v%s", path.Join(kubernetesRepo(mirror), name), v) } // coreDNS returns the images used for CoreDNS @@ -83,17 +77,11 @@ func coreDNS(v semver.Version, mirror string) string { case 11: cv = "1.1.3" } - return path.Join(kubernetesRepo(mirror), "coredns"+":"+cv) + return path.Join(kubernetesRepo(mirror), "coredns:"+cv) } // etcd returns the image used for etcd func etcd(v semver.Version, mirror string) string { - needsArchSuffix := false - ancient := semver.MustParseRange("<1.12.0") - if ancient(v) { - needsArchSuffix = true - } - // Should match `DefaultEtcdVersion` in: // https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/constants/constants.go ev := "3.4.13-0" @@ -116,15 +104,7 @@ func etcd(v semver.Version, mirror string) string { ev = "3.4.9-1" } - return path.Join(kubernetesRepo(mirror), "etcd"+archTag(needsArchSuffix)+ev) -} - -// archTag returns a CPU architecture suffix for images -func archTag(hasTag bool) string { - if runtime.GOARCH == "amd64" && !hasTag { - return ":" - } - return "-" + runtime.GOARCH + ":" + return path.Join(kubernetesRepo(mirror), "etcd:"+ev) } // auxiliary returns images that are helpful for running minikube @@ -139,7 +119,7 @@ func auxiliary(mirror string) []string { // storageProvisioner returns the minikube storage provisioner image func storageProvisioner(mirror string) string { - return path.Join(minikubeRepo(mirror), "storage-provisioner"+archTag(false)+version.GetStorageProvisionerVersion()) + return path.Join(minikubeRepo(mirror), "storage-provisioner:"+version.GetStorageProvisionerVersion()) } // dashboardFrontend returns the image used for the dashboard frontend diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 4b0fc2e108..5baf460eb6 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -576,14 +576,14 @@ func (k *Bootstrapper) restartControlPlane(cfg config.ClusterConfig) error { klog.Infof("restartCluster took %s", time.Since(start)) }() - version, err := util.ParseKubernetesVersion(cfg.KubernetesConfig.KubernetesVersion) + k8sVersion, err := util.ParseKubernetesVersion(cfg.KubernetesConfig.KubernetesVersion) if err != nil { return errors.Wrap(err, "parsing Kubernetes version") } phase := "alpha" controlPlane := "controlplane" - if version.GTE(semver.MustParse("1.13.0")) { + if k8sVersion.GTE(semver.MustParse("1.13.0")) { phase = "init" controlPlane = "control-plane" } @@ -603,7 +603,7 @@ func (k *Bootstrapper) restartControlPlane(cfg config.ClusterConfig) error { } // Save the costly tax of reinstalling Kubernetes if the only issue is a missing kube context - _, err = kubeconfig.UpdateEndpoint(cfg.Name, hostname, port, kubeconfig.PathFromEnv()) + _, err = kubeconfig.UpdateEndpoint(cfg.Name, hostname, port, kubeconfig.PathFromEnv(), kubeconfig.NewExtension()) if err != nil { klog.Warningf("unable to update kubeconfig (cluster will likely require a reset): %v", err) } diff --git a/pkg/minikube/cluster/ip.go b/pkg/minikube/cluster/ip.go index 891705e60c..54a715c3bd 100644 --- a/pkg/minikube/cluster/ip.go +++ b/pkg/minikube/cluster/ip.go @@ -40,6 +40,12 @@ func HostIP(host *host.Host, clusterName string) (net.IP, error) { return oci.RoutableHostIPFromInside(oci.Docker, clusterName, host.Name) case driver.Podman: return oci.RoutableHostIPFromInside(oci.Podman, clusterName, host.Name) + case driver.SSH: + ip, err := host.Driver.GetIP() + if err != nil { + return []byte{}, errors.Wrap(err, "Error getting VM/Host IP address") + } + return net.ParseIP(ip), nil case driver.KVM2: return net.ParseIP("192.168.39.1"), nil case driver.HyperV: diff --git a/pkg/minikube/config/types.go b/pkg/minikube/config/types.go index 00cc1b54d4..fe533bf0a7 100644 --- a/pkg/minikube/config/types.go +++ b/pkg/minikube/config/types.go @@ -66,6 +66,10 @@ type ClusterConfig struct { HostDNSResolver bool // Only used by virtualbox HostOnlyNicType string // Only used by virtualbox NatNicType string // Only used by virtualbox + SSHIPAddress string // Only used by ssh driver + SSHUser string // Only used by ssh driver + SSHKey string // Only used by ssh driver + SSHPort int // Only used by ssh driver KubernetesConfig KubernetesConfig Nodes []Node Addons map[string]bool diff --git a/pkg/minikube/driver/driver.go b/pkg/minikube/driver/driver.go index 261f8be5db..4053a9dfef 100644 --- a/pkg/minikube/driver/driver.go +++ b/pkg/minikube/driver/driver.go @@ -38,6 +38,8 @@ const ( Mock = "mock" // None driver None = "none" + // SSH driver + SSH = "ssh" // KVM2 driver KVM2 = "kvm2" // VirtualBox driver @@ -55,6 +57,8 @@ const ( // AliasKVM is driver name alias for kvm2 AliasKVM = "kvm" + // AliasSSH is driver name alias for ssh + AliasSSH = "generic" ) var ( @@ -96,6 +100,10 @@ func MachineType(name string) string { return "container" } + if IsSSH(name) { + return "bare metal machine" + } + if IsVM(name) { return "VM" } @@ -143,6 +151,11 @@ func BareMetal(name string) bool { return name == None || name == Mock } +// IsSSH checks if the driver is ssh +func IsSSH(name string) bool { + return name == SSH +} + // NeedsPortForward returns true if driver is unable provide direct IP connectivity func NeedsPortForward(name string) bool { if !IsKIC(name) { diff --git a/pkg/minikube/driver/driver_darwin.go b/pkg/minikube/driver/driver_darwin.go index f951177677..1eef33635d 100644 --- a/pkg/minikube/driver/driver_darwin.go +++ b/pkg/minikube/driver/driver_darwin.go @@ -26,7 +26,7 @@ var supportedDrivers = []string{ HyperKit, VMware, Docker, - Podman, + SSH, } func VBoxManagePath() string { diff --git a/pkg/minikube/driver/driver_linux.go b/pkg/minikube/driver/driver_linux.go index 16e7b5e706..a428a9a2bf 100644 --- a/pkg/minikube/driver/driver_linux.go +++ b/pkg/minikube/driver/driver_linux.go @@ -29,6 +29,7 @@ var supportedDrivers = []string{ None, Docker, Podman, + SSH, } // VBoxManagePath returns the path to the VBoxManage command diff --git a/pkg/minikube/driver/driver_test.go b/pkg/minikube/driver/driver_test.go index b18fc36cab..b4fac014eb 100644 --- a/pkg/minikube/driver/driver_test.go +++ b/pkg/minikube/driver/driver_test.go @@ -65,6 +65,7 @@ func TestMachineType(t *testing.T) { Docker: "container", Mock: "bare metal machine", None: "bare metal machine", + SSH: "bare metal machine", KVM2: "VM", VirtualBox: "VM", HyperKit: "VM", diff --git a/pkg/minikube/driver/driver_windows.go b/pkg/minikube/driver/driver_windows.go index 80ff83b825..1ebc9d5f1d 100644 --- a/pkg/minikube/driver/driver_windows.go +++ b/pkg/minikube/driver/driver_windows.go @@ -33,6 +33,7 @@ var supportedDrivers = []string{ HyperV, VMware, Docker, + SSH, } // TODO: medyagh add same check for kic docker diff --git a/pkg/minikube/kubeconfig/extension.go b/pkg/minikube/kubeconfig/extension.go new file mode 100644 index 0000000000..c1a1321bf7 --- /dev/null +++ b/pkg/minikube/kubeconfig/extension.go @@ -0,0 +1,66 @@ +/* +Copyright 2021 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubeconfig + +import ( + "time" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/minikube/pkg/version" +) + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// implementing the runtime.Object internally so we can write extensions to kubeconfig +type Extension struct { + runtime.TypeMeta `json:",inline"` + Version string `json:"version"` + Provider string `json:"provider"` + LastUpdate string `json:"last-update"` +} + +// NewExtension returns a minikube formated kubeconfig's extension block to idenity clusters and contexts +func NewExtension() *Extension { + return &Extension{ + Provider: "minikube.sigs.k8s.io", + Version: version.GetVersion(), + // time format matching other RFC in notify.go + LastUpdate: time.Now().Format(time.RFC1123)} +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Extension. +func (in *Extension) DeepCopy() *Extension { + if in == nil { + return nil + } + out := new(Extension) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Extension) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Extension) DeepCopyInto(out *Extension) { + *out = *in + out.TypeMeta = in.TypeMeta +} diff --git a/pkg/minikube/kubeconfig/kubeconfig.go b/pkg/minikube/kubeconfig/kubeconfig.go index 9496fd7470..25317fce23 100644 --- a/pkg/minikube/kubeconfig/kubeconfig.go +++ b/pkg/minikube/kubeconfig/kubeconfig.go @@ -105,7 +105,7 @@ func Endpoint(contextName string, configPath ...string) (string, int, error) { } // UpdateEndpoint overwrites the IP stored in kubeconfig with the provided IP. -func UpdateEndpoint(contextName string, hostname string, port int, confpath string) (bool, error) { +func UpdateEndpoint(contextName string, hostname string, port int, confpath string, ext *Extension) (bool, error) { if hostname == "" { return false, fmt.Errorf("empty ip") } @@ -136,6 +136,9 @@ func UpdateEndpoint(contextName string, hostname string, port int, confpath stri CertificateAuthority: path.Join(gp, "ca.crt"), KeepContext: false, } + if ext != nil { + kcs.ExtensionCluster = ext + } err = PopulateFromSettings(kcs, cfg) if err != nil { return false, errors.Wrap(err, "populating kubeconfig") diff --git a/pkg/minikube/kubeconfig/kubeconfig_test.go b/pkg/minikube/kubeconfig/kubeconfig_test.go index 5507f8142b..f617b03fe5 100644 --- a/pkg/minikube/kubeconfig/kubeconfig_test.go +++ b/pkg/minikube/kubeconfig/kubeconfig_test.go @@ -410,7 +410,7 @@ func TestUpdateIP(t *testing.T) { t.Parallel() configFilename := tempFile(t, test.existing) defer os.Remove(configFilename) - statusActual, err := UpdateEndpoint("minikube", test.hostname, test.port, configFilename) + statusActual, err := UpdateEndpoint("minikube", test.hostname, test.port, configFilename, nil) if err != nil && !test.err { t.Errorf("Got unexpected error: %v", err) } @@ -430,7 +430,7 @@ func TestUpdateIP(t *testing.T) { t.Fatal(err) } if !configEquals(actual, expected) { - t.Fatal("configs did not match") + t.Fatalf("configs did not match: Actual:\n%+v\n Expected:\n%+v", actual, expected) } }) diff --git a/pkg/minikube/kubeconfig/settings.go b/pkg/minikube/kubeconfig/settings.go index a61f13d170..2d206c390b 100644 --- a/pkg/minikube/kubeconfig/settings.go +++ b/pkg/minikube/kubeconfig/settings.go @@ -23,6 +23,7 @@ import ( "github.com/juju/mutex" "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/clientcmd/api" "k8s.io/klog/v2" "k8s.io/minikube/pkg/util/lock" @@ -54,6 +55,12 @@ type Settings struct { // Should the certificate files be embedded instead of referenced by path EmbedCerts bool + // Extension meta data for the cluster + ExtensionCluster *Extension + + // Extension meta data for the cluster + ExtensionContext *Extension + // kubeConfigFile is the path where the kube config is stored // Only access this with atomic ops kubeConfigFile atomic.Value @@ -83,6 +90,10 @@ func PopulateFromSettings(cfg *Settings, apiCfg *api.Config) error { } else { cluster.CertificateAuthority = cfg.CertificateAuthority } + + if cfg.ExtensionCluster != nil { + cluster.Extensions = map[string]runtime.Object{"cluster_info": cfg.ExtensionCluster.DeepCopy()} + } apiCfg.Clusters[clusterName] = cluster // user @@ -109,6 +120,10 @@ func PopulateFromSettings(cfg *Settings, apiCfg *api.Config) error { context.Cluster = cfg.ClusterName context.Namespace = cfg.Namespace context.AuthInfo = userName + if cfg.ExtensionContext != nil { + context.Extensions = map[string]runtime.Object{"context_info": cfg.ExtensionContext.DeepCopy()} + } + apiCfg.Contexts[contextName] = context // Only set current context to minikube if the user has not used the keepContext flag @@ -138,6 +153,9 @@ func Update(kcs *Settings) error { return err } + ext := NewExtension() + kcs.ExtensionCluster = ext + kcs.ExtensionContext = ext err = PopulateFromSettings(kcs, kcfg) if err != nil { return err diff --git a/pkg/minikube/machine/info.go b/pkg/minikube/machine/info.go index bc507d9d0b..05361182a8 100644 --- a/pkg/minikube/machine/info.go +++ b/pkg/minikube/machine/info.go @@ -17,8 +17,11 @@ limitations under the License. package machine import ( + "errors" "io/ioutil" "os/exec" + "strconv" + "strings" "github.com/docker/machine/libmachine/provision" "github.com/shirou/gopsutil/v3/cpu" @@ -39,8 +42,8 @@ type HostInfo struct { DiskSize int64 } -// CachedHostInfo returns system information such as memory,CPU, DiskSize -func CachedHostInfo() (*HostInfo, error, error, error) { +// LocalHostInfo returns system information such as memory,CPU, DiskSize +func LocalHostInfo() (*HostInfo, error, error, error) { var cpuErr, memErr, diskErr error i, cpuErr := cachedCPUInfo() if cpuErr != nil { @@ -63,6 +66,43 @@ func CachedHostInfo() (*HostInfo, error, error, error) { return &info, cpuErr, memErr, diskErr } +// RemoteHostInfo returns system information such as memory,CPU, DiskSize +func RemoteHostInfo(r command.Runner) (*HostInfo, error, error, error) { + rr, cpuErr := r.RunCmd(exec.Command("nproc")) + if cpuErr != nil { + klog.Warningf("Unable to get CPU info: %v", cpuErr) + } + nproc := rr.Stdout.String() + ncpus, err := strconv.Atoi(strings.TrimSpace(nproc)) + if err != nil { + klog.Warningf("Failed to parse CPU info: %v", err) + } + rr, memErr := r.RunCmd(exec.Command("free", "-m")) + if memErr != nil { + klog.Warningf("Unable to get mem info: %v", memErr) + } + free := rr.Stdout.String() + memory, err := parseMemFree(free) + if err != nil { + klog.Warningf("Unable to parse mem info: %v", err) + } + rr, diskErr := r.RunCmd(exec.Command("df", "-m")) + if diskErr != nil { + klog.Warningf("Unable to get disk info: %v", diskErr) + } + df := rr.Stdout.String() + disksize, err := parseDiskFree(df) + if err != nil { + klog.Warningf("Unable to parse disk info: %v", err) + } + + var info HostInfo + info.CPUs = ncpus + info.Memory = memory + info.DiskSize = disksize + return &info, cpuErr, memErr, diskErr +} + // showLocalOsRelease shows systemd information about the current linux distribution, on the local host func showLocalOsRelease() { osReleaseOut, err := ioutil.ReadFile("/etc/os-release") @@ -150,3 +190,50 @@ func cachedCPUInfo() ([]cpu.InfoStat, error) { } return *cachedCPU, *cachedCPUErr } + +// ParseMemFree parses the output of the `free -m` command +func parseMemFree(out string) (int64, error) { + // total used free shared buff/cache available + //Mem: 1987 706 194 1 1086 1173 + //Swap: 0 0 0 + outlines := strings.Split(out, "\n") + l := len(outlines) + for _, line := range outlines[1 : l-1] { + parsedLine := strings.Fields(line) + if len(parsedLine) < 7 { + continue + } + t, err := strconv.ParseInt(parsedLine[1], 10, 64) + if err != nil { + return 0, err + } + m := strings.Trim(parsedLine[0], ":") + if m == "Mem" { + return t, nil + } + } + return 0, errors.New("no matching data found") +} + +// ParseDiskFree parses the output of the `df -m` command +func parseDiskFree(out string) (int64, error) { + // Filesystem 1M-blocks Used Available Use% Mounted on + // /dev/sda1 39643 3705 35922 10% / + outlines := strings.Split(out, "\n") + l := len(outlines) + for _, line := range outlines[1 : l-1] { + parsedLine := strings.Fields(line) + if len(parsedLine) < 6 { + continue + } + t, err := strconv.ParseInt(parsedLine[1], 10, 64) + if err != nil { + return 0, err + } + m := parsedLine[5] + if m == "/" { + return t, nil + } + } + return 0, errors.New("no matching data found") +} diff --git a/pkg/minikube/machine/machine.go b/pkg/minikube/machine/machine.go index d382041f6a..a465694e0b 100644 --- a/pkg/minikube/machine/machine.go +++ b/pkg/minikube/machine/machine.go @@ -104,7 +104,7 @@ func fastDetectProvisioner(h *host.Host) (libprovision.Provisioner, error) { switch { case driver.IsKIC(d): return provision.NewUbuntuProvisioner(h.Driver), nil - case driver.BareMetal(d): + case driver.BareMetal(d), driver.IsSSH(d): return libprovision.DetectProvisioner(h.Driver) default: return provision.NewBuildrootProvisioner(h.Driver), nil diff --git a/pkg/minikube/machine/start.go b/pkg/minikube/machine/start.go index 301286257b..2fa43e58b6 100644 --- a/pkg/minikube/machine/start.go +++ b/pkg/minikube/machine/start.go @@ -129,7 +129,10 @@ func createHost(api libmachine.API, cfg *config.ClusterConfig, n *config.Node) ( klog.Infof("duration metric: createHost completed in %s", time.Since(start)) }() - showHostInfo(*cfg) + if cfg.Driver != driver.SSH { + showHostInfo(nil, *cfg) + } + def := registry.Driver(cfg.Driver) if def.Empty() { return nil, fmt.Errorf("unsupported/missing driver: %s", cfg.Driver) @@ -163,6 +166,9 @@ func createHost(api libmachine.API, cfg *config.ClusterConfig, n *config.Node) ( return nil, errors.Wrap(err, "creating host") } klog.Infof("duration metric: libmachine.API.Create for %q took %s", cfg.Name, time.Since(cstart)) + if cfg.Driver == driver.SSH { + showHostInfo(h, *cfg) + } if err := postStartSetup(h, *cfg); err != nil { return h, errors.Wrap(err, "post-start") @@ -283,7 +289,7 @@ func postStartSetup(h *host.Host, mc config.ClusterConfig) error { if driver.BareMetal(mc.Driver) { showLocalOsRelease() } - if driver.IsVM(mc.Driver) || driver.IsKIC(mc.Driver) { + if driver.IsVM(mc.Driver) || driver.IsKIC(mc.Driver) || driver.IsSSH(mc.Driver) { logRemoteOsRelease(r) } return syncLocalAssets(r) @@ -314,16 +320,29 @@ func acquireMachinesLock(name string, drv string) (mutex.Releaser, error) { } // showHostInfo shows host information -func showHostInfo(cfg config.ClusterConfig) { +func showHostInfo(h *host.Host, cfg config.ClusterConfig) { machineType := driver.MachineType(cfg.Driver) if driver.BareMetal(cfg.Driver) { - info, cpuErr, memErr, DiskErr := CachedHostInfo() + info, cpuErr, memErr, DiskErr := LocalHostInfo() if cpuErr == nil && memErr == nil && DiskErr == nil { register.Reg.SetStep(register.RunningLocalhost) out.Step(style.StartingNone, "Running on localhost (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...", out.V{"number_of_cpus": info.CPUs, "memory_size": info.Memory, "disk_size": info.DiskSize}) } return } + if driver.IsSSH(cfg.Driver) { + r, err := CommandRunner(h) + if err != nil { + klog.Warningf("error getting command runner: %v", err) + return + } + info, cpuErr, memErr, DiskErr := RemoteHostInfo(r) + if cpuErr == nil && memErr == nil && DiskErr == nil { + register.Reg.SetStep(register.RunningRemotely) + out.Step(style.StartingSSH, "Running remotely (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...", out.V{"number_of_cpus": info.CPUs, "memory_size": info.Memory, "disk_size": info.DiskSize}) + } + return + } if driver.IsKIC(cfg.Driver) { // TODO:medyagh add free disk space on docker machine register.Reg.SetStep(register.CreatingContainer) out.Step(style.StartingVM, "Creating {{.driver_name}} {{.machine_type}} (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB) ...", out.V{"driver_name": cfg.Driver, "number_of_cpus": cfg.CPUs, "memory_size": cfg.Memory, "machine_type": machineType}) diff --git a/pkg/minikube/out/register/register.go b/pkg/minikube/out/register/register.go index ebb6c13a4c..6b810b0e30 100644 --- a/pkg/minikube/out/register/register.go +++ b/pkg/minikube/out/register/register.go @@ -32,6 +32,7 @@ const ( StartingNode RegStep = "Starting Node" PullingBaseImage RegStep = "Pulling Base Image" RunningLocalhost RegStep = "Running on Localhost" + RunningRemotely RegStep = "Running Remotely" LocalOSRelease RegStep = "Local OS Release" CreatingContainer RegStep = "Creating Container" CreatingVM RegStep = "Creating VM" @@ -79,6 +80,7 @@ func init() { LocalOSRelease, CreatingContainer, CreatingVM, + RunningRemotely, PreparingKubernetes, PreparingKubernetesCerts, PreparingKubernetesControlPlane, diff --git a/pkg/minikube/registry/drvs/init.go b/pkg/minikube/registry/drvs/init.go index bca12775c3..06b1115827 100644 --- a/pkg/minikube/registry/drvs/init.go +++ b/pkg/minikube/registry/drvs/init.go @@ -25,6 +25,7 @@ import ( _ "k8s.io/minikube/pkg/minikube/registry/drvs/none" _ "k8s.io/minikube/pkg/minikube/registry/drvs/parallels" _ "k8s.io/minikube/pkg/minikube/registry/drvs/podman" + _ "k8s.io/minikube/pkg/minikube/registry/drvs/ssh" _ "k8s.io/minikube/pkg/minikube/registry/drvs/virtualbox" _ "k8s.io/minikube/pkg/minikube/registry/drvs/vmware" _ "k8s.io/minikube/pkg/minikube/registry/drvs/vmwarefusion" diff --git a/pkg/minikube/registry/drvs/ssh/ssh.go b/pkg/minikube/registry/drvs/ssh/ssh.go new file mode 100644 index 0000000000..4d7446d626 --- /dev/null +++ b/pkg/minikube/registry/drvs/ssh/ssh.go @@ -0,0 +1,73 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ssh + +import ( + "fmt" + + "github.com/docker/machine/libmachine/drivers" + "github.com/pkg/errors" + + "k8s.io/minikube/pkg/drivers/ssh" + "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/driver" + "k8s.io/minikube/pkg/minikube/localpath" + "k8s.io/minikube/pkg/minikube/registry" +) + +func init() { + err := registry.Register(registry.DriverDef{ + Name: driver.SSH, + Alias: []string{driver.AliasSSH}, + Config: configure, + Status: status, + Priority: registry.Fallback, + Init: func() drivers.Driver { return ssh.NewDriver(ssh.Config{}) }, + }) + if err != nil { + panic(fmt.Sprintf("unable to register: %v", err)) + } +} + +func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) { + d := ssh.NewDriver(ssh.Config{ + MachineName: config.MachineName(cc, n), + StorePath: localpath.MiniPath(), + ContainerRuntime: cc.KubernetesConfig.ContainerRuntime, + }) + + if cc.SSHIPAddress == "" { + return nil, errors.Errorf("please provide an IP address") + } + + // We don't want the API server listening on loopback interface, + // even if we might use a tunneled VM port for the SSH service + if cc.SSHIPAddress == "127.0.0.1" || cc.SSHIPAddress == "localhost" { + return nil, errors.Errorf("please provide real IP address") + } + + d.IPAddress = cc.SSHIPAddress + d.SSHUser = cc.SSHUser + d.SSHKey = cc.SSHKey + d.SSHPort = cc.SSHPort + + return d, nil +} + +func status() registry.State { + return registry.State{Installed: true, Healthy: true} +} diff --git a/pkg/minikube/style/style.go b/pkg/minikube/style/style.go index 6707ab4fe1..3adc368609 100644 --- a/pkg/minikube/style/style.go +++ b/pkg/minikube/style/style.go @@ -126,6 +126,7 @@ var Config = map[Enum]Options{ Resetting: {Prefix: "🔄 "}, Shutdown: {Prefix: "🛑 "}, StartingNone: {Prefix: "🤹 "}, + StartingSSH: {Prefix: "🔗 "}, StartingVM: {Prefix: "🔥 ", OmitNewline: true, Spinner: true}, SubStep: {Prefix: " ▪ ", LowPrefix: LowIndent, OmitNewline: true, Spinner: true}, // Indented bullet Tip: {Prefix: "💡 "}, diff --git a/pkg/minikube/style/style_enum.go b/pkg/minikube/style/style_enum.go index 63a6c49e9f..7647fba2ea 100644 --- a/pkg/minikube/style/style_enum.go +++ b/pkg/minikube/style/style_enum.go @@ -81,6 +81,7 @@ const ( Shutdown Sparkle StartingNone + StartingSSH StartingVM Stopped Stopping diff --git a/site/content/en/docs/commands/start.md b/site/content/en/docs/commands/start.md index fe1ae89539..e45bf8ecf2 100644 --- a/site/content/en/docs/commands/start.md +++ b/site/content/en/docs/commands/start.md @@ -26,7 +26,7 @@ minikube start [flags] --apiserver-names strings A set of apiserver names which are used in the generated certificate for kubernetes. This can be used if you want to make the apiserver available from outside the machine --apiserver-port int The apiserver listening port (default 8443) --auto-update-drivers If set, automatically updates drivers to the latest version. Defaults to true. (default true) - --base-image string The base image to use for docker/podman drivers. Intended for local development. (default "gcr.io/k8s-minikube/kicbase:v0.0.16-snapshot1@sha256:dff16232547bb3ac3f2a9e09a42246a96ecf8f40d9a1c5bcf5a37953690954b6") + --base-image string The base image to use for docker/podman drivers. Intended for local development. (default "gcr.io/k8s-minikube/kicbase:v0.0.17@sha256:1cd2e039ec9d418e6380b2fa0280503a72e5b282adea674ee67882f59f4f546e") --cache-images If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --driver=none. (default true) --cni string CNI plug-in to use. Valid options: auto, bridge, calico, cilium, flannel, kindnet, or path to a CNI manifest (default: auto) --container-runtime string The container runtime to be used (docker, cri-o, containerd). (default "docker") @@ -64,7 +64,7 @@ minikube start [flags] --insecure-registry strings Insecure Docker registries to pass to the Docker daemon. The default service CIDR range will automatically be added. --install-addons If set, install addons. Defaults to true. (default true) --interactive Allow user prompts for more information (default true) - --iso-url strings Locations to fetch the minikube ISO from. (default [https://storage.googleapis.com/minikube/iso/minikube-v1.16.0.iso,https://github.com/kubernetes/minikube/releases/download/v1.16.0/minikube-v1.16.0.iso,https://kubernetes.oss-cn-hangzhou.aliyuncs.com/minikube/iso/minikube-v1.16.0.iso]) + --iso-url strings Locations to fetch the minikube ISO from. (default [https://storage.googleapis.com/minikube/iso/minikube-v1.17.0.iso,https://github.com/kubernetes/minikube/releases/download/v1.17.0/minikube-v1.17.0.iso,https://kubernetes.oss-cn-hangzhou.aliyuncs.com/minikube/iso/minikube-v1.17.0.iso]) --keep-context This will keep the existing kubectl context and will create a minikube context. --kubernetes-version string The Kubernetes version that the minikube VM will use (ex: v1.2.3, 'stable' for v1.20.0, 'latest' for v1.20.0). Defaults to 'stable'. --kvm-gpu Enable experimental NVIDIA GPU support in minikube @@ -88,6 +88,10 @@ minikube start [flags] --preload If set, download tarball of preloaded images if available to improve start time. Defaults to true. (default true) --registry-mirror strings Registry mirrors to pass to the Docker daemon --service-cluster-ip-range string The CIDR to be used for service cluster IPs. (default "10.96.0.0/12") + --ssh-ip-address string IP address (ssh driver only) + --ssh-key string SSH key (ssh driver only) + --ssh-port int SSH port (ssh driver only) (default 22) + --ssh-user string SSH user (ssh driver only) (default "root") --trace string Send trace events. Options include: [gcp] --uuid string Provide VM UUID to restore MAC address (hyperkit driver only) --vm Filter to use only VM Drivers diff --git a/site/content/en/docs/drivers/includes/ssh_usage.inc b/site/content/en/docs/drivers/includes/ssh_usage.inc new file mode 100644 index 0000000000..608383fe81 --- /dev/null +++ b/site/content/en/docs/drivers/includes/ssh_usage.inc @@ -0,0 +1,25 @@ +## Requirements + +A Linux VM with the following: + +* systemd or OpenRC +* a container runtime, such as Docker or CRIO + +This VM must also meet the [kubeadm requirements](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/), such as: + +* 2 CPU's +* 2GB RAM +* iptables (in legacy mode) +* conntrack +* crictl +* SELinux permissive +* cgroups v1 (v2 is not yet supported by Kubernetes) + +## Usage + +The ssh driver requires the IP address of the VM to use. + +```shell +minikube start --driver=ssh --ssh-ip-address=vm.example.com +``` + diff --git a/site/content/en/docs/drivers/ssh.md b/site/content/en/docs/drivers/ssh.md new file mode 100644 index 0000000000..50777aed6a --- /dev/null +++ b/site/content/en/docs/drivers/ssh.md @@ -0,0 +1,22 @@ +--- +title: "ssh" +weight: 3 +description: > + Linux ssh (remote) driver +aliases: + - /docs/reference/drivers/ssh +--- + +## Overview + +This document is written for system integrators who wish to run minikube within a customized VM environment. The `ssh` driver allows advanced minikube users to skip VM creation, allowing minikube to be run on a user-supplied VM. + +{{% readfile file="/docs/drivers/includes/ssh_usage.inc" %}} + +## Issues + +* [Full list of open 'ssh' driver issues](https://github.com/kubernetes/minikube/labels/co%2Fssh-driver) + +## Troubleshooting + +* Run `minikube start --alsologtostderr -v=4` to debug crashes diff --git a/test/integration/aaa_download_only_test.go b/test/integration/aaa_download_only_test.go index 46152dda7a..b39d98bae1 100644 --- a/test/integration/aaa_download_only_test.go +++ b/test/integration/aaa_download_only_test.go @@ -194,6 +194,10 @@ func TestDownloadOnlyKic(t *testing.T) { if err != nil { t.Errorf("failed to read tarball file %q: %v", tarball, err) } + + if arm64Platform() { + t.Skip("Skip for arm64 platform. See https://github.com/kubernetes/minikube/issues/10144") + } // Make sure it has the correct checksum checksum := md5.Sum(contents) remoteChecksum, err := ioutil.ReadFile(download.PreloadChecksumPath(constants.DefaultKubernetesVersion, cRuntime)) diff --git a/test/integration/aab_offline_test.go b/test/integration/aab_offline_test.go index d0275b53bf..ced6b5da2d 100644 --- a/test/integration/aab_offline_test.go +++ b/test/integration/aab_offline_test.go @@ -29,20 +29,24 @@ import ( // TestOffline makes sure minikube works without internet, once it the user has already cached the images, This test has to run after TestDownloadOnly! func TestOffline(t *testing.T) { t.Run("group", func(t *testing.T) { - for _, runtime := range []string{"docker", "crio", "containerd"} { - runtime := runtime - t.Run(runtime, func(t *testing.T) { + for _, rt := range []string{"docker", "crio", "containerd"} { + rt := rt + t.Run(rt, func(t *testing.T) { MaybeParallel(t) - if runtime != "docker" && NoneDriver() { + if rt != "docker" && arm64Platform() { + t.Skipf("skipping %s - only docker runtime supported on arm64. See https://github.com/kubernetes/minikube/issues/10144", t.Name()) + } + + if rt != "docker" && NoneDriver() { t.Skipf("skipping %s - incompatible with none driver", t.Name()) } - profile := UniqueProfileName(fmt.Sprintf("offline-%s", runtime)) + profile := UniqueProfileName(fmt.Sprintf("offline-%s", rt)) ctx, cancel := context.WithTimeout(context.Background(), Minutes(15)) defer CleanupWithLogs(t, profile, cancel) - startArgs := []string{"start", "-p", profile, "--alsologtostderr", "-v=1", "--memory=2000", "--wait=true", "--container-runtime", runtime} + startArgs := []string{"start", "-p", profile, "--alsologtostderr", "-v=1", "--memory=2000", "--wait=true", "--container-runtime", rt} startArgs = append(startArgs, StartArgs()...) c := exec.CommandContext(ctx, Target(), startArgs...) env := os.Environ() diff --git a/test/integration/addons_test.go b/test/integration/addons_test.go index 779be84c3b..4046a77986 100644 --- a/test/integration/addons_test.go +++ b/test/integration/addons_test.go @@ -58,10 +58,13 @@ func TestAddons(t *testing.T) { t.Fatalf("Failed setting GOOGLE_CLOUD_PROJECT env var: %v", err) } - args := append([]string{"start", "-p", profile, "--wait=true", "--memory=4000", "--alsologtostderr", "--addons=registry", "--addons=metrics-server", "--addons=helm-tiller", "--addons=olm", "--addons=volumesnapshots", "--addons=csi-hostpath-driver", "--addons=gcp-auth"}, StartArgs()...) + args := append([]string{"start", "-p", profile, "--wait=true", "--memory=4000", "--alsologtostderr", "--addons=registry", "--addons=metrics-server", "--addons=olm", "--addons=volumesnapshots", "--addons=csi-hostpath-driver", "--addons=gcp-auth"}, StartArgs()...) if !NoneDriver() && !(runtime.GOOS == "darwin" && KicDriver()) { // none doesn't support ingress args = append(args, "--addons=ingress") } + if !arm64Platform() { + args = append(args, "--addons=helm-tiller") + } rr, err := Run(t, exec.CommandContext(ctx, Target(), args...)) if err != nil { t.Fatalf("%s failed: %v", rr.Command(), err) @@ -305,6 +308,7 @@ func validateMetricsServerAddon(ctx context.Context, t *testing.T, profile strin } func validateHelmTillerAddon(ctx context.Context, t *testing.T, profile string) { + defer PostMortemLogs(t, profile) client, err := kapi.Client(profile) diff --git a/test/integration/driver_install_or_update_test.go b/test/integration/driver_install_or_update_test.go index 7b1ae0436d..13152e1310 100644 --- a/test/integration/driver_install_or_update_test.go +++ b/test/integration/driver_install_or_update_test.go @@ -42,6 +42,10 @@ func TestKVMDriverInstallOrUpdate(t *testing.T) { t.Skip("Skip if not linux.") } + if arm64Platform() { + t.Skip("Skip if arm64. See https://github.com/kubernetes/minikube/issues/10144") + } + MaybeParallel(t) tests := []struct { diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go index 2342152524..09e99f7fb4 100644 --- a/test/integration/functional_test.go +++ b/test/integration/functional_test.go @@ -361,6 +361,26 @@ func validateExtraConfig(ctx context.Context, t *testing.T, profile string) { } +// imageID returns a docker image id for image `image` and current architecture +// 'image' is supposed to be one commonly used in minikube integration tests, +// like k8s 'pause' +func imageID(image string) string { + ids := map[string]map[string]string{ + "pause": { + "amd64": "0184c1613d929", + "arm64": "3d18732f8686c", + }, + } + + if imgIds, ok := ids[image]; ok { + if id, ok := imgIds[runtime.GOARCH]; ok { + return id + } + panic(fmt.Sprintf("unexpected architecture for image %q: %v", image, runtime.GOARCH)) + } + panic("unexpected image name: " + image) +} + // validateComponentHealth asserts that all Kubernetes components are healthy func validateComponentHealth(ctx context.Context, t *testing.T, profile string) { defer PostMortemLogs(t, profile) @@ -588,10 +608,10 @@ func validateCacheCmd(ctx context.Context, t *testing.T, profile string) { if err != nil { t.Errorf("failed to get images by %q ssh %v", rr.Command(), err) } - if !strings.Contains(rr.Output(), "0184c1613d929") { - t.Errorf("expected sha for pause:3.3 '0184c1613d929' to be in the output but got *%s*", rr.Output()) + pauseID := imageID("pause") + if !strings.Contains(rr.Output(), pauseID) { + t.Errorf("expected sha for pause:3.3 %q to be in the output but got *%s*", pauseID, rr.Output()) } - }) t.Run("cache_reload", func(t *testing.T) { // deleting image inside minikube node manually and expecting reload to bring it back @@ -783,7 +803,15 @@ func validateServiceCmd(ctx context.Context, t *testing.T, profile string) { } }() - rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "create", "deployment", "hello-node", "--image=k8s.gcr.io/echoserver:1.4")) + var rr *RunResult + var err error + // k8s.gcr.io/echoserver is not multi-arch + if arm64Platform() { + rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "create", "deployment", "hello-node", "--image=k8s.gcr.io/echoserver-arm:1.8")) + } else { + rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "create", "deployment", "hello-node", "--image=k8s.gcr.io/echoserver:1.8")) + } + if err != nil { t.Fatalf("failed to create hello-node deployment with this command %q: %v.", rr.Command(), err) } @@ -955,6 +983,10 @@ func validateSSHCmd(ctx context.Context, t *testing.T, profile string) { // validateMySQL validates a minimalist MySQL deployment func validateMySQL(ctx context.Context, t *testing.T, profile string) { + if arm64Platform() { + t.Skip("arm64 is not supported by mysql. Skip the test. See https://github.com/kubernetes/minikube/issues/10144") + } + defer PostMortemLogs(t, profile) rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "replace", "--force", "-f", filepath.Join(*testdataDir, "mysql.yaml"))) diff --git a/test/integration/main_test.go b/test/integration/main_test.go index f0a1b9502c..d6dfda30b2 100644 --- a/test/integration/main_test.go +++ b/test/integration/main_test.go @@ -135,6 +135,15 @@ func GithubActionRunner() bool { return os.Getenv("GITHUB_ACTIONS") == "true" } +func ContainerdContainerRuntime() bool { + return strings.Contains(*startArgs, "--container-runtime=containerd") +} + +// arm64Platform returns true if running on arm64/* platform +func arm64Platform() bool { + return runtime.GOARCH == "arm64" +} + // NeedsPortForward returns access to endpoints with this driver needs port forwarding // (Docker on non-Linux platforms requires ports to be forwarded to 127.0.0.1) func NeedsPortForward() bool { diff --git a/test/integration/net_test.go b/test/integration/net_test.go index 17b52abf06..9df994f57f 100644 --- a/test/integration/net_test.go +++ b/test/integration/net_test.go @@ -34,9 +34,7 @@ import ( func TestNetworkPlugins(t *testing.T) { MaybeParallel(t) - if NoneDriver() { - t.Skip("skipping since test for none driver") - } + validations(t) t.Run("group", func(t *testing.T) { tests := []struct { @@ -73,7 +71,7 @@ func TestNetworkPlugins(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), Minutes(40)) defer CleanupWithLogs(t, profile, cancel) - startArgs := append([]string{"start", "-p", profile, "--memory=1800", "--alsologtostderr", "--wait=true", "--wait-timeout=25m"}, tc.args...) + startArgs := append([]string{"start", "-p", profile, "--memory=1800", "--alsologtostderr", "--wait=true", "--wait-timeout=5m"}, tc.args...) startArgs = append(startArgs, StartArgs()...) t.Run("Start", func(t *testing.T) { @@ -206,3 +204,12 @@ func TestNetworkPlugins(t *testing.T) { } }) } + +func validations(t *testing.T) { + if NoneDriver() { + t.Skip("skipping since test for none driver") + } + if ContainerdContainerRuntime() { + t.Skip("skipping as this test currently times out on containerd") + } +} diff --git a/test/integration/pause_test.go b/test/integration/pause_test.go index c412d62942..cd202745ee 100644 --- a/test/integration/pause_test.go +++ b/test/integration/pause_test.go @@ -29,6 +29,9 @@ import ( ) func TestPause(t *testing.T) { + if ContainerdContainerRuntime() { + t.Skip("skipping as this test currently times out on containerd") + } MaybeParallel(t) type validateFunc func(context.Context, *testing.T, string) diff --git a/test/integration/preload_test.go b/test/integration/preload_test.go index 9364b00171..e0c868f9a0 100644 --- a/test/integration/preload_test.go +++ b/test/integration/preload_test.go @@ -31,6 +31,10 @@ func TestPreload(t *testing.T) { t.Skipf("skipping %s - incompatible with none driver", t.Name()) } + if arm64Platform() { + t.Skipf("skipping %s - not yet supported on arm64. See https://github.com/kubernetes/minikube/issues/10144", t.Name()) + } + profile := UniqueProfileName("test-preload") ctx, cancel := context.WithTimeout(context.Background(), Minutes(40)) defer CleanupWithLogs(t, profile, cancel) diff --git a/test/integration/skaffold_test.go b/test/integration/skaffold_test.go index 48f9b42ed4..d1d1a4a93b 100644 --- a/test/integration/skaffold_test.go +++ b/test/integration/skaffold_test.go @@ -116,8 +116,8 @@ func installSkaffold() (f *os.File, err error) { } tf.Close() - url := "https://storage.googleapis.com/skaffold/releases/latest/skaffold-%s-amd64" - url = fmt.Sprintf(url, runtime.GOOS) + url := "https://storage.googleapis.com/skaffold/releases/latest/skaffold-%s-%s" + url = fmt.Sprintf(url, runtime.GOOS, runtime.GOARCH) if runtime.GOOS == "windows" { url += ".exe" } diff --git a/translations/ko.json b/translations/ko.json index 03c153eb6e..8f3fe07c87 100644 --- a/translations/ko.json +++ b/translations/ko.json @@ -24,7 +24,6 @@ "A set of apiserver IP Addresses which are used in the generated certificate for kubernetes. This can be used if you want to make the apiserver available from outside the machine": "", "A set of apiserver names which are used in the generated certificate for kubernetes. This can be used if you want to make the apiserver available from outside the machine": "", "A set of key=value pairs that describe feature gates for alpha/experimental features.": "", - "Access the Kubernetes dashboard running within the minikube cluster": "minikube 클러스터 내의 쿠버네티스 대시보드에 접근합니다", "Access the kubernetes dashboard running within the minikube cluster": "minikube 클러스터 내의 쿠버네티스 대시보드에 접근합니다", "Access to ports below 1024 may fail on Windows with OpenSSH clients older than v8.1. For more information, see: https://minikube.sigs.k8s.io/docs/handbook/accessing/#access-to-ports-1024-on-windows-requires-root-permission": "", "Add SSH identity key to SSH authentication agent": "", @@ -749,4 +748,4 @@ "{{.prefix}}minikube {{.version}} on {{.platform}}": "{{.prefix}}{{.platform}} 위의 minikube {{.version}}", "{{.type}} is not yet a supported filesystem. We will try anyways!": "", "{{.url}} is not accessible: {{.error}}": "{{.url}} 이 접근 불가능합니다: {{.error}}" -} \ No newline at end of file +} diff --git a/translations/pl.json b/translations/pl.json index 8d0e6a64c6..d52ba3813c 100644 --- a/translations/pl.json +++ b/translations/pl.json @@ -25,7 +25,6 @@ "A set of apiserver IP Addresses which are used in the generated certificate for kubernetes. This can be used if you want to make the apiserver available from outside the machine": "", "A set of apiserver names which are used in the generated certificate for kubernetes. This can be used if you want to make the apiserver available from outside the machine": "", "A set of key=value pairs that describe feature gates for alpha/experimental features.": "", - "Access the Kubernetes dashboard running within the minikube cluster": "", "Access the kubernetes dashboard running within the minikube cluster": "Dostęp do dashboardu uruchomionego w klastrze kubernetesa w minikube", "Add an image to local cache.": "", "Add machine IP to NO_PROXY environment variable": "", @@ -808,4 +807,4 @@ "{{.prefix}}minikube {{.version}} on {{.platform}}": "{{.prefix}}minikube {{.version}} na {{.platform}}", "{{.type}} is not yet a supported filesystem. We will try anyways!": "{{.type}} nie jest wspierany przez system plików. I tak spróbujemy!", "{{.url}} is not accessible: {{.error}}": "" -} \ No newline at end of file +} diff --git a/translations/zh-CN.json b/translations/zh-CN.json index b361ef5bf2..59c8e8787a 100644 --- a/translations/zh-CN.json +++ b/translations/zh-CN.json @@ -1,7 +1,7 @@ { - "\"The '{{.minikube_addon}}' addon is disabled": "", + "\"The '{{.minikube_addon}}' addon is disabled": "'{{.minikube_addon}}' 插件已被禁用", "\"{{.context}}\" context has been updated to point to {{.hostname}}:{{.port}}": "", - "\"{{.machineName}}\" does not exist, nothing to stop": "", + "\"{{.machineName}}\" does not exist, nothing to stop": "\"{{.machineName}}\" 不存在，没有什么可供停止的", "\"{{.minikube_addon}}\" was successfully disabled": "已成功禁用 \"{{.minikube_addon}}\"", "\"{{.name}}\" cluster does not exist. Proceeding ahead with cleanup.": "\"{{.name}}\" 集群不存在，将继续清理", "\"{{.name}}\" profile does not exist": "“{{.name}}”配置文件不存在", @@ -21,9 +21,9 @@ "- Prune unused {{.driver_name}} images, volumes, networks and abandoned containers.\n\n\t\t\t\t{{.driver_name}} system prune --volumes": "", "- Restart your {{.driver_name}} service": "", "A VPN or firewall is interfering with HTTP access to the minikube VM. Alternatively, try a different VM driver: https://minikube.sigs.k8s.io/docs/start/": "VPN 或者防火墙正在干扰对 minikube 虚拟机的 HTTP 访问。或者，您可以使用其它的虚拟机驱动：https://minikube.sigs.k8s.io/docs/start/", - "A firewall is blocking Docker the minikube VM from reaching the image repository. You may need to select --image-repository, or use a proxy.": "", + "A firewall is blocking Docker the minikube VM from reaching the image repository. You may need to select --image-repository, or use a proxy.": "防火墙正在阻止 minikube 虚拟机中的 Docker 访问镜像仓库。您可能需要选择 --image-repository 或使用代理", "A firewall is blocking Docker the minikube VM from reaching the internet. You may need to configure it to use a proxy.": "防火墙正在阻止 minikube 虚拟机中的 Docker 访问互联网。您可能需要对其进行配置为使用代理", - "A firewall is blocking Docker within the minikube VM from reaching the internet. You may need to configure it to use a proxy.": "防火墙正在阻止 minikube 虚拟机中的 Docker 访问互联网，您可能需要对其进行配置为使用代理", + "A firewall is blocking Docker within the minikube VM from reaching the internet. You may need to configure it to use a proxy.": "防火墙正在阻止 minikube 虚拟机中的 Docker 访问互联网。您可能需要对其进行配置为使用代理", "A firewall is interfering with minikube's ability to make outgoing HTTPS requests. You may need to change the value of the HTTPS_PROXY environment variable.": "防火墙正在干扰 minikube 发送 HTTPS 请求的能力，您可能需要改变 HTTPS_PROXY 环境变量的值", "A firewall is likely blocking minikube from reaching the internet. You may need to configure minikube to use a proxy.": "防火墙可能会阻止 minikube 访问互联网。您可能需要将 minikube 配置为使用", "A set of apiserver IP Addresses which are used in the generated certificate for kubernetes. This can be used if you want to make the apiserver available from outside the machine": "一组在为 kubernetes 生成的证书中使用的 apiserver IP 地址。如果您希望将此 apiserver 设置为可从机器外部访问，则可以使用这组 apiserver IP 地址", @@ -32,14 +32,13 @@ "A set of apiserver names which are used in the generated certificate for kubernetes. This can be used if you want to make the apiserver available from outside the machine": "一组在为 kubernetes 生成的证书中使用的 apiserver 名称。如果您希望将此 apiserver 设置为可从机器外部访问，则可以使用这组 apiserver 名称", "A set of key=value pairs that describe configuration that may be passed to different components.\nThe key should be '.' separated, and the first part before the dot is the component to apply the configuration to.\nValid components are: kubelet, kubeadm, apiserver, controller-manager, etcd, proxy, scheduler\nValid kubeadm parameters:": "一组用于描述可传递给不同组件的配置的键值对。\n其中键应以英文句点“.”分隔，英文句点前面的第一个部分是应用该配置的组件。\n有效组件包括：kubelet、kubeadm、apiserver、controller-manager、etcd、proxy、scheduler\n有效 kubeadm 参数包括：", "A set of key=value pairs that describe feature gates for alpha/experimental features.": "一组用于描述 alpha 版功能/实验性功能的功能限制的键值对。", - "Access the Kubernetes dashboard running within the minikube cluster": "", "Access the kubernetes dashboard running within the minikube cluster": "访问在 minikube 集群中运行的 kubernetes dashboard", "Add an image to local cache.": "将 image 添加到本地缓存。", "Add machine IP to NO_PROXY environment variable": "将机器IP添加到环境变量 NO_PROXY 中", "Add or delete an image from the local cache.": "在本地缓存中添加或删除 image。", "Add, delete, or push a local image into minikube": "", "Add, remove, or list additional nodes": "", - "Adding node {{.name}} to cluster {{.cluster}}": "", + "Adding node {{.name}} to cluster {{.cluster}}": "添加节点 {{.name}} 至集群 {{.cluster}}", "Additional help topics": "其他帮助", "Additional mount options, such as cache=fscache": "其他挂载选项，例如：cache=fscache", "Adds a node to the given cluster config, and starts it.": "", @@ -58,8 +57,8 @@ "Another program is using a file required by minikube. If you are using Hyper-V, try stopping the minikube VM from within the Hyper-V manager": "", "Automatically selected the '{{.driver}}' driver": "自动选择 '{{.driver}}' 驱动", "Automatically selected the '{{.driver}}' driver (alternates: {{.alternates}})": "自动选择 '{{.driver}}' 驱动（可选项：{{.alternates}}）", - "Automatically selected the {{.driver}} driver": "", - "Automatically selected the {{.driver}} driver. Other choices: {{.alternates}}": "", + "Automatically selected the {{.driver}} driver": "自动选择 {{.driver}} 驱动", + "Automatically selected the {{.driver}} driver. Other choices: {{.alternates}}": "自动选择 {{.driver}} 驱动。其他选项：{{.alternates}}", "Available Commands": "可用命令", "Basic Commands:": "基本命令：", "Because you are using a Docker driver on {{.operating_system}}, the terminal needs to be open to run it.": "", @@ -72,29 +71,29 @@ "Check if you have unnecessary pods running by running 'kubectl get po -A": "", "Check output of 'journalctl -xeu kubelet', try passing --extra-config=kubelet.cgroup-driver=systemd to minikube start": "检查 'journalctl -xeu kubelet' 的输出，尝试启动 minikube 时添加参数 --extra-config=kubelet.cgroup-driver=systemd", "Check that SELinux is disabled, and that the provided apiserver flags are valid": "检查 SELinux 是否禁用，且提供的 apiserver 标志是否有效", - "Check that libvirt is setup properly": "", + "Check that libvirt is setup properly": "检查 libvirt 是否正确设置", "Check that minikube is running and that you have specified the correct namespace (-n flag) if required.": "检测 minikube 是否正在运行，以及是否根据需要指定了正确的 namespace （-n 标志）", - "Check that the provided apiserver flags are valid": "检查提供的 apiserver 标志是否有效", - "Check that the provided apiserver flags are valid, and that SELinux is disabled": "", + "Check that the provided apiserver flags are valid": "检查提供的 apiserver 标志是有效的", + "Check that the provided apiserver flags are valid, and that SELinux is disabled": "检查提供的 apiserver 标志是有效的，且禁用了 SELinux", "Check that your --kubernetes-version has a leading 'v'. For example: 'v1.1.14'": "检测您的 --kubernetes-version 前面是否有 'v'， 例如：'v1.1.14", "Check that your apiserver flags are valid, or run 'minikube delete'": "请检查您的 apiserver 标志是否有效，或者允许 'minikube delete'", "Check your firewall rules for interference, and run 'virt-host-validate' to check for KVM configuration issues. If you are running minikube within a VM, consider using --driver=none": "", "Check your firewall rules for interference, and run 'virt-host-validate' to check for KVM configuration issues. If you are running minikube within a VM, consider using --vm-driver=none": "检查您的防火墙规则是否存在干扰，然后运行 'virt-host-validate' 以检查 KVM 配置问题，如果在虚拟机中运行minikube，请考虑使用 --vm-driver=none", - "Choose a smaller value for --memory, such as 2000": "", - "ChromeOS is missing the kernel support necessary for running Kubernetes": "", + "Choose a smaller value for --memory, such as 2000": "为 --memory 选择一个更小的值，例如 2000", + "ChromeOS is missing the kernel support necessary for running Kubernetes": "ChromeOS 缺少运行 Kubernetes 所需的内核支持", "Configuration and Management Commands:": "配置和管理命令：", - "Configure a default route on this Linux host, or use another --driver that does not require it": "", + "Configure a default route on this Linux host, or use another --driver that does not require it": "为当前 Linux 主机配置一个默认的路由, 或者使用另一个不需要他的 --driver", "Configure a default route on this Linux host, or use another --vm-driver that does not require it": "为当前 Linux 主机配置一个默认的路由, 或者使用另一个不需要他的 --vm-driver", "Configure an external network switch following the official documentation, then add `--hyperv-virtual-switch=\u003cswitch-name\u003e` to `minikube start`": "根据官方文档配置外部网络交换机，然后添加 `--hyperv-virtual-switch=\u003cswitch-name\u003e` 到 `minikube start`", - "Configure environment to use minikube's Docker daemon": "", - "Configure environment to use minikube's Podman service": "", + "Configure environment to use minikube's Docker daemon": "配置环境以使用 minikube's Docker daemon", + "Configure environment to use minikube's Podman service": "配置环境以使用 minikube's Podman service", "Configures the addon w/ADDON_NAME within minikube (example: minikube addons configure registry-creds). For a list of available addons use: minikube addons list": "在 minikube 中配置插件 w/ADDON_NAME（例如：minikube addons configure registry-creds）。查看相关可用的插件列表，请使用：minikube addons list", "Configuring environment for Kubernetes {{.k8sVersion}} on {{.runtime}} {{.runtimeVersion}}": "开始为Kubernetes {{.k8sVersion}}，{{.runtime}} {{.runtimeVersion}} 配置环境变量", "Configuring local host environment ...": "开始配置本地主机环境...", "Configuring {{.name}} (Container Networking Interface) ...": "", "Confirm that you have a working internet connection and that your VM has not run out of resources by using: 'minikube logs'": "使用 'minikube logs' 确认您的互联网连接正常，并且您的虚拟机没有耗尽资源", "Confirm that you have supplied the correct value to --hyperv-virtual-switch using the 'Get-VMSwitch' command": "使用 'Get-VMSwitch' 命令确认已经为 --hyperv-virtual-switch 提供了正确的值", - "Connect to LoadBalancer services": "", + "Connect to LoadBalancer services": "连接到 LoadBalancer 服务", "Consider creating a cluster with larger memory size using `minikube start --memory SIZE_MB` ": "", "Consider increasing Docker Desktop's memory size.": "", "Could not determine a Google Cloud project, which might be ok.": "", @@ -110,7 +109,7 @@ "Creating {{.driver_name}} VM (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...": "正在创建 {{.driver_name}} 虚拟机（CPUs={{.number_of_cpus}}，Memory={{.memory_size}}MB, Disk={{.disk_size}}MB）...", "Creating {{.driver_name}} {{.machine_type}} (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB) ...": "", "Creating {{.driver_name}} {{.machine_type}} (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...": "", - "Current context is \"{{.context}}\"": "", + "Current context is \"{{.context}}\"": "当前的上下文为 \"{{.context}}\"", "DEPRECATED, use `driver` instead.": "", "DEPRECATED: Replaced by --cni=bridge": "", "Default group id used for the mount": "用于挂载默认的 group id", @@ -121,10 +120,10 @@ "Deletes a local kubernetes cluster": "删除本地的 kubernetes 集群", "Deletes a local kubernetes cluster. This command deletes the VM, and removes all\nassociated files.": "删除本地的 kubernetes 集群。此命令还将删除虚拟机，并删除所有的\n相关文件", "Deletes a local kubernetes cluster. This command deletes the VM, and removes all associated files.": "删除本地 kubernetes 集群。此命令会删除虚拟机并移除所有关联的文件。", - "Deletes a node from a cluster.": "", + "Deletes a node from a cluster.": "从集群中删除节点。", "Deleting \"{{.profile_name}}\" in {{.driver_name}} ...": "正在删除 {{.driver_name}} 中的“{{.profile_name}}”…", - "Deleting container \"{{.name}}\" ...": "", - "Deleting node {{.name}} from cluster {{.cluster}}": "", + "Deleting container \"{{.name}}\" ...": "正在删除容器 \"{{.name}}\" ...", + "Deleting node {{.name}} from cluster {{.cluster}}": "正在从集群 {{.cluster}} 中删除节点 {{.name}}", "Disable checking for the availability of hardware virtualization before the vm is started (virtualbox driver only)": "禁用在启动虚拟机之前检查硬件虚拟化的可用性（仅限 virtualbox 驱动程序）", "Disable dynamic memory in your VM manager, or pass in a larger --memory value": "禁用虚拟机管理器中的动态内存，或者使用 --memory 传入更大的值", "Disables the addon w/ADDON_NAME within minikube (example: minikube addons disable dashboard). For a list of available addons use: minikube addons list": "在 minikube 中禁用插件 w/ADDON_NAME（例如：minikube addons disable dashboard）。查看相关可用的插件列表，请使用：minikube addons list", @@ -216,7 +215,7 @@ "Error getting profiles to delete": "获取待删除配置文件时出错", "Error getting service status": "获取 service status 时出错", "Error getting service with namespace: {{.namespace}} and labels {{.labelName}}:{{.addonName}}: {{.error}}": "使用 namespace: {{.namespace}} 和 labels {{.labelName}}:{{.addonName}} 获取 service 时出错：{{.error}}", - "Error getting ssh client": "", + "Error getting ssh client": "获取 ssh 客户端时出错", "Error getting the host IP address to use from within the VM": "从虚拟机中获取 host IP 地址时出错", "Error killing mount process": "杀死 mount 进程时出错", "Error loading api": "加载 api 时出错", @@ -234,7 +233,7 @@ "Error starting mount": "开启 mount 时出错", "Error unsetting shell variables": "取消设置 shell 变量时出错", "Error while setting kubectl current context : {{.error}}": "设置 kubectl 上下文时出错 ：{{.error}}", - "Error while setting kubectl current context: {{.error}}": "", + "Error while setting kubectl current context: {{.error}}": "设置 kubectl 上下文时出错：{{.error}}", "Error writing mount pid": "写入 mount pid 时出错", "Error: You have selected Kubernetes v{{.new}}, but the existing cluster for your profile is running Kubernetes v{{.old}}. Non-destructive downgrades are not supported, but you can proceed by performing one of the following options:\n\n* Recreate the cluster using Kubernetes v{{.new}}: Run \"minikube delete {{.profile}}\", then \"minikube start {{.profile}} --kubernetes-version={{.new}}\"\n* Create a second cluster with Kubernetes v{{.new}}: Run \"minikube start -p \u003cnew name\u003e --kubernetes-version={{.new}}\"\n* Reuse the existing cluster with Kubernetes v{{.old}} or newer: Run \"minikube start {{.profile}} --kubernetes-version={{.old}}\"": "错误：您已选择 Kubernetes v{{.new}}，但您的配置文件的现有集群正在运行 Kubernetes v{{.old}}。非破坏性降级不受支持，但若要继续操作，您可以执行以下选项之一：\n\n* 使用 Kubernetes v{{.new}} 重新创建现有集群：运行“minikube delete {{.profile}}”，然后运行“minikube start {{.profile}} --kubernetes-version={{.new}}”\n* 使用 Kubernetes v{{.new}} 再创建一个集群：运行“minikube start -p \u003cnew name\u003e --kubernetes-version={{.new}}”\n* 通过 Kubernetes v{{.old}} 或更高版本重复使用现有集群：运行“minikube start {{.profile}} --kubernetes-version={{.old}}”", "Error: You have selected Kubernetes v{{.new}}, but the existing cluster for your profile is running Kubernetes v{{.old}}. Non-destructive downgrades are not supported, but you can proceed by performing one of the following options:\n* Recreate the cluster using Kubernetes v{{.new}}: Run \"minikube delete {{.profile}}\", then \"minikube start {{.profile}} --kubernetes-version={{.new}}\"\n* Create a second cluster with Kubernetes v{{.new}}: Run \"minikube start -p \u003cnew name\u003e --kubernetes-version={{.new}}\"\n* Reuse the existing cluster with Kubernetes v{{.old}} or newer: Run \"minikube start {{.profile}} --kubernetes-version={{.old}}": "错误：您已选择 Kubernetes v{{.new}}，但您的配置文件的现有集群正在运行 Kubernetes v{{.old}}。非破坏性降级不受支持，但若要继续操作，您可以执行以下选项之一：\n* 使用 Kubernetes v{{.new}} 重新创建现有集群：运行“minikube delete {{.profile}}”，然后运行“minikube start {{.profile}} --kubernetes-version={{.new}}”\n* 使用 Kubernetes v{{.new}} 再创建一个集群：运行“minikube start -p \u003cnew name\u003e --kubernetes-version={{.new}}”\n* 通过 Kubernetes v{{.old}} 或更高版本重复使用现有集群：运行“minikube start {{.profile}} --kubernetes-version={{.old}}”", @@ -917,4 +916,4 @@ "{{.prefix}}minikube {{.version}} on {{.platform}}": "{{.platform}} 上的 {{.prefix}}minikube {{.version}}", "{{.type}} is not yet a supported filesystem. We will try anyways!": "", "{{.url}} is not accessible: {{.error}}": "" -} \ No newline at end of file +}