diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml new file mode 100644 index 0000000000..a8cc54413e --- /dev/null +++ b/.github/workflows/master.yml @@ -0,0 +1,537 @@ +name: MasterCI +on: + push: + branches: + - master + paths: + - '**.go' +env: + GOPROXY: https://proxy.golang.org +jobs: + # Runs before all other jobs + # builds the minikube binaries + build_minikube: + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v2 + - name: Download Dependencies + run : go mod download + - name: Build Binaries + run : | + make minikube-linux-amd64 + make e2e-linux-amd64 + cp -r test/integration/testdata ./out + whoami + echo github ref $GITHUB_REF + echo workflow $GITHUB_WORKFLOW + echo home $HOME + echo event name $GITHUB_EVENT_NAME + echo workspace $GITHUB_WORKSPACE + echo "end of debug stuff" + echo $(which jq) + - uses: actions/upload-artifact@v1 + with: + name: minikube_binaries + path: out + lint: + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v2 + - name: Install libvirt + run : | + sudo apt-get update + sudo apt-get install -y libvirt-dev + - name: Download Dependencies + run : go mod download + - name: Lint + env: + TESTSUITE: lintall + run : make test + continue-on-error: false + unit_test: + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v2 + - name: Install libvirt + run : | + sudo apt-get update + sudo apt-get install -y libvirt-dev + - name: Download Dependencies + run : go mod download + - name: Unit Test + env: + TESTSUITE: unittest + run : + make test + continue-on-error: false + # Run the following integration tests after the build_minikube + # They will run in parallel and use the binaries in previous step + functional_test_docker_ubuntu: + needs: [build_minikube] + env: + TIME_ELAPSED: time + JOB_NAME: "functional_test_docker_ubuntu" + GOPOGH_RESULT: "" + SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643 + runs-on: ubuntu-18.04 + steps: + - name: Install kubectl + shell: bash + run: | + curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/amd64/kubectl + sudo install kubectl /usr/local/bin/kubectl + kubectl version --client=true + - name: Docker Info + shell: bash + run: | + echo "--------------------------" + docker version || true + echo "--------------------------" + docker info || true + echo "--------------------------" + docker system df || true + echo "--------------------------" + docker system info || true + echo "--------------------------" + docker ps || true + echo "--------------------------" + - name: Install lz4 + shell: bash + run: | + sudo apt-get update -qq + sudo apt-get -qq -y install liblz4-tool + - name: Install gopogh + shell: bash + run: | + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.1.19/gopogh-linux-amd64 + sudo install gopogh-linux-amd64 /usr/local/bin/gopogh + - name: Download Binaries + uses: actions/download-artifact@v1 + with: + name: minikube_binaries + - name: Run Integration Test + continue-on-error: false + # bash {0} to allow test to continue to next step. in case of + shell: bash {0} + run: | + cd minikube_binaries + mkdir -p report + mkdir -p testhome + chmod a+x e2e-* + chmod a+x minikube-* + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + START_TIME=$(date -u +%s) + KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--vm-driver=docker -test.run TestFunctional -test.timeout=30m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt + END_TIME=$(date -u +%s) + TIME_ELAPSED=$(($END_TIME-$START_TIME)) + min=$((${TIME_ELAPSED}/60)) + sec=$((${TIME_ELAPSED}%60)) + TIME_ELAPSED="${min} min $sec seconds " + echo ::set-env name=TIME_ELAPSED::${TIME_ELAPSED} + - name: Generate HTML Report + shell: bash + run: | + cd minikube_binaries + export PATH=${PATH}:`go env GOPATH`/bin + go tool test2json -t < ./report/testout.txt > ./report/testout.json || true + STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}" -details "${GITHUB_SHA}") || true + echo status: ${STAT} + FailNum=$(echo $STAT | jq '.NumberOfFail') + TestsNum=$(echo $STAT | jq '.NumberOfTests') + GOPOGH_RESULT="${JOB_NAME} : completed with ${FailNum} / ${TestsNum} failures in ${TIME_ELAPSED}" + echo ::set-env name=GOPOGH_RESULT::${GOPOGH_RESULT} + echo ::set-env name=STAT::${STAT} + - uses: actions/upload-artifact@v1 + with: + name: functional_test_docker_ubuntu + path: minikube_binaries/report + - name: The End Result functional_test_docker_ubuntu + shell: bash + run: | + echo ${GOPOGH_RESULT} + numFail=$(echo $STAT | jq '.NumberOfFail') + echo "----------------${numFail} Failures----------------------------" + echo $STAT | jq '.FailedTests' || true + echo "-------------------------------------------------------" + numPass=$(echo $STAT | jq '.NumberOfPass') + echo "*** $numPass Passed ***" + if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi + addons_certs_tests_docker_ubuntu: + runs-on: ubuntu-18.04 + env: + TIME_ELAPSED: time + JOB_NAME: "addons_certs_tests_docker_ubuntu" + GOPOGH_RESULT: "" + SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643 + needs: [build_minikube] + steps: + - name: Install kubectl + shell: bash + run: | + curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/amd64/kubectl + sudo install kubectl /usr/local/bin/kubectl + kubectl version --client=true + - name: Install lz4 + shell: bash + run: | + sudo apt-get update -qq + sudo apt-get -qq -y install liblz4-tool + - name: Docker Info + shell: bash + run: | + echo "--------------------------" + docker version || true + echo "--------------------------" + docker info || true + echo "--------------------------" + docker system df || true + echo "--------------------------" + docker system info || true + echo "--------------------------" + docker ps || true + echo "--------------------------" + - name: Install gopogh + shell: bash + run: | + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.1.19/gopogh-linux-amd64 + sudo install gopogh-linux-amd64 /usr/local/bin/gopogh + - name: Download Binaries + uses: actions/download-artifact@v1 + with: + name: minikube_binaries + - name: Run Integration Test + continue-on-error: true + # bash {0} to allow test to continue to next step. in case of + shell: bash {0} + run: | + cd minikube_binaries + mkdir -p report + mkdir -p testhome + chmod a+x e2e-* + chmod a+x minikube-* + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + START_TIME=$(date -u +%s) + KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--driver=docker -test.run "(TestAddons|TestCertOptions)" -test.timeout=30m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt + END_TIME=$(date -u +%s) + TIME_ELAPSED=$(($END_TIME-$START_TIME)) + min=$((${TIME_ELAPSED}/60)) + sec=$((${TIME_ELAPSED}%60)) + TIME_ELAPSED="${min} min $sec seconds " + echo ::set-env name=TIME_ELAPSED::${TIME_ELAPSED} + - name: Generate HTML Report + shell: bash + run: | + cd minikube_binaries + export PATH=${PATH}:`go env GOPATH`/bin + go tool test2json -t < ./report/testout.txt > ./report/testout.json || true + STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}" -details "${GITHUB_SHA}") || true + echo status: ${STAT} + FailNum=$(echo $STAT | jq '.NumberOfFail') + TestsNum=$(echo $STAT | jq '.NumberOfTests') + GOPOGH_RESULT="${JOB_NAME} : completed with ${FailNum} / ${TestsNum} failures in ${TIME_ELAPSED}" + echo ::set-env name=GOPOGH_RESULT::${GOPOGH_RESULT} + echo ::set-env name=STAT::${STAT} + - uses: actions/upload-artifact@v1 + with: + name: addons_certs_tests_docker_ubuntu + path: minikube_binaries/report + - name: The End Result - addons_certs_tests_docker_ubuntu + shell: bash + run: | + echo ${GOPOGH_RESULT} + numFail=$(echo $STAT | jq '.NumberOfFail') + echo "----------------${numFail} Failures----------------------------" + echo $STAT | jq '.FailedTests' || true + echo "-------------------------------------------------------" + numPass=$(echo $STAT | jq '.NumberOfPass') + echo "*** $numPass Passed ***" + if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi + multinode_pause_tests_docker_ubuntu: + runs-on: ubuntu-18.04 + env: + TIME_ELAPSED: time + JOB_NAME: "multinode_pause_tests_docker_ubuntu" + GOPOGH_RESULT: "" + SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643 + needs: [build_minikube] + steps: + - name: Install kubectl + shell: bash + run: | + curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/amd64/kubectl + sudo install kubectl /usr/local/bin/kubectl + kubectl version --client=true + - name: Install lz4 + shell: bash + run: | + sudo apt-get update -qq + sudo apt-get -qq -y install liblz4-tool + - name: Docker Info + shell: bash + run: | + echo "--------------------------" + docker version || true + echo "--------------------------" + docker info || true + echo "--------------------------" + docker system df || true + echo "--------------------------" + docker system info || true + echo "--------------------------" + docker ps || true + echo "--------------------------" + - name: Install gopogh + shell: bash + run: | + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.1.19/gopogh-linux-amd64 + sudo install gopogh-linux-amd64 /usr/local/bin/gopogh + - name: Download Binaries + uses: actions/download-artifact@v1 + with: + name: minikube_binaries + - name: Run Integration Test + continue-on-error: true + # bash {0} to allow test to continue to next step. in case of + shell: bash {0} + run: | + cd minikube_binaries + mkdir -p report + mkdir -p testhome + chmod a+x e2e-* + chmod a+x minikube-* + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + START_TIME=$(date -u +%s) + KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--driver=docker -test.run "(TestPause|TestMultiNode)" -test.timeout=30m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt + END_TIME=$(date -u +%s) + TIME_ELAPSED=$(($END_TIME-$START_TIME)) + min=$((${TIME_ELAPSED}/60)) + sec=$((${TIME_ELAPSED}%60)) + TIME_ELAPSED="${min} min $sec seconds " + echo ::set-env name=TIME_ELAPSED::${TIME_ELAPSED} + - name: Generate HTML Report + shell: bash + run: | + cd minikube_binaries + export PATH=${PATH}:`go env GOPATH`/bin + go tool test2json -t < ./report/testout.txt > ./report/testout.json || true + STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}" -details "${GITHUB_SHA}") || true + echo status: ${STAT} + FailNum=$(echo $STAT | jq '.NumberOfFail') + TestsNum=$(echo $STAT | jq '.NumberOfTests') + GOPOGH_RESULT="${JOB_NAME} : completed with ${FailNum} / ${TestsNum} failures in ${TIME_ELAPSED}" + echo ::set-env name=GOPOGH_RESULT::${GOPOGH_RESULT} + echo ::set-env name=STAT::${STAT} + - uses: actions/upload-artifact@v1 + with: + name: multinode_pause_tests_docker_ubuntu + path: minikube_binaries/report + - name: The End Result - multinode_pause_tests_docker_ubuntu + shell: bash + run: | + echo ${GOPOGH_RESULT} + numFail=$(echo $STAT | jq '.NumberOfFail') + echo "----------------${numFail} Failures----------------------------" + echo $STAT | jq '.FailedTests' || true + echo "-------------------------------------------------------" + numPass=$(echo $STAT | jq '.NumberOfPass') + echo "*** $numPass Passed ***" + if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi + preload_docker_flags_tests_docker_ubuntu: + runs-on: ubuntu-18.04 + env: + TIME_ELAPSED: time + JOB_NAME: "preload_docker_flags_tests_docker_ubuntu" + GOPOGH_RESULT: "" + SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643 + needs: [build_minikube] + steps: + - name: Install kubectl + shell: bash + run: | + curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/amd64/kubectl + sudo install kubectl /usr/local/bin/kubectl + kubectl version --client=true + - name: Install lz4 + shell: bash + run: | + sudo apt-get update -qq + sudo apt-get -qq -y install liblz4-tool + - name: Docker Info + shell: bash + run: | + echo "--------------------------" + docker version || true + echo "--------------------------" + docker info || true + echo "--------------------------" + docker system df || true + echo "--------------------------" + docker system info || true + echo "--------------------------" + docker ps || true + echo "--------------------------" + - name: Install gopogh + shell: bash + run: | + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.1.19/gopogh-linux-amd64 + sudo install gopogh-linux-amd64 /usr/local/bin/gopogh + - name: Download Binaries + uses: actions/download-artifact@v1 + with: + name: minikube_binaries + - name: Run Integration Test + continue-on-error: true + # bash {0} to allow test to continue to next step. in case of + shell: bash {0} + run: | + cd minikube_binaries + mkdir -p report + mkdir -p testhome + chmod a+x e2e-* + chmod a+x minikube-* + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + START_TIME=$(date -u +%s) + KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--driver=docker -test.run "(TestPreload|TestDockerFlags)" -test.timeout=30m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt + END_TIME=$(date -u +%s) + TIME_ELAPSED=$(($END_TIME-$START_TIME)) + min=$((${TIME_ELAPSED}/60)) + sec=$((${TIME_ELAPSED}%60)) + TIME_ELAPSED="${min} min $sec seconds " + echo ::set-env name=TIME_ELAPSED::${TIME_ELAPSED} + - name: Generate HTML Report + shell: bash + run: | + cd minikube_binaries + export PATH=${PATH}:`go env GOPATH`/bin + go tool test2json -t < ./report/testout.txt > ./report/testout.json || true + STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}" -details "${GITHUB_SHA}") || true + echo status: ${STAT} + FailNum=$(echo $STAT | jq '.NumberOfFail') + TestsNum=$(echo $STAT | jq '.NumberOfTests') + GOPOGH_RESULT="${JOB_NAME} : completed with ${FailNum} / ${TestsNum} failures in ${TIME_ELAPSED}" + echo ::set-env name=GOPOGH_RESULT::${GOPOGH_RESULT} + echo ::set-env name=STAT::${STAT} + - uses: actions/upload-artifact@v1 + with: + name: preload_docker_flags_tests_docker_ubuntu + path: minikube_binaries/report + - name: The End Result - preload_docker_flags_tests_docker_ubuntu + shell: bash + run: | + echo ${GOPOGH_RESULT} + numFail=$(echo $STAT | jq '.NumberOfFail') + echo "----------------${numFail} Failures----------------------------" + echo $STAT | jq '.FailedTests' || true + echo "-------------------------------------------------------" + numPass=$(echo $STAT | jq '.NumberOfPass') + echo "*** $numPass Passed ***" + if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi + functional_baremetal_ubuntu18_04: + needs: [build_minikube] + env: + TIME_ELAPSED: time + JOB_NAME: "functional_baremetal_ubuntu18_04" + GOPOGH_RESULT: "" + SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643 + runs-on: ubuntu-18.04 + steps: + - name: Install kubectl + shell: bash + run: | + curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/amd64/kubectl + sudo install kubectl /usr/local/bin/kubectl + kubectl version --client=true + # conntrack is required for kubernetes 1.18 and higher + # socat is required for kubectl port forward which is used in some tests such as validateHelmTillerAddon + - name: Install tools for none + shell: bash + run: | + sudo apt-get update -qq + sudo apt-get -qq -y install conntrack + sudo apt-get -qq -y install socat + VERSION="v1.17.0" + curl -L https://github.com/kubernetes-sigs/cri-tools/releases/download/$VERSION/crictl-${VERSION}-linux-amd64.tar.gz --output crictl-${VERSION}-linux-amd64.tar.gz + sudo tar zxvf crictl-$VERSION-linux-amd64.tar.gz -C /usr/local/bin + - name: Install gopogh + shell: bash + run: | + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.1.19/gopogh-linux-amd64 + sudo install gopogh-linux-amd64 /usr/local/bin/gopogh + - name: Download Binaries + uses: actions/download-artifact@v1 + with: + name: minikube_binaries + - name: Run Integration Test + continue-on-error: true + # bash {0} to allow test to continue to next step. in case of + shell: bash {0} + run: | + cd minikube_binaries + mkdir -p report + mkdir -p testhome + chmod a+x e2e-* + chmod a+x minikube-* + START_TIME=$(date -u +%s) + KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome sudo -E ./e2e-linux-amd64 -minikube-start-args=--driver=none -test.timeout=35m -test.run TestFunctional -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt + END_TIME=$(date -u +%s) + TIME_ELAPSED=$(($END_TIME-$START_TIME)) + min=$((${TIME_ELAPSED}/60)) + sec=$((${TIME_ELAPSED}%60)) + TIME_ELAPSED="${min} min $sec seconds " + echo ::set-env name=TIME_ELAPSED::${TIME_ELAPSED} + - name: Generate HTML Report + shell: bash + run: | + cd minikube_binaries + export PATH=${PATH}:`go env GOPATH`/bin + go tool test2json -t < ./report/testout.txt > ./report/testout.json || true + STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}" -details "${GITHUB_SHA}") || true + echo status: ${STAT} + FailNum=$(echo $STAT | jq '.NumberOfFail') + TestsNum=$(echo $STAT | jq '.NumberOfTests') + GOPOGH_RESULT="${JOB_NAME} : completed with ${FailNum} / ${TestsNum} failures in ${TIME_ELAPSED}" + echo ::set-env name=GOPOGH_RESULT::${GOPOGH_RESULT} + echo ::set-env name=STAT::${STAT} + - uses: actions/upload-artifact@v1 + with: + name: none_ubuntu18_04 + path: minikube_binaries/report + - name: The End Result - None on Ubuntu 18:04 + shell: bash + run: | + echo ${GOPOGH_RESULT} + numFail=$(echo $STAT | jq '.NumberOfFail') + echo "----------------${numFail} Failures----------------------------" + echo $STAT | jq '.FailedTests' || true + echo "-------------------------------------------------------" + numPass=$(echo $STAT | jq '.NumberOfPass') + echo "*** $numPass Passed ***" + if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi + # After all integration tests finished + # collect all the reports and upload them + upload_all_reports: + if: always() + needs: [functional_test_docker_ubuntu, addons_certs_tests_docker_ubuntu, multinode_pause_tests_docker_ubuntu, preload_docker_flags_tests_docker_ubuntu, functional_baremetal_ubuntu18_04] + runs-on: ubuntu-18.04 + steps: + - name: download all reports + uses: actions/download-artifact@v2-preview + - name: upload all reports + shell: bash {0} + continue-on-error: true + run: | + mkdir -p all_reports + ls -lah + cp -r ./functional_test_docker_ubuntu ./all_reports/ + cp -r ./addons_certs_tests_docker_ubuntu ./all_reports/ + cp -r ./multinode_pause_tests_docker_ubuntu ./all_reports/ + cp -r ./preload_docker_flags_tests_docker_ubuntu ./all_reports/ + cp -r ./functional_baremetal_ubuntu18_04 ./all_reports/ + - uses: actions/upload-artifact@v1 + with: + name: all_reports + path: all_reports \ No newline at end of file diff --git a/.github/workflows/main.yml b/.github/workflows/pr.yml similarity index 69% rename from .github/workflows/main.yml rename to .github/workflows/pr.yml index 38a5b5cd62..de141079a8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/pr.yml @@ -1,5 +1,8 @@ name: CI -on: [pull_request] +on: + pull_request: + paths: + - '**.go' env: GOPROXY: https://proxy.golang.org jobs: @@ -15,8 +18,6 @@ jobs: run : | make minikube-linux-amd64 make e2e-linux-amd64 - make minikube-windows-amd64.exe - make e2e-windows-amd64.exe cp -r test/integration/testdata ./out whoami echo github ref $GITHUB_REF @@ -62,15 +63,15 @@ jobs: make test continue-on-error: false # Run the following integration tests after the build_minikube - # They will run in parallel and use the binaries in previous step - functional_test_docker_ubuntu_16_04: + # They will run in parallel and use the binaries in previous step + functional_test_docker_ubuntu: needs: [build_minikube] env: TIME_ELAPSED: time - JOB_NAME: "Docker_Ubuntu_16_04" + JOB_NAME: "functional_test_docker_ubuntu" GOPOGH_RESULT: "" SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643 - runs-on: ubuntu-16.04 + runs-on: ubuntu-18.04 steps: - name: Install kubectl shell: bash @@ -116,6 +117,8 @@ jobs: mkdir -p testhome chmod a+x e2e-* chmod a+x minikube-* + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld START_TIME=$(date -u +%s) KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--vm-driver=docker -test.run TestFunctional -test.timeout=30m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt END_TIME=$(date -u +%s) @@ -139,9 +142,9 @@ jobs: echo ::set-env name=STAT::${STAT} - uses: actions/upload-artifact@v1 with: - name: docker_ubuntu_16_04 + name: functional_test_docker_ubuntu path: minikube_binaries/report - - name: The End Result Docker on ubuntu 16:04 + - name: The End Result functional_test_docker_ubuntu shell: bash run: | echo ${GOPOGH_RESULT} @@ -152,11 +155,11 @@ jobs: numPass=$(echo $STAT | jq '.NumberOfPass') echo "*** $numPass Passed ***" if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi - pause_test_docker_ubuntu_18_04: + addons_certs_tests_docker_ubuntu: runs-on: ubuntu-18.04 env: TIME_ELAPSED: time - JOB_NAME: "Docker_Ubuntu_18_04" + JOB_NAME: "addons_certs_tests_docker_ubuntu" GOPOGH_RESULT: "" SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643 needs: [build_minikube] @@ -205,8 +208,10 @@ jobs: mkdir -p testhome chmod a+x e2e-* chmod a+x minikube-* + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld START_TIME=$(date -u +%s) - KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--driver=docker -test.run TestPause -test.timeout=30m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt + KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--driver=docker -test.run "(TestAddons|TestCertOptions)" -test.timeout=30m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt END_TIME=$(date -u +%s) TIME_ELAPSED=$(($END_TIME-$START_TIME)) min=$((${TIME_ELAPSED}/60)) @@ -228,9 +233,9 @@ jobs: echo ::set-env name=STAT::${STAT} - uses: actions/upload-artifact@v1 with: - name: docker_ubuntu_18_04 + name: addons_certs_tests_docker_ubuntu path: minikube_binaries/report - - name: The End Result - Docker On Ubuntu 18:04 + - name: The End Result - addons_certs_tests_docker_ubuntu shell: bash run: | echo ${GOPOGH_RESULT} @@ -241,14 +246,14 @@ jobs: numPass=$(echo $STAT | jq '.NumberOfPass') echo "*** $numPass Passed ***" if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi - baremetal_ubuntu16_04: - needs: [build_minikube] + multinode_pause_tests_docker_ubuntu: + runs-on: ubuntu-18.04 env: TIME_ELAPSED: time - JOB_NAME: "None_Ubuntu_16_04" + JOB_NAME: "multinode_pause_tests_docker_ubuntu" GOPOGH_RESULT: "" SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643 - runs-on: ubuntu-16.04 + needs: [build_minikube] steps: - name: Install kubectl shell: bash @@ -256,17 +261,25 @@ jobs: curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/amd64/kubectl sudo install kubectl /usr/local/bin/kubectl kubectl version --client=true - # conntrack is required for kubernetes 1.18 and higher - # socat is required for kubectl port forward which is used in some tests such as validateHelmTillerAddon - - name: Install tools for none + - name: Install lz4 shell: bash run: | sudo apt-get update -qq - sudo apt-get -qq -y install conntrack - sudo apt-get -qq -y install socat - VERSION="v1.17.0" - curl -L https://github.com/kubernetes-sigs/cri-tools/releases/download/$VERSION/crictl-${VERSION}-linux-amd64.tar.gz --output crictl-${VERSION}-linux-amd64.tar.gz - sudo tar zxvf crictl-$VERSION-linux-amd64.tar.gz -C /usr/local/bin + sudo apt-get -qq -y install liblz4-tool + - name: Docker Info + shell: bash + run: | + echo "--------------------------" + docker version || true + echo "--------------------------" + docker info || true + echo "--------------------------" + docker system df || true + echo "--------------------------" + docker system info || true + echo "--------------------------" + docker ps || true + echo "--------------------------" - name: Install gopogh shell: bash run: | @@ -286,8 +299,10 @@ jobs: mkdir -p testhome chmod a+x e2e-* chmod a+x minikube-* + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld START_TIME=$(date -u +%s) - KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome sudo -E ./e2e-linux-amd64 -minikube-start-args=--driver=none -test.timeout=35m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt + KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--driver=docker -test.run "(TestPause|TestMultiNode)" -test.timeout=30m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt END_TIME=$(date -u +%s) TIME_ELAPSED=$(($END_TIME-$START_TIME)) min=$((${TIME_ELAPSED}/60)) @@ -309,9 +324,9 @@ jobs: echo ::set-env name=STAT::${STAT} - uses: actions/upload-artifact@v1 with: - name: none_ubuntu16_04 + name: multinode_pause_tests_docker_ubuntu path: minikube_binaries/report - - name: The End Result - None On Ubuntu 16:04 + - name: The End Result - multinode_pause_tests_docker_ubuntu shell: bash run: | echo ${GOPOGH_RESULT} @@ -322,11 +337,102 @@ jobs: numPass=$(echo $STAT | jq '.NumberOfPass') echo "*** $numPass Passed ***" if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi - baremetal_ubuntu18_04: + preload_docker_flags_tests_docker_ubuntu: + runs-on: ubuntu-18.04 + env: + TIME_ELAPSED: time + JOB_NAME: "preload_docker_flags_tests_docker_ubuntu" + GOPOGH_RESULT: "" + SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643 + needs: [build_minikube] + steps: + - name: Install kubectl + shell: bash + run: | + curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/amd64/kubectl + sudo install kubectl /usr/local/bin/kubectl + kubectl version --client=true + - name: Install lz4 + shell: bash + run: | + sudo apt-get update -qq + sudo apt-get -qq -y install liblz4-tool + - name: Docker Info + shell: bash + run: | + echo "--------------------------" + docker version || true + echo "--------------------------" + docker info || true + echo "--------------------------" + docker system df || true + echo "--------------------------" + docker system info || true + echo "--------------------------" + docker ps || true + echo "--------------------------" + - name: Install gopogh + shell: bash + run: | + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.1.19/gopogh-linux-amd64 + sudo install gopogh-linux-amd64 /usr/local/bin/gopogh + - name: Download Binaries + uses: actions/download-artifact@v1 + with: + name: minikube_binaries + - name: Run Integration Test + continue-on-error: true + # bash {0} to allow test to continue to next step. in case of + shell: bash {0} + run: | + cd minikube_binaries + mkdir -p report + mkdir -p testhome + chmod a+x e2e-* + chmod a+x minikube-* + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + START_TIME=$(date -u +%s) + KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--driver=docker -test.run "(TestPreload|TestDockerFlags)" -test.timeout=30m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt + END_TIME=$(date -u +%s) + TIME_ELAPSED=$(($END_TIME-$START_TIME)) + min=$((${TIME_ELAPSED}/60)) + sec=$((${TIME_ELAPSED}%60)) + TIME_ELAPSED="${min} min $sec seconds " + echo ::set-env name=TIME_ELAPSED::${TIME_ELAPSED} + - name: Generate HTML Report + shell: bash + run: | + cd minikube_binaries + export PATH=${PATH}:`go env GOPATH`/bin + go tool test2json -t < ./report/testout.txt > ./report/testout.json || true + STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}" -details "${GITHUB_SHA}") || true + echo status: ${STAT} + FailNum=$(echo $STAT | jq '.NumberOfFail') + TestsNum=$(echo $STAT | jq '.NumberOfTests') + GOPOGH_RESULT="${JOB_NAME} : completed with ${FailNum} / ${TestsNum} failures in ${TIME_ELAPSED}" + echo ::set-env name=GOPOGH_RESULT::${GOPOGH_RESULT} + echo ::set-env name=STAT::${STAT} + - uses: actions/upload-artifact@v1 + with: + name: preload_docker_flags_tests_docker_ubuntu + path: minikube_binaries/report + - name: The End Result - preload_docker_flags_tests_docker_ubuntu + shell: bash + run: | + echo ${GOPOGH_RESULT} + numFail=$(echo $STAT | jq '.NumberOfFail') + echo "----------------${numFail} Failures----------------------------" + echo $STAT | jq '.FailedTests' || true + echo "-------------------------------------------------------" + numPass=$(echo $STAT | jq '.NumberOfPass') + echo "*** $numPass Passed ***" + if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi + functional_baremetal_ubuntu18_04: needs: [build_minikube] env: TIME_ELAPSED: time - JOB_NAME: "None_Ubuntu_18_04" + JOB_NAME: "functional_baremetal_ubuntu18_04" GOPOGH_RESULT: "" SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643 runs-on: ubuntu-18.04 @@ -368,7 +474,7 @@ jobs: chmod a+x e2e-* chmod a+x minikube-* START_TIME=$(date -u +%s) - KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome sudo -E ./e2e-linux-amd64 -minikube-start-args=--driver=none -test.timeout=35m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt + KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome sudo -E ./e2e-linux-amd64 -minikube-start-args=--driver=none -test.timeout=35m -test.v -timeout-multiplier=1.5 -test.run TestFunctional -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt END_TIME=$(date -u +%s) TIME_ELAPSED=$(($END_TIME-$START_TIME)) min=$((${TIME_ELAPSED}/60)) @@ -403,53 +509,26 @@ jobs: numPass=$(echo $STAT | jq '.NumberOfPass') echo "*** $numPass Passed ***" if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi - # After all 4 integration tests finished - # collect all the reports and upload + # After all integration tests finished + # collect all the reports and upload them upload_all_reports: if: always() - needs: [functional_test_docker_ubuntu_16_04, pause_test_docker_ubuntu_18_04,baremetal_ubuntu16_04, baremetal_ubuntu18_04] + needs: [functional_test_docker_ubuntu, addons_certs_tests_docker_ubuntu, multinode_pause_tests_docker_ubuntu, preload_docker_flags_tests_docker_ubuntu, functional_baremetal_ubuntu18_04] runs-on: ubuntu-18.04 steps: - - name: Download Results docker_ubuntu_16_04 - uses: actions/download-artifact@v1 - with: - name: docker_ubuntu_16_04 - - name: cp docker_ubuntu_16_04 to all_report - continue-on-error: true + - name: download all reports + uses: actions/download-artifact@v2-preview + - name: upload all reports shell: bash {0} + continue-on-error: true run: | mkdir -p all_reports - cp -r docker_ubuntu_16_04 ./all_reports/ - - name: Download Results docker_ubuntu_18_04 - uses: actions/download-artifact@v1 - with: - name: docker_ubuntu_18_04 - - name: cp docker_ubuntu_18_04 to all_report - continue-on-error: true - shell: bash {0} - run: | - mkdir -p all_reports - cp -r docker_ubuntu_18_04 ./all_reports/ - - name: Download Results none_ubuntu16_04 - uses: actions/download-artifact@v1 - with: - name: none_ubuntu16_04 - - name: cp none_ubuntu16_04 to all_report - continue-on-error: true - shell: bash {0} - run: | - mkdir -p all_reports - cp -r none_ubuntu16_04 ./all_reports/ - - name: Download Results none_ubuntu18_04 - uses: actions/download-artifact@v1 - with: - name: none_ubuntu18_04 - - name: Copy none_ubuntu18_04 to all_report - continue-on-error: true - shell: bash {0} - run: | - mkdir -p all_reports - cp -r none_ubuntu18_04 ./all_reports/ + ls -lah + cp -r ./functional_test_docker_ubuntu ./all_reports/ + cp -r ./addons_certs_tests_docker_ubuntu ./all_reports/ + cp -r ./multinode_pause_tests_docker_ubuntu ./all_reports/ + cp -r ./preload_docker_flags_tests_docker_ubuntu ./all_reports/ + cp -r ./functional_baremetal_ubuntu18_04 ./all_reports/ - uses: actions/upload-artifact@v1 with: name: all_reports diff --git a/deploy/iso/minikube-iso/board/coreos/minikube/linux_defconfig b/deploy/iso/minikube-iso/board/coreos/minikube/linux_defconfig index 9611b6a9ac..21f310fb0c 100644 --- a/deploy/iso/minikube-iso/board/coreos/minikube/linux_defconfig +++ b/deploy/iso/minikube-iso/board/coreos/minikube/linux_defconfig @@ -114,6 +114,7 @@ CONFIG_TCP_CONG_ADVANCED=y CONFIG_TCP_MD5SIG=y CONFIG_INET6_AH=y CONFIG_INET6_ESP=y +CONFIG_IPV6_MULTIPLE_TABLES=y CONFIG_NETLABEL=y CONFIG_NETFILTER=y CONFIG_NETFILTER_NETLINK_ACCT=y @@ -351,6 +352,7 @@ CONFIG_NETCONSOLE=y CONFIG_TUN=y CONFIG_VETH=y CONFIG_VIRTIO_NET=y +CONFIG_NET_VRF=m CONFIG_AMD8111_ETH=m CONFIG_PCNET32=m CONFIG_PCMCIA_NMCLAN=m diff --git a/deploy/iso/minikube-iso/package/automount/minikube-automount b/deploy/iso/minikube-iso/package/automount/minikube-automount index 17fc8b0279..77e5fda1fe 100755 --- a/deploy/iso/minikube-iso/package/automount/minikube-automount +++ b/deploy/iso/minikube-iso/package/automount/minikube-automount @@ -109,6 +109,10 @@ if [ -n "$BOOT2DOCKER_DATA" ]; then mkdir /var/log mount --bind /mnt/$PARTNAME/var/log /var/log + mkdir -p /mnt/$PARTNAME/var/tmp + mkdir /var/tmp + mount --bind /mnt/$PARTNAME/var/tmp /var/tmp + mkdir -p /mnt/$PARTNAME/var/lib/kubelet mkdir /var/lib/kubelet mount --bind /mnt/$PARTNAME/var/lib/kubelet /var/lib/kubelet diff --git a/deploy/iso/minikube-iso/package/podman/podman.mk b/deploy/iso/minikube-iso/package/podman/podman.mk index 8781c0b0b9..fb12281c9d 100644 --- a/deploy/iso/minikube-iso/package/podman/podman.mk +++ b/deploy/iso/minikube-iso/package/podman/podman.mk @@ -29,6 +29,8 @@ endef define PODMAN_INSTALL_TARGET_CMDS $(INSTALL) -Dm755 $(@D)/bin/podman $(TARGET_DIR)/usr/bin/podman + $(INSTALL) -d -m 755 $(TARGET_DIR)/etc/cni/net.d/ + $(INSTALL) -m 644 cni/87-podman-bridge.conflist $(TARGET_DIR)/etc/cni/net.d/87-podman-bridge.conflist endef $(eval $(generic-package)) diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/api_server.go b/pkg/minikube/bootstrapper/bsutil/kverify/api_server.go index 6aa6763a26..146eabc7c6 100644 --- a/pkg/minikube/bootstrapper/bsutil/kverify/api_server.go +++ b/pkg/minikube/bootstrapper/bsutil/kverify/api_server.go @@ -166,7 +166,15 @@ func APIServerStatus(cr command.Runner, hostname string, port int) (state.State, rr, err = cr.RunCmd(exec.Command("sudo", "cat", path.Join("/sys/fs/cgroup/freezer", fparts[2], "freezer.state"))) if err != nil { - glog.Errorf("unable to get freezer state: %s", rr.Stderr.String()) + // example error from github action: + // cat: /sys/fs/cgroup/freezer/actions_job/e62ef4349cc5a70f4b49f8a150ace391da6ad6df27073c83ecc03dbf81fde1ce/kubepods/burstable/poda1de58db0ce81d19df7999f6808def1b/5df53230fe3483fd65f341923f18a477fda92ae9cd71061168130ef164fe479c/freezer.state: No such file or directory\n"* + // TODO: #7770 investigate how to handle this error better. + if strings.Contains(rr.Stderr.String(), "freezer.state: No such file or directory\n") { + glog.Infof("unable to get freezer state (might be okay and be related to #770): %s", rr.Stderr.String()) + } else { + glog.Warningf("unable to get freezer state : %s", rr.Stderr.String()) + } + return apiServerHealthz(hostname, port) } diff --git a/site/content/en/docs/contrib/testing.en.md b/site/content/en/docs/contrib/testing.en.md index bc8e02eae5..2f88cf99d9 100644 --- a/site/content/en/docs/contrib/testing.en.md +++ b/site/content/en/docs/contrib/testing.en.md @@ -16,8 +16,9 @@ make test ### Integration Tests -Integration tests are currently run manually. -To run them, build the binary and run the tests: +#### The basics + +From the minikube root directory, build the binary and run the tests: ```shell make integration @@ -29,6 +30,34 @@ You may find it useful to set various options to test only a particular test aga env TEST_ARGS="-minikube-start-args=--driver=hyperkit -test.run TestStartStop" make integration ``` +#### Quickly iterating on a single test + +Run a single test on an active cluster: + +```shell +make integration -e TEST_ARGS="-test.run TestFunctional/parallel/MountCmd --profile=minikube --cleanup=false" +``` + +WARNING: For this to work repeatedly, the test must be written so that it cleans up after itself. + +The `--cleanup=false` test arg ensures that the cluster will not be deleted after the test is run. + +See [main.go](https://github.com/kubernetes/minikube/blob/master/test/integration/main.go) for details. + +#### Disabling parallelism + +```shell +make integration -e TEST_ARGS="-test.parallel=1" +``` + +#### Testing philosophy + +- Tests should be so simple as to be correct by inspection +- Readers should need to read only the test body to understand the test +- Top-to-bottom readability is more important than code de-duplication + +Tests are typically read with a great air of skepticism, because chances are they are being read only when things are broken. + ### Conformance Tests These are Kubernetes tests that run against an arbitrary cluster and exercise a wide range of Kubernetes features. diff --git a/site/content/en/docs/drivers/docker.md b/site/content/en/docs/drivers/docker.md index b26f6a879f..ad5949d0ce 100644 --- a/site/content/en/docs/drivers/docker.md +++ b/site/content/en/docs/drivers/docker.md @@ -33,3 +33,8 @@ The Docker driver allows you to install Kubernetes into an existing Docker insta - On macOS or Windows, you may need to restart Docker for Desktop if a command gets hung - Run `--alsologtostderr -v=1` for extra debugging information + +- On Linux, if you want to run MySQL pod, you need to disable AppArmor for mysql profile + + If your docker has [AppArmor](https://wiki.ubuntu.com/AppArmor) enabled, running mysql in privileged mode with docker driver will have the issue [#7401](https://github.com/kubernetes/minikube/issues/7401). + There is a workaround - see [moby/moby#7512](https://github.com/moby/moby/issues/7512#issuecomment-61787845). diff --git a/test/integration/README.md b/test/integration/README.md index 7af031f6ea..8a62883dde 100644 --- a/test/integration/README.md +++ b/test/integration/README.md @@ -1,29 +1 @@ -# Integration tests - -## The basics - -To run all tests from the minikube root directory: - -`make integration` - -## Quickly iterating on a single test - -Run a single test on an active cluster: - -`make integration -e TEST_ARGS="-test.run TestFunctional/parallel/MountCmd --profile=minikube --cleanup=false"` - -WARNING: For this to work repeatedly, the test must be written so that it cleans up after itself. - -See `main.go` for details. - -## Disabling parallelism - -`make integration -e TEST_ARGS="-test.parallel=1"` - -## Testing philosophy - -- Tests should be so simple as to be correct by inspection -- Readers should need to read only the test body to understand the test -- Top-to-bottom readability is more important than code de-duplication - -Tests are typically read with a great air of skepticism, because chances are they are being read only when things are broken. +This document has moved to https://minikube.sigs.k8s.io/docs/contrib/testing/#integration-tests diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go index 5f1ef0124b..2f41ea89ef 100644 --- a/test/integration/functional_test.go +++ b/test/integration/functional_test.go @@ -670,7 +670,7 @@ func validateServiceCmd(ctx context.Context, t *testing.T, profile string) { t.Fatalf("failed to get service url. args %q : %v", rr.Command(), err) } if rr.Stderr.String() != "" { - t.Errorf("expected stderr to be empty but got *%q*", rr.Stderr) + t.Errorf("expected stderr to be empty but got *%q* . args %q", rr.Stderr, rr.Command()) } endpoint := strings.TrimSpace(rr.Stdout.String()) diff --git a/test/integration/multinode_test.go b/test/integration/multinode_test.go index 0502cd08e3..efa662964a 100644 --- a/test/integration/multinode_test.go +++ b/test/integration/multinode_test.go @@ -30,11 +30,6 @@ func TestMultiNode(t *testing.T) { t.Skip("none driver does not support multinode") } - // TODO: remove this skip once docker multinode is fixed - if KicDriver() { - t.Skip("docker driver multinode is currently broken :(") - } - profile := UniqueProfileName("multinode") ctx, cancel := context.WithTimeout(context.Background(), Minutes(30)) defer CleanupWithLogs(t, profile, cancel)