diff --git a/.github/workflows/iso.yml b/.github/workflows/iso.yml index 07e03d7197..d06a930eee 100644 --- a/.github/workflows/iso.yml +++ b/.github/workflows/iso.yml @@ -82,7 +82,7 @@ jobs: - name: Install gopogh shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-linux-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64 sudo install gopogh-linux-amd64 /usr/local/bin/gopogh sudo apt-get install -y jq - name: Run Integration Test diff --git a/.github/workflows/kic_image.yml b/.github/workflows/kic_image.yml index b3ddee81b2..cadf209cab 100644 --- a/.github/workflows/kic_image.yml +++ b/.github/workflows/kic_image.yml @@ -60,7 +60,7 @@ jobs: - name: Install gopogh shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-linux-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64 sudo install gopogh-linux-amd64 /usr/local/bin/gopogh sudo apt-get install -y jq rm -f gopogh-linux-amd64 || true diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 73d297e07e..d89d6156bf 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -120,7 +120,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-linux-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64 sudo install gopogh-linux-amd64 /usr/local/bin/gopogh - name: Download Binaries uses: actions/download-artifact@v1 @@ -205,7 +205,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-darwin-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-darwin-amd64 sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh - name: Install docker shell: bash @@ -350,7 +350,7 @@ jobs: continue-on-error: true shell: powershell run: | - (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe") + (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe") choco install -y kubernetes-cli choco install -y jq choco install -y caffeine @@ -487,7 +487,7 @@ jobs: shell: powershell run: | $ErrorActionPreference = "SilentlyContinue" - (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe") + (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe") choco install -y kubernetes-cli choco install -y jq choco install -y caffeine @@ -592,7 +592,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-linux-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64 sudo install gopogh-linux-amd64 /usr/local/bin/gopogh - name: Download Binaries uses: actions/download-artifact@v1 @@ -689,7 +689,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-linux-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64 sudo install gopogh-linux-amd64 /usr/local/bin/gopogh - name: Download Binaries uses: actions/download-artifact@v1 @@ -771,7 +771,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-darwin-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-darwin-amd64 sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh - name: Install docker shell: bash @@ -883,7 +883,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-linux-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64 sudo install gopogh-linux-amd64 /usr/local/bin/gopogh - name: Download Binaries uses: actions/download-artifact@v1 @@ -967,7 +967,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-darwin-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-darwin-amd64 sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh - name: Download Binaries uses: actions/download-artifact@v1 @@ -1074,7 +1074,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-linux-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64 sudo install gopogh-linux-amd64 /usr/local/bin/gopogh - name: Download Binaries uses: actions/download-artifact@v1 @@ -1156,7 +1156,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-darwin-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-darwin-amd64 sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh - name: Download Binaries uses: actions/download-artifact@v1 diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 8e0021e663..c84945b0df 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -118,7 +118,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-linux-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64 sudo install gopogh-linux-amd64 /usr/local/bin/gopogh - name: Download Binaries uses: actions/download-artifact@v1 @@ -203,7 +203,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-darwin-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-darwin-amd64 sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh - name: Install docker shell: bash @@ -348,7 +348,7 @@ jobs: continue-on-error: true shell: powershell run: | - (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe") + (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe") choco install -y kubernetes-cli choco install -y jq choco install -y caffeine @@ -485,7 +485,7 @@ jobs: shell: powershell run: | $ErrorActionPreference = "SilentlyContinue" - (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe") + (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe") choco install -y kubernetes-cli choco install -y jq choco install -y caffeine @@ -590,7 +590,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-linux-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64 sudo install gopogh-linux-amd64 /usr/local/bin/gopogh - name: Download Binaries uses: actions/download-artifact@v1 @@ -687,7 +687,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-linux-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64 sudo install gopogh-linux-amd64 /usr/local/bin/gopogh - name: Download Binaries uses: actions/download-artifact@v1 @@ -769,7 +769,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-darwin-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-darwin-amd64 sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh - name: Install docker shell: bash @@ -881,7 +881,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-linux-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64 sudo install gopogh-linux-amd64 /usr/local/bin/gopogh - name: Download Binaries uses: actions/download-artifact@v1 @@ -965,7 +965,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-darwin-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-darwin-amd64 sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh - name: Download Binaries uses: actions/download-artifact@v1 @@ -1072,7 +1072,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-linux-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64 sudo install gopogh-linux-amd64 /usr/local/bin/gopogh - name: Download Binaries uses: actions/download-artifact@v1 @@ -1154,7 +1154,7 @@ jobs: shell: bash run: | - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-darwin-amd64 + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-darwin-amd64 sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh - name: Download Binaries uses: actions/download-artifact@v1 diff --git a/Makefile b/Makefile index 836d7f78b8..3688f9730a 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ RPM_VERSION ?= $(DEB_VERSION) GO_VERSION ?= 1.15.2 INSTALL_SIZE ?= $(shell du out/minikube-windows-amd64.exe | cut -f1) -BUILDROOT_BRANCH ?= 2020.02.7 +BUILDROOT_BRANCH ?= 2020.02.8 REGISTRY?=gcr.io/k8s-minikube REGISTRY_GH?=docker.pkg.github.com/kubernetes/minikube @@ -58,7 +58,7 @@ MINIKUBE_BUCKET ?= minikube/releases MINIKUBE_UPLOAD_LOCATION := gs://${MINIKUBE_BUCKET} MINIKUBE_RELEASES_URL=https://github.com/kubernetes/minikube/releases/download -KERNEL_VERSION ?= 4.19.150 +KERNEL_VERSION ?= 4.19.157 # latest from https://github.com/golangci/golangci-lint/releases GOLINT_VERSION ?= v1.30.0 # Limit number of default jobs, to avoid the CI builds running out of memory @@ -92,7 +92,7 @@ SHA512SUM=$(shell command -v sha512sum || echo "shasum -a 512") GVISOR_TAG ?= latest # storage provisioner tag to push changes to -STORAGE_PROVISIONER_TAG ?= v3 +STORAGE_PROVISIONER_TAG ?= v4 STORAGE_PROVISIONER_MANIFEST ?= $(REGISTRY)/storage-provisioner:$(STORAGE_PROVISIONER_TAG) STORAGE_PROVISIONER_IMAGE ?= $(REGISTRY)/storage-provisioner-$(GOARCH):$(STORAGE_PROVISIONER_TAG) @@ -462,6 +462,11 @@ out/minikube_$(DEB_VERSION)-0_%.deb: out/minikube-linux-% chmod 0755 out/minikube_$(DEB_VERSION)/DEBIAN sed -E -i 's/--VERSION--/'$(DEB_VERSION)'/g' out/minikube_$(DEB_VERSION)/DEBIAN/control sed -E -i 's/--ARCH--/'$*'/g' out/minikube_$(DEB_VERSION)/DEBIAN/control + if [ "$*" = "amd64" ]; then \ + sed -E -i 's/--RECOMMENDS--/virtualbox/' out/minikube_$(DEB_VERSION)/DEBIAN/control; \ + else \ + sed -E -i '/Recommends: --RECOMMENDS--/d' out/minikube_$(DEB_VERSION)/DEBIAN/control; \ + fi mkdir -p out/minikube_$(DEB_VERSION)/usr/bin cp $< out/minikube_$(DEB_VERSION)/usr/bin/minikube fakeroot dpkg-deb --build out/minikube_$(DEB_VERSION) $@ @@ -767,6 +772,11 @@ out/mkcmp: out/performance-bot: GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o $@ cmd/performance/pr-bot/bot.go +.PHONY: out/metrics-collector +out/metrics-collector: + GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o $@ hack/metrics/*.go + + .PHONY: compare compare: out/mkcmp out/minikube mv out/minikube out/$(CURRENT_GIT_BRANCH).minikube diff --git a/cmd/minikube/cmd/config/config.go b/cmd/minikube/cmd/config/config.go index 08f4608f29..5869c1d175 100644 --- a/cmd/minikube/cmd/config/config.go +++ b/cmd/minikube/cmd/config/config.go @@ -92,8 +92,8 @@ var settings = []Setting{ }, { name: "memory", - set: SetInt, - validations: []setFn{IsPositive}, + set: SetString, + validations: []setFn{IsValidDiskSize}, callbacks: []setFn{RequiresRestartMsg}, }, { diff --git a/cmd/minikube/cmd/config/configure.go b/cmd/minikube/cmd/config/configure.go index 5b4d5301b6..b5800cf2fb 100644 --- a/cmd/minikube/cmd/config/configure.go +++ b/cmd/minikube/cmd/config/configure.go @@ -19,6 +19,7 @@ package config import ( "io/ioutil" "net" + "regexp" "github.com/spf13/cobra" "k8s.io/minikube/pkg/minikube/config" @@ -204,6 +205,22 @@ var addonsConfigureCmd = &cobra.Command{ cfg.KubernetesConfig.LoadBalancerEndIP = AskForStaticValidatedValue("-- Enter Load Balancer End IP: ", validator) } + if err := config.SaveProfile(profile, cfg); err != nil { + out.ErrT(style.Fatal, "Failed to save config {{.profile}}", out.V{"profile": profile}) + } + case "ingress": + profile := ClusterFlagValue() + _, cfg := mustload.Partial(profile) + + validator := func(s string) bool { + format := regexp.MustCompile("^.+/.+$") + return format.MatchString(s) + } + + if cfg.KubernetesConfig.CustomIngressCert == "" { + cfg.KubernetesConfig.CustomIngressCert = AskForStaticValidatedValue("-- Enter custom cert(format is \"namespace/secret\"): ", validator) + } + if err := config.SaveProfile(profile, cfg); err != nil { out.ErrT(style.Fatal, "Failed to save config {{.profile}}", out.V{"profile": profile}) } diff --git a/cmd/minikube/cmd/config/set_test.go b/cmd/minikube/cmd/config/set_test.go index 5d73523fff..7469b3798a 100644 --- a/cmd/minikube/cmd/config/set_test.go +++ b/cmd/minikube/cmd/config/set_test.go @@ -38,6 +38,10 @@ func TestSetNotAllowed(t *testing.T) { if err == nil || err.Error() != "run validations for \"driver\" with value of \"123456\": [driver \"123456\" is not supported]" { t.Fatalf("Set did not return error for unallowed value: %+v", err) } + err = Set("memory", "10a") + if err == nil || err.Error() != "run validations for \"memory\" with value of \"10a\": [invalid disk size: invalid size: '10a']" { + t.Fatalf("Set did not return error for unallowed value: %+v", err) + } } func TestSetOK(t *testing.T) { diff --git a/cmd/minikube/cmd/delete.go b/cmd/minikube/cmd/delete.go index a19d9493dd..60a638ef94 100644 --- a/cmd/minikube/cmd/delete.go +++ b/cmd/minikube/cmd/delete.go @@ -260,7 +260,7 @@ func deletePossibleKicLeftOver(cname string, driverName string) { klog.Warningf("error deleting volumes (might be okay).\nTo see the list of volumes run: 'docker volume ls'\n:%v", errs) } - errs = oci.DeleteKICNetworks() + errs = oci.DeleteKICNetworks(bin) if errs != nil { klog.Warningf("error deleting leftover networks (might be okay).\nTo see the list of networks: 'docker network ls'\n:%v", errs) } diff --git a/cmd/minikube/cmd/node_start.go b/cmd/minikube/cmd/node_start.go index f2740dda63..0e8bea7d31 100644 --- a/cmd/minikube/cmd/node_start.go +++ b/cmd/minikube/cmd/node_start.go @@ -27,6 +27,7 @@ import ( "k8s.io/minikube/pkg/minikube/mustload" "k8s.io/minikube/pkg/minikube/node" "k8s.io/minikube/pkg/minikube/out" + "k8s.io/minikube/pkg/minikube/out/register" "k8s.io/minikube/pkg/minikube/reason" "k8s.io/minikube/pkg/minikube/style" ) @@ -54,6 +55,7 @@ var nodeStartCmd = &cobra.Command{ os.Exit(0) } + register.Reg.SetStep(register.InitialSetup) r, p, m, h, err := node.Provision(cc, n, false, viper.GetBool(deleteOnFailure)) if err != nil { exit.Error(reason.GuestNodeProvision, "provisioning host for node", err) diff --git a/cmd/minikube/cmd/podman-env.go b/cmd/minikube/cmd/podman-env.go index a0290d78e9..970811e930 100644 --- a/cmd/minikube/cmd/podman-env.go +++ b/cmd/minikube/cmd/podman-env.go @@ -29,6 +29,7 @@ import ( "github.com/docker/machine/libmachine/drivers" "github.com/docker/machine/libmachine/ssh" "github.com/spf13/cobra" + "k8s.io/minikube/pkg/drivers/kic/oci" "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/driver" @@ -39,9 +40,27 @@ import ( "k8s.io/minikube/pkg/minikube/shell" ) -var podmanEnv1Tmpl = fmt.Sprintf("{{ .Prefix }}%s{{ .Delimiter }}{{ .VarlinkBridge }}{{ .Suffix }}{{ .Prefix }}%s{{ .Delimiter }}{{ .MinikubePodmanProfile }}{{ .Suffix }}{{ .UsageHint }}", constants.PodmanVarlinkBridgeEnv, constants.MinikubeActivePodmanEnv) +var podmanEnv1Tmpl = fmt.Sprintf( + "{{ .Prefix }}%s{{ .Delimiter }}{{ .VarlinkBridge }}{{ .Suffix }}"+ + "{{ .Prefix }}%s{{ .Delimiter }}{{ .MinikubePodmanProfile }}{{ .Suffix }}"+ + "{{ .UsageHint }}", + constants.PodmanVarlinkBridgeEnv, + constants.MinikubeActivePodmanEnv) -var podmanEnv2Tmpl = fmt.Sprintf("{{ .Prefix }}%s{{ .Delimiter }}{{ .ContainerHost }}{{ .Suffix }}{{ if .ContainerSSHKey }}{{ .Prefix }}%s{{ .Delimiter }}{{ .ContainerSSHKey}}{{ .Suffix }}{{ end }}{{ .Prefix }}%s{{ .Delimiter }}{{ .MinikubePodmanProfile }}{{ .Suffix }}{{ .UsageHint }}", constants.PodmanContainerHostEnv, constants.PodmanContainerSSHKeyEnv, constants.MinikubeActivePodmanEnv) +var podmanEnv2Tmpl = fmt.Sprintf( + "{{ .Prefix }}%s{{ .Delimiter }}{{ .ContainerHost }}{{ .Suffix }}"+ + "{{ if .ContainerSSHKey }}"+ + "{{ .Prefix }}%s{{ .Delimiter }}{{ .ContainerSSHKey}}{{ .Suffix }}"+ + "{{ end }}"+ + "{{ if .ExistingContainerHost }}"+ + "{{ .Prefix }}%s{{ .Delimiter }}{{ .ExistingContainerHost }}{{ .Suffix }}"+ + "{{ end }}"+ + "{{ .Prefix }}%s{{ .Delimiter }}{{ .MinikubePodmanProfile }}{{ .Suffix }}"+ + "{{ .UsageHint }}", + constants.PodmanContainerHostEnv, + constants.PodmanContainerSSHKeyEnv, + constants.ExistingContainerHostEnv, + constants.MinikubeActivePodmanEnv) // PodmanShellConfig represents the shell config for Podman type PodmanShellConfig struct { @@ -50,6 +69,8 @@ type PodmanShellConfig struct { ContainerHost string ContainerSSHKey string MinikubePodmanProfile string + + ExistingContainerHost string } var podmanUnset bool @@ -65,6 +86,9 @@ func podmanShellCfgSet(ec PodmanEnvConfig, envMap map[string]string) *PodmanShel s.VarlinkBridge = envMap[constants.PodmanVarlinkBridgeEnv] s.ContainerHost = envMap[constants.PodmanContainerHostEnv] s.ContainerSSHKey = envMap[constants.PodmanContainerSSHKeyEnv] + + s.ExistingContainerHost = envMap[constants.ExistingContainerHostEnv] + s.MinikubePodmanProfile = envMap[constants.MinikubeActivePodmanEnv] return s @@ -259,6 +283,13 @@ func podmanEnvVars(ec PodmanEnvConfig) map[string]string { for k, v := range env0 { env[k] = v } + if os.Getenv(constants.MinikubeActivePodmanEnv) == "" { + e := constants.PodmanContainerHostEnv + if v := oci.InitialEnv(e); v != "" { + key := constants.ExistingContainerHostEnv + env[key] = v + } + } return env } diff --git a/cmd/minikube/cmd/stop.go b/cmd/minikube/cmd/stop.go index 9d63808a8c..8f54dfae0e 100644 --- a/cmd/minikube/cmd/stop.go +++ b/cmd/minikube/cmd/stop.go @@ -46,6 +46,7 @@ var ( stopAll bool keepActive bool scheduledStopDuration time.Duration + cancelScheduledStop bool ) // stopCmd represents the stop command @@ -60,6 +61,8 @@ func init() { stopCmd.Flags().BoolVar(&stopAll, "all", false, "Set flag to stop all profiles (clusters)") stopCmd.Flags().BoolVar(&keepActive, "keep-context-active", false, "keep the kube-context active after cluster is stopped. Defaults to false.") stopCmd.Flags().DurationVar(&scheduledStopDuration, "schedule", 0*time.Second, "Set flag to stop cluster after a set amount of time (e.g. --schedule=5m)") + stopCmd.Flags().BoolVar(&cancelScheduledStop, "cancel-scheduled", false, "cancel any existing scheduled stop requests") + if err := stopCmd.Flags().MarkHidden("schedule"); err != nil { klog.Info("unable to mark --schedule flag as hidden") } @@ -97,6 +100,11 @@ func runStop(cmd *cobra.Command, args []string) { // Kill any existing scheduled stops schedule.KillExisting(profilesToStop) + if cancelScheduledStop { + register.Reg.SetStep(register.Done) + out.Step(style.Stopped, `All existing scheduled stops cancelled`) + return + } if scheduledStopDuration != 0 { if err := schedule.Daemonize(profilesToStop, scheduledStopDuration); err != nil { diff --git a/deploy/addons/ingress/ingress-dp.yaml.tmpl b/deploy/addons/ingress/ingress-dp.yaml.tmpl index 56f9dc3443..fda40ff2ba 100644 --- a/deploy/addons/ingress/ingress-dp.yaml.tmpl +++ b/deploy/addons/ingress/ingress-dp.yaml.tmpl @@ -65,6 +65,9 @@ spec: - --validating-webhook=:8443 - --validating-webhook-certificate=/usr/local/certificates/cert - --validating-webhook-key=/usr/local/certificates/key + {{if .CustomIngressCert}} + - --default-ssl-certificate={{ .CustomIngressCert }} + {{end}} securityContext: capabilities: drop: diff --git a/deploy/iso/minikube-iso/board/coreos/minikube/users b/deploy/iso/minikube-iso/board/coreos/minikube/users index 8e967e11f1..cdff9ff1f9 100644 --- a/deploy/iso/minikube-iso/board/coreos/minikube/users +++ b/deploy/iso/minikube-iso/board/coreos/minikube/users @@ -1 +1 @@ -docker 1000 docker 1000 =tcuser /home/docker /bin/bash wheel,vboxsf - +docker 1000 docker 1000 =tcuser /home/docker /bin/bash wheel,vboxsf,podman - diff --git a/deploy/iso/minikube-iso/configs/minikube_defconfig b/deploy/iso/minikube-iso/configs/minikube_defconfig index d420e19419..0008346e16 100644 --- a/deploy/iso/minikube-iso/configs/minikube_defconfig +++ b/deploy/iso/minikube-iso/configs/minikube_defconfig @@ -21,7 +21,7 @@ BR2_GLOBAL_PATCH_DIR="$(BR2_EXTERNAL_MINIKUBE_PATH)/board/coreos/minikube/patche BR2_LINUX_KERNEL=y BR2_LINUX_KERNEL_LATEST_VERSION=n BR2_LINUX_KERNEL_CUSTOM_VERSION=y -BR2_LINUX_KERNEL_CUSTOM_VERSION_VALUE="4.19.150" +BR2_LINUX_KERNEL_CUSTOM_VERSION_VALUE="4.19.157" BR2_LINUX_KERNEL_BZIMAGE=y BR2_LINUX_KERNEL_LZ4=y BR2_LINUX_KERNEL_USE_CUSTOM_CONFIG=y diff --git a/deploy/iso/minikube-iso/package/Config.in b/deploy/iso/minikube-iso/package/Config.in index 23539745f0..954fbac175 100644 --- a/deploy/iso/minikube-iso/package/Config.in +++ b/deploy/iso/minikube-iso/package/Config.in @@ -1,7 +1,6 @@ menu "System tools" source "$BR2_EXTERNAL_MINIKUBE_PATH/package/runc-master/Config.in" source "$BR2_EXTERNAL_MINIKUBE_PATH/package/podman/Config.in" - source "$BR2_EXTERNAL_MINIKUBE_PATH/package/varlink/Config.in" source "$BR2_EXTERNAL_MINIKUBE_PATH/package/conmon/Config.in" source "$BR2_EXTERNAL_MINIKUBE_PATH/package/crio-bin/Config.in" source "$BR2_EXTERNAL_MINIKUBE_PATH/package/crictl-bin/Config.in" diff --git a/deploy/iso/minikube-iso/package/containerd-bin/containerd-bin.hash b/deploy/iso/minikube-iso/package/containerd-bin/containerd-bin.hash index a5d89c8404..d9eeaac5a5 100644 --- a/deploy/iso/minikube-iso/package/containerd-bin/containerd-bin.hash +++ b/deploy/iso/minikube-iso/package/containerd-bin/containerd-bin.hash @@ -6,3 +6,4 @@ sha256 a0965e1492fca558629826f1aa89a9675de3d451cec67540400b30c0bf6ac387 v1.2.10. sha256 318886ea1efdec36f088fd6a0a0fe2b2f0ebdfd0066bdb4bd284bad12abc0a41 v1.2.12.tar.gz sha256 0811057ab67b78ce911416e793edaeb14b3f1e105d67b8e67b6302e0eab572e4 v1.2.13.tar.gz sha256 d30d59e143697aa4f0960205b3f5ac59c573b332f20507740ef2dc0fb5ae8ded v1.3.7.tar.gz +sha256 9244212589c84b12262769dca6fb985c0c680cb5259c8904b29c511d81fd62d0 v1.3.9.tar.gz diff --git a/deploy/iso/minikube-iso/package/containerd-bin/containerd-bin.mk b/deploy/iso/minikube-iso/package/containerd-bin/containerd-bin.mk index 828eb81092..7cd6ec47f3 100644 --- a/deploy/iso/minikube-iso/package/containerd-bin/containerd-bin.mk +++ b/deploy/iso/minikube-iso/package/containerd-bin/containerd-bin.mk @@ -3,8 +3,8 @@ # containerd # ################################################################################ -CONTAINERD_BIN_VERSION = v1.3.7 -CONTAINERD_BIN_COMMIT = 8fba4e9a7d01810a393d5d25a3621dc101981175 +CONTAINERD_BIN_VERSION = v1.3.9 +CONTAINERD_BIN_COMMIT = ea765aba0d05254012b0b9e595e995c09186427f CONTAINERD_BIN_SITE = https://github.com/containerd/containerd/archive CONTAINERD_BIN_SOURCE = $(CONTAINERD_BIN_VERSION).tar.gz CONTAINERD_BIN_DEPENDENCIES = host-go libgpgme diff --git a/deploy/iso/minikube-iso/package/crictl-bin/crictl-bin.hash b/deploy/iso/minikube-iso/package/crictl-bin/crictl-bin.hash index c2b9da6eff..6b52b763dd 100644 --- a/deploy/iso/minikube-iso/package/crictl-bin/crictl-bin.hash +++ b/deploy/iso/minikube-iso/package/crictl-bin/crictl-bin.hash @@ -3,3 +3,4 @@ sha256 9bdbea7a2b382494aff2ff014da328a042c5aba9096a7772e57fdf487e5a1d51 crictl- sha256 c3b71be1f363e16078b51334967348aab4f72f46ef64a61fe7754e029779d45a crictl-v1.15.0-linux-amd64.tar.gz sha256 19fed421710fccfe58f5573383bb137c19438a9056355556f1a15da8d23b3ad1 crictl-v1.16.1-linux-amd64.tar.gz sha256 7b72073797f638f099ed19550d52e9b9067672523fc51b746e65d7aa0bafa414 crictl-v1.17.0-linux-amd64.tar.gz +sha256 876dd2b3d0d1c2590371f940fb1bf1fbd5f15aebfbe456703ee465d959700f4a crictl-v1.18.0-linux-amd64.tar.gz diff --git a/deploy/iso/minikube-iso/package/crictl-bin/crictl-bin.mk b/deploy/iso/minikube-iso/package/crictl-bin/crictl-bin.mk index 1f7ba0e2b4..4cd00305a6 100644 --- a/deploy/iso/minikube-iso/package/crictl-bin/crictl-bin.mk +++ b/deploy/iso/minikube-iso/package/crictl-bin/crictl-bin.mk @@ -4,7 +4,7 @@ # ################################################################################ -CRICTL_BIN_VERSION = v1.17.0 +CRICTL_BIN_VERSION = v1.18.0 CRICTL_BIN_SITE = https://github.com/kubernetes-sigs/cri-tools/releases/download/$(CRICTL_BIN_VERSION) CRICTL_BIN_SOURCE = crictl-$(CRICTL_BIN_VERSION)-linux-amd64.tar.gz CRICTL_BIN_STRIP_COMPONENTS = 0 diff --git a/deploy/iso/minikube-iso/package/docker-bin/docker-bin.hash b/deploy/iso/minikube-iso/package/docker-bin/docker-bin.hash index 35ea54f47c..94b7580224 100644 --- a/deploy/iso/minikube-iso/package/docker-bin/docker-bin.hash +++ b/deploy/iso/minikube-iso/package/docker-bin/docker-bin.hash @@ -22,3 +22,4 @@ sha256 7c1576a0bc749418d1423d2b78c8920b5d61f849789904612862dd118742e82b docker- sha256 0f4336378f61ed73ed55a356ac19e46699a995f2aff34323ba5874d131548b9e docker-19.03.11.tgz sha256 88de1b87b8a2582fe827154899475a72fb707c5793cfb39d2a24813ba1f31197 docker-19.03.12.tgz sha256 ddb13aff1fcdcceb710bf71a210169b9c1abfd7420eeaf42cf7975f8fae2fcc8 docker-19.03.13.tgz +sha256 9f1ec28e357a8f18e9561129239caf9c0807d74756e21cc63637c7fdeaafe847 docker-19.03.14.tgz diff --git a/deploy/iso/minikube-iso/package/docker-bin/docker-bin.mk b/deploy/iso/minikube-iso/package/docker-bin/docker-bin.mk index 71c537eaed..78dd22ad7b 100644 --- a/deploy/iso/minikube-iso/package/docker-bin/docker-bin.mk +++ b/deploy/iso/minikube-iso/package/docker-bin/docker-bin.mk @@ -4,7 +4,7 @@ # ################################################################################ -DOCKER_BIN_VERSION = 19.03.13 +DOCKER_BIN_VERSION = 19.03.14 DOCKER_BIN_SITE = https://download.docker.com/linux/static/stable/x86_64 DOCKER_BIN_SOURCE = docker-$(DOCKER_BIN_VERSION).tgz diff --git a/deploy/iso/minikube-iso/package/podman/override.conf b/deploy/iso/minikube-iso/package/podman/override.conf new file mode 100644 index 0000000000..b762370a4a --- /dev/null +++ b/deploy/iso/minikube-iso/package/podman/override.conf @@ -0,0 +1,4 @@ +[Socket] +SocketMode=0660 +SocketUser=root +SocketGroup=podman diff --git a/deploy/iso/minikube-iso/package/podman/podman.conf b/deploy/iso/minikube-iso/package/podman/podman.conf new file mode 100644 index 0000000000..8e31190ab8 --- /dev/null +++ b/deploy/iso/minikube-iso/package/podman/podman.conf @@ -0,0 +1 @@ +d /run/podman 0770 root podman diff --git a/deploy/iso/minikube-iso/package/podman/podman.hash b/deploy/iso/minikube-iso/package/podman/podman.hash index 29f2e1a338..04fbbf474d 100644 --- a/deploy/iso/minikube-iso/package/podman/podman.hash +++ b/deploy/iso/minikube-iso/package/podman/podman.hash @@ -1 +1,3 @@ sha256 a16846fe076aaf2c9ea2e854c3baba9fb838d916be7fb4b5be332e6c92d907d4 v1.9.3.tar.gz +sha256 5ebaa6e0dbd7fd1863f70d2bc71dc8a94e195c3339c17e3cac4560c9ec5747f8 v2.1.1.tar.gz +sha256 ec5473e51fa28f29af323473fc484f742dc7df23d06d8ba9f217f13382893a71 v2.2.0.tar.gz diff --git a/deploy/iso/minikube-iso/package/podman/podman.mk b/deploy/iso/minikube-iso/package/podman/podman.mk index a5c297b7f2..f95aeb8929 100644 --- a/deploy/iso/minikube-iso/package/podman/podman.mk +++ b/deploy/iso/minikube-iso/package/podman/podman.mk @@ -1,12 +1,15 @@ -PODMAN_DUMMY = DUMMY -PODMAN_VERSION = v1.9.3 -PODMAN_COMMIT = 5d44534fff6877b1cb15b760242279ae6293154c +PODMAN_VERSION = v2.2.0 +PODMAN_COMMIT = db1d2ff111ee9b012779ff3a5279a982520ccda4 PODMAN_SITE = https://github.com/containers/podman/archive PODMAN_SOURCE = $(PODMAN_VERSION).tar.gz PODMAN_LICENSE = Apache-2.0 PODMAN_LICENSE_FILES = LICENSE PODMAN_DEPENDENCIES = host-go +ifeq ($(BR2_INIT_SYSTEMD),y) +# need libsystemd for journal +PODMAN_DEPENDENCIES += systemd +endif PODMAN_GOPATH = $(@D)/_output PODMAN_BIN_ENV = \ @@ -17,11 +20,24 @@ PODMAN_BIN_ENV = \ PATH=$(PODMAN_GOPATH)/bin:$(BR_PATH) +define PODMAN_USERS + - -1 podman -1 - - - - - +endef + +define PODMAN_MOD_VENDOR_MAKEFILE + # "build flag -mod=vendor only valid when using modules" + sed -e 's|-mod=vendor ||' -i $(@D)/Makefile +endef + +PODMAN_POST_EXTRACT_HOOKS += PODMAN_MOD_VENDOR_MAKEFILE + define PODMAN_CONFIGURE_CMDS + mkdir -p $(PODMAN_GOPATH) && mv $(@D)/vendor $(PODMAN_GOPATH)/src + mkdir -p $(PODMAN_GOPATH)/src/github.com/containers - ln -sf $(@D) $(PODMAN_GOPATH)/src/github.com/containers/libpod - mkdir -p $(PODMAN_GOPATH)/src/github.com/varlink - ln -sf $(@D)/vendor/github.com/varlink/go $(PODMAN_GOPATH)/src/github.com/varlink/go + ln -sf $(@D) $(PODMAN_GOPATH)/src/github.com/containers/podman + + ln -sf $(@D) $(PODMAN_GOPATH)/src/github.com/containers/podman/v2 endef define PODMAN_BUILD_CMDS @@ -35,4 +51,21 @@ define PODMAN_INSTALL_TARGET_CMDS $(INSTALL) -m 644 $(@D)/cni/87-podman-bridge.conflist $(TARGET_DIR)/etc/cni/net.d/87-podman-bridge.conflist endef +define PODMAN_INSTALL_INIT_SYSTEMD + $(INSTALL) -D -m 644 \ + $(@D)/contrib/systemd/system/podman.service \ + $(TARGET_DIR)/usr/lib/systemd/system/podman.service + $(INSTALL) -D -m 644 \ + $(@D)/contrib/systemd/system/podman.socket \ + $(TARGET_DIR)/usr/lib/systemd/system/podman.socket + + # Allow running podman-remote as a user in the group "podman" + $(INSTALL) -D -m 644 \ + $(PODMAN_PKGDIR)/override.conf \ + $(TARGET_DIR)/usr/lib/systemd/system/podman.socket.d/override.conf + $(INSTALL) -D -m 644 \ + $(PODMAN_PKGDIR)/podman.conf \ + $(TARGET_DIR)/usr/lib/tmpfiles.d/podman.conf +endef + $(eval $(generic-package)) diff --git a/deploy/iso/minikube-iso/package/varlink/Config.in b/deploy/iso/minikube-iso/package/varlink/Config.in deleted file mode 100644 index ef047e1560..0000000000 --- a/deploy/iso/minikube-iso/package/varlink/Config.in +++ /dev/null @@ -1,3 +0,0 @@ -config BR2_PACKAGE_VARLINK - bool "varlink" - default y diff --git a/deploy/iso/minikube-iso/package/varlink/varlink.hash b/deploy/iso/minikube-iso/package/varlink/varlink.hash deleted file mode 100644 index 64b07416e6..0000000000 --- a/deploy/iso/minikube-iso/package/varlink/varlink.hash +++ /dev/null @@ -1,4 +0,0 @@ -sha256 3857f109574750403b233b5fdf73f1852d8decc33dac8f73bd49f2003b69ad22 16.tar.gz -sha256 0dcb451f32033154c56710c216e67f245923fe2b011321271f6670e5a2285ce6 17.tar.gz -sha256 7a32543643116ad105da4ddb2f8030de7dcad1cdb3feb1a214ae5e7b65a6a198 18.tar.gz -sha256 0e316138ef6abc34363b05d0caf6df2e389a93b832e8d971e3ae64b48ba96133 19.tar.gz diff --git a/deploy/iso/minikube-iso/package/varlink/varlink.mk b/deploy/iso/minikube-iso/package/varlink/varlink.mk deleted file mode 100644 index 06a26e788f..0000000000 --- a/deploy/iso/minikube-iso/package/varlink/varlink.mk +++ /dev/null @@ -1,9 +0,0 @@ -VARLINK_VERSION = 19 -VARLINK_SITE = https://github.com/varlink/libvarlink/archive -VARLINK_SOURCE = $(VARLINK_VERSION).tar.gz -VARLINK_LICENSE = Apache-2.0 -VARLINK_LICENSE_FILES = LICENSE - -VARLINK_NEEDS_HOST_PYTHON = python3 - -$(eval $(meson-package)) diff --git a/deploy/kicbase/Dockerfile b/deploy/kicbase/Dockerfile index 695a112e04..a832c847fc 100644 --- a/deploy/kicbase/Dockerfile +++ b/deploy/kicbase/Dockerfile @@ -19,7 +19,7 @@ # start from ubuntu 20.04, this image is reasonably small as a starting point # for a kubernetes node image, it doesn't contain much we don't need -FROM ubuntu:focal-20200925 +FROM ubuntu:focal-20201106 ARG BUILDKIT_VERSION="v0.7.2" @@ -123,19 +123,26 @@ RUN export ARCH=$(dpkg --print-architecture | sed 's/ppc64el/ppc64le/' | sed 's/ RUN sh -c "echo 'deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_20.04/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list" && \ curl -LO https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_20.04/Release.key && \ apt-key add - < Release.key && \ - clean-install containers-common catatonit conmon containernetworking-plugins cri-tools podman-plugins varlink + clean-install containers-common catatonit conmon containernetworking-plugins cri-tools podman-plugins # install cri-o based on https://github.com/cri-o/cri-o/blob/release-1.18/README.md#installing-cri-o -RUN sh -c "echo 'deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/1.18/xUbuntu_20.04/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:1.18.list" && \ - curl -LO https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/1.18/xUbuntu_20.04/Release.key && \ +RUN sh -c "echo 'deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/1.18:/1.18.4/xUbuntu_20.04/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:1.18.list" && \ + curl -LO https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/1.18:/1.18.4/xUbuntu_20.04/Release.key && \ apt-key add - < Release.key && \ clean-install cri-o cri-o-runc # install podman -RUN sh -c "echo 'deb https://dl.bintray.com/afbjorklund/podman focal main' > /etc/apt/sources.list.d/podman.list" && \ - curl -L https://bintray.com/user/downloadSubjectPublicKey?username=afbjorklund -o afbjorklund-public.key.asc && \ - apt-key add - < afbjorklund-public.key.asc && \ - clean-install podman=1.9.3~1 +RUN sh -c "echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_20.04/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list" && \ + curl -LO https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_20.04/Release.key && \ + apt-key add - < Release.key && \ + clean-install podman && \ + addgroup --system podman && \ + mkdir -p /etc/systemd/system/podman.socket.d && \ + printf "[Socket]\nSocketMode=0660\nSocketUser=root\nSocketGroup=podman\n" \ + > /etc/systemd/system/podman.socket.d/override.conf && \ + mkdir -p /etc/tmpfiles.d && \ + echo "d /run/podman 0770 root podman" > /etc/tmpfiles.d/podman.conf && \ + systemd-tmpfiles --create # automount service COPY automount/minikube-automount /usr/sbin/minikube-automount @@ -153,7 +160,7 @@ RUN ln -fs /usr/lib/systemd/system/minikube-scheduled-stop.service \ # disable non-docker runtimes by default RUN systemctl disable containerd && systemctl disable crio && rm /etc/crictl.yaml # enable docker which is default -RUN systemctl enable docker +RUN systemctl enable docker.service && systemctl enable podman.socket # making SSH work for docker container # based on https://github.com/rastasheep/ubuntu-sshd/blob/master/18.04/Dockerfile RUN mkdir /var/run/sshd @@ -168,6 +175,7 @@ EXPOSE 22 # create docker user for minikube ssh. to match VM using "docker" as username RUN adduser --ingroup docker --disabled-password --gecos '' docker RUN adduser docker sudo +RUN adduser docker podman RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers USER docker RUN mkdir /home/docker/.ssh diff --git a/deploy/kicbase/entrypoint b/deploy/kicbase/entrypoint index fea0343c35..bf6c54df84 100755 --- a/deploy/kicbase/entrypoint +++ b/deploy/kicbase/entrypoint @@ -19,6 +19,16 @@ set -o nounset set -o pipefail set -x +configure_proxy() { + # ensure all processes receive the proxy settings by default + # https://www.freedesktop.org/software/systemd/man/systemd-system.conf.html + mkdir -p /etc/systemd/system.conf.d/ + cat </etc/systemd/system.conf.d/proxy-default-environment.conf +[Manager] +DefaultEnvironment="HTTP_PROXY=${HTTP_PROXY:-}" "HTTPS_PROXY=${HTTPS_PROXY:-}" "NO_PROXY=${NO_PROXY:-}" +EOF +} + update-alternatives() { echo "retryable update-alternatives: $*" local args=$* @@ -48,7 +58,7 @@ fix_mount() { # This is a workaround to an AUFS bug that might cause `Text file # busy` on `mount` command below. See more details in # https://github.com/moby/moby/issues/9547 - if [[ "$(stat -f -c %T /bin/mount)" == 'aufs' ]]; then + if [[ "$(stat -f -c %T "$(which mount)")" == 'aufs' ]]; then echo 'INFO: detected aufs, calling sync' >&2 sync fi @@ -174,16 +184,6 @@ fix_kmsg() { fi } -configure_proxy() { - # ensure all processes receive the proxy settings by default - # https://www.freedesktop.org/software/systemd/man/systemd-system.conf.html - mkdir -p /etc/systemd/system.conf.d/ - cat </etc/systemd/system.conf.d/proxy-default-environment.conf -[Manager] -DefaultEnvironment="HTTP_PROXY=${HTTP_PROXY:-}" "HTTPS_PROXY=${HTTPS_PROXY:-}" "NO_PROXY=${NO_PROXY:-}" -EOF -} - select_iptables() { # based on: https://github.com/kubernetes/kubernetes/blob/ffe93b3979486feb41a0f85191bdd189cbd56ccc/build/debian-iptables/iptables-wrapper local mode=nft @@ -229,7 +229,7 @@ enable_network_magic(){ sed -e "s/${docker_embedded_dns_ip}/${docker_host_ip}/g" /etc/resolv.conf.original >/etc/resolv.conf # fixup IPs in manifests ... - curr_ipv4="$( (getent ahostsv4 $(hostname) | head -n1 | cut -d' ' -f1) || true)" + curr_ipv4="$( (getent ahostsv4 "$(hostname)" | head -n1 | cut -d' ' -f1) || true)" echo "INFO: Detected IPv4 address: ${curr_ipv4}" >&2 if [ -f /kind/old-ipv4 ]; then old_ipv4=$(cat /kind/old-ipv4) @@ -249,7 +249,7 @@ enable_network_magic(){ fi # do IPv6 - curr_ipv6="$( (getent ahostsv6 $(hostname) | head -n1 | cut -d' ' -f1) || true)" + curr_ipv6="$( (getent ahostsv6 "$(hostname)" | head -n1 | cut -d' ' -f1) || true)" echo "INFO: Detected IPv6 address: ${curr_ipv6}" >&2 if [ -f /kind/old-ipv6 ]; then old_ipv6=$(cat /kind/old-ipv6) @@ -269,15 +269,16 @@ enable_network_magic(){ } # run pre-init fixups -select_iptables +# NOTE: it's important that we do configure* first in this order to avoid races +configure_proxy fix_kmsg fix_mount retryable_fix_cgroup_mounts fix_machine_id fix_product_name fix_product_uuid -configure_proxy +select_iptables enable_network_magic # we want the command (expected to be systemd) to be PID1, so exec to it -exec "$@" \ No newline at end of file +exec "$@" diff --git a/go.sum b/go.sum index dc22280309..1ce4cf5266 100644 --- a/go.sum +++ b/go.sum @@ -45,6 +45,7 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 code.gitea.io/sdk/gitea v0.12.0/go.mod h1:z3uwDV/b9Ls47NGukYM9XhnHtqPh/J+t40lsUrR6JDY= contrib.go.opencensus.io/exporter/aws v0.0.0-20181029163544-2befc13012d0/go.mod h1:uu1P0UCM/6RbsMrgPa98ll8ZcHM858i/AD06a9aLRCA= contrib.go.opencensus.io/exporter/ocagent v0.5.0/go.mod h1:ImxhfLRpxoYiSq891pBrLVhN+qmP8BTVvdH2YLs7Gl0= +contrib.go.opencensus.io/exporter/stackdriver v0.12.1 h1:Dll2uFfOVI3fa8UzsHyP6z0M6fEc9ZTAMo+Y3z282Xg= contrib.go.opencensus.io/exporter/stackdriver v0.12.1/go.mod h1:iwB6wGarfphGGe/e5CWqyUk/cLzKnWsOKPVW3no6OTw= contrib.go.opencensus.io/integrations/ocsql v0.1.4/go.mod h1:8DsSdjz3F+APR+0z0WkU1aRorQCFfRxvqjUUPMbF3fE= contrib.go.opencensus.io/resource v0.1.1/go.mod h1:F361eGI91LCmW1I/Saf+rX0+OFcigGlFvXwEGEnkRLA= @@ -92,7 +93,6 @@ github.com/DataDog/sketches-go v0.0.1/go.mod h1:Q5DbzQ+3AkgGwymQO7aZFNP7ns2lZKGt github.com/Djarvur/go-err113 v0.0.0-20200410182137-af658d038157/go.mod h1:4UJr5HIiMZrwgkSPdsjy2uOQExX/WEILpIrO9UPGuXs= github.com/Djarvur/go-err113 v0.1.0/go.mod h1:4UJr5HIiMZrwgkSPdsjy2uOQExX/WEILpIrO9UPGuXs= github.com/GoogleCloudPlatform/cloudsql-proxy v0.0.0-20191009163259-e802c2cb94ae/go.mod h1:mjwGPas4yKduTyubHvD1Atl9r1rUq8DfVy+gkVvZ+oo= -github.com/GoogleCloudPlatform/opentelemetry-operations-go v0.13.0 h1:Gx9IxOjR9ug5Ad8YbafxJ6N2g6oDj/Q419tYHdd1od4= github.com/GoogleCloudPlatform/k8s-cloud-provider v0.0.0-20190822182118-27a4ced34534/go.mod h1:iroGtC8B3tQiqtds1l+mgk/BBOrxbqjH+eUfFQYRc14= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v0.13.0 h1:fjKUtfldCPIF4nIzAAj3LzP8Lrd3DuRIMiFdOsj4fLc= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v0.13.0/go.mod h1:q/paYxLXKVhwfC3lzLfhtL54fAx14wzMN9DundQOBMc= @@ -148,6 +148,7 @@ github.com/alonyb/spinner v1.12.1/go.mod h1:QOuQk7x+EaDASo80FEXwlwiA+j/PPIcX3FSc github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/apex/log v1.1.4/go.mod h1:AlpoD9aScyQfJDVHmLMEcx4oU6LqzkWp4Mg9GdAcEvQ= +github.com/apex/log v1.3.0 h1:1fyfbPvUwD10nMoh3hY6MXzvZShJQn9/ck7ATgAt5pA= github.com/apex/log v1.3.0/go.mod h1:jd8Vpsr46WAe3EZSQ/IUMs2qQD/GOycT5rPWCO1yGcs= github.com/apex/logs v0.0.4/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo= github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE= @@ -214,6 +215,7 @@ github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QH github.com/cenkalti/backoff/v4 v4.1.0 h1:c8LkOFQTzuO0WBM/ae5HdGQuZPfPxp7lqBRwQRm4fSc= github.com/cenkalti/backoff/v4 v4.1.0/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/prettybench v0.0.0-20150116022406-03b8cfe5406c/go.mod h1:Xe6ZsFhtM8HrDku0pxJ3/Lr51rwykrzgFwpmTzleatY= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= @@ -557,6 +559,7 @@ github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPg github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible h1:xmapqc1AyLoB+ddYT6r04bD9lIjlOqGaREovi0SzFaE= github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0 h1:pMen7vLs8nvgEYhywH3KDWJIJTeEr2ULsVWHWYHQyBs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f h1:Jnx61latede7zDD3DiiP4gmNz33uK0U5HDUaF0a/HVQ= @@ -587,6 +590,7 @@ github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTV github.com/googleapis/gnostic v0.2.2/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.3.0 h1:CcQijm0XKekKjP/YCz28LXVSpgguuB+nCxaSjCe09y0= github.com/googleapis/gnostic v0.3.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleinterns/cloud-operations-api-mock v0.0.0-20200709193332-a1e58c29bdd3 h1:eHv/jVY/JNop1xg2J9cBb4EzyMpWZoNCP1BslSAIkOI= github.com/googleinterns/cloud-operations-api-mock v0.0.0-20200709193332-a1e58c29bdd3/go.mod h1:h/KNeRx7oYU4SpA4SoY7W2/NxDKEEVuwA6j9A27L4OI= github.com/gookit/color v1.2.4/go.mod h1:AhIE+pS6D4Ql0SQWbBeXPHw7gY0/sjHoA4s/n1KB7xg= github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= @@ -1107,6 +1111,7 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= @@ -1192,7 +1197,6 @@ go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4 h1:LYy1Hy3MJdrCdMwwzxA/dRok4ejH+RwNGbuoD9fCjto= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opentelemetry.io v0.1.0 h1:EANZoRCOP+A3faIlw/iN6YEWoYb1vleZRKm1EvH8T48= go.opentelemetry.io/otel v0.13.0 h1:2isEnyzjjJZq6r2EKMsFj4TxiQiexsM04AVhwbR/oBA= go.opentelemetry.io/otel v0.13.0/go.mod h1:dlSNewoRYikTkotEnxdmuBHgzT+k/idJSfDv/FxEnOY= go.opentelemetry.io/otel/sdk v0.13.0 h1:4VCfpKamZ8GtnepXxMRurSpHpMKkcxhtO33z1S4rGDQ= @@ -1505,6 +1509,7 @@ golang.org/x/tools v0.0.0-20200527183253-8e7acdbce89d h1:SR+e35rACZFBohNb4Om1ibX golang.org/x/tools v0.0.0-20200527183253-8e7acdbce89d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200701151220-7cb253f4c4f8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200713011307-fd294ab11aed h1:+qzWo37K31KxduIYaBeMqJ8MUOyTayOQKpH9aDPLMSY= golang.org/x/tools v0.0.0-20200713011307-fd294ab11aed/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA= @@ -1663,6 +1668,7 @@ gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.1.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= diff --git a/hack/conformance_tests.sh b/hack/conformance_tests.sh index b4509e8e9c..439649449f 100755 --- a/hack/conformance_tests.sh +++ b/hack/conformance_tests.sh @@ -27,28 +27,30 @@ set -ex -o pipefail readonly PROFILE_NAME="k8sconformance" readonly MINIKUBE=${1:-./out/minikube} shift || true +readonly START_ARGS=${@:-} # Requires a fully running Kubernetes cluster. "${MINIKUBE}" delete -p "${PROFILE_NAME}" || true -"${MINIKUBE}" start -p "${PROFILE_NAME}" --wait=all +"${MINIKUBE}" start -p "${PROFILE_NAME}" ${START_ARGS} --wait=all --nodes=2 kubectl --context "${PROFILE_NAME}" get pods --all-namespaces "${MINIKUBE}" status -p "${PROFILE_NAME}" -go get -u -v github.com/vmware-tanzu/sonobuoy +curl -LO https://github.com/vmware-tanzu/sonobuoy/releases/download/v0.19.0/sonobuoy_0.19.0_linux_amd64.tar.gz || true +tar -xzf sonobuoy_0.19.0_linux_amd64.tar.gz -sonobuoy run --wait +./sonobuoy run --mode=certified-conformance --wait --alsologtostderr outdir="$(mktemp -d)" -sonobuoy retrieve "${outdir}" +./sonobuoy retrieve "${outdir}" cwd=$(pwd) cd "${outdir}" mkdir ./results; tar xzf *.tar.gz -C ./results -version=$(${MINIKUBE} version | cut -d" " -f3) +version=$(${cwd}/${MINIKUBE} version | cut -d" " -f3) -mkdir "minikube-${version}" +mkdir -p "minikube-${version}" cd "minikube-${version}" cat <PRODUCT.yaml @@ -67,6 +69,6 @@ cat <README.md ./hack/conformance_tests.sh $MINIKUBE $START_ARGS EOF -cp ../results/plugins/e2e/results/* . +cp -r ../results/plugins/e2e/results/global/* . cd .. cp -r "minikube-${version}" "${cwd}" diff --git a/hack/jenkins/build/minikube_build_upload_latest.sh b/hack/jenkins/build/minikube_build_upload_latest.sh index b311e0e8ad..b6b50cc86c 100755 --- a/hack/jenkins/build/minikube_build_upload_latest.sh +++ b/hack/jenkins/build/minikube_build_upload_latest.sh @@ -28,9 +28,18 @@ WANT_GOLANG_VERSION=$(grep '^GO_VERSION' Makefile | awk '{ print $3 }') declare -rx GOPATH=/var/lib/jenkins/go +GIT_COMMIT_AT_HEAD=$(git rev-parse HEAD | xargs) +MINIKUBE_LATEST_COMMIT=$(gsutil stat gs://minikube/latest/minikube-linux-amd64 | grep commit | awk '{ print $2 }' | xargs) + +if [ "$GIT_COMMIT_AT_HEAD" = "$MINIKUBE_LATEST_COMMIT" ]; then + echo "The current uploaded binary is already latest, skipping build" + exit 0 +fi + make cross && failed=$? || failed=$? if [[ "${failed}" -ne 0 ]]; then echo "build failed" exit "${failed}" fi gsutil cp out/minikube-* "gs://${bucket}" +gsutil setmeta -r -h "x-goog-meta-commit:$GIT_COMMIT_AT_HEAD" "gs://${bucket}" diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh index 0f8cec2a20..aebdb5cea0 100755 --- a/hack/jenkins/common.sh +++ b/hack/jenkins/common.sh @@ -347,9 +347,9 @@ fi echo ">> Installing gopogh" if [ "$(uname)" != "Darwin" ]; then - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-linux-amd64 && sudo install gopogh-linux-amd64 /usr/local/bin/gopogh + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64 && sudo install gopogh-linux-amd64 /usr/local/bin/gopogh else - curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-darwin-amd64 && sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh + curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-darwin-amd64 && sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh fi echo ">> Running gopogh" diff --git a/hack/jenkins/release_build_and_upload.sh b/hack/jenkins/release_build_and_upload.sh index 493eabb81a..c2f15d9645 100755 --- a/hack/jenkins/release_build_and_upload.sh +++ b/hack/jenkins/release_build_and_upload.sh @@ -48,7 +48,9 @@ env BUILD_IN_DOCKER=y \ all \ out/minikube-installer.exe \ "out/minikube_${DEB_VERSION}-0_amd64.deb" \ + "out/minikube_${DEB_VERSION}-0_arm64.deb" \ "out/minikube-${RPM_VERSION}-0.x86_64.rpm" \ + "out/minikube-${RPM_VERSION}-0.aarch64.rpm" \ "out/docker-machine-driver-kvm2_${DEB_VERSION}-0_amd64.deb" \ "out/docker-machine-driver-kvm2-${RPM_VERSION}-0.x86_64.rpm" @@ -56,7 +58,9 @@ make checksum # unversioned names to avoid updating upstream Kubernetes documentation each release cp "out/minikube_${DEB_VERSION}-0_amd64.deb" out/minikube_latest_amd64.deb +cp "out/minikube_${DEB_VERSION}-0_arm64.deb" out/minikube_latest_arm64.deb cp "out/minikube-${RPM_VERSION}-0.x86_64.rpm" out/minikube-latest.x86_64.rpm +cp "out/minikube-${RPM_VERSION}-0.aarch64.rpm" out/minikube-latest.aarch64.rpm gsutil -m cp out/* "gs://$BUCKET/releases/$TAGNAME/" diff --git a/hack/metrics/README.md b/hack/metrics/README.md new file mode 100644 index 0000000000..03dd6187bd --- /dev/null +++ b/hack/metrics/README.md @@ -0,0 +1,12 @@ +This script runs `minikube start` in a loop and measures how long it takes. +It exports this data to Stackdriver via the OpenTelemetry API. + +To run this script, run: + +``` +MINIKUBE_GCP_PROJECT_ID= go run hack/metrics/*.go +``` + +This script is used to track minikube performance and prevent regressions. + +_Note: this script will export data to both Cloud Monitoring and Cloud Trace in the provided GCP project_ diff --git a/hack/metrics/metrics.go b/hack/metrics/metrics.go new file mode 100644 index 0000000000..e3e85078e0 --- /dev/null +++ b/hack/metrics/metrics.go @@ -0,0 +1,171 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "flag" + "fmt" + "log" + "os" + "os/exec" + "strings" + "time" + + _ "cloud.google.com/go/storage" + "contrib.go.opencensus.io/exporter/stackdriver" + "github.com/pkg/errors" + "go.opencensus.io/stats" + "go.opencensus.io/stats/view" + "go.opencensus.io/trace" + pkgtrace "k8s.io/minikube/pkg/trace" +) + +const ( + customMetricName = "custom.googleapis.com/minikube/start_time" + profile = "cloud-monitoring" +) + +var ( + // The task latency in seconds + latencyS = stats.Float64("repl/start_time", "start time in seconds", "s") + labels string + tmpFile string +) + +func main() { + if err := execute(); err != nil { + fmt.Println(err) + os.Exit(1) + } +} + +func init() { + flag.StringVar(&labels, "label", "", "Comma separated list of labels to add to metrics in key:value format") + flag.StringVar(&tmpFile, "file", "/tmp/minikube", "Download minikube to this file for testing") + flag.Parse() +} + +func execute() error { + projectID := os.Getenv(pkgtrace.ProjectEnvVar) + if projectID == "" { + return fmt.Errorf("metrics collector requires a valid GCP project id set via the %s env variable", pkgtrace.ProjectEnvVar) + } + ctx := context.Background() + if err := downloadMinikube(ctx, tmpFile); err != nil { + return errors.Wrap(err, "downloading minikube") + } + + // Register the view. It is imperative that this step exists, + // otherwise recorded metrics will be dropped and never exported. + v := &view.View{ + Name: customMetricName, + Measure: latencyS, + Description: "minikube start time", + Aggregation: view.LastValue(), + } + if err := view.Register(v); err != nil { + return errors.Wrap(err, "registering view") + } + for _, cr := range []string{"docker", "containerd", "crio"} { + if err := exportMinikubeStart(ctx, projectID, cr); err != nil { + log.Printf("error exporting minikube start data for runtime %v: %v", cr, err) + } + } + return nil +} + +func exportMinikubeStart(ctx context.Context, projectID, containerRuntime string) error { + sd, err := getExporter(projectID, containerRuntime) + if err != nil { + return errors.Wrap(err, "getting stackdriver exporter") + } + // Register it as a trace exporter + trace.RegisterExporter(sd) + + if err := sd.StartMetricsExporter(); err != nil { + return errors.Wrap(err, "starting metric exporter") + } + // track minikube start time and record it to metrics collector + st, err := minikubeStartTime(ctx, projectID, tmpFile, containerRuntime) + if err != nil { + return errors.Wrap(err, "collecting start time") + } + fmt.Printf("Latency: %f\n", st) + stats.Record(ctx, latencyS.M(st)) + time.Sleep(30 * time.Second) + sd.Flush() + sd.StopMetricsExporter() + trace.UnregisterExporter(sd) + return nil +} + +func getExporter(projectID, containerRuntime string) (*stackdriver.Exporter, error) { + return stackdriver.NewExporter(stackdriver.Options{ + ProjectID: projectID, + // MetricPrefix helps uniquely identify your metrics. + MetricPrefix: "minikube_start", + // ReportingInterval sets the frequency of reporting metrics + // to stackdriver backend. + ReportingInterval: 1 * time.Second, + DefaultMonitoringLabels: getLabels(containerRuntime), + }) +} + +func getLabels(containerRuntime string) *stackdriver.Labels { + l := &stackdriver.Labels{} + l.Set("container-runtime", containerRuntime, "container-runtime") + if labels == "" { + return l + } + separated := strings.Split(labels, ",") + for _, s := range separated { + sep := strings.Split(s, ":") + if len(sep) != 2 { + continue + } + log.Printf("Adding label %s=%s to metrics...", sep[0], sep[1]) + l.Set(sep[0], sep[1], "") + } + return l +} + +func minikubeStartTime(ctx context.Context, projectID, minikubePath, containerRuntime string) (float64, error) { + defer deleteMinikube(ctx, minikubePath) + + cmd := exec.CommandContext(ctx, minikubePath, "start", "--driver=docker", "-p", profile, "--memory=2000", "--trace=gcp", fmt.Sprintf("--container-runtime=%s", containerRuntime)) + cmd.Env = append(os.Environ(), fmt.Sprintf("%s=%s", pkgtrace.ProjectEnvVar, projectID)) + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + + t := time.Now() + log.Printf("Running [%v]....", cmd.Args) + if err := cmd.Run(); err != nil { + return 0, errors.Wrapf(err, "running %v", cmd.Args) + } + totalTime := time.Since(t).Seconds() + return totalTime, nil +} + +func deleteMinikube(ctx context.Context, minikubePath string) { + cmd := exec.CommandContext(ctx, minikubePath, "delete", "-p", profile) + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + log.Printf("error deleting: %v", err) + } +} diff --git a/hack/metrics/minikube.go b/hack/metrics/minikube.go new file mode 100644 index 0000000000..5b9d649205 --- /dev/null +++ b/hack/metrics/minikube.go @@ -0,0 +1,101 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "fmt" + "io/ioutil" + "os" + "os/exec" + "runtime" + "strings" + + "log" + + "cloud.google.com/go/storage" + "github.com/pkg/errors" +) + +const ( + bucketName = "minikube" +) + +// download minikube latest to file +func downloadMinikube(ctx context.Context, minikubePath string) error { + client, err := storage.NewClient(ctx) + if err != nil { + return errors.Wrap(err, "creating client") + } + obj := client.Bucket("minikube").Object(fmt.Sprintf("latest/%s", binary())) + + if localMinikubeIsLatest(ctx, minikubePath, obj) { + log.Print("local minikube is latest, skipping download...") + return nil + } + + os.Remove(minikubePath) + // download minikube binary from GCS + rc, err := obj.NewReader(ctx) + if err != nil { + return errors.Wrap(err, "gcs new reader") + } + defer rc.Close() + + data, err := ioutil.ReadAll(rc) + if err != nil { + return errors.Wrap(err, "ioutil read all") + } + log.Printf("downloading gs://%s/%s to %v", bucketName, binary(), minikubePath) + if err := ioutil.WriteFile(minikubePath, data, 0777); err != nil { + return errors.Wrap(err, "writing minikubePath") + } + if err := os.Chmod(minikubePath, 0700); err != nil { + return errors.Wrap(err, "chmod") + } + return nil +} + +// localMinikubeIsLatest returns true if the local version of minikube +// matches the latest version in GCS +func localMinikubeIsLatest(ctx context.Context, minikubePath string, obj *storage.ObjectHandle) bool { + log.Print("checking if local minikube is latest...") + attrs, err := obj.Attrs(ctx) + if err != nil { + log.Printf("error getting %s object attrs: %v", obj.ObjectName(), err) + return false + } + gcsMinikubeVersion, ok := attrs.Metadata["commit"] + if !ok { + log.Printf("there is no commit: %v", attrs.Metadata) + return false + } + currentMinikubeVersion, err := exec.Command(minikubePath, "version", "--output=json").Output() + if err != nil { + log.Printf("error running [%s version]: %v", minikubePath, err) + return false + } + return strings.Contains(string(currentMinikubeVersion), gcsMinikubeVersion) +} + +func binary() string { + b := fmt.Sprintf("minikube-%s-amd64", runtime.GOOS) + if runtime.GOOS == "windows" { + b += ".exe" + } + return b +} diff --git a/hack/update/github.go b/hack/update/github.go index 8371296ddf..897a36a404 100644 --- a/hack/update/github.go +++ b/hack/update/github.go @@ -221,7 +221,7 @@ func GHReleases(ctx context.Context, owner, repo string) (stable, latest string, return "", "", err } for _, rl := range rls { - ver := rl.GetName() + ver := rl.GetTagName() if !semver.IsValid(ver) { continue } diff --git a/hack/update/gopogh_version/update_gopogh_version.go b/hack/update/gopogh_version/update_gopogh_version.go new file mode 100644 index 0000000000..605fdff3b1 --- /dev/null +++ b/hack/update/gopogh_version/update_gopogh_version.go @@ -0,0 +1,108 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Script expects the following env variables: + - UPDATE_TARGET=: optional - if unset/absent, default option is "fs"; valid options are: + - "fs" - update only local filesystem repo files [default] + - "gh" - update only remote GitHub repo files and create PR (if one does not exist already) + - "all" - update local and remote repo files and create PR (if one does not exist already) + - GITHUB_TOKEN=: GitHub [personal] access token + - note: GITHUB_TOKEN is required if UPDATE_TARGET is "gh" or "all" +*/ + +package main + +import ( + "context" + "time" + + "golang.org/x/mod/semver" + "k8s.io/klog/v2" + + "k8s.io/minikube/hack/update" +) + +const ( + // default context timeout + cxTimeout = 300 * time.Second +) + +var ( + schema = map[string]update.Item{ + ".github/workflows/iso.yml": { + Replace: map[string]string{ + `(?U)https://github.com/medyagh/gopogh/releases/download/.*/gopogh`: `https://github.com/medyagh/gopogh/releases/download/{{.StableVersion}}/gopogh`, + }, + }, + ".github/workflows/kic_image.yml": { + Replace: map[string]string{ + `(?U)https://github.com/medyagh/gopogh/releases/download/.*/gopogh`: `https://github.com/medyagh/gopogh/releases/download/{{.StableVersion}}/gopogh`, + }, + }, + ".github/workflows/master.yml": { + Replace: map[string]string{ + `(?U)https://github.com/medyagh/gopogh/releases/download/.*/gopogh`: `https://github.com/medyagh/gopogh/releases/download/{{.StableVersion}}/gopogh`, + }, + }, + ".github/workflows/pr.yml": { + Replace: map[string]string{ + `(?U)https://github.com/medyagh/gopogh/releases/download/.*/gopogh`: `https://github.com/medyagh/gopogh/releases/download/{{.StableVersion}}/gopogh`, + }, + }, + "hack/jenkins/common.sh": { + Replace: map[string]string{ + `(?U)https://github.com/medyagh/gopogh/releases/download/.*/gopogh`: `https://github.com/medyagh/gopogh/releases/download/{{.StableVersion}}/gopogh`, + }, + }, + } + + // PR data + prBranchPrefix = "update-gopogh-version_" // will be appended with first 7 characters of the PR commit SHA + prTitle = `update_gopogh_version: {stable: "{{.StableVersion}}"}` + prIssue = 9850 +) + +// Data holds stable gopogh version in semver format. +type Data struct { + StableVersion string `json:"stableVersion"` +} + +func main() { + // set a context with defined timeout + ctx, cancel := context.WithTimeout(context.Background(), cxTimeout) + defer cancel() + + // get gopogh stable version from https://github.com/medyagh/gopogh + stable, err := gopoghVersion(ctx, "medyagh", "gopogh") + if err != nil || stable == "" { + klog.Fatalf("Unable to get gopogh stable version: %v", err) + } + data := Data{StableVersion: stable} + klog.Infof("gopogh stable version: %s", data.StableVersion) + + update.Apply(ctx, schema, data, prBranchPrefix, prTitle, prIssue) +} + +// gopoghVersion returns gopogh stable version in semver format. +func gopoghVersion(ctx context.Context, owner, repo string) (stable string, err error) { + // get Kubernetes versions from GitHub Releases + stable, _, err = update.GHReleases(ctx, owner, repo) + if err != nil || !semver.IsValid(stable) { + return "", err + } + return stable, nil +} diff --git a/hack/update/update.go b/hack/update/update.go index 8cf74d01d8..dbf7061999 100644 --- a/hack/update/update.go +++ b/hack/update/update.go @@ -88,8 +88,7 @@ func (i *Item) apply(data interface{}) error { if i.Content == nil { return fmt.Errorf("unable to update content: nothing to update") } - org := string(i.Content) - str := org + str := string(i.Content) for src, dst := range i.Replace { out, err := ParseTmpl(dst, data, "") if err != nil { diff --git a/installers/linux/deb/minikube_deb_template/DEBIAN/control b/installers/linux/deb/minikube_deb_template/DEBIAN/control index ad9d368ba0..3b29fe8094 100644 --- a/installers/linux/deb/minikube_deb_template/DEBIAN/control +++ b/installers/linux/deb/minikube_deb_template/DEBIAN/control @@ -3,7 +3,7 @@ Version: --VERSION-- Section: base Priority: optional Architecture: --ARCH-- -Recommends: virtualbox +Recommends: --RECOMMENDS-- Maintainer: Thomas Strömberg Description: Minikube minikube is a tool that makes it easy to run Kubernetes locally. diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go index 4c47a478cf..156079675d 100644 --- a/pkg/drivers/kic/kic.go +++ b/pkg/drivers/kic/kic.go @@ -320,7 +320,7 @@ func (d *Driver) Remove() error { return fmt.Errorf("expected no container ID be found for %q after delete. but got %q", d.MachineName, id) } - if err := oci.RemoveNetwork(d.NodeConfig.ClusterName); err != nil { + if err := oci.RemoveNetwork(d.OCIBinary, d.NodeConfig.ClusterName); err != nil { klog.Warningf("failed to remove network (which might be okay) %s: %v", d.NodeConfig.ClusterName, err) } return nil diff --git a/pkg/drivers/kic/oci/env.go b/pkg/drivers/kic/oci/env.go index 8bedcf4d0c..0cc3ab4e12 100644 --- a/pkg/drivers/kic/oci/env.go +++ b/pkg/drivers/kic/oci/env.go @@ -25,6 +25,7 @@ import ( var initialEnvs = make(map[string]string) func init() { + // docker for _, env := range constants.DockerDaemonEnvs { if v, set := os.LookupEnv(env); set { initialEnvs[env] = v @@ -34,6 +35,15 @@ func init() { initialEnvs[exEnv] = v } } + // podman + env := constants.PodmanContainerHostEnv + if v, set := os.LookupEnv(env); set { + initialEnvs[env] = v + } + exEnv := constants.ExistingContainerHostEnv + if v, set := os.LookupEnv(exEnv); set { + initialEnvs[exEnv] = v + } } // InitialEnv returns the value of the environment variable env before any environment changes made by minikube diff --git a/pkg/drivers/kic/oci/errors.go b/pkg/drivers/kic/oci/errors.go index 6089d88cb7..ce4aa00be4 100644 --- a/pkg/drivers/kic/oci/errors.go +++ b/pkg/drivers/kic/oci/errors.go @@ -85,14 +85,15 @@ func LogContainerDebug(ociBin string, name string) string { } else { klog.Infof("postmortem docker info: %+v", di) } - logDockerNetworkInspect(name) + logDockerNetworkInspect(ociBin, name) } else { pi, err := podmanSystemInfo() if err != nil { - klog.Warningf("couldn't get postmortem info, failed to to run podman info: %v", err) + klog.Warningf("couldn't get postmortem podman info: %v", err) } else { klog.Infof("postmortem podman info: %+v", pi) } + logDockerNetworkInspect(ociBin, name) } if rr.Stdout.Len() == 0 { diff --git a/pkg/drivers/kic/oci/network.go b/pkg/drivers/kic/oci/network.go index a3c7edb94f..de2b742adf 100644 --- a/pkg/drivers/kic/oci/network.go +++ b/pkg/drivers/kic/oci/network.go @@ -35,7 +35,7 @@ import ( func RoutableHostIPFromInside(ociBin string, clusterName string, containerName string) (net.IP, error) { if ociBin == Docker { if runtime.GOOS == "linux" { - info, err := dockerNetworkInspect(clusterName) + info, err := containerNetworkInspect(ociBin, clusterName) if err != nil { if errors.Is(err, ErrNetworkNotFound) { klog.Infof("The container %s is not attached to a network, this could be because the cluster was created by minikube 0 { args = append(args, "-o") - args = append(args, fmt.Sprintf("com.docker.network.driver.mtu=%d", mtu)) - } + args = append(args, "--ip-masq") + args = append(args, "-o") + args = append(args, "--icc") - rr, err := runCmd(exec.Command(Docker, args...)) + // adding MTU option because #9528 + if mtu > 0 { + args = append(args, "-o") + args = append(args, fmt.Sprintf("com.docker.network.driver.mtu=%d", mtu)) + } + + args = append(args, fmt.Sprintf("--label=%s=%s", CreatedByLabelKey, "true")) + } + args = append(args, name) + + rr, err := runCmd(exec.Command(ociBin, args...)) if err != nil { // Pool overlaps with other one on this address space if strings.Contains(rr.Output(), "Pool overlaps") { @@ -135,6 +137,16 @@ type netInfo struct { mtu int } +func containerNetworkInspect(ociBin string, name string) (netInfo, error) { + if ociBin == Docker { + return dockerNetworkInspect(name) + } + if ociBin == Podman { + return podmanNetworkInspect(name) + } + return netInfo{}, fmt.Errorf("%s unknown", ociBin) +} + // networkInspect is only used to unmarshal the docker network inspect output and translate it to netInfo type networkInspect struct { Name string @@ -153,7 +165,7 @@ func dockerNetworkInspect(name string) (netInfo, error) { cmd := exec.Command(Docker, "network", "inspect", name, "--format", `{"Name": "{{.Name}}","Driver": "{{.Driver}}","Subnet": "{{range .IPAM.Config}}{{.Subnet}}{{end}}","Gateway": "{{range .IPAM.Config}}{{.Gateway}}{{end}}","MTU": {{(index .Options "com.docker.network.driver.mtu")}},{{$first := true}} "ContainerIPs": [{{range $k,$v := .Containers }}{{if $first}}{{$first = false}}{{else}}, {{end}}"{{$v.IPv4Address}}"{{end}}]}`) rr, err := runCmd(cmd) if err != nil { - logDockerNetworkInspect(name) + logDockerNetworkInspect(Docker, name) if strings.Contains(rr.Output(), "No such network") { return info, ErrNetworkNotFound @@ -177,8 +189,39 @@ func dockerNetworkInspect(name string) (netInfo, error) { return info, nil } -func logDockerNetworkInspect(name string) { - cmd := exec.Command(Docker, "network", "inspect", name) +func podmanNetworkInspect(name string) (netInfo, error) { + var info = netInfo{name: name} + cmd := exec.Command(Podman, "network", "inspect", name, "--format", `{{(index .IPAM.Config 0).Subnet}},{{(index .IPAM.Config 0).Gateway}}`) + rr, err := runCmd(cmd) + if err != nil { + logDockerNetworkInspect(Podman, name) + if strings.Contains(rr.Output(), "No such network") { + + return info, ErrNetworkNotFound + } + return info, err + } + + // results looks like 172.17.0.0/16,172.17.0.1,1500 + vals := strings.Split(strings.TrimSpace(rr.Stdout.String()), ",") + if len(vals) == 0 { + return info, fmt.Errorf("empty list network inspect: %q", rr.Output()) + } + + if len(vals) > 0 { + info.gateway = net.ParseIP(vals[1]) + } + + _, info.subnet, err = net.ParseCIDR(vals[0]) + if err != nil { + return info, errors.Wrapf(err, "parse subnet for %s", name) + } + + return info, nil +} + +func logDockerNetworkInspect(ociBin string, name string) { + cmd := exec.Command(ociBin, "network", "inspect", name) klog.Infof("running %v to gather additional debugging logs...", cmd.Args) rr, err := runCmd(cmd) if err != nil { @@ -188,11 +231,11 @@ func logDockerNetworkInspect(name string) { } // RemoveNetwork removes a network -func RemoveNetwork(name string) error { - if !networkExists(name) { +func RemoveNetwork(ociBin string, name string) error { + if !networkExists(ociBin, name) { return nil } - rr, err := runCmd(exec.Command(Docker, "network", "remove", name)) + rr, err := runCmd(exec.Command(ociBin, "network", "rm", name)) if err != nil { if strings.Contains(rr.Output(), "No such network") { return ErrNetworkNotFound @@ -206,8 +249,8 @@ func RemoveNetwork(name string) error { return err } -func networkExists(name string) bool { - _, err := dockerNetworkInspect(name) +func networkExists(ociBin string, name string) bool { + _, err := containerNetworkInspect(ociBin, name) if err != nil && !errors.Is(err, ErrNetworkNotFound) { // log unexpected error klog.Warningf("Error inspecting docker network %s: %v", name, err) } @@ -216,12 +259,8 @@ func networkExists(name string) bool { // networkNamesByLabel returns all network names created by a label func networkNamesByLabel(ociBin string, label string) ([]string, error) { - if ociBin != Docker { - return nil, fmt.Errorf("%s not supported", ociBin) - } - // docker network ls --filter='label=created_by.minikube.sigs.k8s.io=true' --format '{{.Name}}' - rr, err := runCmd(exec.Command(Docker, "network", "ls", fmt.Sprintf("--filter=label=%s", label), "--format", "{{.Name}}")) + rr, err := runCmd(exec.Command(ociBin, "network", "ls", fmt.Sprintf("--filter=label=%s", label), "--format", "{{.Name}}")) if err != nil { return nil, err } @@ -235,14 +274,14 @@ func networkNamesByLabel(ociBin string, label string) ([]string, error) { } // DeleteKICNetworks deletes all networks created by kic -func DeleteKICNetworks() []error { +func DeleteKICNetworks(ociBin string) []error { var errs []error - ns, err := networkNamesByLabel(Docker, CreatedByLabelKey+"=true") + ns, err := networkNamesByLabel(ociBin, CreatedByLabelKey) if err != nil { return []error{errors.Wrap(err, "list all volume")} } for _, n := range ns { - err := RemoveNetwork(n) + err := RemoveNetwork(ociBin, n) if err != nil { errs = append(errs, err) } diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go index 087d6224f4..63c45fc283 100644 --- a/pkg/drivers/kic/oci/oci.go +++ b/pkg/drivers/kic/oci/oci.go @@ -146,6 +146,12 @@ func CreateContainerNode(p CreateParams) error { // label th enode wuth the node ID "--label", p.NodeLabel, } + // to provide a static IP + if p.Network != "" && p.IP != "" { + runArgs = append(runArgs, "--network", p.Network) + runArgs = append(runArgs, "--ip", p.IP) + } + memcgSwap := true if runtime.GOOS == "linux" { if _, err := os.Stat("/sys/fs/cgroup/memory/memsw.limit_in_bytes"); os.IsNotExist(err) { @@ -170,11 +176,6 @@ func CreateContainerNode(p CreateParams) error { virtualization = "podman" // VIRTUALIZATION_PODMAN } if p.OCIBinary == Docker { - // to provide a static IP for docker - if p.Network != "" && p.IP != "" { - runArgs = append(runArgs, "--network", p.Network) - runArgs = append(runArgs, "--ip", p.IP) - } runArgs = append(runArgs, "--volume", fmt.Sprintf("%s:/var", p.Name)) // ignore apparmore github actions docker: https://github.com/kubernetes/minikube/issues/7624 runArgs = append(runArgs, "--security-opt", "apparmor=unconfined") diff --git a/pkg/generate/docs.go b/pkg/generate/docs.go index dd8cb02fce..fefc720d8e 100644 --- a/pkg/generate/docs.go +++ b/pkg/generate/docs.go @@ -17,19 +17,16 @@ limitations under the License. package generate import ( - "bufio" "bytes" "fmt" "io" "io/ioutil" "os" "path/filepath" - "strings" "time" "github.com/pkg/errors" "github.com/spf13/cobra" - "github.com/spf13/cobra/doc" "k8s.io/klog/v2" "k8s.io/minikube/pkg/minikube/out" ) @@ -65,49 +62,71 @@ func DocForCommand(command *cobra.Command) (string, error) { if err := writeSubcommands(command, buf); err != nil { return "", errors.Wrap(err, "writing subcommands") } - return removeHelpText(buf), nil + return buf.String(), nil } -// after every command, cobra automatically appends -// ### SEE ALSO +// GenMarkdown creates markdown output. +func GenMarkdown(cmd *cobra.Command, w io.Writer) error { + return GenMarkdownCustom(cmd, w, func(s string) string { return s }) +} -// * [minikube addons](minikube_addons.md) - Modify minikube's Kubernetes addons +// GenMarkdownCustom creates custom markdown output. +func GenMarkdownCustom(cmd *cobra.Command, w io.Writer, linkHandler func(string) string) error { + cmd.InitDefaultHelpCmd() + cmd.InitDefaultHelpFlag() -// ###### Auto generated by spf13/cobra on 1-Apr-2020 -// help text which is unnecessary info after every subcommand -// This function removes that text. -func removeHelpText(buffer *bytes.Buffer) string { - beginningHelpText := "### SEE ALSO" - endHelpText := "###### Auto generated by spf13/cobra" - scanner := bufio.NewScanner(buffer) - includeLine := true + buf := new(bytes.Buffer) + name := cmd.CommandPath() - final := bytes.NewBuffer([]byte{}) - for scanner.Scan() { - line := scanner.Text() - if strings.Contains(line, beginningHelpText) { - includeLine = false - continue - } - if strings.Contains(line, endHelpText) { - includeLine = true - continue - } - if !includeLine { - continue - } - // scanner strips the ending newline - if _, err := final.WriteString(line + "\n"); err != nil { - klog.Warningf("error removing help text: %v", err) - break - } + short := cmd.Short + long := cmd.Long + if len(long) == 0 { + long = short } - return final.String() + + buf.WriteString("## " + name + "\n\n") + buf.WriteString(short + "\n\n") + buf.WriteString("### Synopsis\n\n") + buf.WriteString(long + "\n\n") + + if cmd.Runnable() { + buf.WriteString(fmt.Sprintf("```shell\n%s\n```\n\n", cmd.UseLine())) + } + + if len(cmd.Example) > 0 { + buf.WriteString("### Examples\n\n") + buf.WriteString(fmt.Sprintf("```\n%s\n```\n\n", cmd.Example)) + } + + if err := printOptions(buf, cmd); err != nil { + return err + } + _, err := buf.WriteTo(w) + return err +} + +func printOptions(buf *bytes.Buffer, cmd *cobra.Command) error { + flags := cmd.NonInheritedFlags() + flags.SetOutput(buf) + if flags.HasAvailableFlags() { + buf.WriteString("### Options\n\n```\n") + flags.PrintDefaults() + buf.WriteString("```\n\n") + } + + parentFlags := cmd.InheritedFlags() + parentFlags.SetOutput(buf) + if parentFlags.HasAvailableFlags() { + buf.WriteString("### Options inherited from parent commands\n\n```\n") + parentFlags.PrintDefaults() + buf.WriteString("```\n\n") + } + return nil } // writeSubcommands recursively appends all subcommands to the doc func writeSubcommands(command *cobra.Command, w io.Writer) error { - if err := doc.GenMarkdown(command, w); err != nil { + if err := GenMarkdown(command, w); err != nil { return errors.Wrapf(err, "getting markdown custom") } if !command.HasSubCommands() { diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go index 609b103818..85aa09d225 100644 --- a/pkg/minikube/assets/addons.go +++ b/pkg/minikube/assets/addons.go @@ -489,6 +489,7 @@ func GenerateTemplateData(cfg config.KubernetesConfig) interface{} { ImageRepository string LoadBalancerStartIP string LoadBalancerEndIP string + CustomIngressCert string StorageProvisionerVersion string }{ Arch: a, @@ -496,6 +497,7 @@ func GenerateTemplateData(cfg config.KubernetesConfig) interface{} { ImageRepository: cfg.ImageRepository, LoadBalancerStartIP: cfg.LoadBalancerStartIP, LoadBalancerEndIP: cfg.LoadBalancerEndIP, + CustomIngressCert: cfg.CustomIngressCert, StorageProvisionerVersion: version.GetStorageProvisionerVersion(), } diff --git a/pkg/minikube/bootstrapper/images/images_test.go b/pkg/minikube/bootstrapper/images/images_test.go index fd3743a8a8..5831e450c6 100644 --- a/pkg/minikube/bootstrapper/images/images_test.go +++ b/pkg/minikube/bootstrapper/images/images_test.go @@ -24,7 +24,7 @@ import ( func TestAuxiliary(t *testing.T) { want := []string{ - "gcr.io/k8s-minikube/storage-provisioner:v3", + "gcr.io/k8s-minikube/storage-provisioner:v4", "docker.io/kubernetesui/dashboard:v2.0.3", "docker.io/kubernetesui/metrics-scraper:v1.0.4", } @@ -36,7 +36,7 @@ func TestAuxiliary(t *testing.T) { func TestAuxiliaryMirror(t *testing.T) { want := []string{ - "test.mirror/storage-provisioner:v3", + "test.mirror/storage-provisioner:v4", "test.mirror/dashboard:v2.0.3", "test.mirror/metrics-scraper:v1.0.4", } diff --git a/pkg/minikube/bootstrapper/images/kubeadm_test.go b/pkg/minikube/bootstrapper/images/kubeadm_test.go index 36f976d9e7..81d37bf8ed 100644 --- a/pkg/minikube/bootstrapper/images/kubeadm_test.go +++ b/pkg/minikube/bootstrapper/images/kubeadm_test.go @@ -37,7 +37,7 @@ func TestKubeadmImages(t *testing.T) { "k8s.gcr.io/coredns:1.6.5", "k8s.gcr.io/etcd:3.4.3-0", "k8s.gcr.io/pause:3.1", - "gcr.io/k8s-minikube/storage-provisioner:v3", + "gcr.io/k8s-minikube/storage-provisioner:v4", "docker.io/kubernetesui/dashboard:v2.0.3", "docker.io/kubernetesui/metrics-scraper:v1.0.4", }}, @@ -49,7 +49,7 @@ func TestKubeadmImages(t *testing.T) { "mirror.k8s.io/coredns:1.6.2", "mirror.k8s.io/etcd:3.3.15-0", "mirror.k8s.io/pause:3.1", - "mirror.k8s.io/storage-provisioner:v3", + "mirror.k8s.io/storage-provisioner:v4", "mirror.k8s.io/dashboard:v2.0.3", "mirror.k8s.io/metrics-scraper:v1.0.4", }}, @@ -61,7 +61,7 @@ func TestKubeadmImages(t *testing.T) { "k8s.gcr.io/coredns:1.3.1", "k8s.gcr.io/etcd:3.3.10", "k8s.gcr.io/pause:3.1", - "gcr.io/k8s-minikube/storage-provisioner:v3", + "gcr.io/k8s-minikube/storage-provisioner:v4", "docker.io/kubernetesui/dashboard:v2.0.3", "docker.io/kubernetesui/metrics-scraper:v1.0.4", }}, @@ -73,7 +73,7 @@ func TestKubeadmImages(t *testing.T) { "k8s.gcr.io/coredns:1.3.1", "k8s.gcr.io/etcd:3.3.10", "k8s.gcr.io/pause:3.1", - "gcr.io/k8s-minikube/storage-provisioner:v3", + "gcr.io/k8s-minikube/storage-provisioner:v4", "docker.io/kubernetesui/dashboard:v2.0.3", "docker.io/kubernetesui/metrics-scraper:v1.0.4", }}, @@ -85,7 +85,7 @@ func TestKubeadmImages(t *testing.T) { "k8s.gcr.io/coredns:1.2.6", "k8s.gcr.io/etcd:3.2.24", "k8s.gcr.io/pause:3.1", - "gcr.io/k8s-minikube/storage-provisioner:v3", + "gcr.io/k8s-minikube/storage-provisioner:v4", "docker.io/kubernetesui/dashboard:v2.0.3", "docker.io/kubernetesui/metrics-scraper:v1.0.4", }}, @@ -97,7 +97,7 @@ func TestKubeadmImages(t *testing.T) { "k8s.gcr.io/coredns:1.2.2", "k8s.gcr.io/etcd:3.2.24", "k8s.gcr.io/pause:3.1", - "gcr.io/k8s-minikube/storage-provisioner:v3", + "gcr.io/k8s-minikube/storage-provisioner:v4", "docker.io/kubernetesui/dashboard:v2.0.3", "docker.io/kubernetesui/metrics-scraper:v1.0.4", }}, diff --git a/pkg/minikube/config/types.go b/pkg/minikube/config/types.go index 1d19604bc4..d95e9f7c85 100644 --- a/pkg/minikube/config/types.go +++ b/pkg/minikube/config/types.go @@ -92,6 +92,7 @@ type KubernetesConfig struct { ImageRepository string LoadBalancerStartIP string // currently only used by MetalLB addon LoadBalancerEndIP string // currently only used by MetalLB addon + CustomIngressCert string // used by Ingress addon ExtraOptions ExtraOptionSlice ShouldLoadCachedImages bool diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index 1b3e794bed..5853a5893a 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -93,6 +93,9 @@ const ( ExistingDockerCertPathEnv = MinikubeExistingPrefix + "DOCKER_CERT_PATH" // ExistingDockerTLSVerifyEnv is used to save original docker environment ExistingDockerTLSVerifyEnv = MinikubeExistingPrefix + "DOCKER_TLS_VERIFY" + + // ExistingContainerHostEnv is used to save original podman environment + ExistingContainerHostEnv = MinikubeExistingPrefix + "CONTAINER_HOST" ) var ( diff --git a/pkg/minikube/cruntime/containerd_test.go b/pkg/minikube/cruntime/containerd_test.go index 7fb4846f38..bd7227fc77 100644 --- a/pkg/minikube/cruntime/containerd_test.go +++ b/pkg/minikube/cruntime/containerd_test.go @@ -27,7 +27,7 @@ func TestAddRepoTagToImageName(t *testing.T) { }{ {"kubernetesui/dashboard:v2.0.3", "docker.io/kubernetesui/dashboard:v2.0.3"}, {"kubernetesui/metrics-scraper:v1.0.4", "docker.io/kubernetesui/metrics-scraper:v1.0.4"}, - {"gcr.io/k8s-minikube/storage-provisioner:v3", "gcr.io/k8s-minikube/storage-provisioner:v3"}, + {"gcr.io/k8s-minikube/storage-provisioner:v4", "gcr.io/k8s-minikube/storage-provisioner:v4"}, } for _, tc := range tests { t.Run(tc.imgName, func(t *testing.T) { diff --git a/pkg/minikube/download/preload.go b/pkg/minikube/download/preload.go index 07682aea3b..7d907fbd41 100644 --- a/pkg/minikube/download/preload.go +++ b/pkg/minikube/download/preload.go @@ -41,7 +41,7 @@ const ( // PreloadVersion is the current version of the preloaded tarball // // NOTE: You may need to bump this version up when upgrading auxiliary docker images - PreloadVersion = "v6" + PreloadVersion = "v7" // PreloadBucket is the name of the GCS bucket where preloaded volume tarballs exist PreloadBucket = "minikube-preloaded-volume-tarballs" ) diff --git a/pkg/minikube/node/cache.go b/pkg/minikube/node/cache.go index bc9c3df668..a47b55a7e0 100644 --- a/pkg/minikube/node/cache.go +++ b/pkg/minikube/node/cache.go @@ -124,6 +124,7 @@ func beginDownloadKicBaseImage(g *errgroup.Group, cc *config.ClusterConfig, down baseImg := cc.KicBaseImage if baseImg == kic.BaseImage && len(cc.KubernetesConfig.ImageRepository) != 0 { baseImg = strings.Replace(baseImg, "gcr.io/k8s-minikube", cc.KubernetesConfig.ImageRepository, 1) + cc.KicBaseImage = baseImg } var finalImg string // If we end up using a fallback image, notify the user diff --git a/pkg/minikube/out/register/register.go b/pkg/minikube/out/register/register.go index dc07c15c17..985d81463d 100644 --- a/pkg/minikube/out/register/register.go +++ b/pkg/minikube/out/register/register.go @@ -24,6 +24,7 @@ import ( "k8s.io/minikube/pkg/trace" ) +// If you add a new step here, please also add it to register.Reg registry inside the init() function const ( InitialSetup RegStep = "Initial Minikube Setup" SelectingDriver RegStep = "Selecting Driver" @@ -70,6 +71,7 @@ func init() { SelectingDriver, DownloadingArtifacts, StartingNode, + PullingBaseImage, RunningLocalhost, LocalOSRelease, CreatingContainer, diff --git a/pkg/trace/gcp.go b/pkg/trace/gcp.go index 8a4bf57b0d..212d198fb5 100644 --- a/pkg/trace/gcp.go +++ b/pkg/trace/gcp.go @@ -31,7 +31,8 @@ import ( ) const ( - projectEnvVar = "MINIKUBE_GCP_PROJECT_ID" + // ProjectEnvVar is the name of the env variable that the user must pass in their GCP project ID through + ProjectEnvVar = "MINIKUBE_GCP_PROJECT_ID" // this is the name of the parent span to help identify it // in the Cloud Trace UI. parentSpanName = "minikube start" @@ -72,9 +73,9 @@ func (t *gcpTracer) Cleanup() { } func initGCPTracer() (*gcpTracer, error) { - projectID := os.Getenv(projectEnvVar) + projectID := os.Getenv(ProjectEnvVar) if projectID == "" { - return nil, fmt.Errorf("GCP tracer requires a valid GCP project id set via the %s env variable", projectEnvVar) + return nil, fmt.Errorf("GCP tracer requires a valid GCP project id set via the %s env variable", ProjectEnvVar) } _, flush, err := texporter.InstallNewPipeline( diff --git a/site/assets/scss/_variables_project.scss b/site/assets/scss/_variables_project.scss index 7bc0efdb22..536b3b4a01 100644 --- a/site/assets/scss/_variables_project.scss +++ b/site/assets/scss/_variables_project.scss @@ -180,7 +180,7 @@ section.td-box--height-auto { // Allow code tags to span most of a window length (default is 80%) pre { max-width: 99% !important; - font-family: 'Inconsolata', monospace !important; + font-family: 'SFMono-Regular', Menlo, Monaco, Consolas, 'liberation mono', 'courier new', monospace !important; font-size: 13px !important; } @@ -197,6 +197,17 @@ div.td-content { padding: 0.5em !important; margin-top: 1.25em; margin-bottom: 1.25em; + background-color: $gray-100; + max-width: 99% !important; + + pre { + background-color: inherit !important; + padding: 0 !important; + + code { + font-family: inherit !important; + } + } } } @@ -214,3 +225,6 @@ div.td-content { max-width: 460px; } +div.code-toolbar > .toolbar { + top: -.3em !important; +} diff --git a/site/config.toml b/site/config.toml index 7a9c6cf087..3e6c1da9db 100644 --- a/site/config.toml +++ b/site/config.toml @@ -92,6 +92,9 @@ github_subdir = "site" # enabling local search https://www.docsy.dev/docs/adding-content/navigation/#configure-local-search-with-lunr offlineSearch = true +# Enable syntax highlighting and copy buttons on code blocks with Prism +prism_syntax_highlighting = true + # User interface configuration [params.ui] # Enable to show the side bar menu in its compact state. diff --git a/site/content/en/docs/commands/addons.md b/site/content/en/docs/commands/addons.md index 621179f56f..c558656d0d 100644 --- a/site/content/en/docs/commands/addons.md +++ b/site/content/en/docs/commands/addons.md @@ -13,7 +13,7 @@ Enable or disable a minikube addon addons modifies minikube addons files using subcommands like "minikube addons enable dashboard" -``` +```shell minikube addons SUBCOMMAND [flags] ``` @@ -46,7 +46,7 @@ Configures the addon w/ADDON_NAME within minikube (example: minikube addons conf Configures the addon w/ADDON_NAME within minikube (example: minikube addons configure registry-creds). For a list of available addons use: minikube addons list -``` +```shell minikube addons configure ADDON_NAME [flags] ``` @@ -79,7 +79,7 @@ Disables the addon w/ADDON_NAME within minikube (example: minikube addons disabl Disables the addon w/ADDON_NAME within minikube (example: minikube addons disable dashboard). For a list of available addons use: minikube addons list -``` +```shell minikube addons disable ADDON_NAME [flags] ``` @@ -112,7 +112,7 @@ Enables the addon w/ADDON_NAME within minikube (example: minikube addons enable Enables the addon w/ADDON_NAME within minikube (example: minikube addons enable dashboard). For a list of available addons use: minikube addons list -``` +```shell minikube addons enable ADDON_NAME [flags] ``` @@ -146,7 +146,7 @@ Help about any command Help provides help for any command in the application. Simply type addons help [path to command] for full details. -``` +```shell minikube addons help [command] [flags] ``` @@ -179,7 +179,7 @@ Lists all available minikube addons as well as their current statuses (enabled/d Lists all available minikube addons as well as their current statuses (enabled/disabled) -``` +```shell minikube addons list [flags] ``` @@ -218,7 +218,7 @@ Opens the addon w/ADDON_NAME within minikube (example: minikube addons open dash Opens the addon w/ADDON_NAME within minikube (example: minikube addons open dashboard). For a list of available addons use: minikube addons list -``` +```shell minikube addons open ADDON_NAME [flags] ``` diff --git a/site/content/en/docs/commands/cache.md b/site/content/en/docs/commands/cache.md index 3917b19b70..c5694102cb 100644 --- a/site/content/en/docs/commands/cache.md +++ b/site/content/en/docs/commands/cache.md @@ -42,7 +42,7 @@ Add an image to local cache. Add an image to local cache. -``` +```shell minikube cache add [flags] ``` @@ -75,7 +75,7 @@ Delete an image from the local cache. Delete an image from the local cache. -``` +```shell minikube cache delete [flags] ``` @@ -109,7 +109,7 @@ Help about any command Help provides help for any command in the application. Simply type cache help [path to command] for full details. -``` +```shell minikube cache help [command] [flags] ``` @@ -142,7 +142,7 @@ List all available images from the local cache. List all available images from the local cache. -``` +```shell minikube cache list [flags] ``` @@ -182,7 +182,7 @@ reload cached images. reloads images previously added using the 'cache add' subcommand -``` +```shell minikube cache reload [flags] ``` diff --git a/site/content/en/docs/commands/completion.md b/site/content/en/docs/commands/completion.md index 7738a2e216..0b10bf41d9 100644 --- a/site/content/en/docs/commands/completion.md +++ b/site/content/en/docs/commands/completion.md @@ -34,7 +34,7 @@ Outputs minikube shell completion for the given shell (bash, zsh or fish) Note for fish users: [2] please refer to this docs for more details https://fishshell.com/docs/current/#tab-completion -``` +```shell minikube completion SHELL [flags] ``` diff --git a/site/content/en/docs/commands/config.md b/site/content/en/docs/commands/config.md index 8e66d3f193..40c9c9571f 100644 --- a/site/content/en/docs/commands/config.md +++ b/site/content/en/docs/commands/config.md @@ -43,7 +43,7 @@ Configurable fields: * embed-certs * native-ssh -``` +```shell minikube config SUBCOMMAND [flags] ``` @@ -79,7 +79,7 @@ Acceptable fields: * driver -``` +```shell minikube config defaults PROPERTY_NAME [flags] ``` @@ -118,7 +118,7 @@ Gets the value of PROPERTY_NAME from the minikube config file Returns the value of PROPERTY_NAME from the minikube config file. Can be overwritten at runtime by flags or environmental variables. -``` +```shell minikube config get PROPERTY_NAME [flags] ``` @@ -152,7 +152,7 @@ Help about any command Help provides help for any command in the application. Simply type config help [path to command] for full details. -``` +```shell minikube config help [command] [flags] ``` @@ -186,7 +186,7 @@ Sets an individual value in a minikube config file Sets the PROPERTY_NAME config value to PROPERTY_VALUE These values can be overwritten by flags or environment variables at runtime. -``` +```shell minikube config set PROPERTY_NAME PROPERTY_VALUE [flags] ``` @@ -219,7 +219,7 @@ unsets an individual value in a minikube config file unsets PROPERTY_NAME from the minikube config file. Can be overwritten by flags or environmental variables -``` +```shell minikube config unset PROPERTY_NAME [flags] ``` @@ -252,7 +252,7 @@ Display values currently set in the minikube config file Display values currently set in the minikube config file. -``` +```shell minikube config view [flags] ``` diff --git a/site/content/en/docs/commands/dashboard.md b/site/content/en/docs/commands/dashboard.md index 8df23e2c93..a4cba95564 100644 --- a/site/content/en/docs/commands/dashboard.md +++ b/site/content/en/docs/commands/dashboard.md @@ -13,7 +13,7 @@ Access the Kubernetes dashboard running within the minikube cluster Access the Kubernetes dashboard running within the minikube cluster -``` +```shell minikube dashboard [flags] ``` diff --git a/site/content/en/docs/commands/delete.md b/site/content/en/docs/commands/delete.md index 34cea14518..65310fdea5 100644 --- a/site/content/en/docs/commands/delete.md +++ b/site/content/en/docs/commands/delete.md @@ -14,7 +14,7 @@ Deletes a local Kubernetes cluster Deletes a local Kubernetes cluster. This command deletes the VM, and removes all associated files. -``` +```shell minikube delete [flags] ``` diff --git a/site/content/en/docs/commands/docker-env.md b/site/content/en/docs/commands/docker-env.md index 7ce914a063..39b939d880 100644 --- a/site/content/en/docs/commands/docker-env.md +++ b/site/content/en/docs/commands/docker-env.md @@ -13,7 +13,7 @@ Configure environment to use minikube's Docker daemon Sets up docker env variables; similar to '$(docker-machine env)'. -``` +```shell minikube docker-env [flags] ``` diff --git a/site/content/en/docs/commands/help.md b/site/content/en/docs/commands/help.md index 6f25e68ca8..3bc45fe4e9 100644 --- a/site/content/en/docs/commands/help.md +++ b/site/content/en/docs/commands/help.md @@ -14,7 +14,7 @@ Help about any command Help provides help for any command in the application. Simply type minikube help [path to command] for full details. -``` +```shell minikube help [command] [flags] ``` diff --git a/site/content/en/docs/commands/ip.md b/site/content/en/docs/commands/ip.md index 0a14345642..044e4435d7 100644 --- a/site/content/en/docs/commands/ip.md +++ b/site/content/en/docs/commands/ip.md @@ -13,7 +13,7 @@ Retrieves the IP address of the running cluster Retrieves the IP address of the running cluster, and writes it to STDOUT. -``` +```shell minikube ip [flags] ``` diff --git a/site/content/en/docs/commands/kubectl.md b/site/content/en/docs/commands/kubectl.md index a787c52b5b..f3e231b35c 100644 --- a/site/content/en/docs/commands/kubectl.md +++ b/site/content/en/docs/commands/kubectl.md @@ -17,7 +17,7 @@ Examples: minikube kubectl -- --help minikube kubectl -- get pods --namespace kube-system -``` +```shell minikube kubectl [flags] ``` diff --git a/site/content/en/docs/commands/logs.md b/site/content/en/docs/commands/logs.md index 1734f8b7b9..f84208137c 100644 --- a/site/content/en/docs/commands/logs.md +++ b/site/content/en/docs/commands/logs.md @@ -13,7 +13,7 @@ Returns logs to debug a local Kubernetes cluster Gets the logs of the running instance, used for debugging minikube, not user code. -``` +```shell minikube logs [flags] ``` diff --git a/site/content/en/docs/commands/mount.md b/site/content/en/docs/commands/mount.md index 8d9decd590..3d6409c178 100644 --- a/site/content/en/docs/commands/mount.md +++ b/site/content/en/docs/commands/mount.md @@ -13,7 +13,7 @@ Mounts the specified directory into minikube Mounts the specified directory into minikube. -``` +```shell minikube mount [flags] : ``` diff --git a/site/content/en/docs/commands/node.md b/site/content/en/docs/commands/node.md index e7c22ec26c..79778a7dfa 100644 --- a/site/content/en/docs/commands/node.md +++ b/site/content/en/docs/commands/node.md @@ -13,7 +13,7 @@ Add, remove, or list additional nodes Operations on nodes -``` +```shell minikube node [flags] ``` @@ -46,7 +46,7 @@ Adds a node to the given cluster. Adds a node to the given cluster config, and starts it. -``` +```shell minikube node add [flags] ``` @@ -87,7 +87,7 @@ Deletes a node from a cluster. Deletes a node from a cluster. -``` +```shell minikube node delete [flags] ``` @@ -121,7 +121,7 @@ Help about any command Help provides help for any command in the application. Simply type node help [path to command] for full details. -``` +```shell minikube node help [command] [flags] ``` @@ -154,7 +154,7 @@ List nodes. List existing minikube nodes. -``` +```shell minikube node list [flags] ``` @@ -187,7 +187,7 @@ Starts a node. Starts an existing stopped node in a cluster. -``` +```shell minikube node start [flags] ``` @@ -226,7 +226,7 @@ Stops a node in a cluster. Stops a node in a cluster. -``` +```shell minikube node stop [flags] ``` diff --git a/site/content/en/docs/commands/pause.md b/site/content/en/docs/commands/pause.md index 26938f43a2..b6463f1ff9 100644 --- a/site/content/en/docs/commands/pause.md +++ b/site/content/en/docs/commands/pause.md @@ -13,7 +13,7 @@ pause Kubernetes pause Kubernetes -``` +```shell minikube pause [flags] ``` diff --git a/site/content/en/docs/commands/podman-env.md b/site/content/en/docs/commands/podman-env.md index ef94b796b3..4ea853f8e1 100644 --- a/site/content/en/docs/commands/podman-env.md +++ b/site/content/en/docs/commands/podman-env.md @@ -13,7 +13,7 @@ Configure environment to use minikube's Podman service Sets up podman env variables; similar to '$(podman-machine env)'. -``` +```shell minikube podman-env [flags] ``` diff --git a/site/content/en/docs/commands/profile.md b/site/content/en/docs/commands/profile.md index eb1ea697a3..9f26ea434f 100644 --- a/site/content/en/docs/commands/profile.md +++ b/site/content/en/docs/commands/profile.md @@ -13,7 +13,7 @@ Get or list the current profiles (clusters) profile sets the current minikube profile, or gets the current profile if no arguments are provided. This is used to run and manage multiple minikube instance. You can return to the default minikube profile by running `minikube profile default` -``` +```shell minikube profile [MINIKUBE_PROFILE_NAME]. You can return to the default minikube profile by running `minikube profile default` [flags] ``` @@ -47,7 +47,7 @@ Help about any command Help provides help for any command in the application. Simply type profile help [path to command] for full details. -``` +```shell minikube profile help [command] [flags] ``` @@ -80,7 +80,7 @@ Lists all minikube profiles. Lists all valid minikube profiles and detects all possible invalid profiles. -``` +```shell minikube profile list [flags] ``` diff --git a/site/content/en/docs/commands/service.md b/site/content/en/docs/commands/service.md index c387e9f637..e10846d670 100644 --- a/site/content/en/docs/commands/service.md +++ b/site/content/en/docs/commands/service.md @@ -13,7 +13,7 @@ Returns a URL to connect to a service Returns the Kubernetes URL for a service in your local cluster. In the case of multiple URLs they will be printed one at a time. -``` +```shell minikube service [flags] SERVICE ``` @@ -58,7 +58,7 @@ Help about any command Help provides help for any command in the application. Simply type service help [path to command] for full details. -``` +```shell minikube service help [command] [flags] ``` @@ -92,7 +92,7 @@ Lists the URLs for the services in your local cluster Lists the URLs for the services in your local cluster -``` +```shell minikube service list [flags] ``` diff --git a/site/content/en/docs/commands/ssh-key.md b/site/content/en/docs/commands/ssh-key.md index 89d84187f0..5abba114ad 100644 --- a/site/content/en/docs/commands/ssh-key.md +++ b/site/content/en/docs/commands/ssh-key.md @@ -13,7 +13,7 @@ Retrieve the ssh identity key path of the specified cluster Retrieve the ssh identity key path of the specified cluster. -``` +```shell minikube ssh-key [flags] ``` diff --git a/site/content/en/docs/commands/ssh.md b/site/content/en/docs/commands/ssh.md index 3a00f98d1e..bf37dedc52 100644 --- a/site/content/en/docs/commands/ssh.md +++ b/site/content/en/docs/commands/ssh.md @@ -13,7 +13,7 @@ Log into the minikube environment (for debugging) Log into or run a command on a machine with SSH; similar to 'docker-machine ssh'. -``` +```shell minikube ssh [flags] ``` diff --git a/site/content/en/docs/commands/start.md b/site/content/en/docs/commands/start.md index 52715a96b2..d24e0f719e 100644 --- a/site/content/en/docs/commands/start.md +++ b/site/content/en/docs/commands/start.md @@ -13,7 +13,7 @@ Starts a local Kubernetes cluster Starts a local Kubernetes cluster -``` +```shell minikube start [flags] ``` diff --git a/site/content/en/docs/commands/status.md b/site/content/en/docs/commands/status.md index 07eb6da745..f4902dab0b 100644 --- a/site/content/en/docs/commands/status.md +++ b/site/content/en/docs/commands/status.md @@ -15,7 +15,7 @@ Gets the status of a local Kubernetes cluster. Exit status contains the status of minikube's VM, cluster and Kubernetes encoded on it's bits in this order from right to left. Eg: 7 meaning: 1 (for minikube NOK) + 2 (for cluster NOK) + 4 (for Kubernetes NOK) -``` +```shell minikube status [flags] ``` diff --git a/site/content/en/docs/commands/stop.md b/site/content/en/docs/commands/stop.md index 9f54204ab1..34db14a875 100644 --- a/site/content/en/docs/commands/stop.md +++ b/site/content/en/docs/commands/stop.md @@ -13,7 +13,7 @@ Stops a running local Kubernetes cluster Stops a local Kubernetes cluster. This command stops the underlying VM or container, but keeps user data intact. The cluster can be started again with the "start" command. -``` +```shell minikube stop [flags] ``` @@ -21,6 +21,7 @@ minikube stop [flags] ``` --all Set flag to stop all profiles (clusters) + --cancel-scheduled cancel any existing scheduled stop requests --keep-context-active keep the kube-context active after cluster is stopped. Defaults to false. -o, --output string Format to print stdout in. Options include: [text,json] (default "text") ``` diff --git a/site/content/en/docs/commands/tunnel.md b/site/content/en/docs/commands/tunnel.md index 370e8128dc..62cc151026 100644 --- a/site/content/en/docs/commands/tunnel.md +++ b/site/content/en/docs/commands/tunnel.md @@ -13,7 +13,7 @@ Connect to LoadBalancer services tunnel creates a route to services deployed with type LoadBalancer and sets their Ingress to their ClusterIP. for a detailed example see https://minikube.sigs.k8s.io/docs/tasks/loadbalancer -``` +```shell minikube tunnel [flags] ``` diff --git a/site/content/en/docs/commands/unpause.md b/site/content/en/docs/commands/unpause.md index aabcedf80b..5ee5c52a7b 100644 --- a/site/content/en/docs/commands/unpause.md +++ b/site/content/en/docs/commands/unpause.md @@ -13,7 +13,7 @@ unpause Kubernetes unpause Kubernetes -``` +```shell minikube unpause [flags] ``` diff --git a/site/content/en/docs/commands/update-check.md b/site/content/en/docs/commands/update-check.md index b3d98bb1bb..f4a66c7602 100644 --- a/site/content/en/docs/commands/update-check.md +++ b/site/content/en/docs/commands/update-check.md @@ -13,7 +13,7 @@ Print current and latest version number Print current and latest version number -``` +```shell minikube update-check [flags] ``` diff --git a/site/content/en/docs/commands/update-context.md b/site/content/en/docs/commands/update-context.md index b981dbeba5..2725cb81d2 100644 --- a/site/content/en/docs/commands/update-context.md +++ b/site/content/en/docs/commands/update-context.md @@ -14,7 +14,7 @@ Update kubeconfig in case of an IP or port change Retrieves the IP address of the running cluster, checks it with IP in kubeconfig, and corrects kubeconfig if incorrect. -``` +```shell minikube update-context [flags] ``` diff --git a/site/content/en/docs/commands/version.md b/site/content/en/docs/commands/version.md index d1fc7e06aa..fc223a2e77 100644 --- a/site/content/en/docs/commands/version.md +++ b/site/content/en/docs/commands/version.md @@ -13,7 +13,7 @@ Print the version of minikube Print the version of minikube. -``` +```shell minikube version [flags] ``` diff --git a/site/content/en/docs/contrib/addons.en.md b/site/content/en/docs/contrib/addons.en.md index b4325e4eaf..1305f88a59 100644 --- a/site/content/en/docs/contrib/addons.en.md +++ b/site/content/en/docs/contrib/addons.en.md @@ -9,23 +9,33 @@ description: > To create an addon, first fork the minikube repository, and check out your fork: -`git clone git@github.com:/minikube.git` +```shell +git clone git@github.com:/minikube.git +``` Then go into the source directory: -`cd minikube` +```shell +cd minikube +``` Create a subdirectory: -`mkdir deploy/addons/` +```shell +mkdir deploy/addons/ +``` Add your manifest YAML's to the directory you have created: -`cp *.yaml deploy/addons/` +```shell +cp *.yaml deploy/addons/ +``` Note: If the addon never needs authentication to GCP, then consider adding the following label to the pod's yaml: -`gcp-auth-skip-secret: "true"` +```yaml +gcp-auth-skip-secret: "true" +``` To make the addon appear in `minikube addons list`, add it to `pkg/addons/config.go`. Here is the entry used by the `registry` addon, which will work for any addon which does not require custom code: diff --git a/site/content/en/docs/contrib/documentation.en.md b/site/content/en/docs/contrib/documentation.en.md index 23e8408c9d..2a3036956b 100644 --- a/site/content/en/docs/contrib/documentation.en.md +++ b/site/content/en/docs/contrib/documentation.en.md @@ -22,7 +22,9 @@ Use Github's repositories and markdown editor as described by [Kubernetes's gene To serve documentation pages locally, clone the `minikube` repository and run: -`make site` +```shell +make site +``` Notes : @@ -33,7 +35,9 @@ Notes : We recommend installing [markdownlint](https://github.com/markdownlint/markdownlint) to find issues with your markdown file. Once installed, you can use this handy target: -`make mdlint` +```shell +make mdlint +``` ## Style Guidelines diff --git a/site/content/en/docs/contrib/json_output.en.md b/site/content/en/docs/contrib/json_output.en.md index 888d819aa3..398e356baa 100644 --- a/site/content/en/docs/contrib/json_output.en.md +++ b/site/content/en/docs/contrib/json_output.en.md @@ -12,7 +12,7 @@ You may need to add logs to the registry if the `TestJSONOutput` integration tes ### Background minikube provides JSON output for `minikube start`, accesible via the `--output` flag: -``` +```shell minikube start --output json ``` @@ -61,7 +61,7 @@ You will need to add your new log in two places: Finally, set your new step in the cod by placing this line before you call `out.T`: -``` +```go register.Reg.SetStep(register.MyNewStep) ``` diff --git a/site/content/en/docs/contrib/releasing/gvisor.md b/site/content/en/docs/contrib/releasing/gvisor.md index 9d3c302c4c..0b1cec2b47 100644 --- a/site/content/en/docs/contrib/releasing/gvisor.md +++ b/site/content/en/docs/contrib/releasing/gvisor.md @@ -30,4 +30,6 @@ The image is located at `gcr.io/k8s-minikube/gvisor-addon` ## Updating the gVisor image -`make push-gvisor-addon-image` +```shell +make push-gvisor-addon-image +``` diff --git a/site/content/en/docs/drivers/vmware.md b/site/content/en/docs/drivers/vmware.md index 7351904e4e..81b2ba3010 100644 --- a/site/content/en/docs/drivers/vmware.md +++ b/site/content/en/docs/drivers/vmware.md @@ -10,15 +10,15 @@ aliases: The vmware driver supports virtualization across all VMware based hypervisors. {{% tabs %}} -{{% tab "macOS" %}} +{{% mactab %}} {{% readfile file="/docs/drivers/includes/vmware_macos_usage.inc" %}} -{{% /tab %}} -{{% tab "Linux" %}} +{{% /mactab %}} +{{% linuxtab %}} No documentation is available yet. -{{% /tab %}} -{{% tab "Windows" %}} +{{% /linuxtab %}} +{{% windowstab %}} No documentation is available yet. -{{% /tab %}} +{{% /windowstab %}} {{% /tabs %}} ## Issues diff --git a/site/content/en/docs/faq/_index.md b/site/content/en/docs/faq/_index.md index 0539c92f28..8a49549276 100644 --- a/site/content/en/docs/faq/_index.md +++ b/site/content/en/docs/faq/_index.md @@ -22,15 +22,18 @@ Alternatively, configure `sudo` to never prompt for the commands issued by minik minikube's bootstrapper, [Kubeadm](https://github.com/kubernetes/kubeadm) verifies a list of features on the host system before installing Kubernetes. in case you get this error, and you still want to try minikube anyways despite your system's limitation you can skip the verification by starting minikube with this extra option: -`minikube start --extra-config kubeadm.ignore-preflight-errors=SystemVerification` +```shell +minikube start --extra-config kubeadm.ignore-preflight-errors=SystemVerification +``` ## what is the resource allocation for Knative Setup using minikube? Please allocate sufficient resources for Knative setup using minikube, especially when you run a minikube cluster on your local machine. We recommend allocating at least 6 CPUs and 8G memory. -`minikube start --cpus 6 --memory 8000` +```shell +minikube start --cpus 6 --memory 8000 +``` ## Do I need to install kubectl locally? No, minikube comes with built-in kubectl [see minikube's kubectl documentation]({{< ref "docs/handbook/kubectl.md" >}}). - diff --git a/site/content/en/docs/handbook/accessing.md b/site/content/en/docs/handbook/accessing.md index 6289cc8588..41c612bca1 100644 --- a/site/content/en/docs/handbook/accessing.md +++ b/site/content/en/docs/handbook/accessing.md @@ -24,7 +24,9 @@ A NodePort service is the most basic way to get external traffic directly to you We also have a shortcut for fetching the minikube IP and a service's `NodePort`: -`minikube service --url $SERVICE` +```shell +minikube service --url $SERVICE +``` ## Getting the NodePort using kubectl @@ -32,13 +34,17 @@ The minikube VM is exposed to the host system via a host-only IP address, that c To determine the NodePort for your service, you can use a `kubectl` command like this (note that `nodePort` begins with lowercase `n` in JSON output): -`kubectl get service $SERVICE --output='jsonpath="{.spec.ports[0].nodePort}"'` +```shell +kubectl get service $SERVICE --output='jsonpath="{.spec.ports[0].nodePort}"' +``` ### Increasing the NodePort range By default, minikube only exposes ports 30000-32767. If this does not work for you, you can adjust the range by using: -`minikube start --extra-config=apiserver.service-node-port-range=1-65535` +```shell +minikube start --extra-config=apiserver.service-node-port-range=1-65535 +``` This flag also accepts a comma separated list of ports and port ranges. @@ -57,7 +63,7 @@ Services of type `LoadBalancer` can be exposed via the `minikube tunnel` command #### Run tunnel in a separate terminal it will ask for password. -``` +```shell minikube tunnel ``` @@ -88,16 +94,16 @@ Status: #### Create a kubernetes deployment -``` +```shell kubectl create deployment hello-minikube1 --image=k8s.gcr.io/echoserver:1.4 ``` #### Create a kubernetes service type LoadBalancer -``` +```shell kubectl expose deployment hello-minikube1 --type=LoadBalancer --port=8080 ``` ### Check external IP -``` +```shell kubectl get svc ``` 
diff --git a/site/content/en/docs/handbook/addons/gcp-auth.md b/site/content/en/docs/handbook/addons/gcp-auth.md
index 19aabd3920..801a702e9f 100644
--- a/site/content/en/docs/handbook/addons/gcp-auth.md
+++ b/site/content/en/docs/handbook/addons/gcp-auth.md
@@ -8,8 +8,11 @@ date: 2020-07-15
 If you have a containerized GCP app with a Kubernetes yaml, you can automatically add your credentials to all your deployed pods dynamically with this minikube addon. You just need to have a credentials file, which can be generated with `gcloud auth application-default login`. If you already have a json credentials file you want specify, use the GOOGLE_APPLICATION_CREDENTIALS environment variable.
 
 - Start a cluster:
-```
+```shell
 minikube start
+```
+
+```
 😄  minikube v1.12.0 on Darwin 10.15.5
 ✨  Automatically selected the docker driver. Other choices: hyperkit, virtualbox
 👍  Starting control plane node minikube in cluster minikube
@@ -21,8 +24,11 @@ minikube start
 ```
 
 - Enable the `gcp-auth` addon:
-```
+```shell
 minikube addons enable gcp-auth
+```
+
+```
 🔎  Verifying gcp-auth addon...
 📌  Your GCP credentials will now be mounted into every pod created in the minikube cluster.
 📌  If you don't want credential mounted into a specific pod, add a label with the `gcp-auth-skip-secret` key to your pod configuration.
@@ -30,14 +36,17 @@ minikube addons enable gcp-auth
 ```
 
 - For credentials in an arbitrary path:
-```
+```shell
 export GOOGLE_APPLICATION_CREDENTIALS=.json 
 minikube addons enable gcp-auth
 ```
 
 - Deploy your GCP app as normal:
-```
+```shell
 kubectl apply -f test.yaml
+```
+
+```
 deployment.apps/pytest created
 ```
 
diff --git a/site/content/en/docs/handbook/config.md b/site/content/en/docs/handbook/config.md
index 9012eb1444..b2168b6850 100644
--- a/site/content/en/docs/handbook/config.md
+++ b/site/content/en/docs/handbook/config.md
@@ -49,7 +49,9 @@ This flag is repeated, so you can pass it several times with several different v
 
 By default, minikube installs the latest stable version of Kubernetes that was available at the time of the minikube release. You may select a different Kubernetes release by using the `--kubernetes-version` flag, for example:
 
-`minikube start --kubernetes-version=v1.11.10`
+```shell
+minikube start --kubernetes-version=v1.11.10
+```
   
 minikube follows the [Kubernetes Version and Version Skew Support Policy](https://kubernetes.io/docs/setup/version-skew-policy/), so we guarantee support for the latest build for the last 3 minor Kubernetes releases. When practical, minikube aims to support older releases as well so that users can emulate legacy environments.
 
diff --git a/site/content/en/docs/handbook/controls.md b/site/content/en/docs/handbook/controls.md
index 18866c8500..baa402de67 100755
--- a/site/content/en/docs/handbook/controls.md
+++ b/site/content/en/docs/handbook/controls.md
@@ -10,40 +10,60 @@ aliases:
 
 Start a cluster by running:
 
-`minikube start`
+```shell
+minikube start
+```
 
 Access the Kubernetes Dashboard running within the minikube cluster:
 
-`minikube dashboard`
+```shell
+minikube dashboard
+```
 
 Once started, you can interact with your cluster using `kubectl`, just like any other Kubernetes cluster. For instance, starting a server:
 
-`kubectl create deployment hello-minikube --image=k8s.gcr.io/echoserver:1.4`
+```shell
+kubectl create deployment hello-minikube --image=k8s.gcr.io/echoserver:1.4
+```
 
 Exposing a service as a NodePort
 
-`kubectl expose deployment hello-minikube --type=NodePort --port=8080`
+```shell
+kubectl expose deployment hello-minikube --type=NodePort --port=8080
+```
 
 minikube makes it easy to open this exposed endpoint in your browser:
 
-`minikube service hello-minikube`
+```shell
+minikube service hello-minikube
+```
 
 Upgrade your cluster:
 
-`minikube start --kubernetes-version=latest`
+```shell
+minikube start --kubernetes-version=latest
+```
 
 Start a second local cluster (_note: This will not work if minikube is using the bare-metal/none driver_):
 
-`minikube start -p cluster2`
+```shell
+minikube start -p cluster2
+```
 
 Stop your local cluster:
 
-`minikube stop`
+```shell
+minikube stop
+```
 
 Delete your local cluster:
 
-`minikube delete`
+```shell
+minikube delete
+```
 
 Delete all local clusters and profiles
 
-`minikube delete --all`
+```shell
+minikube delete --all
+```
diff --git a/site/content/en/docs/handbook/deploying.md b/site/content/en/docs/handbook/deploying.md
index 521cb949f2..701ff2507e 100644
--- a/site/content/en/docs/handbook/deploying.md
+++ b/site/content/en/docs/handbook/deploying.md
@@ -10,7 +10,7 @@ aliases:
 
 ## kubectl
 
-```
+```shell
 kubectl create deployment hello-minikube1 --image=k8s.gcr.io/echoserver:1.4
 kubectl expose deployment hello-minikube1 --type=LoadBalancer --port=8080
 ```
diff --git a/site/content/en/docs/handbook/filesync.md b/site/content/en/docs/handbook/filesync.md
index 9b83831f48..355e2ab51a 100644
--- a/site/content/en/docs/handbook/filesync.md
+++ b/site/content/en/docs/handbook/filesync.md
@@ -22,7 +22,7 @@ Place files to be synced in `$MINIKUBE_HOME/files`
 
 For example, running the following will result in the deployment of a custom /etc/resolv.conf:
 
-```
+```shell
 mkdir -p ~/.minikube/files/etc
 echo nameserver 8.8.8.8 > ~/.minikube/files/etc/resolv.conf
 minikube start
diff --git a/site/content/en/docs/handbook/kubectl.md b/site/content/en/docs/handbook/kubectl.md
index 24cab2ff66..e4948013e3 100644
--- a/site/content/en/docs/handbook/kubectl.md
+++ b/site/content/en/docs/handbook/kubectl.md
@@ -12,29 +12,41 @@ inside minikube when the `minikube start` command is executed.
 
 However if `kubectl` is not installed locally, minikube already includes kubectl which can be used like this:
 
-`minikube kubectl -- `
+```shell
+minikube kubectl -- 
+```
 
 You can also `alias kubectl="minikube kubectl --"` for easier usage.
 
 Alternatively, you can create a symbolic link to minikube's binary named 'kubectl'.
 
-`ln -s $(which minikube) /usr/local/bin/kubectl`
+```shell
+ln -s $(which minikube) /usr/local/bin/kubectl
+```
 
 Get pods
 
-`minikube kubectl -- get pods`
+```shell
+minikube kubectl -- get pods
+```
 
 Creating a deployment inside kubernetes cluster
 
-`minikube kubectl -- create deployment hello-minikube --image=k8s.gcr.io/echoserver:1.4`
+```shell
+minikube kubectl -- create deployment hello-minikube --image=k8s.gcr.io/echoserver:1.4
+```
 
 Exposing the deployment with a NodePort service
 
-`minikube kubectl -- expose deployment hello-minikube --type=NodePort --port=8080`
+```shell
+minikube kubectl -- expose deployment hello-minikube --type=NodePort --port=8080
+```
 
 For more help
 
-`minikube kubectl -- --help`
+```shell
+minikube kubectl -- --help
+```
 
 ### Shell autocompletion
 
diff --git a/site/content/en/docs/handbook/mount.md b/site/content/en/docs/handbook/mount.md
index 40033f5bcd..b0f548c8c9 100644
--- a/site/content/en/docs/handbook/mount.md
+++ b/site/content/en/docs/handbook/mount.md
@@ -14,13 +14,13 @@ aliases:
 
 To mount a directory from the host into the guest using the `mount` subcommand:
 
-```
+```shell
 minikube mount :
 ```
 
 For example, this would mount your home directory to appear as /host within the minikube VM:
 
-```
+```shell
 minikube mount $HOME:/host
 ```
 
diff --git a/site/content/en/docs/handbook/pushing.md b/site/content/en/docs/handbook/pushing.md
index a7419a58b2..5a43517bdc 100644
--- a/site/content/en/docs/handbook/pushing.md
+++ b/site/content/en/docs/handbook/pushing.md
@@ -136,7 +136,7 @@ eval $(minikube podman-env)
 You should now be able to use podman client on the command line on your host machine talking to the podman service inside the minikube VM:
 
 {{% tabs %}}
-{{% tab "Linux" %}}
+{{% linuxtab %}}
 
 ```shell
 podman-remote help
@@ -146,8 +146,8 @@ podman-remote help
 Note: On Linux the remote client is called "podman-remote", while the local program is called "podman".
 {{% /pageinfo %}}
 
-{{% /tab %}}
-{{% tab "macOS" %}}
+{{% /linuxtab %}}
+{{% mactab %}}
 
 ```shell
 podman help
@@ -157,8 +157,8 @@ podman help
 Note: On macOS the remote client is called "podman", since there is no local "podman" program available.
 {{% /pageinfo %}}
 
-{{% /tab %}}
-{{% tab "Windows" %}}
+{{% /mactab %}}
+{{% windowstab %}}
 
 ```shell
 podman help
@@ -168,7 +168,7 @@ podman help
 Note: On Windows the remote client is called "podman", since there is no local "podman" program available.
 {{% /pageinfo %}}
 
-{{% /tab %}}
+{{% /windowstab %}}
 {{% /tabs %}}
 
 
diff --git a/site/content/en/docs/handbook/registry.md b/site/content/en/docs/handbook/registry.md
index 0b04a6f364..2a63905b9d 100644
--- a/site/content/en/docs/handbook/registry.md
+++ b/site/content/en/docs/handbook/registry.md
@@ -50,7 +50,7 @@ Quick guide for configuring minikube and docker on macOS, enabling docker to pus
 
 The first step is to enable the registry addon:
 
-```
+```shell
 minikube addons enable registry
 ```
 
@@ -58,13 +58,13 @@ When enabled, the registry addon exposes its port 5000 on the minikube's virtual
 
 In order to make docker accept pushing images to this registry, we have to redirect port 5000 on the docker virtual machine over to port 5000 on the minikube machine. We can (ab)use docker's network configuration to instantiate a container on the docker's host, and run socat there:
 
-```
+```shell
 docker run --rm -it --network=host alpine ash -c "apk add socat && socat TCP-LISTEN:5000,reuseaddr,fork TCP:$(minikube ip):5000"
 ```
 
 Once socat is running it's possible to push images to the minikube registry:
 
-```
+```shell
 docker tag my/image localhost:5000/myimage
 docker push localhost:5000/myimage
 ```
@@ -77,7 +77,7 @@ Quick guide for configuring minikube and docker on Windows, enabling docker to p
 
 The first step is to enable the registry addon:
 
-```
+```shell
 minikube addons enable registry
 ```
 
@@ -86,7 +86,7 @@ When enabled, the registry addon exposes its port 5000 on the minikube's virtual
 In order to make docker accept pushing images to this registry, we have to redirect port 5000 on the docker virtual machine over to port 5000 on the minikube machine. Unfortunately, the docker vm cannot directly see the IP address of the minikube vm. To fix this, you will have to add one more level of redirection. 
 
 Use kubectl port-forward to map your local workstation to the minikube vm
-```
+```shell
 kubectl port-forward --namespace kube-system  5000:5000
 ```
 
@@ -94,13 +94,13 @@ On your local machine you should now be able to reach the minikube registry by u
 
 From this point we can (ab)use docker's network configuration to instantiate a container on the docker's host, and run socat there to redirect traffic going to the docker vm's port 5000 to port 5000 on your host workstation.
 
-```
+```shell
 docker run --rm -it --network=host alpine ash -c "apk add socat && socat TCP-LISTEN:5000,reuseaddr,fork TCP:host.docker.internal:5000"
 ```
 
 Once socat is running it's possible to push images to the minikube registry from your local workstation:
 
-```
+```shell
 docker tag my/image localhost:5000/myimage
 docker push localhost:5000/myimage
 ```
diff --git a/site/content/en/docs/handbook/troubleshooting.md b/site/content/en/docs/handbook/troubleshooting.md
index bc44b458c7..ea627a6956 100644
--- a/site/content/en/docs/handbook/troubleshooting.md
+++ b/site/content/en/docs/handbook/troubleshooting.md
@@ -21,7 +21,9 @@ Example:
 
 minikube stores post-mortem INFO logs in the temporary directory of your system. On macOS or Linux, it's easy to get a list of recent INFO logs: 
 
-`find $TMPDIR -mtime -1 -type f -name "*minikube*INFO*" -ls  2>/dev/null`
+```shell
+find $TMPDIR -mtime -1 -type f -name "*minikube*INFO*" -ls  2>/dev/null
+```
 
 For instance, this shows:
 
@@ -29,7 +31,9 @@ For instance, this shows:
 
 These are plain text log files: you may rename them to ".log" and then drag/drop them into a GitHub issue for further analysis by the minikube team. You can quickly inspect the final lines of any of these logs via:
   
-`tail -n 10 `
+```shell
+tail -n 10 
+```
 
 for example, this shows:
 
@@ -83,4 +87,3 @@ minikube logs --problems
 ```
 
 This will attempt to surface known errors, such as invalid configuration flags. If nothing interesting shows up, try `minikube logs`.
-
diff --git a/site/content/en/docs/handbook/untrusted_certs.md b/site/content/en/docs/handbook/untrusted_certs.md
index 049f9075d7..c17646ba7b 100644
--- a/site/content/en/docs/handbook/untrusted_certs.md
+++ b/site/content/en/docs/handbook/untrusted_certs.md
@@ -17,7 +17,7 @@ You may install the Root Certificate into the minikube cluster to access these c
 
 You will need a corporate X.509 Root Certificate in PEM format. If it's in DER format, convert it:
 
-```
+```shell
 openssl x509 -inform der -in my_company.cer -out my_company.pem
 ```
 
diff --git a/site/content/en/docs/start/_index.md b/site/content/en/docs/start/_index.md
index d3ede45804..820334d9d2 100644
--- a/site/content/en/docs/start/_index.md
+++ b/site/content/en/docs/start/_index.md
@@ -22,33 +22,59 @@ All you need is Docker (or similarly compatible) container or a Virtual Machine
 
1Installation

 
 {{% tabs %}}
-{{% tab "Linux" %}}
+{{% linuxtab %}}
 
-For Linux users, we provide 3 easy download options:
+For Linux users, we provide 3 easy download options (for each architecture):
+
+### x86
+
+#### Binary download
 
-### Binary download
 
 ```shell
  curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64
  sudo install minikube-linux-amd64 /usr/local/bin/minikube
 ```
 
-### Debian package
+#### Debian package
 
 ```shell
 curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube_latest_amd64.deb
 sudo dpkg -i minikube_latest_amd64.deb
 ```
 
-### RPM package
+#### RPM package
 
 ```shell
 curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-latest.x86_64.rpm
 sudo rpm -ivh minikube-latest.x86_64.rpm
 ```
 
-{{% /tab %}}
-{{% tab "macOS" %}}
+### ARM
+
+#### Binary download
+
+```shell
+ curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-linux-arm64
+ sudo install minikube-linux-arm64 /usr/local/bin/minikube
+```
+
+#### Debian package
+
+```shell
+curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube_latest_arm64.deb
+sudo dpkg -i minikube_latest_arm64.deb
+```
+
+#### RPM package
+
+```shell
+curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-latest.aarch64.rpm
+sudo rpm -ivh minikube-latest.aarch64.rpm
+```
+
+{{% /linuxtab %}}
+{{% mactab %}}
 
 If the [Brew Package Manager](https://brew.sh/) installed:
 
@@ -58,7 +84,7 @@ brew install minikube
 
 If `which minikube` fails after installation via brew, you may have to remove the minikube cask and link the binary:
 
-```
+```shell
 brew cask remove minikube
 brew link minikube
 ```
@@ -70,8 +96,8 @@ curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-darwin
 sudo install minikube-darwin-amd64 /usr/local/bin/minikube
 ```
 
-{{% /tab %}}
-{{% tab "Windows" %}}
+{{% /mactab %}}
+{{% windowstab %}}
 
 If the [Chocolatey Package Manager](https://chocolatey.org/) is installed, use it to install minikube:
 
@@ -81,7 +107,7 @@ choco install minikube
 
 Otherwise, download and run the [Windows installer](https://storage.googleapis.com/minikube/releases/latest/minikube-installer.exe)
 
-{{% /tab %}}
+{{% /windowstab %}}
 {{% /tabs %}}
 
 
2Start your cluster

@@ -160,7 +186,9 @@ minikube tunnel
 
 To find the routable IP, run this command and examine the `EXTERNAL-IP` column:
 
-`kubectl get services balanced`
+```shell
+kubectl get services balanced
+```
 
 Your deployment is now available at <EXTERNAL-IP>:8080
 
diff --git a/site/content/en/docs/tutorials/ambassador_ingress_controller.md b/site/content/en/docs/tutorials/ambassador_ingress_controller.md
index 30057a7e10..7d6cb3693d 100644
--- a/site/content/en/docs/tutorials/ambassador_ingress_controller.md
+++ b/site/content/en/docs/tutorials/ambassador_ingress_controller.md
@@ -84,7 +84,10 @@ spec:
           serviceName: hello-minikube
           servicePort: 8080
 ```
-Run the command: `kubectl apply -f hello-ingress.yaml`
+Run the command: 
+```shell
+kubectl apply -f hello-ingress.yaml
+```
 
 That's it! You can now access your service via Ambassador:
 ```shell script
@@ -120,7 +123,10 @@ spec:
   prefix: /hello-mapping/
   service: mapping-minikube.default:8080
 ```
-Run the command: `kubectl apply -f hello-mapping.yaml`
+Run the command: 
+```shell
+kubectl apply -f hello-mapping.yaml
+```
 
 That's it! You can now access your service via Ambassador:
 ```shell script
@@ -128,4 +134,4 @@ curl http://
 ```
 
 **Note:** Read more about mappings in Ambassador's
-[documentation](https://www.getambassador.io/docs/latest/topics/using/mappings/).
\ No newline at end of file
+[documentation](https://www.getambassador.io/docs/latest/topics/using/mappings/).
diff --git a/site/content/en/docs/tutorials/custom_cert_ingress.md b/site/content/en/docs/tutorials/custom_cert_ingress.md
new file mode 100644
index 0000000000..1f419e6207
--- /dev/null
+++ b/site/content/en/docs/tutorials/custom_cert_ingress.md
@@ -0,0 +1,44 @@
+---
+title: "How to use custom TLS certificate with ingress addon"
+linkTitle: "Using custom TLS certificate with ingress addon"
+weight: 1
+date: 2020-11-30
+---
+
+## Overview
+
+- This tutorial will show you how to configure custom TLS certificatate for ingress addon.
+
+## Tutorial
+
+- Start minikube
+```
+$ minikube start
+```
+
+- Create TLS secret which contains custom certificate and private key
+```
+$ kubectl -n kube-system create secret tls mkcert --key key.pem --cert cert.pem
+```
+
+- Configure ingress addon
+```
+$ minikube addons configure ingress
+-- Enter custom cert(format is "namespace/secret"): kube-system/mkcert
+✅  ingress was successfully configured
+```
+
+- Enable ingress addon (disable first when already enabled)
+```
+$ minikube addons disable ingress
+🌑  "The 'ingress' addon is disabled
+
+$ minikube addons enable ingress
+🔎  Verifying ingress addon...
+🌟  The 'ingress' addon is enabled
+```
+- Verify if custom certificate was enabled
+```
+$ kubectl -n kube-system get deployment ingress-nginx-controller -o yaml | grep "kube-system"
+- --default-ssl-certificate=kube-system/mkcert
+```
\ No newline at end of file
diff --git a/site/content/en/docs/tutorials/multi_node.md b/site/content/en/docs/tutorials/multi_node.md
index 99a7e4e352..4872dc67ab 100644
--- a/site/content/en/docs/tutorials/multi_node.md
+++ b/site/content/en/docs/tutorials/multi_node.md
@@ -17,8 +17,10 @@ date: 2019-11-24
 ## Tutorial
 
 - Start a cluster with 2 nodes in the driver of your choice:
-```
+```shell
 minikube start --nodes 2 -p multinode-demo
+```
+```
 😄  [multinode-demo] minikube v1.10.1 on Darwin 10.15.4
 ✨  Automatically selected the hyperkit driver
 👍  Starting control plane node multinode-demo in cluster multinode-demo
@@ -40,16 +42,20 @@ To track progress on multi-node clusters, see https://github.com/kubernetes/mini
 ```
 
 - Get the list of your nodes:
-```
+```shell
 kubectl get nodes
+```
+```
 NAME                 STATUS   ROLES    AGE   VERSION
 multinode-demo       Ready    master   72s   v1.18.2
 multinode-demo-m02   Ready       33s   v1.18.2
 ```
 
 - You can also check the status of your nodes:
+```shell
+minikube status -p multinode-demo
+```
 ```
-$ minikube status -p multinode-demo
 multinode-demo
 type: Control Plane
 host: Running
@@ -64,33 +70,44 @@ kubelet: Running
 ```
 
 - Deploy our hello world deployment:
-```
+```shell
 kubectl apply -f hello-deployment.yaml
+```
+```
 deployment.apps/hello created
-
+```
+```shell
 kubectl rollout status deployment/hello
+```
+```
 deployment "hello" successfully rolled out
 ```
 
 
 - Deploy our hello world service, which just spits back the IP address the request was served from:
-```
+```shell
 kubectl apply -f hello-svc.yaml
+```
+```
 service/hello created
 ```
 
 
 - Check out the IP addresses of our pods, to note for future reference
-```
+```shell
 kubectl get pods -o wide
+```
+```
 NAME                    READY   STATUS    RESTARTS   AGE   IP           NODE             NOMINATED NODE   READINESS GATES
 hello-c7b8df44f-qbhxh   1/1     Running   0          31s   10.244.0.3   multinode-demo              
 hello-c7b8df44f-xv4v6   1/1     Running   0          31s   10.244.0.2   multinode-demo              
 ```
 
 - Look at our service, to know what URL to hit
-```
+```shell
 minikube service list -p multinode-demo
+```
+```
 |-------------|------------|--------------|-----------------------------|
 |  NAMESPACE  |    NAME    | TARGET PORT  |             URL             |
 |-------------|------------|--------------|-----------------------------|
@@ -101,8 +118,10 @@ minikube service list -p multinode-demo
 ```
 
 - Let's hit the URL a few times and see what comes back
-```
+```shell
 curl  http://192.168.64.226:31000
+```
+```
 Hello from hello-c7b8df44f-qbhxh (10.244.0.3)
 
 curl  http://192.168.64.226:31000
diff --git a/site/content/en/docs/tutorials/telemetry.md b/site/content/en/docs/tutorials/telemetry.md
index a41bafc520..5c36284be2 100644
--- a/site/content/en/docs/tutorials/telemetry.md
+++ b/site/content/en/docs/tutorials/telemetry.md
@@ -14,7 +14,7 @@ Currently, minikube supports the following exporters for tracing data:
 
 To collect trace data with minikube and the Stackdriver exporter, run:
 
-```
+```shell
 MINIKUBE_GCP_PROJECT_ID= minikube start --output json --trace gcp
 ```
 
diff --git a/site/content/en/docs/tutorials/using_psp.md b/site/content/en/docs/tutorials/using_psp.md
index 38123c73c0..55974c8a80 100644
--- a/site/content/en/docs/tutorials/using_psp.md
+++ b/site/content/en/docs/tutorials/using_psp.md
@@ -20,7 +20,9 @@ This tutorial explains how to start minikube with Pod Security Policies (PSP) en
 Start minikube with the `PodSecurityPolicy` admission controller and the
 `pod-security-policy` addon enabled.
 
-`minikube start --extra-config=apiserver.enable-admission-plugins=PodSecurityPolicy --addons=pod-security-policy`
+```shell
+minikube start --extra-config=apiserver.enable-admission-plugins=PodSecurityPolicy --addons=pod-security-policy
+```
 
 The `pod-security-policy` addon must be enabled along with the admission
 controller to prevent issues during bootstrap.
@@ -35,12 +37,16 @@ the policies that addon enables must be separately applied to the cluster.
 Before starting minikube, you need to give it the PSP YAMLs in order to allow minikube to bootstrap.
 
 Create the directory:
-`mkdir -p ~/.minikube/files/etc/kubernetes/addons`
+```shell
+mkdir -p ~/.minikube/files/etc/kubernetes/addons
+```
 
 Copy the YAML below into this file: `~/.minikube/files/etc/kubernetes/addons/psp.yaml`
 
 Now start minikube:
-`minikube start --extra-config=apiserver.enable-admission-plugins=PodSecurityPolicy`
+```shell
+minikube start --extra-config=apiserver.enable-admission-plugins=PodSecurityPolicy
+```
 
 ```yaml
 ---
@@ -191,7 +197,7 @@ Next, apply the YAML shown above to the cluster.
 Finally, stop the cluster and then restart it with the admission controller
 enabled.
 
-```
+```shell
 minikube start
 kubectl apply -f /path/to/psp.yaml
 minikube stop
diff --git a/site/layouts/partials/hooks/head-end.html b/site/layouts/partials/hooks/head-end.html
index ed11ad10a1..bf082de3bd 100644
--- a/site/layouts/partials/hooks/head-end.html
+++ b/site/layouts/partials/hooks/head-end.html
@@ -1,5 +1,5 @@
 
-
+
 
 
 
\ No newline at end of file
diff --git a/site/layouts/partials/sidebar-tree.html b/site/layouts/partials/sidebar-tree.html
index 029abae4c5..b1527f944f 100644
--- a/site/layouts/partials/sidebar-tree.html
+++ b/site/layouts/partials/sidebar-tree.html
@@ -12,7 +12,7 @@
     
   
   {{ end }}
-