From ada0fc173cc072d41975f57e6e65725222454576 Mon Sep 17 00:00:00 2001 From: "Jituri, Pranav" Date: Tue, 17 Nov 2020 00:30:06 +0530 Subject: [PATCH 01/21] Fixed sudo error for Windows --- pkg/minikube/tunnel/kic/ssh_conn.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/pkg/minikube/tunnel/kic/ssh_conn.go b/pkg/minikube/tunnel/kic/ssh_conn.go index d850e92bef..1bf4890c8d 100644 --- a/pkg/minikube/tunnel/kic/ssh_conn.go +++ b/pkg/minikube/tunnel/kic/ssh_conn.go @@ -19,6 +19,7 @@ package kic import ( "fmt" "os/exec" + "runtime" "github.com/phayes/freeport" v1 "k8s.io/api/core/v1" @@ -39,7 +40,7 @@ func createSSHConn(name, sshPort, sshKey string, svc *v1.Service) *sshConn { sshArgs := []string{ // TODO: document the options here "-o", "UserKnownHostsFile=/dev/null", - "-o", "StrictHostKeyChecking no", + "-o", "StrictHostKeyChecking=no", "-N", "docker@127.0.0.1", "-p", sshPort, @@ -66,8 +67,7 @@ func createSSHConn(name, sshPort, sshKey string, svc *v1.Service) *sshConn { } command := "ssh" - - if askForSudo { + if askForSudo && runtime.GOOS != "windows" { out.Step( style.Warning, "The service {{.service}} requires privileged ports to be exposed: {{.ports}}", @@ -79,7 +79,7 @@ func createSSHConn(name, sshPort, sshKey string, svc *v1.Service) *sshConn { command = "sudo" sshArgs = append([]string{"ssh"}, sshArgs...) } - + out.Step(style.Command,"Command - [{{.command}}], Arguments - [{{.args}}]",out.V{"command": command, "args":sshArgs}) cmd := exec.Command(command, sshArgs...) return &sshConn{ @@ -94,7 +94,7 @@ func createSSHConnWithRandomPorts(name, sshPort, sshKey string, svc *v1.Service) sshArgs := []string{ // TODO: document the options here "-o", "UserKnownHostsFile=/dev/null", - "-o", "StrictHostKeyChecking no", + "-o", "StrictHostKeyChecking=no", "-N", "docker@127.0.0.1", "-p", sshPort, @@ -139,7 +139,8 @@ func (c *sshConn) startAndWait() error { } // we ignore wait error because the process will be killed - _ = c.cmd.Wait() + err = c.cmd.Wait() + out.Step(style.Running,"Wait Error - [{{.err}}]",out.V{"err": err}) return nil } From 0def75cbc98b4fc45a13b7288621c075f41f212a Mon Sep 17 00:00:00 2001 From: "Jituri, Pranav" Date: Wed, 18 Nov 2020 01:46:28 +0530 Subject: [PATCH 02/21] Work in progress for SSH key file fix --- pkg/drivers/common.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/drivers/common.go b/pkg/drivers/common.go index 808367c204..f96e7e6cdd 100644 --- a/pkg/drivers/common.go +++ b/pkg/drivers/common.go @@ -109,6 +109,11 @@ func MakeDiskImage(d *drivers.BaseDriver, boot2dockerURL string, diskSize int) e if err := ssh.GenerateSSHKey(keyPath); err != nil { return errors.Wrap(err, "generate ssh key") } + + // Make the key file read only. + if err := os.Chmod(keyPath,0400); err != nil { + return errors.Wrap(err, "chmod dir windows") + } diskPath := GetDiskPath(d) klog.Infof("Creating raw disk image: %s...", diskPath) From bf7b09f484d995ffc7bdf7a0bd0b03205a25bcfb Mon Sep 17 00:00:00 2001 From: "Jituri, Pranav" Date: Sat, 21 Nov 2020 00:27:19 +0530 Subject: [PATCH 03/21] Fix permissions for key file on Windows --- pkg/drivers/common.go | 5 ----- pkg/drivers/kic/kic.go | 29 +++++++++++++++++++++++++++++ pkg/minikube/tunnel/kic/ssh_conn.go | 3 +-- 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/pkg/drivers/common.go b/pkg/drivers/common.go index f96e7e6cdd..808367c204 100644 --- a/pkg/drivers/common.go +++ b/pkg/drivers/common.go @@ -109,11 +109,6 @@ func MakeDiskImage(d *drivers.BaseDriver, boot2dockerURL string, diskSize int) e if err := ssh.GenerateSSHKey(keyPath); err != nil { return errors.Wrap(err, "generate ssh key") } - - // Make the key file read only. - if err := os.Chmod(keyPath,0400); err != nil { - return errors.Wrap(err, "chmod dir windows") - } diskPath := GetDiskPath(d) klog.Infof("Creating raw disk image: %s...", diskPath) diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go index 10c5aebdcc..b0d7edcdbd 100644 --- a/pkg/drivers/kic/kic.go +++ b/pkg/drivers/kic/kic.go @@ -20,6 +20,8 @@ import ( "fmt" "net" "os/exec" + "context" + "runtime" "strconv" "strings" "sync" @@ -205,6 +207,33 @@ func (d *Driver) prepareSSH() error { return errors.Wrapf(err, "apply authorized_keys file ownership, output %s", rr.Output()) } + if runtime.GOOS == "windows" { + path, _ := exec.LookPath("powershell") + ctx, cancel := context.WithTimeout(context.Background(), 8*time.Second) + defer cancel() + + klog.Infof("ensuring only current user has permissions to key file located at : %s...", keyPath) + + // Get the SID of the current user + currentUserSidCmd := exec.CommandContext(ctx, path, "-NoProfile","-NonInteractive","([System.Security.Principal.WindowsIdentity]::GetCurrent()).User.Value") + currentUserSidOut, currentUserSidErr := currentUserSidCmd.CombinedOutput() + if currentUserSidErr != nil { + return errors.Wrap(currentUserSidErr, "unable to determine current user's SID") + } + + icaclsArguments := fmt.Sprintf(`"%s" /grant:r *%s:F /inheritancelevel:r`, keyPath, strings.TrimSpace(string(currentUserSidOut))) + icaclsCmd := exec.CommandContext(ctx, path, "-NoProfile","-NonInteractive","icacls.exe", icaclsArguments) + icaclsCmdOut, icaclsCmdErr := icaclsCmd.CombinedOutput() + + if icaclsCmdErr != nil { + return errors.Wrap(icaclsCmdErr, "unable to execute icacls to set permissions") + } + + if !strings.Contains(string(icaclsCmdOut),"Successfully processed 1 files; Failed processing 0 files") { + return errors.Errorf("icacls failed applying permissions - %s, output - [%s]", icaclsCmdErr, strings.TrimSpace(string(icaclsCmdOut))) + } + } + return nil } diff --git a/pkg/minikube/tunnel/kic/ssh_conn.go b/pkg/minikube/tunnel/kic/ssh_conn.go index 1bf4890c8d..93a0801998 100644 --- a/pkg/minikube/tunnel/kic/ssh_conn.go +++ b/pkg/minikube/tunnel/kic/ssh_conn.go @@ -139,8 +139,7 @@ func (c *sshConn) startAndWait() error { } // we ignore wait error because the process will be killed - err = c.cmd.Wait() - out.Step(style.Running,"Wait Error - [{{.err}}]",out.V{"err": err}) + _ = c.cmd.Wait() return nil } From 197f2a25ae8b18d4aa52f9a3a9020607bb4d4683 Mon Sep 17 00:00:00 2001 From: "Jituri, Pranav" Date: Sat, 21 Nov 2020 00:30:05 +0530 Subject: [PATCH 04/21] Removed unused debugging lines --- pkg/drivers/kic/kic.go | 2 +- pkg/minikube/tunnel/kic/ssh_conn.go | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go index b0d7edcdbd..592d7be0aa 100644 --- a/pkg/drivers/kic/kic.go +++ b/pkg/drivers/kic/kic.go @@ -230,7 +230,7 @@ func (d *Driver) prepareSSH() error { } if !strings.Contains(string(icaclsCmdOut),"Successfully processed 1 files; Failed processing 0 files") { - return errors.Errorf("icacls failed applying permissions - %s, output - [%s]", icaclsCmdErr, strings.TrimSpace(string(icaclsCmdOut))) + return errors.Errorf("icacls failed applying permissions - err - [%s], output - [%s]", icaclsCmdErr, strings.TrimSpace(string(icaclsCmdOut))) } } diff --git a/pkg/minikube/tunnel/kic/ssh_conn.go b/pkg/minikube/tunnel/kic/ssh_conn.go index 93a0801998..f1d132aea5 100644 --- a/pkg/minikube/tunnel/kic/ssh_conn.go +++ b/pkg/minikube/tunnel/kic/ssh_conn.go @@ -79,7 +79,6 @@ func createSSHConn(name, sshPort, sshKey string, svc *v1.Service) *sshConn { command = "sudo" sshArgs = append([]string{"ssh"}, sshArgs...) } - out.Step(style.Command,"Command - [{{.command}}], Arguments - [{{.args}}]",out.V{"command": command, "args":sshArgs}) cmd := exec.Command(command, sshArgs...) return &sshConn{ From 118d2009e7d4731c48e1392d1b8cbf6c88ad6927 Mon Sep 17 00:00:00 2001 From: "Jituri, Pranav" Date: Sat, 21 Nov 2020 01:05:40 +0530 Subject: [PATCH 05/21] Enable the NGINX ingress addon for windows and kic --- pkg/addons/addons.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/addons/addons.go b/pkg/addons/addons.go index 12e80fabd5..9a110cd464 100644 --- a/pkg/addons/addons.go +++ b/pkg/addons/addons.go @@ -145,14 +145,18 @@ func enableOrDisableAddon(cc *config.ClusterConfig, name string, val string) err // to match both ingress and ingress-dns addons if strings.HasPrefix(name, "ingress") && enable { - if driver.IsKIC(cc.Driver) && runtime.GOOS != "linux" { - exit.Message(reason.Usage, `Due to networking limitations of driver {{.driver_name}} on {{.os_name}}, {{.addon_name}} addon is not supported. + if driver.IsKIC(cc.Driver) { + if runtime.GOOS == "windows" { + out.Step(style.Tip,`After the addon is enabled, please run "minikube tunnel" and your ingress resources would be available at "127.0.0.1"`) + } else if runtime.GOOS != "linux" { + exit.Message(reason.Usage, `Due to networking limitations of driver {{.driver_name}} on {{.os_name}}, {{.addon_name}} addon is not supported. Alternatively to use this addon you can use a vm-based driver: 'minikube start --vm=true' To track the update on this work in progress feature please check: https://github.com/kubernetes/minikube/issues/7332`, out.V{"driver_name": cc.Driver, "os_name": runtime.GOOS, "addon_name": name}) + } } else if driver.BareMetal(cc.Driver) { exit.Message(reason.Usage, `Due to networking limitations of driver {{.driver_name}}, {{.addon_name}} addon is not supported. Try using a different driver.`, out.V{"driver_name": cc.Driver, "addon_name": name}) From 61ba2b625279d39debcd8c8529def336af5ae18d Mon Sep 17 00:00:00 2001 From: "Jituri, Pranav" Date: Sat, 21 Nov 2020 18:39:42 +0530 Subject: [PATCH 06/21] Revert "Enable the NGINX ingress addon for windows and kic" This reverts commit 118d2009 --- pkg/addons/addons.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pkg/addons/addons.go b/pkg/addons/addons.go index 9a110cd464..12e80fabd5 100644 --- a/pkg/addons/addons.go +++ b/pkg/addons/addons.go @@ -145,18 +145,14 @@ func enableOrDisableAddon(cc *config.ClusterConfig, name string, val string) err // to match both ingress and ingress-dns addons if strings.HasPrefix(name, "ingress") && enable { - if driver.IsKIC(cc.Driver) { - if runtime.GOOS == "windows" { - out.Step(style.Tip,`After the addon is enabled, please run "minikube tunnel" and your ingress resources would be available at "127.0.0.1"`) - } else if runtime.GOOS != "linux" { - exit.Message(reason.Usage, `Due to networking limitations of driver {{.driver_name}} on {{.os_name}}, {{.addon_name}} addon is not supported. + if driver.IsKIC(cc.Driver) && runtime.GOOS != "linux" { + exit.Message(reason.Usage, `Due to networking limitations of driver {{.driver_name}} on {{.os_name}}, {{.addon_name}} addon is not supported. Alternatively to use this addon you can use a vm-based driver: 'minikube start --vm=true' To track the update on this work in progress feature please check: https://github.com/kubernetes/minikube/issues/7332`, out.V{"driver_name": cc.Driver, "os_name": runtime.GOOS, "addon_name": name}) - } } else if driver.BareMetal(cc.Driver) { exit.Message(reason.Usage, `Due to networking limitations of driver {{.driver_name}}, {{.addon_name}} addon is not supported. Try using a different driver.`, out.V{"driver_name": cc.Driver, "addon_name": name}) From 74cdda9e9a5bb0ef8b3d75815b15a508a780d9ce Mon Sep 17 00:00:00 2001 From: "Jituri, Pranav" Date: Wed, 9 Dec 2020 16:54:19 +0530 Subject: [PATCH 07/21] Resolve review comments --- pkg/drivers/kic/kic.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go index 592d7be0aa..ded2622a95 100644 --- a/pkg/drivers/kic/kic.go +++ b/pkg/drivers/kic/kic.go @@ -218,19 +218,19 @@ func (d *Driver) prepareSSH() error { currentUserSidCmd := exec.CommandContext(ctx, path, "-NoProfile","-NonInteractive","([System.Security.Principal.WindowsIdentity]::GetCurrent()).User.Value") currentUserSidOut, currentUserSidErr := currentUserSidCmd.CombinedOutput() if currentUserSidErr != nil { - return errors.Wrap(currentUserSidErr, "unable to determine current user's SID") - } + klog.Warningf("unable to determine current user's SID. minikube tunnel may not work.") + } else { + icaclsArguments := fmt.Sprintf(`"%s" /grant:r *%s:F /inheritancelevel:r`, keyPath, strings.TrimSpace(string(currentUserSidOut))) + icaclsCmd := exec.CommandContext(ctx, path, "-NoProfile","-NonInteractive","icacls.exe", icaclsArguments) + icaclsCmdOut, icaclsCmdErr := icaclsCmd.CombinedOutput() - icaclsArguments := fmt.Sprintf(`"%s" /grant:r *%s:F /inheritancelevel:r`, keyPath, strings.TrimSpace(string(currentUserSidOut))) - icaclsCmd := exec.CommandContext(ctx, path, "-NoProfile","-NonInteractive","icacls.exe", icaclsArguments) - icaclsCmdOut, icaclsCmdErr := icaclsCmd.CombinedOutput() + if icaclsCmdErr != nil { + return errors.Wrap(icaclsCmdErr, "unable to execute icacls to set permissions") + } - if icaclsCmdErr != nil { - return errors.Wrap(icaclsCmdErr, "unable to execute icacls to set permissions") - } - - if !strings.Contains(string(icaclsCmdOut),"Successfully processed 1 files; Failed processing 0 files") { - return errors.Errorf("icacls failed applying permissions - err - [%s], output - [%s]", icaclsCmdErr, strings.TrimSpace(string(icaclsCmdOut))) + if !strings.Contains(string(icaclsCmdOut),"Successfully processed 1 files; Failed processing 0 files") { + return errors.Errorf("icacls failed applying permissions - err - [%s], output - [%s]", icaclsCmdErr, strings.TrimSpace(string(icaclsCmdOut))) + } } } From 789610bbafab75340c873bf623e4a49b5c9b6460 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Fri, 11 Dec 2020 10:58:46 -0800 Subject: [PATCH 08/21] Log warning if br_netfilter cannot be enabled rather than fatally exiting --- pkg/minikube/cruntime/crio.go | 20 ++++++++++++++++++++ pkg/minikube/cruntime/cruntime.go | 19 ------------------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/pkg/minikube/cruntime/crio.go b/pkg/minikube/cruntime/crio.go index 6169ad9107..8483752d86 100644 --- a/pkg/minikube/cruntime/crio.go +++ b/pkg/minikube/cruntime/crio.go @@ -109,6 +109,26 @@ func (r *CRIO) Active() bool { return r.Init.Active("crio") } +// enableIPForwarding configures IP forwarding, which is handled normally by Docker +// Context: https://github.com/kubernetes/kubeadm/issues/1062 +func enableIPForwarding(cr CommandRunner) error { + // The bridge-netfilter module enables iptables rules to work on Linux bridges + // NOTE: br_netfilter isn't available in WSL2, but forwarding works fine there anyways + c := exec.Command("sudo", "sysctl", "net.bridge.bridge-nf-call-iptables") + if rr, err := cr.RunCmd(c); err != nil { + klog.Infof("couldn't verify netfilter by %q which might be okay. error: %v", rr.Command(), err) + c = exec.Command("sudo", "modprobe", "br_netfilter") + if _, err := cr.RunCmd(c); err != nil { + klog.Warningf("%q failed, which may be ok: %v", rr.Command(), err) + } + } + c = exec.Command("sudo", "sh", "-c", "echo 1 > /proc/sys/net/ipv4/ip_forward") + if _, err := cr.RunCmd(c); err != nil { + return errors.Wrapf(err, "ip_forward") + } + return nil +} + // Enable idempotently enables CRIO on a host func (r *CRIO) Enable(disOthers, _ bool) error { if disOthers { diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go index 332dc1be4c..b6c67c8a13 100644 --- a/pkg/minikube/cruntime/cruntime.go +++ b/pkg/minikube/cruntime/cruntime.go @@ -22,7 +22,6 @@ import ( "os/exec" "github.com/blang/semver" - "github.com/pkg/errors" "k8s.io/klog/v2" "k8s.io/minikube/pkg/minikube/assets" "k8s.io/minikube/pkg/minikube/command" @@ -210,21 +209,3 @@ func disableOthers(me Manager, cr CommandRunner) error { } return nil } - -// enableIPForwarding configures IP forwarding, which is handled normally by Docker -// Context: https://github.com/kubernetes/kubeadm/issues/1062 -func enableIPForwarding(cr CommandRunner) error { - c := exec.Command("sudo", "sysctl", "net.bridge.bridge-nf-call-iptables") - if rr, err := cr.RunCmd(c); err != nil { - klog.Infof("couldn't verify netfilter by %q which might be okay. error: %v", rr.Command(), err) - c = exec.Command("sudo", "modprobe", "br_netfilter") - if _, err := cr.RunCmd(c); err != nil { - return errors.Wrapf(err, "br_netfilter") - } - } - c = exec.Command("sudo", "sh", "-c", "echo 1 > /proc/sys/net/ipv4/ip_forward") - if _, err := cr.RunCmd(c); err != nil { - return errors.Wrapf(err, "ip_forward") - } - return nil -} From f3be305abb7c609130b6957b2b63ae924113770f Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Fri, 11 Dec 2020 12:11:26 -0800 Subject: [PATCH 09/21] bump iso on head for snapshot4 --- Makefile | 2 +- site/content/en/docs/commands/config.md | 2 +- site/content/en/docs/commands/start.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index c01dd24ce5..5f2a7f35a9 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ KUBERNETES_VERSION ?= $(shell egrep "DefaultKubernetesVersion =" pkg/minikube/co KIC_VERSION ?= $(shell egrep "Version =" pkg/drivers/kic/types.go | cut -d \" -f2) # Default to .0 for higher cache hit rates, as build increments typically don't require new ISO versions -ISO_VERSION ?= v1.16.0-snapshot1 +ISO_VERSION ?= v1.16.0-snapshot4 # Dashes are valid in semver, but not Linux packaging. Use ~ to delimit alpha/beta DEB_VERSION ?= $(subst -,~,$(RAW_VERSION)) RPM_VERSION ?= $(DEB_VERSION) diff --git a/site/content/en/docs/commands/config.md b/site/content/en/docs/commands/config.md index 556be2b56b..40c9c9571f 100644 --- a/site/content/en/docs/commands/config.md +++ b/site/content/en/docs/commands/config.md @@ -11,7 +11,7 @@ Modify persistent configuration values ### Synopsis -config modifies minikube config files using subcommands like "minikube config set driver kvm2" +config modifies minikube config files using subcommands like "minikube config set driver kvm" Configurable fields: * driver diff --git a/site/content/en/docs/commands/start.md b/site/content/en/docs/commands/start.md index c4572e0fce..b5bf841feb 100644 --- a/site/content/en/docs/commands/start.md +++ b/site/content/en/docs/commands/start.md @@ -64,7 +64,7 @@ minikube start [flags] --insecure-registry strings Insecure Docker registries to pass to the Docker daemon. The default service CIDR range will automatically be added. --install-addons If set, install addons. Defaults to true. (default true) --interactive Allow user prompts for more information (default true) - --iso-url strings Locations to fetch the minikube ISO from. (default [https://storage.googleapis.com/minikube/iso/minikube-v1.16.0-snapshot1.iso,https://github.com/kubernetes/minikube/releases/download/v1.16.0-snapshot1/minikube-v1.16.0-snapshot1.iso,https://kubernetes.oss-cn-hangzhou.aliyuncs.com/minikube/iso/minikube-v1.16.0-snapshot1.iso]) + --iso-url strings Locations to fetch the minikube ISO from. (default [https://storage.googleapis.com/minikube/iso/minikube-v1.16.0-snapshot4.iso,https://github.com/kubernetes/minikube/releases/download/v1.16.0-snapshot4/minikube-v1.16.0-snapshot4.iso,https://kubernetes.oss-cn-hangzhou.aliyuncs.com/minikube/iso/minikube-v1.16.0-snapshot4.iso]) --keep-context This will keep the existing kubectl context and will create a minikube context. --kubernetes-version string The Kubernetes version that the minikube VM will use (ex: v1.2.3, 'stable' for v1.20.0, 'latest' for v1.20.0). Defaults to 'stable'. --kvm-gpu Enable experimental NVIDIA GPU support in minikube From dbdbad6e699c56108d27f7f2f9ca341ef85986fd Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Fri, 11 Dec 2020 12:43:18 -0800 Subject: [PATCH 10/21] Remove obsolete mention of kvm driver, fixes generate-docs issue --- cmd/minikube/cmd/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/minikube/cmd/config/config.go b/cmd/minikube/cmd/config/config.go index 5869c1d175..7c650c3dc9 100644 --- a/cmd/minikube/cmd/config/config.go +++ b/cmd/minikube/cmd/config/config.go @@ -181,7 +181,7 @@ var settings = []Setting{ var ConfigCmd = &cobra.Command{ Use: "config SUBCOMMAND [flags]", Short: "Modify persistent configuration values", - Long: `config modifies minikube config files using subcommands like "minikube config set driver kvm" + Long: `config modifies minikube config files using subcommands like "minikube config set driver kvm2" Configurable fields: ` + "\n\n" + configurableFields(), Run: func(cmd *cobra.Command, args []string) { if err := cmd.Help(); err != nil { From 12893a6121d39ae4c3d2f2f7b4d9cfb3fa9f605f Mon Sep 17 00:00:00 2001 From: Steven Powell Date: Fri, 11 Dec 2020 14:53:43 -0700 Subject: [PATCH 11/21] Fixed output on back to back spinner steps --- pkg/minikube/out/out.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/minikube/out/out.go b/pkg/minikube/out/out.go index df6cadc156..50f09600f5 100644 --- a/pkg/minikube/out/out.go +++ b/pkg/minikube/out/out.go @@ -138,6 +138,10 @@ func spinnerString(format string, a ...interface{}) { } klog.Infof(format, a...) + // if spin is active from a previous step, it will stop spinner displaying + if spin.Active() { + spin.Stop() + } _, err := fmt.Fprintf(outFile, format, a...) if err != nil { klog.Errorf("Fprintf failed: %v", err) From a18166a243d26f5498e1a540d45b49ddfb0d3eea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sat, 12 Dec 2020 09:47:39 +0100 Subject: [PATCH 12/21] Upgrade buildkit from 0.7.2 to 0.8.0 Also include the custom runc version --- .../package/buildkit-bin/buildkit-bin.hash | 1 + .../package/buildkit-bin/buildkit-bin.mk | 13 ++++++++++--- deploy/kicbase/Dockerfile | 2 +- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/deploy/iso/minikube-iso/package/buildkit-bin/buildkit-bin.hash b/deploy/iso/minikube-iso/package/buildkit-bin/buildkit-bin.hash index bfc3e3ad46..e658bb8341 100644 --- a/deploy/iso/minikube-iso/package/buildkit-bin/buildkit-bin.hash +++ b/deploy/iso/minikube-iso/package/buildkit-bin/buildkit-bin.hash @@ -1 +1,2 @@ sha256 33bcaa49b31bc3a277ac75d32fce3f5442d39f53a1799b8624e985279b579f74 buildkit-v0.7.2.linux-amd64.tar.gz +sha256 28005748fae926edf8c93b7cb1df53ec49df65dec67105b94e7fb9c513fa78a4 buildkit-v0.8.0.linux-amd64.tar.gz diff --git a/deploy/iso/minikube-iso/package/buildkit-bin/buildkit-bin.mk b/deploy/iso/minikube-iso/package/buildkit-bin/buildkit-bin.mk index 78f7f7ea38..639c0bf281 100644 --- a/deploy/iso/minikube-iso/package/buildkit-bin/buildkit-bin.mk +++ b/deploy/iso/minikube-iso/package/buildkit-bin/buildkit-bin.mk @@ -4,17 +4,24 @@ # ################################################################################ -BUILDKIT_BIN_VERSION = v0.7.2 +BUILDKIT_BIN_VERSION = v0.8.0 +BUILDKIT_BIN_COMMIT = 73fe4736135645a342abc7b587bba0994cccf0f9 BUILDKIT_BIN_SITE = https://github.com/moby/buildkit/releases/download/$(BUILDKIT_BIN_VERSION) BUILDKIT_BIN_SOURCE = buildkit-$(BUILDKIT_BIN_VERSION).linux-amd64.tar.gz +# https://github.com/opencontainers/runc.git +BUILDKIT_RUNC_VERSION = 939ad4e3fcfa1ab531458355a73688c6f4ee5003 + define BUILDKIT_BIN_INSTALL_TARGET_CMDS $(INSTALL) -D -m 0755 \ $(@D)/buildctl \ - $(TARGET_DIR)/usr/bin/buildctl + $(TARGET_DIR)/usr/bin + $(INSTALL) -D -m 0755 \ + $(@D)/buildkit-runc \ + $(TARGET_DIR)/usr/sbin $(INSTALL) -D -m 0755 \ $(@D)/buildkitd \ - $(TARGET_DIR)/usr/sbin/buildkitd + $(TARGET_DIR)/usr/sbin endef $(eval $(generic-package)) diff --git a/deploy/kicbase/Dockerfile b/deploy/kicbase/Dockerfile index 77fd2c3ee7..45cd7e2e92 100644 --- a/deploy/kicbase/Dockerfile +++ b/deploy/kicbase/Dockerfile @@ -21,7 +21,7 @@ # for a kubernetes node image, it doesn't contain much we don't need FROM ubuntu:focal-20201106 -ARG BUILDKIT_VERSION="v0.7.2" +ARG BUILDKIT_VERSION="v0.8.0" # copy in static files (configs, scripts) COPY 10-network-security.conf /etc/sysctl.d/10-network-security.conf From a5c16653249d2d48baaec5e8aedf40c7dd7bb9b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sat, 12 Dec 2020 18:49:41 +0100 Subject: [PATCH 13/21] Run generate-docs again for the driver rename The iso bump reverted the previous fix again --- site/content/en/docs/commands/config.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/content/en/docs/commands/config.md b/site/content/en/docs/commands/config.md index 40c9c9571f..556be2b56b 100644 --- a/site/content/en/docs/commands/config.md +++ b/site/content/en/docs/commands/config.md @@ -11,7 +11,7 @@ Modify persistent configuration values ### Synopsis -config modifies minikube config files using subcommands like "minikube config set driver kvm" +config modifies minikube config files using subcommands like "minikube config set driver kvm2" Configurable fields: * driver From 636370265f83a310c21e9a93835dbf871486ce52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sun, 13 Dec 2020 11:09:58 +0100 Subject: [PATCH 14/21] Add some more docs about building images Add some more examples on how to use podman for cri-o runtime, and how to use buildctl (with buildkitd) for containerd runtime. --- site/content/en/docs/handbook/pushing.md | 38 +++++++++++++++++++++--- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/site/content/en/docs/handbook/pushing.md b/site/content/en/docs/handbook/pushing.md index 0afec3daf1..d23a5659fd 100644 --- a/site/content/en/docs/handbook/pushing.md +++ b/site/content/en/docs/handbook/pushing.md @@ -31,6 +31,7 @@ Here is a comparison table to help you choose: ## 1. Pushing directly to the in-cluster Docker daemon (docker-env) +This is similar to podman-env but only for Docker runtime. When using a container or VM driver (all drivers except none), you can reuse the Docker daemon inside minikube cluster. this means you don't have to build on your host machine and push the image into a docker registry. You can just build inside the same docker daemon as minikube which speeds up local experiments. @@ -137,6 +138,12 @@ You should now be able to use podman client on the command line on your host mac podman-remote help ``` +now you can 'build' against the storage inside minikube. which is instantly accessible to kubernetes cluster. + +```shell +podman-remote build -t my_image . +``` + {{% pageinfo color="info" %}} Note: On Linux the remote client is called "podman-remote", while the local program is called "podman". {{% /pageinfo %}} @@ -148,6 +155,12 @@ Note: On Linux the remote client is called "podman-remote", while the local prog podman help ``` +now you can 'build' against the storage inside minikube. which is instantly accessible to kubernetes cluster. + +```shell +podman build -t my_image . +``` + {{% pageinfo color="info" %}} Note: On macOS the remote client is called "podman", since there is no local "podman" program available. {{% /pageinfo %}} @@ -155,10 +168,16 @@ Note: On macOS the remote client is called "podman", since there is no local "po {{% /mactab %}} {{% windowstab %}} +now you can 'build' against the storage inside minikube. which is instantly accessible to kubernetes cluster. + ```shell podman help ``` +```shell +podman build -t my_image . +``` + {{% pageinfo color="info" %}} Note: On Windows the remote client is called "podman", since there is no local "podman" program available. {{% /pageinfo %}} @@ -200,8 +219,10 @@ docker push $(minikube ip):5000/test-img ## 5. Building images inside of minikube using SSH -Use `minikube ssh` to run commands inside the minikube node, and run the `docker build` directly there. -Any command you run there will run against the same daemon that kubernetes cluster is using. +Use `minikube ssh` to run commands inside the minikube node, and run the build command directly there. +Any command you run there will run against the same daemon / storage that kubernetes cluster is using. + +For Docker, use: ```shell docker build @@ -209,15 +230,24 @@ docker build For more information on the `docker build` command, read the [Docker documentation](https://docs.docker.com/engine/reference/commandline/build/) (docker.com). -For Podman, use: +For CRI-O, use: ```shell sudo podman build ``` -For more information on the `podman build` command, read the [Podman documentation](https://github.com/containers/libpod/blob/master/docs/source/markdown/podman-build.1.md) (podman.io). +For more information on the `podman build` command, read the [Podman documentation](https://github.com/containers/podman/blob/master/docs/source/markdown/podman-build.1.md) (podman.io). + +For Containerd, use: + +```shell +sudo buildctl build +``` + +For more information on the `buildctl build` command, read the [Buildkit documentation](https://github.com/moby/buildkit#quick-start) (mobyproject.org). to exit minikube ssh and come back to your terminal type: + ```shell exit ``` From 80c3c64be04bd6c459fdfdbfa3ec6465e2941a39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sun, 13 Dec 2020 12:34:27 +0100 Subject: [PATCH 15/21] Fix minor typo in build of the project name itself --- site/content/en/docs/handbook/pushing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/content/en/docs/handbook/pushing.md b/site/content/en/docs/handbook/pushing.md index 0afec3daf1..25285b1b27 100644 --- a/site/content/en/docs/handbook/pushing.md +++ b/site/content/en/docs/handbook/pushing.md @@ -2,7 +2,7 @@ title: "Pushing images" weight: 5 description: > - comparing 5 ways to push your image into a minikiube cluster. + comparing 5 ways to push your image into a minikube cluster. aliases: - /docs/tasks/building - /docs/tasks/caching From 2b5ad68359524e83750bfc925f99d62256fdf4b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sun, 13 Dec 2020 15:57:00 +0100 Subject: [PATCH 16/21] Add persistent storage for /var/lib/buildkit Mostly to not fill up the tmpfs (RAM), most of the contents are expendable (although the cache can of course be useful) --- deploy/iso/minikube-iso/package/automount/minikube-automount | 4 ++++ site/content/en/docs/handbook/persistent_volumes.md | 3 +++ 2 files changed, 7 insertions(+) diff --git a/deploy/iso/minikube-iso/package/automount/minikube-automount b/deploy/iso/minikube-iso/package/automount/minikube-automount index f1a8484bee..478280ec15 100755 --- a/deploy/iso/minikube-iso/package/automount/minikube-automount +++ b/deploy/iso/minikube-iso/package/automount/minikube-automount @@ -105,6 +105,10 @@ if [ -n "$BOOT2DOCKER_DATA" ]; then mkdir -p /var/lib/containerd mount --bind /mnt/$PARTNAME/var/lib/containerd /var/lib/containerd + mkdir -p /mnt/$PARTNAME/var/lib/buildkit + mkdir -p /var/lib/buildkit + mount --bind /mnt/$PARTNAME/var/lib/buildkit /var/lib/buildkit + mkdir -p /mnt/$PARTNAME/var/lib/containers mkdir -p /var/lib/containers mount --bind /mnt/$PARTNAME/var/lib/containers /var/lib/containers diff --git a/site/content/en/docs/handbook/persistent_volumes.md b/site/content/en/docs/handbook/persistent_volumes.md index bb05a7944b..8f4966b5c7 100644 --- a/site/content/en/docs/handbook/persistent_volumes.md +++ b/site/content/en/docs/handbook/persistent_volumes.md @@ -18,6 +18,9 @@ minikube is configured to persist files stored under the following directories, * `/data` * `/var/lib/minikube` * `/var/lib/docker` +* `/var/lib/containerd` +* `/var/lib/buildkit` +* `/var/lib/containers` * `/tmp/hostpath_pv` * `/tmp/hostpath-provisioner` From 720a32d7d5b9adf3b3f39a29808e9a146a5e1181 Mon Sep 17 00:00:00 2001 From: Andrea Spadaccini Date: Sun, 13 Dec 2020 16:17:00 +0000 Subject: [PATCH 17/21] Add mention of the Windows Package Manager MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update the installation guide to mention winget, the Windows Package Manager. Minikube is in the repository, and the command I listed works correctly: andre@IRIDIUM-MMXX  C:\Windows\System32  [16:11] ❯ winget search minikube Name Id Version ------------------------------------ minikube Kubernetes.minikube 1.15.1 andre@IRIDIUM-MMXX  C:\Windows\System32  [16:12] ❯ winget install minikube Found minikube [Kubernetes.minikube] This application is licensed to you by its owner. Microsoft is not responsible for, nor does it grant any licences to, third-party packages. Downloading https://storage.googleapis.com/minikube/releases/latest/minikube-installer.exe ██████████████████████████████ 23.7 MB / 23.7 MB Successfully verified installer hash Starting package install... Successfully installed --- site/content/en/docs/start/_index.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/site/content/en/docs/start/_index.md b/site/content/en/docs/start/_index.md index 15b7b91882..85f2497263 100644 --- a/site/content/en/docs/start/_index.md +++ b/site/content/en/docs/start/_index.md @@ -98,12 +98,22 @@ sudo install minikube-darwin-amd64 /usr/local/bin/minikube {{% /mactab %}} {{% windowstab %}} -If the [Chocolatey Package Manager](https://chocolatey.org/) is installed, use it to install minikube: +### Windows Package Manager + +If the [Windows Package Manager](https://docs.microsoft.com/en-us/windows/package-manager/) is installed, use the following command to install minikube: + +```shell +winget install minikube +``` + +### Chocolatey +If the [Chocolatey Package Manager](https://chocolatey.org/) is installed, use the following command: ```shell choco install minikube ``` +### Stand-alone Windows Installer Otherwise, download and run the [Windows installer](https://storage.googleapis.com/minikube/releases/latest/minikube-installer.exe) {{% /windowstab %}} From 375d25db07afefc36302e52b634a11931f4809bd Mon Sep 17 00:00:00 2001 From: AUT0R3V <63261637+Aut0R3V@users.noreply.github.com> Date: Mon, 14 Dec 2020 15:12:12 +0530 Subject: [PATCH 18/21] Update start_flags.go Using slicing technique in the place of an Array for line 141. --- cmd/minikube/cmd/start_flags.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/minikube/cmd/start_flags.go b/cmd/minikube/cmd/start_flags.go index d5e7276c0c..3237156881 100644 --- a/cmd/minikube/cmd/start_flags.go +++ b/cmd/minikube/cmd/start_flags.go @@ -138,7 +138,7 @@ func initMinikubeFlags() { startCmd.Flags().String(containerRuntime, "docker", fmt.Sprintf("The container runtime to be used (%s).", strings.Join(cruntime.ValidRuntimes(), ", "))) startCmd.Flags().Bool(createMount, false, "This will start the mount daemon and automatically mount files into minikube.") startCmd.Flags().String(mountString, constants.DefaultMountDir+":/minikube-host", "The argument to pass the minikube mount command on start.") - startCmd.Flags().StringArrayVar(&config.AddonList, "addons", nil, "Enable addons. see `minikube addons list` for a list of valid addon names.") + startCmd.Flags().StringSliceVar(&config.AddonList, "addons", nil, "Enable addons. see `minikube addons list` for a list of valid addon names.") startCmd.Flags().String(criSocket, "", "The cri socket path to be used.") startCmd.Flags().String(networkPlugin, "", "Kubelet network plug-in to use (default: auto)") startCmd.Flags().Bool(enableDefaultCNI, false, "DEPRECATED: Replaced by --cni=bridge") From 72443e9b3ec3127c29000cb05e244c9469b8fd13 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Mon, 14 Dec 2020 11:43:42 -0800 Subject: [PATCH 19/21] release notes: fix whitespace before contributors list --- hack/release_notes.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hack/release_notes.sh b/hack/release_notes.sh index dabfc67153..8933f3475c 100755 --- a/hack/release_notes.sh +++ b/hack/release_notes.sh @@ -39,5 +39,6 @@ recent=$(git describe --abbrev=0) "${DIR}/release-notes" kubernetes minikube --since $recent -echo "Thank you to our contributors for this release! " +echo "Thank you to our contributors for this release!" +echo "" git log "$recent".. --format="%aN" --reverse | sort | uniq | awk '{printf "- %s\n", $0 }' From ce17200bd81258f1f13b7ff0f64403d612b648f2 Mon Sep 17 00:00:00 2001 From: "Jituri, Pranav" Date: Tue, 15 Dec 2020 18:09:43 +0530 Subject: [PATCH 20/21] address review comment --- pkg/drivers/kic/kic.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go index ded2622a95..df11094299 100644 --- a/pkg/drivers/kic/kic.go +++ b/pkg/drivers/kic/kic.go @@ -229,7 +229,7 @@ func (d *Driver) prepareSSH() error { } if !strings.Contains(string(icaclsCmdOut),"Successfully processed 1 files; Failed processing 0 files") { - return errors.Errorf("icacls failed applying permissions - err - [%s], output - [%s]", icaclsCmdErr, strings.TrimSpace(string(icaclsCmdOut))) + klog.Errorf("icacls failed applying permissions - err - [%s], output - [%s]", icaclsCmdErr, strings.TrimSpace(string(icaclsCmdOut))) } } } From a8463f34534add9feaa23852af60eac2526eb257 Mon Sep 17 00:00:00 2001 From: "Jituri, Pranav" Date: Tue, 15 Dec 2020 18:25:10 +0530 Subject: [PATCH 21/21] fix linting errors --- pkg/drivers/kic/kic.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go index df11094299..2d78bff52c 100644 --- a/pkg/drivers/kic/kic.go +++ b/pkg/drivers/kic/kic.go @@ -17,10 +17,10 @@ limitations under the License. package kic import ( + "context" "fmt" "net" "os/exec" - "context" "runtime" "strconv" "strings" @@ -215,20 +215,20 @@ func (d *Driver) prepareSSH() error { klog.Infof("ensuring only current user has permissions to key file located at : %s...", keyPath) // Get the SID of the current user - currentUserSidCmd := exec.CommandContext(ctx, path, "-NoProfile","-NonInteractive","([System.Security.Principal.WindowsIdentity]::GetCurrent()).User.Value") + currentUserSidCmd := exec.CommandContext(ctx, path, "-NoProfile", "-NonInteractive", "([System.Security.Principal.WindowsIdentity]::GetCurrent()).User.Value") currentUserSidOut, currentUserSidErr := currentUserSidCmd.CombinedOutput() if currentUserSidErr != nil { klog.Warningf("unable to determine current user's SID. minikube tunnel may not work.") } else { icaclsArguments := fmt.Sprintf(`"%s" /grant:r *%s:F /inheritancelevel:r`, keyPath, strings.TrimSpace(string(currentUserSidOut))) - icaclsCmd := exec.CommandContext(ctx, path, "-NoProfile","-NonInteractive","icacls.exe", icaclsArguments) + icaclsCmd := exec.CommandContext(ctx, path, "-NoProfile", "-NonInteractive", "icacls.exe", icaclsArguments) icaclsCmdOut, icaclsCmdErr := icaclsCmd.CombinedOutput() if icaclsCmdErr != nil { return errors.Wrap(icaclsCmdErr, "unable to execute icacls to set permissions") } - if !strings.Contains(string(icaclsCmdOut),"Successfully processed 1 files; Failed processing 0 files") { + if !strings.Contains(string(icaclsCmdOut), "Successfully processed 1 files; Failed processing 0 files") { klog.Errorf("icacls failed applying permissions - err - [%s], output - [%s]", icaclsCmdErr, strings.TrimSpace(string(icaclsCmdOut))) } }