diff --git a/cmd/minikube/cmd/docker-env.go b/cmd/minikube/cmd/docker-env.go index 5f34398ddd..414af71490 100644 --- a/cmd/minikube/cmd/docker-env.go +++ b/cmd/minikube/cmd/docker-env.go @@ -30,6 +30,7 @@ import ( "github.com/spf13/cobra" "k8s.io/klog/v2" + "k8s.io/minikube/pkg/drivers/kic/oci" "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/constants" @@ -43,7 +44,31 @@ import ( "k8s.io/minikube/pkg/minikube/sysinit" ) -var dockerEnvTmpl = fmt.Sprintf("{{ .Prefix }}%s{{ .Delimiter }}{{ .DockerTLSVerify }}{{ .Suffix }}{{ .Prefix }}%s{{ .Delimiter }}{{ .DockerHost }}{{ .Suffix }}{{ .Prefix }}%s{{ .Delimiter }}{{ .DockerCertPath }}{{ .Suffix }}{{ .Prefix }}%s{{ .Delimiter }}{{ .MinikubeDockerdProfile }}{{ .Suffix }}{{ if .NoProxyVar }}{{ .Prefix }}{{ .NoProxyVar }}{{ .Delimiter }}{{ .NoProxyValue }}{{ .Suffix }}{{end}}{{ .UsageHint }}", constants.DockerTLSVerifyEnv, constants.DockerHostEnv, constants.DockerCertPathEnv, constants.MinikubeActiveDockerdEnv) +var dockerSetEnvTmpl = fmt.Sprintf( + "{{ .Prefix }}%s{{ .Delimiter }}{{ .DockerTLSVerify }}{{ .Suffix }}"+ + "{{ .Prefix }}%s{{ .Delimiter }}{{ .DockerHost }}{{ .Suffix }}"+ + "{{ .Prefix }}%s{{ .Delimiter }}{{ .DockerCertPath }}{{ .Suffix }}"+ + "{{ if .ExistingDockerTLSVerify }}"+ + "{{ .Prefix }}%s{{ .Delimiter }}{{ .ExistingDockerTLSVerify }}{{ .Suffix }}"+ + "{{ end }}"+ + "{{ if .ExistingDockerHost }}"+ + "{{ .Prefix }}%s{{ .Delimiter }}{{ .ExistingDockerHost }}{{ .Suffix }}"+ + "{{ end }}"+ + "{{ if .ExistingDockerCertPath }}"+ + "{{ .Prefix }}%s{{ .Delimiter }}{{ .ExistingDockerCertPath }}{{ .Suffix }}"+ + "{{ end }}"+ + "{{ .Prefix }}%s{{ .Delimiter }}{{ .MinikubeDockerdProfile }}{{ .Suffix }}"+ + "{{ if .NoProxyVar }}"+ + "{{ .Prefix }}{{ .NoProxyVar }}{{ .Delimiter }}{{ .NoProxyValue }}{{ .Suffix }}"+ + "{{ end }}"+ + "{{ .UsageHint }}", + constants.DockerTLSVerifyEnv, + constants.DockerHostEnv, + constants.DockerCertPathEnv, + constants.ExistingDockerTLSVerifyEnv, + constants.ExistingDockerHostEnv, + constants.ExistingDockerCertPathEnv, + constants.MinikubeActiveDockerdEnv) // DockerShellConfig represents the shell config for Docker type DockerShellConfig struct { @@ -54,6 +79,10 @@ type DockerShellConfig struct { MinikubeDockerdProfile string NoProxyVar string NoProxyValue string + + ExistingDockerCertPath string + ExistingDockerHost string + ExistingDockerTLSVerify string } var ( @@ -81,6 +110,11 @@ func dockerShellCfgSet(ec DockerEnvConfig, envMap map[string]string) *DockerShel s.DockerCertPath = envMap[constants.DockerCertPathEnv] s.DockerHost = envMap[constants.DockerHostEnv] s.DockerTLSVerify = envMap[constants.DockerTLSVerifyEnv] + + s.ExistingDockerCertPath = envMap[constants.ExistingDockerCertPathEnv] + s.ExistingDockerHost = envMap[constants.ExistingDockerHostEnv] + s.ExistingDockerTLSVerify = envMap[constants.ExistingDockerTLSVerifyEnv] + s.MinikubeDockerdProfile = envMap[constants.MinikubeActiveDockerdEnv] if ec.noProxy { @@ -134,8 +168,17 @@ var dockerEnvCmd = &cobra.Command{ Short: "Configure environment to use minikube's Docker daemon", Long: `Sets up docker env variables; similar to '$(docker-machine env)'.`, Run: func(cmd *cobra.Command, args []string) { + var err error + + shl := shell.ForceShell + if shl == "" { + shl, err = shell.Detect() + if err != nil { + exit.Error(reason.InternalShellDetect, "Error detecting shell", err) + } + } sh := shell.EnvConfig{ - Shell: shell.ForceShell, + Shell: shl, } if dockerUnset { @@ -167,7 +210,6 @@ var dockerEnvCmd = &cobra.Command{ mustRestartDocker(cname, co.CP.Runner) } - var err error port := constants.DockerDaemonPort if driver.NeedsPortForward(driverName) { port, err = oci.ForwardedPort(driverName, cname, port) @@ -186,13 +228,6 @@ var dockerEnvCmd = &cobra.Command{ noProxy: noProxy, } - if ec.Shell == "" { - ec.Shell, err = shell.Detect() - if err != nil { - exit.Error(reason.InternalShellDetect, "Error detecting shell", err) - } - } - dockerPath, err := exec.LookPath("docker") if err != nil { klog.Warningf("Unable to find docker in path - skipping connectivity check: %v", err) @@ -228,7 +263,7 @@ type DockerEnvConfig struct { // dockerSetScript writes out a shell-compatible 'docker-env' script func dockerSetScript(ec DockerEnvConfig, w io.Writer) error { envVars := dockerEnvVars(ec) - return shell.SetScript(ec.EnvConfig, w, dockerEnvTmpl, dockerShellCfgSet(ec, envVars)) + return shell.SetScript(ec.EnvConfig, w, dockerSetEnvTmpl, dockerShellCfgSet(ec, envVars)) } // dockerSetScript writes out a shell-compatible 'docker-env unset' script @@ -246,7 +281,6 @@ func dockerUnsetScript(ec DockerEnvConfig, w io.Writer) error { vars = append(vars, k) } } - return shell.UnsetScript(ec.EnvConfig, w, vars) } @@ -257,14 +291,21 @@ func dockerURL(ip string, port int) string { // dockerEnvVars gets the necessary docker env variables to allow the use of minikube's docker daemon func dockerEnvVars(ec DockerEnvConfig) map[string]string { - env := map[string]string{ + rt := map[string]string{ constants.DockerTLSVerifyEnv: "1", constants.DockerHostEnv: dockerURL(ec.hostIP, ec.port), constants.DockerCertPathEnv: ec.certsDir, constants.MinikubeActiveDockerdEnv: ec.profile, } - - return env + if os.Getenv(constants.MinikubeActiveDockerdEnv) == "" { + for _, env := range constants.DockerDaemonEnvs { + if v := oci.InitialEnv(env); v != "" { + key := constants.MinikubeExistingPrefix + env + rt[key] = v + } + } + } + return rt } // dockerEnvVarsList gets the necessary docker env variables to allow the use of minikube's docker daemon to be used in a exec.Command diff --git a/cmd/minikube/cmd/docker-env_test.go b/cmd/minikube/cmd/docker-env_test.go index aaa1d97f5b..d34e51213e 100644 --- a/cmd/minikube/cmd/docker-env_test.go +++ b/cmd/minikube/cmd/docker-env_test.go @@ -52,7 +52,10 @@ export MINIKUBE_ACTIVE_DOCKERD="dockerdriver" # To point your shell to minikube's docker-daemon, run: # eval $(minikube -p dockerdriver docker-env) `, - `unset DOCKER_TLS_VERIFY DOCKER_HOST DOCKER_CERT_PATH MINIKUBE_ACTIVE_DOCKERD + `unset DOCKER_TLS_VERIFY; +unset DOCKER_HOST; +unset DOCKER_CERT_PATH; +unset MINIKUBE_ACTIVE_DOCKERD; `, }, { @@ -67,7 +70,10 @@ export MINIKUBE_ACTIVE_DOCKERD="bash" # To point your shell to minikube's docker-daemon, run: # eval $(minikube -p bash docker-env) `, - `unset DOCKER_TLS_VERIFY DOCKER_HOST DOCKER_CERT_PATH MINIKUBE_ACTIVE_DOCKERD + `unset DOCKER_TLS_VERIFY; +unset DOCKER_HOST; +unset DOCKER_CERT_PATH; +unset MINIKUBE_ACTIVE_DOCKERD; `, }, { @@ -82,7 +88,10 @@ export MINIKUBE_ACTIVE_DOCKERD="ipv6" # To point your shell to minikube's docker-daemon, run: # eval $(minikube -p ipv6 docker-env) `, - `unset DOCKER_TLS_VERIFY DOCKER_HOST DOCKER_CERT_PATH MINIKUBE_ACTIVE_DOCKERD + `unset DOCKER_TLS_VERIFY; +unset DOCKER_HOST; +unset DOCKER_CERT_PATH; +unset MINIKUBE_ACTIVE_DOCKERD; `, }, { @@ -115,7 +124,10 @@ $Env:MINIKUBE_ACTIVE_DOCKERD = "powershell" # & minikube -p powershell docker-env | Invoke-Expression `, - `Remove-Item Env:\\DOCKER_TLS_VERIFY Env:\\DOCKER_HOST Env:\\DOCKER_CERT_PATH Env:\\MINIKUBE_ACTIVE_DOCKERD + `Remove-Item Env:\\DOCKER_TLS_VERIFY +Remove-Item Env:\\DOCKER_HOST +Remove-Item Env:\\DOCKER_CERT_PATH +Remove-Item Env:\\MINIKUBE_ACTIVE_DOCKERD `, }, { @@ -167,7 +179,11 @@ export NO_PROXY="127.0.0.1" # eval $(minikube -p bash-no-proxy docker-env) `, - `unset DOCKER_TLS_VERIFY DOCKER_HOST DOCKER_CERT_PATH MINIKUBE_ACTIVE_DOCKERD NO_PROXY + `unset DOCKER_TLS_VERIFY; +unset DOCKER_HOST; +unset DOCKER_CERT_PATH; +unset MINIKUBE_ACTIVE_DOCKERD; +unset NO_PROXY; `, }, { @@ -184,7 +200,11 @@ export no_proxy="127.0.0.1" # eval $(minikube -p bash-no-proxy-lower docker-env) `, - `unset DOCKER_TLS_VERIFY DOCKER_HOST DOCKER_CERT_PATH MINIKUBE_ACTIVE_DOCKERD no_proxy + `unset DOCKER_TLS_VERIFY; +unset DOCKER_HOST; +unset DOCKER_CERT_PATH; +unset MINIKUBE_ACTIVE_DOCKERD; +unset no_proxy; `, }, { @@ -200,7 +220,11 @@ $Env:no_proxy = "192.168.0.1" # & minikube -p powershell-no-proxy-idempotent docker-env | Invoke-Expression `, - `Remove-Item Env:\\DOCKER_TLS_VERIFY Env:\\DOCKER_HOST Env:\\DOCKER_CERT_PATH Env:\\MINIKUBE_ACTIVE_DOCKERD Env:\\no_proxy + `Remove-Item Env:\\DOCKER_TLS_VERIFY +Remove-Item Env:\\DOCKER_HOST +Remove-Item Env:\\DOCKER_CERT_PATH +Remove-Item Env:\\MINIKUBE_ACTIVE_DOCKERD +Remove-Item Env:\\no_proxy `, }, { @@ -217,7 +241,11 @@ export NO_PROXY="192.168.0.1,10.0.0.4,127.0.0.1" # eval $(minikube -p sh-no-proxy-add docker-env) `, - `unset DOCKER_TLS_VERIFY DOCKER_HOST DOCKER_CERT_PATH MINIKUBE_ACTIVE_DOCKERD NO_PROXY + `unset DOCKER_TLS_VERIFY; +unset DOCKER_HOST; +unset DOCKER_CERT_PATH; +unset MINIKUBE_ACTIVE_DOCKERD; +unset NO_PROXY; `, }, { @@ -229,7 +257,10 @@ DOCKER_HOST=tcp://127.0.0.1:32842 DOCKER_CERT_PATH=/certs MINIKUBE_ACTIVE_DOCKERD=noneshell `, - `DOCKER_TLS_VERIFY DOCKER_HOST DOCKER_CERT_PATH MINIKUBE_ACTIVE_DOCKERD + `DOCKER_TLS_VERIFY +DOCKER_HOST +DOCKER_CERT_PATH +MINIKUBE_ACTIVE_DOCKERD `, }, } diff --git a/cmd/minikube/cmd/podman-env_test.go b/cmd/minikube/cmd/podman-env_test.go index 6dbb1eb329..7215fe9a1f 100644 --- a/cmd/minikube/cmd/podman-env_test.go +++ b/cmd/minikube/cmd/podman-env_test.go @@ -49,7 +49,8 @@ export MINIKUBE_ACTIVE_PODMAN="bash" # To point your shell to minikube's podman service, run: # eval $(minikube -p bash podman-env) `, - `unset PODMAN_VARLINK_BRIDGE MINIKUBE_ACTIVE_PODMAN + `unset PODMAN_VARLINK_BRIDGE; +unset MINIKUBE_ACTIVE_PODMAN; `, }, { @@ -62,7 +63,9 @@ export MINIKUBE_ACTIVE_PODMAN="bash" # To point your shell to minikube's podman service, run: # eval $(minikube -p bash podman-env) `, - `unset CONTAINER_HOST CONTAINER_SSHKEY MINIKUBE_ACTIVE_PODMAN + `unset CONTAINER_HOST; +unset CONTAINER_SSHKEY; +unset MINIKUBE_ACTIVE_PODMAN; `, }, } diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go index 10c5aebdcc..4c47a478cf 100644 --- a/pkg/drivers/kic/kic.go +++ b/pkg/drivers/kic/kic.go @@ -30,6 +30,7 @@ import ( "github.com/docker/machine/libmachine/state" "github.com/pkg/errors" "k8s.io/klog/v2" + pkgdrivers "k8s.io/minikube/pkg/drivers" "k8s.io/minikube/pkg/drivers/kic/oci" "k8s.io/minikube/pkg/minikube/assets" @@ -93,22 +94,30 @@ func (d *Driver) Create() error { klog.Infof("calculated static IP %q for the %q container", ip.String(), d.NodeConfig.MachineName) params.IP = ip.String() } + drv := d.DriverName() + listAddr := oci.DefaultBindIPV4 + if oci.IsExternalDaemonHost(drv) { + out.WarningT("Listening to 0.0.0.0 on external docker host {{.host}}. Please be advised", + out.V{"host": oci.DaemonHost(drv)}) + listAddr = "0.0.0.0" + } // control plane specific options - params.PortMappings = append(params.PortMappings, oci.PortMapping{ - ListenAddress: oci.DefaultBindIPV4, - ContainerPort: int32(params.APIServerPort), - }, + params.PortMappings = append(params.PortMappings, oci.PortMapping{ - ListenAddress: oci.DefaultBindIPV4, + ListenAddress: listAddr, + ContainerPort: int32(params.APIServerPort), + }, + oci.PortMapping{ + ListenAddress: listAddr, ContainerPort: constants.SSHPort, }, oci.PortMapping{ - ListenAddress: oci.DefaultBindIPV4, + ListenAddress: listAddr, ContainerPort: constants.DockerDaemonPort, }, oci.PortMapping{ - ListenAddress: oci.DefaultBindIPV4, + ListenAddress: listAddr, ContainerPort: constants.RegistryAddonPort, }, ) @@ -224,12 +233,12 @@ func (d *Driver) GetIP() (string, error) { // GetExternalIP returns an IP which is accessible from outside func (d *Driver) GetExternalIP() (string, error) { - return oci.DefaultBindIPV4, nil + return oci.DaemonHost(d.DriverName()), nil } // GetSSHHostname returns hostname for use with ssh func (d *Driver) GetSSHHostname() (string, error) { - return oci.DefaultBindIPV4, nil + return oci.DaemonHost(d.DriverName()), nil } // GetSSHPort returns port for use with ssh diff --git a/pkg/drivers/kic/oci/cli_runner.go b/pkg/drivers/kic/oci/cli_runner.go index eb7b5abdb4..b141305f7d 100644 --- a/pkg/drivers/kic/oci/cli_runner.go +++ b/pkg/drivers/kic/oci/cli_runner.go @@ -27,6 +27,7 @@ import ( "time" "k8s.io/klog/v2" + "k8s.io/minikube/pkg/minikube/out" "k8s.io/minikube/pkg/minikube/style" ) diff --git a/pkg/drivers/kic/oci/env.go b/pkg/drivers/kic/oci/env.go new file mode 100644 index 0000000000..8bedcf4d0c --- /dev/null +++ b/pkg/drivers/kic/oci/env.go @@ -0,0 +1,48 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package oci + +import ( + "os" + + "k8s.io/minikube/pkg/minikube/constants" +) + +var initialEnvs = make(map[string]string) + +func init() { + for _, env := range constants.DockerDaemonEnvs { + if v, set := os.LookupEnv(env); set { + initialEnvs[env] = v + } + exEnv := constants.MinikubeExistingPrefix + env + if v, set := os.LookupEnv(exEnv); set { + initialEnvs[exEnv] = v + } + } +} + +// InitialEnv returns the value of the environment variable env before any environment changes made by minikube +func InitialEnv(env string) string { + return initialEnvs[env] +} + +// LookupInitialEnv returns the value of the environment variable env before any environment changes made by minikube +func LookupInitialEnv(env string) (string, bool) { + v, set := initialEnvs[env] + return v, set +} diff --git a/pkg/drivers/kic/oci/network.go b/pkg/drivers/kic/oci/network.go index 9ca08fda28..a3c7edb94f 100644 --- a/pkg/drivers/kic/oci/network.go +++ b/pkg/drivers/kic/oci/network.go @@ -164,6 +164,5 @@ func dockerContainerIP(name string) (string, string, error) { if len(ips) != 2 { return "", "", errors.Errorf("container addresses should have 2 values, got %d values: %+v", len(ips), ips) } - return ips[0], ips[1], nil } diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go index 96783e58c2..087d6224f4 100644 --- a/pkg/drivers/kic/oci/oci.go +++ b/pkg/drivers/kic/oci/oci.go @@ -17,26 +17,26 @@ limitations under the License. package oci import ( - "context" - "os" - "time" - "bufio" "bytes" + "context" + "fmt" + "net/url" + "os" + "os/exec" + "runtime" + "strconv" + "strings" + "time" "github.com/docker/machine/libmachine/state" "github.com/pkg/errors" "k8s.io/klog/v2" + "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/out" "k8s.io/minikube/pkg/util/retry" - - "fmt" - "os/exec" - "runtime" - "strconv" - "strings" ) // DeleteContainersByLabel deletes all containers that have a specific label @@ -521,18 +521,29 @@ func ListContainersByLabel(ociBin string, label string, warnSlow ...bool) ([]str // PointToHostDockerDaemon will unset env variables that point to docker inside minikube // to make sure it points to the docker daemon installed by user. func PointToHostDockerDaemon() error { - p := os.Getenv(constants.MinikubeActiveDockerdEnv) - if p != "" { + + if p := os.Getenv(constants.MinikubeActiveDockerdEnv); p != "" { klog.Infof("shell is pointing to dockerd inside minikube. will unset to use host") + for _, e := range constants.DockerDaemonEnvs { + if err := resetEnv(e); err != nil { + return err + } + } } - for i := range constants.DockerDaemonEnvs { - e := constants.DockerDaemonEnvs[i] - err := os.Setenv(e, "") - if err != nil { - return errors.Wrapf(err, "resetting %s env", e) - } + return nil +} +func resetEnv(key string) error { + v := os.Getenv(constants.MinikubeExistingPrefix + key) + if v == "" { + if err := os.Unsetenv(key); err != nil { + return errors.Wrapf(err, "resetting %s env", key) + } + return nil + } + if err := os.Setenv(key, v); err != nil { + return errors.Wrapf(err, "resetting %s env", key) } return nil } @@ -628,3 +639,36 @@ func iptablesFileExists(ociBin string, nameOrID string) bool { } return true } + +// DaemonHost returns the ip/hostname where OCI daemon service for driver is running +// For Podman it's always DefaultBindIPV4 +// For Docker return the host part of DOCKER_HOST environment variable if set +// or DefaultBindIPV4 otherwise +func DaemonHost(driver string) string { + if driver != Docker { + return DefaultBindIPV4 + } + if dh := os.Getenv(constants.DockerHostEnv); dh != "" { + if u, err := url.Parse(dh); err == nil { + if u.Host != "" { + return u.Hostname() + } + } + } + return DefaultBindIPV4 +} + +// IsExternalDaemonHost returns whether or not the OCI runtime is running on an external/virtual host +// For Podman driver it's always false for now +// For Docker driver return true if DOCKER_HOST is set to a URI, and the URI contains a host item +func IsExternalDaemonHost(driver string) bool { + if driver != Docker { + return false + } + if dh := os.Getenv(constants.DockerHostEnv); dh != "" { + if u, err := url.Parse(dh); err == nil { + return u.Host != "" + } + } + return false +} diff --git a/pkg/drivers/kic/oci/oci_test.go b/pkg/drivers/kic/oci/oci_test.go new file mode 100644 index 0000000000..cbe6b98cc0 --- /dev/null +++ b/pkg/drivers/kic/oci/oci_test.go @@ -0,0 +1,97 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package oci + +import ( + "os" + "testing" +) + +func TestPointToHostDockerDaemonEmpty(t *testing.T) { + _ = os.Setenv("DOCKER_HOST", "foo_host") + _ = os.Setenv("DOCKER_CERT_PATH", "foo_cert_path") + _ = os.Setenv("DOCKER_TLS_VERIFY", "foo_tls_verify") + _ = os.Setenv("MINIKUBE_ACTIVE_DOCKERD", "minikube") + + _ = os.Unsetenv("MINIKUBE_EXISTING_DOCKER_HOST") + _ = os.Unsetenv("MINIKUBE_EXISTING_DOCKER_CERT_PATH") + _ = os.Unsetenv("MINIKUBE_EXISTING_DOCKER_TLS_VERIFY") + + if err := PointToHostDockerDaemon(); err != nil { + t.Fatalf("failed to set docker environment: got %v", err) + } + + for _, key := range []string{ + "DOCKER_HOST", "DOCKER_CERT_PATH", "DOCKER_TLS_VERIFY", + } { + if v, set := os.LookupEnv(key); set { + t.Errorf("%v env variable should not be set. got: %v", key, v) + } + } +} + +func TestPointToHostDockerDaemon(t *testing.T) { + _ = os.Setenv("DOCKER_HOST", "foo_host") + _ = os.Setenv("DOCKER_CERT_PATH", "foo_cert_path") + _ = os.Setenv("DOCKER_TLS_VERIFY", "foo_tls_verify") + + _ = os.Setenv("MINIKUBE_EXISTING_DOCKER_HOST", "bar_host") + _ = os.Setenv("MINIKUBE_EXISTING_DOCKER_CERT_PATH", "bar_cert_path") + _ = os.Setenv("MINIKUBE_EXISTING_DOCKER_TLS_VERIFY", "bar_tls_verify") + + if err := PointToHostDockerDaemon(); err != nil { + t.Fatalf("failed to set docker environment: got %v", err) + } + + expected := []struct { + key, value string + }{ + {"DOCKER_HOST", "bar_host"}, + {"DOCKER_CERT_PATH", "bar_cert_path"}, + {"DOCKER_TLS_VERIFY", "bar_tls_verify"}, + } + for _, exp := range expected { + if v := os.Getenv(exp.key); v != exp.value { + t.Errorf("invalid %v env variable. got: %v, want: %v", exp.value, v, exp.value) + } + } +} + +func TestDaemonHost(t *testing.T) { + tests := []struct { + driver string + dockerHost string + expectedAddr string + expectedExternal bool + }{ + {"", "", "127.0.0.1", false}, + {"docker", "tcp://1.1.1.1:2222/foo", "1.1.1.1", true}, + {"podman", "tcp://1.1.1.1:2222/foo", "127.0.0.1", false}, + {"_invalid_", "tcp://1.1.1.1:2222/foo", "127.0.0.1", false}, + {"docker", "unix:///var/run/something", "127.0.0.1", false}, + {"docker", "tcp://127.0.0.1/foo", "127.0.0.1", true}, + } + for _, test := range tests { + _ = os.Setenv("DOCKER_HOST", test.dockerHost) + if v := IsExternalDaemonHost(test.driver); v != test.expectedExternal { + t.Errorf("invalid result of IsExternalDaemonHost. got: %v, want: %v", v, test.expectedExternal) + } + if v := DaemonHost(test.driver); v != test.expectedAddr { + t.Errorf("invalid oci daemon host. got: %v, want: %v", v, test.expectedAddr) + } + } +} diff --git a/pkg/drivers/kic/oci/test b/pkg/drivers/kic/oci/test deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/pkg/minikube/bootstrapper/certs.go b/pkg/minikube/bootstrapper/certs.go index f5181b8ce7..90fd9870fa 100644 --- a/pkg/minikube/bootstrapper/certs.go +++ b/pkg/minikube/bootstrapper/certs.go @@ -194,9 +194,13 @@ func generateProfileCerts(k8s config.KubernetesConfig, n config.Node, ccs CACert return nil, errors.Wrap(err, "getting service cluster ip") } - apiServerIPs := append( - k8s.APIServerIPs, - []net.IP{net.ParseIP(n.IP), serviceIP, net.ParseIP(oci.DefaultBindIPV4), net.ParseIP("10.0.0.1")}...) + apiServerIPs := append(k8s.APIServerIPs, + net.ParseIP(n.IP), serviceIP, net.ParseIP(oci.DefaultBindIPV4), net.ParseIP("10.0.0.1")) + + if v := oci.DaemonHost(k8s.ContainerRuntime); v != oci.DefaultBindIPV4 { + apiServerIPs = append(apiServerIPs, net.ParseIP(v)) + } + apiServerNames := append(k8s.APIServerNames, k8s.APIServerName, constants.ControlPlaneAlias) apiServerAlternateNames := append( apiServerNames, diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index 4fa3f8b79c..99c5128a40 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -79,6 +79,16 @@ const ( MinikubeForceSystemdEnv = "MINIKUBE_FORCE_SYSTEMD" // TestDiskUsedEnv is used in integration tests for insufficient storage with 'minikube status' TestDiskUsedEnv = "MINIKUBE_TEST_STORAGE_CAPACITY" + + // MinikubeExistingPrefix is used to save the original environment when executing docker-env + MinikubeExistingPrefix = "MINIKUBE_EXISTING_" + + // ExistingDockerHostEnv is used to save original docker environment + ExistingDockerHostEnv = MinikubeExistingPrefix + "DOCKER_HOST" + // ExistingDockerCertPathEnv is used to save original docker environment + ExistingDockerCertPathEnv = MinikubeExistingPrefix + "DOCKER_CERT_PATH" + // ExistingDockerTLSVerifyEnv is used to save original docker environment + ExistingDockerTLSVerifyEnv = MinikubeExistingPrefix + "DOCKER_TLS_VERIFY" ) var ( @@ -94,6 +104,9 @@ var ( // DockerDaemonEnvs is list of docker-daemon related environment variables. DockerDaemonEnvs = [3]string{DockerHostEnv, DockerTLSVerifyEnv, DockerCertPathEnv} + // ExistingDockerDaemonEnvs is list of docker-daemon related environment variables. + ExistingDockerDaemonEnvs = [3]string{ExistingDockerHostEnv, ExistingDockerTLSVerifyEnv, ExistingDockerCertPathEnv} + // PodmanRemoteEnvs is list of podman-remote related environment variables. PodmanRemoteEnvs = [1]string{PodmanVarlinkBridgeEnv} diff --git a/pkg/minikube/driver/driver.go b/pkg/minikube/driver/driver.go index 7a5ff92828..91927b54a0 100644 --- a/pkg/minikube/driver/driver.go +++ b/pkg/minikube/driver/driver.go @@ -151,8 +151,14 @@ func NeedsRoot(name string) bool { // NeedsPortForward returns true if driver is unable provide direct IP connectivity func NeedsPortForward(name string) bool { + if !IsKIC(name) { + return false + } + if oci.IsExternalDaemonHost(name) { + return true + } // Docker for Desktop - return IsKIC(name) && (runtime.GOOS == "darwin" || runtime.GOOS == "windows" || IsMicrosoftWSL()) + return runtime.GOOS == "darwin" || runtime.GOOS == "windows" || IsMicrosoftWSL() } // IsMicrosoftWSL will return true if process is running in WSL in windows diff --git a/pkg/minikube/driver/endpoint.go b/pkg/minikube/driver/endpoint.go index 1f8ac92ff9..e72e10c9fc 100644 --- a/pkg/minikube/driver/endpoint.go +++ b/pkg/minikube/driver/endpoint.go @@ -29,7 +29,8 @@ import ( func ControlPlaneEndpoint(cc *config.ClusterConfig, cp *config.Node, driverName string) (string, net.IP, int, error) { if NeedsPortForward(driverName) { port, err := oci.ForwardedPort(cc.Driver, cc.Name, cp.Port) - hostname := oci.DefaultBindIPV4 + hostname := oci.DaemonHost(driverName) + ip := net.ParseIP(hostname) if ip == nil { return hostname, ip, port, fmt.Errorf("failed to parse ip for %q", hostname) diff --git a/pkg/minikube/shell/shell.go b/pkg/minikube/shell/shell.go index 5d562ba653..f224c41c9c 100644 --- a/pkg/minikube/shell/shell.go +++ b/pkg/minikube/shell/shell.go @@ -24,12 +24,21 @@ import ( "io" "os" "runtime" - "strings" "text/template" "github.com/docker/machine/libmachine/shell" + + "k8s.io/minikube/pkg/minikube/constants" ) +var unsetEnvTmpl = "{{ $root := .}}" + + "{{ range .Unset }}" + + "{{ $root.UnsetPrefix }}{{ . }}{{ $root.UnsetDelimiter }}{{ $root.UnsetSuffix }}" + + "{{ end }}" + + "{{ range .Set }}" + + "{{ $root.SetPrefix }}{{ .Env }}{{ $root.SetDelimiter }}{{ .Value }}{{ $root.SetSuffix }}" + + "{{ end }}" + // Config represents the shell config type Config struct { Prefix string @@ -107,7 +116,7 @@ REM @FOR /f "tokens=*" %%i IN ('%s') DO @%%i suffix: "\"\n", delimiter: "=\"", unsetPrefix: "unset ", - unsetSuffix: "\n", + unsetSuffix: ";\n", unsetDelimiter: "", usageHint: func(s ...interface{}) string { return fmt.Sprintf(` @@ -181,24 +190,46 @@ func SetScript(ec EnvConfig, w io.Writer, envTmpl string, data interface{}) erro return tmpl.Execute(w, data) } +type unsetConfigItem struct { + Env, Value string +} +type unsetConfig struct { + Set []unsetConfigItem + Unset []string + SetPrefix string + SetDelimiter string + SetSuffix string + UnsetPrefix string + UnsetDelimiter string + UnsetSuffix string +} + // UnsetScript writes out a shell-compatible unset script func UnsetScript(ec EnvConfig, w io.Writer, vars []string) error { - var sb strings.Builder shellCfg := ec.getShell() - pfx, sfx, delim := shellCfg.unsetPrefix, shellCfg.unsetSuffix, shellCfg.unsetDelimiter - switch ec.Shell { - case "cmd", "emacs", "fish": - break - case "powershell": - vars = []string{strings.Join(vars, " Env:\\\\")} - default: - vars = []string{strings.Join(vars, " ")} + cfg := unsetConfig{ + SetPrefix: shellCfg.prefix, + SetDelimiter: shellCfg.delimiter, + SetSuffix: shellCfg.suffix, + UnsetPrefix: shellCfg.unsetPrefix, + UnsetDelimiter: shellCfg.unsetDelimiter, + UnsetSuffix: shellCfg.unsetSuffix, } - for _, v := range vars { - if _, err := sb.WriteString(fmt.Sprintf("%s%s%s%s", pfx, v, delim, sfx)); err != nil { - return err + var tempUnset []string + for _, env := range vars { + exEnv := constants.MinikubeExistingPrefix + env + if v := os.Getenv(exEnv); v == "" { + cfg.Unset = append(cfg.Unset, env) + } else { + cfg.Set = append(cfg.Set, unsetConfigItem{ + Env: env, + Value: v, + }) + tempUnset = append(tempUnset, exEnv) } } - _, err := w.Write([]byte(sb.String())) - return err + cfg.Unset = append(cfg.Unset, tempUnset...) + + tmpl := template.Must(template.New("unsetEnv").Parse(unsetEnvTmpl)) + return tmpl.Execute(w, &cfg) } diff --git a/pkg/minikube/shell/shell_test.go b/pkg/minikube/shell/shell_test.go index 43dcdcaa71..cf8de26c26 100644 --- a/pkg/minikube/shell/shell_test.go +++ b/pkg/minikube/shell/shell_test.go @@ -87,16 +87,19 @@ func TestUnsetScript(t *testing.T) { ec EnvConfig expected string }{ - {[]string{"baz", "bar"}, EnvConfig{""}, `unset baz bar`}, - {[]string{"baz", "bar"}, EnvConfig{"bash"}, `unset baz bar`}, - {[]string{"baz", "bar"}, EnvConfig{"powershell"}, `Remove-Item Env:\\baz Env:\\bar`}, + {[]string{"baz", "bar"}, EnvConfig{""}, `unset baz; +unset bar;`}, + {[]string{"baz", "bar"}, EnvConfig{"bash"}, `unset baz; +unset bar;`}, + {[]string{"baz", "bar"}, EnvConfig{"powershell"}, `Remove-Item Env:\\baz +Remove-Item Env:\\bar`}, {[]string{"baz", "bar"}, EnvConfig{"cmd"}, `SET baz= SET bar=`}, {[]string{"baz", "bar"}, EnvConfig{"fish"}, `set -e baz; set -e bar;`}, {[]string{"baz", "bar"}, EnvConfig{"emacs"}, `(setenv "baz" nil) (setenv "bar" nil)`}, - {[]string{"baz", "bar"}, EnvConfig{"none"}, `baz bar`}, + {[]string{"baz", "bar"}, EnvConfig{"none"}, "baz\nbar"}, } for _, tc := range testCases { tc := tc diff --git a/pkg/provision/provision.go b/pkg/provision/provision.go index fe6e05c93d..713f94ea4a 100644 --- a/pkg/provision/provision.go +++ b/pkg/provision/provision.go @@ -36,6 +36,7 @@ import ( "github.com/docker/machine/libmachine/swarm" "github.com/pkg/errors" "k8s.io/klog/v2" + "k8s.io/minikube/pkg/minikube/assets" "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/config" @@ -96,12 +97,17 @@ func configureAuth(p miniProvisioner) error { return errors.Wrap(err, "error getting ip during provisioning") } + hostIP, err := driver.GetSSHHostname() + if err != nil { + return errors.Wrap(err, "error getting ssh hostname during provisioning") + } + if err := copyHostCerts(authOptions); err != nil { return err } // The Host IP is always added to the certificate's SANs list - hosts := append(authOptions.ServerCertSANs, ip, "localhost", "127.0.0.1", "minikube", machineName) + hosts := append(authOptions.ServerCertSANs, ip, hostIP, "localhost", "127.0.0.1", "minikube", machineName) klog.Infof("generating server cert: %s ca-key=%s private-key=%s org=%s san=%s", authOptions.ServerCertPath, authOptions.CaCertPath,