From 3d1b44055adad7e03143a6f957c5b2d808d258a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A2=D0=BE=D0=B2=D0=B0=D1=80=D0=B8=D1=89=20=D0=BF=D1=80?= =?UTF-8?q?=D0=BE=D0=B3=D1=80=D0=B0=D0=BC=D0=BC=D0=B8=D1=81=D1=82?= <2962928213@qq.com> Date: Sun, 2 Jul 2023 02:52:41 +0800 Subject: [PATCH] Add auto add-host and ssh-agent for docker-env --- cmd/minikube/cmd/docker-env.go | 22 ++++++- cmd/minikube/cmd/ssh-host.go | 108 ++++++++++++++++---------------- pkg/minikube/reason/reason.go | 2 + test/integration/docker_test.go | 48 +++++--------- 4 files changed, 92 insertions(+), 88 deletions(-) diff --git a/cmd/minikube/cmd/docker-env.go b/cmd/minikube/cmd/docker-env.go index 449408924c..daf294f283 100644 --- a/cmd/minikube/cmd/docker-env.go +++ b/cmd/minikube/cmd/docker-env.go @@ -48,6 +48,7 @@ import ( "k8s.io/minikube/pkg/minikube/out" "k8s.io/minikube/pkg/minikube/reason" "k8s.io/minikube/pkg/minikube/shell" + "k8s.io/minikube/pkg/minikube/sshagent" "k8s.io/minikube/pkg/minikube/sysinit" pkgnetwork "k8s.io/minikube/pkg/network" kconst "k8s.io/minikube/third_party/kubeadm/app/constants" @@ -296,6 +297,14 @@ docker-cli install instructions: https://minikube.sigs.k8s.io/docs/tutorials/doc } cname := ClusterFlagValue() + + // start the ssh-agent + // this must be done before the cluster config is loaded + // otherwise we won't be able to get SSH_AUTH_SOCK and SSH_AGENT_PID from cluster config. + if err := sshagent.Start(cname); err != nil { + exit.Message(reason.SshAgentStart, err.Error()) + } + co := mustload.Running(cname) driverName := co.CP.Host.DriverName @@ -316,6 +325,7 @@ docker-cli install instructions: https://minikube.sigs.k8s.io/docs/tutorials/doc // for the sake of docker-env command, start nerdctl and nerdctld if cr == constants.Containerd { out.WarningT("Using the docker-env command with the containerd runtime is a highly experimental feature, please provide feedback or contribute to make it better") + startNerdctld() // docker-env on containerd depends on nerdctld (https://github.com/afbjorklund/nerdctld) as "docker" daeomn @@ -328,6 +338,10 @@ docker-cli install instructions: https://minikube.sigs.k8s.io/docs/tutorials/doc out.WarningT("Please ensure you have executed 'ssh-agent bash' and 'minikube ssh-host --append-known' in this shell before using docker-env on containerd. Ignore this message if you have done it") } + // set the ssh-agent envs for current process + os.Setenv("SSH_AUTH_SOCK", co.Config.SSHAuthSock) + os.Setenv("SSH_AGENT_PID", strconv.Itoa(co.Config.SSHAgentPID)) + r := co.CP.Runner if cr == constants.Docker { @@ -400,14 +414,18 @@ docker-cli install instructions: https://minikube.sigs.k8s.io/docs/tutorials/doc if err != nil { exit.Error(reason.IfSSHClient, "Error with ssh-add", err) } - cmd := exec.Command(path, d.GetSSHKeyPath()) cmd.Stderr = os.Stderr + cmd.Env = append(cmd.Env, fmt.Sprintf("SSH_AUTH_SOCK=%s", co.Config.SSHAuthSock)) + cmd.Env = append(cmd.Env, fmt.Sprintf("SSH_AGENT_PID=%d", co.Config.SSHAgentPID)) err = cmd.Run() if err != nil { exit.Error(reason.IfSSHClient, "Error with ssh-add", err) } } + + // eventually, run something similar to ssh --append-known + appendKnownHelper(nodeName, true) }, } @@ -558,6 +576,8 @@ func dockerEnvVars(ec DockerEnvConfig) map[string]string { envSSH := map[string]string{ constants.DockerHostEnv: sshURL(ec.username, ec.hostname, ec.sshport), constants.MinikubeActiveDockerdEnv: ec.profile, + constants.SSHAuthSock: ec.sshAuthSock, + constants.SSHAgentPID: agentPID, } var rt map[string]string diff --git a/cmd/minikube/cmd/ssh-host.go b/cmd/minikube/cmd/ssh-host.go index 1755c30ea7..ffec41d3dc 100644 --- a/cmd/minikube/cmd/ssh-host.go +++ b/cmd/minikube/cmd/ssh-host.go @@ -45,69 +45,71 @@ var sshHostCmd = &cobra.Command{ Short: "Retrieve the ssh host key of the specified node", Long: "Retrieve the ssh host key of the specified node.", Run: func(cmd *cobra.Command, args []string) { - cname := ClusterFlagValue() - co := mustload.Running(cname) - if co.CP.Host.DriverName == driver.None { - exit.Message(reason.Usage, "'none' driver does not support 'minikube ssh-host' command") - } + appendKnownHelper(nodeName, appendKnown) + }, +} - var err error - var n *config.Node - if nodeName == "" { - n = co.CP.Node - } else { - n, _, err = node.Retrieve(*co.Config, nodeName) - if err != nil { - exit.Message(reason.GuestNodeRetrieve, "Node {{.nodeName}} does not exist.", out.V{"nodeName": nodeName}) - } - } +func appendKnownHelper(nodeName string, appendKnown bool) { + cname := ClusterFlagValue() + co := mustload.Running(cname) + if co.CP.Host.DriverName == driver.None { + exit.Message(reason.Usage, "'none' driver does not support 'minikube ssh-host' command") + } - scanArgs := []string{"-t", "rsa"} - - keys, err := machine.RunSSHHostCommand(co.API, *co.Config, *n, "ssh-keyscan", scanArgs) + var err error + var n *config.Node + if nodeName == "" { + n = co.CP.Node + } else { + n, _, err = node.Retrieve(*co.Config, nodeName) if err != nil { - // This is typically due to a non-zero exit code, so no need for flourish. - out.ErrLn("ssh-keyscan: %v", err) - // It'd be nice if we could pass up the correct error code here :( + exit.Message(reason.GuestNodeRetrieve, "Node {{.nodeName}} does not exist.", out.V{"nodeName": nodeName}) + } + } + + scanArgs := []string{"-t", "rsa"} + + keys, err := machine.RunSSHHostCommand(co.API, *co.Config, *n, "ssh-keyscan", scanArgs) + if err != nil { + // This is typically due to a non-zero exit code, so no need for flourish. + out.ErrLn("ssh-keyscan: %v", err) + // It'd be nice if we could pass up the correct error code here :( + os.Exit(1) + } + + if appendKnown { + addr, port, err := machine.GetSSHHostAddrPort(co.API, *co.Config, *n) + if err != nil { + out.ErrLn("GetSSHHostAddrPort: %v", err) os.Exit(1) } - if appendKnown { - addr, port, err := machine.GetSSHHostAddrPort(co.API, *co.Config, *n) - if err != nil { - out.ErrLn("GetSSHHostAddrPort: %v", err) - os.Exit(1) - } - - host := addr - if port != 22 { - host = fmt.Sprintf("[%s]:%d", addr, port) - } - knownHosts := filepath.Join(homedir.HomeDir(), ".ssh", "known_hosts") - - fmt.Fprintf(os.Stderr, "Host added: %s (%s)\n", knownHosts, host) - if sshutil.KnownHost(host, knownHosts) { - return - } - - f, err := os.OpenFile(knownHosts, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) - if err != nil { - out.ErrLn("OpenFile: %v", err) - os.Exit(1) - } - defer f.Close() - - _, err = f.WriteString(keys) - if err != nil { - out.ErrLn("WriteString: %v", err) - os.Exit(1) - } + host := addr + if port != 22 { + host = fmt.Sprintf("[%s]:%d", addr, port) + } + knownHosts := filepath.Join(homedir.HomeDir(), ".ssh", "known_hosts") + fmt.Fprintf(os.Stderr, "Host added: %s (%s)\n", knownHosts, host) + if sshutil.KnownHost(host, knownHosts) { return } - fmt.Printf("%s", keys) - }, + f, err := os.OpenFile(knownHosts, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) + if err != nil { + out.ErrLn("OpenFile: %v", err) + os.Exit(1) + } + defer f.Close() + + _, err = f.WriteString(keys) + if err != nil { + out.ErrLn("WriteString: %v", err) + os.Exit(1) + } + + return + } } func init() { diff --git a/pkg/minikube/reason/reason.go b/pkg/minikube/reason/reason.go index 170953f1e7..5c93e594e6 100644 --- a/pkg/minikube/reason/reason.go +++ b/pkg/minikube/reason/reason.go @@ -428,6 +428,8 @@ var ( RuntimeEnable = Kind{ID: "RUNTIME_ENABLE", ExitCode: ExRuntimeError} // minikube failed to cache images for the current container runtime RuntimeCache = Kind{ID: "RUNTIME_CACHE", ExitCode: ExRuntimeError} + // minikube failed to start an ssh-agent when executing docker-env + SshAgentStart = Kind{ID: "SSH_AGENT_START", ExitCode: ExRuntimeError} // service check timed out while starting minikube dashboard SvcCheckTimeout = Kind{ID: "SVC_CHECK_TIMEOUT", ExitCode: ExSvcTimeout} diff --git a/test/integration/docker_test.go b/test/integration/docker_test.go index 86aa7a3cbb..77638f9395 100644 --- a/test/integration/docker_test.go +++ b/test/integration/docker_test.go @@ -184,35 +184,15 @@ func TestDockerEnvContainerd(t *testing.T) { } time.Sleep(time.Second * 10) - // if we are in a shell, we need to run 'ssh-agent bash' to enable the ssh-agent - // but if we are in an integration test, we don't have a bash, so we need to get the environment variables set by ssh-agent, and use them in the following tests - result, err := Run(t, exec.CommandContext(ctx, "ssh-agent")) - if err != nil { - t.Errorf("failed to execute ssh-agent bash, error: %v, output: %s", err, result.Output()) - } - output := result.Output() - // get SSH_AUTH_SOCK - groups := regexp.MustCompile(`SSH_AUTH_SOCK=(\S*);`).FindStringSubmatch(output) - if len(groups) < 2 { - t.Errorf("failed to acquire SSH_AUTH_SOCK, output is %s", output) - } - sshAuthSock := groups[1] - // get SSH_AGENT_PID - groups = regexp.MustCompile(`SSH_AGENT_PID=(\S*);`).FindStringSubmatch(output) - if len(groups) < 2 { - t.Errorf("failed to acquire SSH_AUTH_SOCK, output is %s", output) - } - sshAgentPid := groups[1] - // execute 'minikube docker-env --ssh-host --ssh-add' and extract the 'DOCKER_HOST' environment value - cmd = exec.CommandContext(ctx, "/bin/bash", "-c", fmt.Sprintf("SSH_AUTH_SOCK=%s SSH_AGENT_PID=%s %s docker-env --ssh-host --ssh-add -p %s", sshAuthSock, sshAgentPid, Target(), profile)) - result, err = Run(t, cmd) + cmd = exec.CommandContext(ctx, "/bin/bash", "-c", fmt.Sprintf("%s docker-env --ssh-host --ssh-add -p %s", Target(), profile)) + result, err := Run(t, cmd) if err != nil { t.Errorf("failed to execute minikube docker-env --ssh-host --ssh-add, error: %v, output: %s", err, result.Output()) } - output = result.Output() - groups = regexp.MustCompile(`DOCKER_HOST="(\S*)"`).FindStringSubmatch(output) + output := result.Output() + groups := regexp.MustCompile(`DOCKER_HOST="(\S*)"`).FindStringSubmatch(output) if len(groups) < 2 { t.Errorf("failed to acquire SSH_AUTH_SOCK, output is %s", output) } @@ -221,19 +201,19 @@ func TestDockerEnvContainerd(t *testing.T) { if len(segments) < 3 { t.Errorf("failed to acquire dockerHost, output is %s", dockerHost) } - port := segments[2] - // clear remaining keys - clearResult, err := Run(t, exec.CommandContext(ctx, "ssh-keygen", "-R", "[127.0.0.1]:"+port)) - if err != nil { - t.Logf("failed to clear duplicate keys: %q : %v", clearResult.Command(), err) + // get SSH_AUTH_SOCK + groups = regexp.MustCompile(`SSH_AUTH_SOCK=(\S*)`).FindStringSubmatch(output) + if len(groups) < 2 { + t.Errorf("failed to acquire SSH_AUTH_SOCK, output is %s", output) } - - // execute 'minikube ssh-host --append-known' - result, err = Run(t, exec.CommandContext(ctx, Target(), "ssh-host", "--append-known", "-p", profile)) - if err != nil { - t.Errorf("failed to execute 'minikube ssh-host --append-known', error: %v, output: %s", err, result.Output()) + sshAuthSock := groups[1] + // get SSH_AGENT_PID + groups = regexp.MustCompile(`SSH_AGENT_PID=(\S*)`).FindStringSubmatch(output) + if len(groups) < 2 { + t.Errorf("failed to acquire SSH_AUTH_SOCK, output is %s", output) } + sshAgentPid := groups[1] cmd = exec.CommandContext(ctx, "/bin/bash", "-c", fmt.Sprintf("SSH_AUTH_SOCK=%s SSH_AGENT_PID=%s DOCKER_HOST=%s docker version", sshAuthSock, sshAgentPid, dockerHost))