diff --git a/pkg/agent/config/config.go b/pkg/agent/config/config.go index f074cbd104a..54957987639 100644 --- a/pkg/agent/config/config.go +++ b/pkg/agent/config/config.go @@ -27,6 +27,8 @@ import ( "github.com/k3s-io/k3s/pkg/clientaccess" "github.com/k3s-io/k3s/pkg/daemons/config" "github.com/k3s-io/k3s/pkg/daemons/control/deps" + "github.com/k3s-io/k3s/pkg/daemons/executor" + "github.com/k3s-io/k3s/pkg/signals" "github.com/k3s-io/k3s/pkg/spegel" "github.com/k3s-io/k3s/pkg/util" "github.com/k3s-io/k3s/pkg/util/errors" @@ -515,54 +517,13 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N os.Setenv("NODE_NAME", nodeName) - // Ensure that the kubelet's server certificate is valid for all configured node IPs. Note - // that in the case of an external CCM, additional IPs may be added by the infra provider - // that the cert will not be valid for, as they are not present in the list collected here. - nodeExternalAndInternalIPs := append(nodeIPs, nodeExternalIPs...) - - // Ask the server to sign our kubelet server cert. - if err := getKubeletServingCert(nodeName, nodeExternalAndInternalIPs, servingKubeletCert, servingKubeletKey, newNodePasswordFile, info); err != nil { - return nil, errors.WithMessage(err, servingKubeletCert) - } - - // Ask the server to sign our kubelet client cert. - if err := getKubeletClientCert(clientKubeletCert, clientKubeletKey, nodeName, nodeIPs, newNodePasswordFile, info); err != nil { - return nil, errors.WithMessage(err, clientKubeletCert) - } - - // Generate a kubeconfig for the kubelet. kubeconfigKubelet := filepath.Join(envInfo.DataDir, "agent", "kubelet.kubeconfig") - if err := deps.KubeConfig(kubeconfigKubelet, apiServerURL, serverCAFile, clientKubeletCert, clientKubeletKey); err != nil { - return nil, err - } - clientKubeProxyCert := filepath.Join(envInfo.DataDir, "agent", "client-kube-proxy.crt") clientKubeProxyKey := filepath.Join(envInfo.DataDir, "agent", "client-kube-proxy.key") - - // Ask the server to sign our kube-proxy client cert. - if err := getClientCert(clientKubeProxyCert, clientKubeProxyKey, info); err != nil { - return nil, errors.WithMessage(err, clientKubeProxyCert) - } - - // Generate a kubeconfig for kube-proxy. kubeconfigKubeproxy := filepath.Join(envInfo.DataDir, "agent", "kubeproxy.kubeconfig") - if err := deps.KubeConfig(kubeconfigKubeproxy, apiServerURL, serverCAFile, clientKubeProxyCert, clientKubeProxyKey); err != nil { - return nil, err - } - clientK3sControllerCert := filepath.Join(envInfo.DataDir, "agent", "client-"+version.Program+"-controller.crt") clientK3sControllerKey := filepath.Join(envInfo.DataDir, "agent", "client-"+version.Program+"-controller.key") - - // Ask the server to sign our agent controller client cert. - if err := getClientCert(clientK3sControllerCert, clientK3sControllerKey, info); err != nil { - return nil, errors.WithMessage(err, clientK3sControllerCert) - } - - // Generate a kubeconfig for the agent controller. kubeconfigK3sController := filepath.Join(envInfo.DataDir, "agent", version.Program+"controller.kubeconfig") - if err := deps.KubeConfig(kubeconfigK3sController, apiServerURL, serverCAFile, clientK3sControllerCert, clientK3sControllerKey); err != nil { - return nil, err - } // Ensure kubelet config dir exists kubeletConfigDir := filepath.Join(envInfo.DataDir, "agent", "etc", "kubelet.conf.d") @@ -769,6 +730,53 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N return nil, err } + // allow executor to do additional configuration; this is the last chance to modify nodeConfig before certs are signed + if err := executor.Prepare(ctx, nodeConfig, *envInfo); err != nil { + err = errors.WithMessage(err, "failed to prepare configuration") + signals.RequestShutdown(err) + return nil, err + } + + // Ensure that the kubelet's server certificate is valid for all configured node IPs. Note + // that in the case of an external CCM, additional IPs may be added by the infra provider + // that the cert will not be valid for, as they are not present in the list collected here. + nodeExternalAndInternalIPs := append(nodeConfig.AgentConfig.NodeIPs, nodeConfig.AgentConfig.NodeExternalIPs...) + + // Ask the server to sign our kubelet server cert. + if err := getKubeletServingCert(nodeConfig.AgentConfig.NodeName, nodeExternalAndInternalIPs, servingKubeletCert, servingKubeletKey, newNodePasswordFile, info); err != nil { + return nil, errors.WithMessage(err, servingKubeletCert) + } + + // Ask the server to sign our kubelet client cert. + if err := getKubeletClientCert(clientKubeletCert, clientKubeletKey, nodeConfig.AgentConfig.NodeName, nodeConfig.AgentConfig.NodeIPs, newNodePasswordFile, info); err != nil { + return nil, errors.WithMessage(err, clientKubeletCert) + } + + // Generate a kubeconfig for the kubelet. + if err := deps.KubeConfig(kubeconfigKubelet, apiServerURL, serverCAFile, clientKubeletCert, clientKubeletKey); err != nil { + return nil, err + } + + // Ask the server to sign our kube-proxy client cert. + if err := getClientCert(clientKubeProxyCert, clientKubeProxyKey, info); err != nil { + return nil, errors.WithMessage(err, clientKubeProxyCert) + } + + // Generate a kubeconfig for kube-proxy. + if err := deps.KubeConfig(kubeconfigKubeproxy, apiServerURL, serverCAFile, clientKubeProxyCert, clientKubeProxyKey); err != nil { + return nil, err + } + + // Ask the server to sign our agent controller client cert. + if err := getClientCert(clientK3sControllerCert, clientK3sControllerKey, info); err != nil { + return nil, errors.WithMessage(err, clientK3sControllerCert) + } + + // Generate a kubeconfig for the agent controller. + if err := deps.KubeConfig(kubeconfigK3sController, apiServerURL, serverCAFile, clientK3sControllerCert, clientK3sControllerKey); err != nil { + return nil, err + } + return nodeConfig, nil } diff --git a/pkg/cli/agent/agent.go b/pkg/cli/agent/agent.go index 99439c195a5..f221b810279 100644 --- a/pkg/cli/agent/agent.go +++ b/pkg/cli/agent/agent.go @@ -23,7 +23,6 @@ import ( "github.com/k3s-io/k3s/pkg/util/mux" "github.com/k3s-io/k3s/pkg/util/permissions" "github.com/k3s-io/k3s/pkg/version" - "github.com/k3s-io/k3s/pkg/vpn" "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" "k8s.io/klog/v2" @@ -72,7 +71,7 @@ func Run(clx *cli.Context) (rerr error) { } if cmds.AgentConfig.TokenFile != "" { - token, err := util.ReadFile(cmds.AgentConfig.TokenFile) + token, err := util.ReadFile(ctx, cmds.AgentConfig.TokenFile) if err != nil { return err } @@ -111,20 +110,6 @@ func Run(clx *cli.Context) (rerr error) { cfg.DataDir = dataDir go cmds.WriteCoverage(ctx) - if cfg.VPNAuthFile != "" { - cfg.VPNAuth, err = util.ReadFile(cfg.VPNAuthFile) - if err != nil { - return err - } - } - - // Starts the VPN in the agent if config was set up - if cfg.VPNAuth != "" { - err := vpn.StartVPN(cfg.VPNAuth) - if err != nil { - return err - } - } // Until the agent is run and retrieves config from the server, we won't know // if the embedded registry is enabled. If it is not enabled, these are not diff --git a/pkg/cli/server/server.go b/pkg/cli/server/server.go index baa3047c2dd..a633bca0960 100644 --- a/pkg/cli/server/server.go +++ b/pkg/cli/server/server.go @@ -117,21 +117,6 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont } } - if cmds.AgentConfig.VPNAuthFile != "" { - cmds.AgentConfig.VPNAuth, err = util.ReadFile(cmds.AgentConfig.VPNAuthFile) - if err != nil { - return err - } - } - - // Starts the VPN in the server if config was set up - if cmds.AgentConfig.VPNAuth != "" { - err := vpn.StartVPN(cmds.AgentConfig.VPNAuth) - if err != nil { - return err - } - } - serverConfig := server.Config{} serverConfig.DisableAgent = cfg.DisableAgent serverConfig.ControlConfig.Runtime = config.NewRuntime() @@ -139,13 +124,13 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont serverConfig.ControlConfig.AgentToken = cfg.AgentToken serverConfig.ControlConfig.JoinURL = cfg.ServerURL if cfg.AgentTokenFile != "" { - serverConfig.ControlConfig.AgentToken, err = util.ReadFile(cfg.AgentTokenFile) + serverConfig.ControlConfig.AgentToken, err = util.ReadFile(ctx, cfg.AgentTokenFile) if err != nil { return err } } if cfg.TokenFile != "" { - serverConfig.ControlConfig.Token, err = util.ReadFile(cfg.TokenFile) + serverConfig.ControlConfig.Token, err = util.ReadFile(ctx, cfg.TokenFile) if err != nil { return err } @@ -298,60 +283,14 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont } serverConfig.ControlConfig.ServerNodeName = nodeName serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, "127.0.0.1", "::1", "localhost", nodeName) + serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, cmds.AgentConfig.NodeExternalIP.Value()...) + if shortName := strings.SplitN(nodeName, ".", 2)[0]; shortName != nodeName { + serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, shortName) + } for _, ip := range nodeIPs { serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, ip.String()) } - // if not set, try setting advertise-ip from agent VPN - if cmds.AgentConfig.VPNAuth != "" { - vpnInfo, err := vpn.GetInfo(cmds.AgentConfig.VPNAuth) - if err != nil { - return err - } - - // If we are in ipv6-only mode, we should pass the ipv6 address. Otherwise, ipv4 - if utilsnet.IsIPv6(nodeIPs[0]) { - if vpnInfo.IPv6Address != nil { - logrus.Infof("Changed advertise-address to %v due to VPN", vpnInfo.IPv6Address) - if serverConfig.ControlConfig.AdvertiseIP != "" { - logrus.Warn("Conflict in the config detected. VPN integration overwrites advertise-address but the config is setting the advertise-address parameter") - } - serverConfig.ControlConfig.AdvertiseIP = vpnInfo.IPv6Address.String() - } else { - return errors.New("tailscale does not provide an ipv6 address") - } - } else { - // We are in dual-stack or ipv4-only mode - if vpnInfo.IPv4Address != nil { - logrus.Infof("Changed advertise-address to %v due to VPN", vpnInfo.IPv4Address) - if serverConfig.ControlConfig.AdvertiseIP != "" { - logrus.Warn("Conflict in the config detected. VPN integration overwrites advertise-address but the config is setting the advertise-address parameter") - } - serverConfig.ControlConfig.AdvertiseIP = vpnInfo.IPv4Address.String() - } else { - return errors.New("tailscale does not provide an ipv4 address") - } - } - logrus.Warn("Etcd IP (PrivateIP) remains the local IP. Running etcd traffic over VPN is not recommended due to performance issues") - } else { - // if not set, try setting advertise-ip from agent node-external-ip - if serverConfig.ControlConfig.AdvertiseIP == "" && len(cmds.AgentConfig.NodeExternalIP.Value()) != 0 { - serverConfig.ControlConfig.AdvertiseIP = util.GetFirstValidIPString(cmds.AgentConfig.NodeExternalIP.Value()) - } - - // if not set, try setting advertise-ip from agent node-ip - if serverConfig.ControlConfig.AdvertiseIP == "" && len(cmds.AgentConfig.NodeIP.Value()) != 0 { - serverConfig.ControlConfig.AdvertiseIP = util.GetFirstValidIPString(cmds.AgentConfig.NodeIP.Value()) - } - } - - // if we ended up with any advertise-ips, ensure they're added to the SAN list; - // note that kube-apiserver does not support dual-stack advertise-ip as of 1.21.0: - // https://github.com/kubernetes/kubeadm/issues/1612#issuecomment-772583989 - if serverConfig.ControlConfig.AdvertiseIP != "" { - serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, serverConfig.ControlConfig.AdvertiseIP) - } - // configure ClusterIPRanges. Use default 10.42.0.0/16 or fd00:42::/56 if user did not set it _, defaultClusterCIDR, defaultServiceCIDR, _ := util.GetDefaultAddresses(nodeIPs[0]) if len(cmds.ServerConfig.ClusterCIDR.Value()) == 0 { @@ -613,6 +552,51 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont } } + // try setting advertise-ip from agent VPN + if vpnInfo, _ := vpn.GetInfoFromExecutor(); vpnInfo != nil { + // If we are in ipv6-only mode, we should pass the ipv6 address. Otherwise, ipv4 + if utilsnet.IsIPv6(nodeIPs[0]) { + if vpnInfo.IPv6Address != nil { + logrus.Infof("Changed advertise-address to %v due to VPN", vpnInfo.IPv6Address) + if serverConfig.ControlConfig.AdvertiseIP != "" { + logrus.Warn("Conflict in the config detected. VPN integration overwrites advertise-address but the config is setting the advertise-address parameter") + } + serverConfig.ControlConfig.AdvertiseIP = vpnInfo.IPv6Address.String() + } else { + return errors.New("tailscale does not provide an ipv6 address") + } + } else { + // We are in dual-stack or ipv4-only mode + if vpnInfo.IPv4Address != nil { + logrus.Infof("Changed advertise-address to %v due to VPN", vpnInfo.IPv4Address) + if serverConfig.ControlConfig.AdvertiseIP != "" { + logrus.Warn("Conflict in the config detected. VPN integration overwrites advertise-address but the config is setting the advertise-address parameter") + } + serverConfig.ControlConfig.AdvertiseIP = vpnInfo.IPv4Address.String() + } else { + return errors.New("tailscale does not provide an ipv4 address") + } + } + logrus.Warn("Etcd IP (PrivateIP) remains the local IP. Running etcd traffic over VPN is not recommended due to performance issues") + } else { + // if not set, try setting advertise-ip from agent node-external-ip + if serverConfig.ControlConfig.AdvertiseIP == "" && len(cmds.AgentConfig.NodeExternalIP.Value()) != 0 { + serverConfig.ControlConfig.AdvertiseIP = util.GetFirstValidIPString(cmds.AgentConfig.NodeExternalIP.Value()) + } + + // if not set, try setting advertise-ip from agent node-ip + if serverConfig.ControlConfig.AdvertiseIP == "" && len(cmds.AgentConfig.NodeIP.Value()) != 0 { + serverConfig.ControlConfig.AdvertiseIP = util.GetFirstValidIPString(cmds.AgentConfig.NodeIP.Value()) + } + } + + // if we ended up with any advertise-ips, ensure they're added to the SAN list; + // note that kube-apiserver does not support dual-stack advertise-ip as of 1.21.0: + // https://github.com/kubernetes/kubeadm/issues/1612#issuecomment-772583989 + if serverConfig.ControlConfig.AdvertiseIP != "" { + serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, serverConfig.ControlConfig.AdvertiseIP) + } + go cmds.WriteCoverage(ctx) serverConfig.ControlConfig.Runtime.StartupHooksWg = &sync.WaitGroup{} diff --git a/pkg/daemons/executor/executor.go b/pkg/daemons/executor/executor.go index 7b95dbf1b2c..2ae5df177d9 100644 --- a/pkg/daemons/executor/executor.go +++ b/pkg/daemons/executor/executor.go @@ -28,6 +28,7 @@ var ( // The enableMaintenance flag enables attempts to perform corrective maintenance during the test process. type TestFunc func(ctx context.Context, enableMaintenance bool) error +// Executor is a set of functions for bootstrapping a node and starting the CRI, CNI, and Kubernetes components type Executor interface { Bootstrap(ctx context.Context, nodeConfig *daemonconfig.Node, cfg cmds.Agent) error Kubelet(ctx context.Context, args []string) error @@ -49,6 +50,12 @@ type Executor interface { IsSelfHosted() bool } +// PreparingExecutor is an optional interface that Executors may implement to modify node configuration +// and CLI flags before config is retrieved from the server. +type PreparingExecutor interface { + Prepare(ctx context.Context, nodeConfig *daemonconfig.Node, cfg cmds.Agent) error +} + type ETCDSocketOpts struct { ReuseAddress bool `json:"reuse-address,omitempty"` ReusePort bool `json:"reuse-port,omitempty"` @@ -155,6 +162,10 @@ func Set(driver Executor) { executor = driver } +func Get() Executor { + return executor +} + func Bootstrap(ctx context.Context, nodeConfig *daemonconfig.Node, cfg cmds.Agent) error { if executor == nil { return ErrNotInitialized @@ -278,6 +289,13 @@ func IsSelfHosted() bool { return executor.IsSelfHosted() } +func Prepare(ctx context.Context, nodeConfig *daemonconfig.Node, cfg cmds.Agent) error { + if ex, ok := executor.(PreparingExecutor); ok { + return ex.Prepare(ctx, nodeConfig, cfg) + } + return nil +} + func CloseIfNilErr(err error, ch chan struct{}) error { if err == nil { close(ch) diff --git a/pkg/executor/embed/embed.go b/pkg/executor/embed/embed.go index 25c86bfd3ae..433b6f2fa69 100644 --- a/pkg/executor/embed/embed.go +++ b/pkg/executor/embed/embed.go @@ -57,12 +57,76 @@ func init() { // explicit type check var _ executor.Executor = &Embedded{} +var _ executor.PreparingExecutor = &Embedded{} +var _ vpn.InfoProvider = &Embedded{} type Embedded struct { apiServerReady <-chan struct{} etcdReady chan struct{} criReady chan struct{} nodeConfig *daemonconfig.Node + vpnInfo *vpn.Info +} + +// Prepare modifies the node config prior to downloading client and server certificates from the server. +// If node IPs or names need to be modified, it should be done here so that the kubelet client and serving certs are valid. +func (e *Embedded) Prepare(ctx context.Context, nodeConfig *daemonconfig.Node, cfg cmds.Agent) error { + // If there is a VPN, we must start it early to overwrite NodeIP and flannel interface + var err error + if cfg.VPNAuthFile != "" { + cfg.VPNAuth, err = util.ReadFile(ctx, cfg.VPNAuthFile) + if err != nil { + return errors.WithMessage(err, "failed to read vpn-auth-file") + } + } + + if cfg.VPNAuth != "" { + err = vpn.StartVPN(cfg.VPNAuth) + if err != nil { + return err + } + + e.vpnInfo, err = vpn.GetInfo(cfg.VPNAuth) + if err != nil { + return err + } + + // Pass ipv4, ipv6 or both depending on nodeIPs mode + nodeIPs := nodeConfig.AgentConfig.NodeIPs + var vpnIPs []net.IP + if utilsnet.IsIPv4(nodeIPs[0]) && e.vpnInfo.IPv4Address != nil { + vpnIPs = append(vpnIPs, e.vpnInfo.IPv4Address) + if e.vpnInfo.IPv6Address != nil { + vpnIPs = append(vpnIPs, e.vpnInfo.IPv6Address) + } + } else if utilsnet.IsIPv6(nodeIPs[0]) && e.vpnInfo.IPv6Address != nil { + vpnIPs = append(vpnIPs, e.vpnInfo.IPv6Address) + if e.vpnInfo.IPv4Address != nil { + vpnIPs = append(vpnIPs, e.vpnInfo.IPv4Address) + } + } else { + return fmt.Errorf("address family mismatch when assigning VPN addresses to node: node=%v, VPN ipv4=%v ipv6=%v", nodeIPs, e.vpnInfo.IPv4Address, e.vpnInfo.IPv6Address) + } + + // Overwrite nodeip and flannel interface and throw a warning if user explicitly set those parameters + if len(vpnIPs) != 0 { + logrus.Infof("Node-ip changed to %v due to VPN", vpnIPs) + if len(cfg.NodeIP.Value()) != 0 { + logrus.Warn("VPN provider overrides configured node-ip parameter") + } + if len(cfg.NodeExternalIP.Value()) != 0 { + logrus.Warn("VPN provider overrides node-external-ip parameter") + } + nodeIPs = vpnIPs + nodeConfig.AgentConfig.NodeIPs = vpnIPs + nodeConfig.AgentConfig.NodeIP = vpnIPs[0].String() + nodeConfig.Flannel.Iface, err = net.InterfaceByName(e.vpnInfo.Interface) + if err != nil { + return errors.WithMessagef(err, "unable to find vpn interface: %s", e.vpnInfo.Interface) + } + } + } + return err } func (e *Embedded) Bootstrap(ctx context.Context, nodeConfig *daemonconfig.Node, cfg cmds.Agent) error { @@ -100,50 +164,6 @@ func (e *Embedded) Bootstrap(ctx context.Context, nodeConfig *daemonconfig.Node, } } - // If there is a VPN, we must overwrite NodeIP and flannel interface - var vpnInfo *vpn.Info - if cfg.VPNAuth != "" { - vpnInfo, err = vpn.GetInfo(cfg.VPNAuth) - if err != nil { - return err - } - - // Pass ipv4, ipv6 or both depending on nodeIPs mode - nodeIPs := nodeConfig.AgentConfig.NodeIPs - var vpnIPs []net.IP - if utilsnet.IsIPv4(nodeIPs[0]) && vpnInfo.IPv4Address != nil { - vpnIPs = append(vpnIPs, vpnInfo.IPv4Address) - if vpnInfo.IPv6Address != nil { - vpnIPs = append(vpnIPs, vpnInfo.IPv6Address) - } - } else if utilsnet.IsIPv6(nodeIPs[0]) && vpnInfo.IPv6Address != nil { - vpnIPs = append(vpnIPs, vpnInfo.IPv6Address) - if vpnInfo.IPv4Address != nil { - vpnIPs = append(vpnIPs, vpnInfo.IPv4Address) - } - } else { - return fmt.Errorf("address family mismatch when assigning VPN addresses to node: node=%v, VPN ipv4=%v ipv6=%v", nodeIPs, vpnInfo.IPv4Address, vpnInfo.IPv6Address) - } - - // Overwrite nodeip and flannel interface and throw a warning if user explicitly set those parameters - if len(vpnIPs) != 0 { - logrus.Infof("Node-ip changed to %v due to VPN", vpnIPs) - if len(cfg.NodeIP.Value()) != 0 { - logrus.Warn("VPN provider overrides configured node-ip parameter") - } - if len(cfg.NodeExternalIP.Value()) != 0 { - logrus.Warn("VPN provider overrides node-external-ip parameter") - } - nodeIPs = vpnIPs - nodeConfig.AgentConfig.NodeIPs = vpnIPs - nodeConfig.AgentConfig.NodeIP = vpnIPs[0].String() - nodeConfig.Flannel.Iface, err = net.InterfaceByName(vpnInfo.Interface) - if err != nil { - return errors.WithMessagef(err, "unable to find vpn interface: %s", vpnInfo.Interface) - } - } - } - // set paths for embedded flannel if enabled hostLocal, err := exec.LookPath("host-local") if err != nil { @@ -160,8 +180,8 @@ func (e *Embedded) Bootstrap(ctx context.Context, nodeConfig *daemonconfig.Node, nodeConfig.AgentConfig.CNIConfDir = filepath.Join(cfg.DataDir, "agent", "etc", "cni", "net.d") // It does not make sense to use VPN without its flannel backend - if cfg.VPNAuth != "" { - nodeConfig.Flannel.Backend = vpnInfo.ProviderName + if e.vpnInfo != nil { + nodeConfig.Flannel.Backend = e.vpnInfo.ProviderName } } @@ -419,3 +439,10 @@ func (e *Embedded) CRIReadyChan() <-chan struct{} { func (e Embedded) IsSelfHosted() bool { return false } + +func (e Embedded) GetVPNInfo() (*vpn.Info, error) { + if e.vpnInfo != nil { + return e.vpnInfo, nil + } + return nil, errors.New("VPN info not set") +} diff --git a/pkg/util/file.go b/pkg/util/file.go index bdba4da27d3..061120679db 100644 --- a/pkg/util/file.go +++ b/pkg/util/file.go @@ -1,7 +1,7 @@ package util import ( - "errors" + "context" "os" "os/user" "path/filepath" @@ -10,6 +10,7 @@ import ( "time" "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/util/wait" ) func SetFileModeForPath(name string, mode os.FileMode) error { @@ -41,25 +42,23 @@ func SetFileModeForFile(file *os.File, mode os.FileMode) error { return file.Chmod(mode) } -// ReadFile reads from a file -func ReadFile(path string) (string, error) { +// ReadFile waits for a file to exist, then returns its trimmed contents as a string +func ReadFile(ctx context.Context, path string) (string, error) { if path == "" { return "", nil } - - for start := time.Now(); time.Since(start) < 4*time.Minute; { - vpnBytes, err := os.ReadFile(path) + var trimmed string + return trimmed, wait.PollUntilContextTimeout(ctx, 2*time.Second, 4*time.Minute, true, func(ctx context.Context) (bool, error) { + b, err := os.ReadFile(path) if err == nil { - return strings.TrimSpace(string(vpnBytes)), nil + trimmed = strings.TrimSpace(string(b)) + return true, nil } else if os.IsNotExist(err) { - logrus.Infof("Waiting for %s to be available\n", path) - time.Sleep(2 * time.Second) - } else { - return "", err + logrus.Infof("Waiting for file %q to be created\n", path) + return false, nil } - } - - return "", errors.New("Timeout while trying to read the file") + return false, err + }) } // AtomicWrite firsts writes data to a temp file, then renames to the destination file. diff --git a/pkg/vpn/vpn.go b/pkg/vpn/vpn.go index 09c2649ec73..2ce700dcbaf 100644 --- a/pkg/vpn/vpn.go +++ b/pkg/vpn/vpn.go @@ -8,6 +8,7 @@ import ( "net/url" "strings" + "github.com/k3s-io/k3s/pkg/daemons/executor" "github.com/k3s-io/k3s/pkg/util" "github.com/k3s-io/k3s/pkg/util/errors" @@ -18,6 +19,10 @@ const ( tailscaleIf = "tailscale0" ) +type InfoProvider interface { + GetVPNInfo() (*Info, error) +} + type TailscaleOutput struct { TailscaleIPs []string `json:"TailscaleIPs"` BackendState string `json:"BackendState"` @@ -72,7 +77,10 @@ func StartVPN(vpnAuthConfigFile string) error { logrus.Debugf("Flags passed to tailscale up: %v", args) output, err := util.ExecCommand("tailscale", args) if err != nil { - return errors.WithMessage(err, "tailscale up failed: "+output) + if output != "" { + return errors.WithMessagef(err, "tailscale up failed (%q)", output) + } + return errors.WithMessage(err, "tailscale up failed") } logrus.Debugf("Output from tailscale up: %v", output) return nil @@ -94,6 +102,14 @@ func GetInfo(vpnAuth string) (*Info, error) { return nil, nil } +func GetInfoFromExecutor() (*Info, error) { + ex := executor.Get() + if provider, ok := ex.(InfoProvider); ok { + return provider.GetVPNInfo() + } + return nil, errors.New("executor does not provide VPN info") +} + // getVPNAuthInfo returns the required authInfo object func getVPNAuthInfo(vpnAuth string) (vpnCliAuthInfo, error) { var authInfo vpnCliAuthInfo diff --git a/tests/e2e/rootless/rootless_test.go b/tests/e2e/rootless/rootless_test.go index 3e2c44e29de..78436e6a12f 100644 --- a/tests/e2e/rootless/rootless_test.go +++ b/tests/e2e/rootless/rootless_test.go @@ -228,7 +228,7 @@ func SaveRootlessJournalLogs(nodes []e2e.VagrantNode) error { return err } defer lf.Close() - cmd := "vagrant ssh --no-tty " + node.Name + " -c \"journalctl -u --user k3s-rootless --no-pager\"" + cmd := "vagrant ssh --no-tty " + node.Name + " -c \"journalctl --user -u k3s-rootless --no-pager\"" logs, err := tests.RunCommand(cmd) if err != nil { return err