k3s-io
/
k3s
mirror of https://github.com/k3s-io/k3s.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Fix embedded excutor VPN config injection 

Allow the executor to modify node config before certs are generated, and use this to add VPN node IPs to kubelet serving cert
pull/13902/head^2
Brad Davidson 2026-04-13 20:45:38 +00:00 committed by Brad Davidson
parent 4b18631007
commit f891548e32
8 changed files with 223 additions and 186 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

90
pkg/agent/config/config.go
View File

@ -27,6 +27,8 @@ import (
"github.com/k3s-io/k3s/pkg/clientaccess"
"github.com/k3s-io/k3s/pkg/daemons/config"
"github.com/k3s-io/k3s/pkg/daemons/control/deps"
"github.com/k3s-io/k3s/pkg/daemons/executor"
"github.com/k3s-io/k3s/pkg/signals"
"github.com/k3s-io/k3s/pkg/spegel"
"github.com/k3s-io/k3s/pkg/util"
"github.com/k3s-io/k3s/pkg/util/errors"
@ -515,54 +517,13 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
os.Setenv("NODE_NAME", nodeName)
// Ensure that the kubelet's server certificate is valid for all configured node IPs. Note
// that in the case of an external CCM, additional IPs may be added by the infra provider
// that the cert will not be valid for, as they are not present in the list collected here.
nodeExternalAndInternalIPs := append(nodeIPs, nodeExternalIPs...)
// Ask the server to sign our kubelet server cert.
if err := getKubeletServingCert(nodeName, nodeExternalAndInternalIPs, servingKubeletCert, servingKubeletKey, newNodePasswordFile, info); err != nil {
return nil, errors.WithMessage(err, servingKubeletCert)
}
// Ask the server to sign our kubelet client cert.
if err := getKubeletClientCert(clientKubeletCert, clientKubeletKey, nodeName, nodeIPs, newNodePasswordFile, info); err != nil {
return nil, errors.WithMessage(err, clientKubeletCert)
}
// Generate a kubeconfig for the kubelet.
kubeconfigKubelet := filepath.Join(envInfo.DataDir, "agent", "kubelet.kubeconfig")
if err := deps.KubeConfig(kubeconfigKubelet, apiServerURL, serverCAFile, clientKubeletCert, clientKubeletKey); err != nil {
return nil, err
}
clientKubeProxyCert := filepath.Join(envInfo.DataDir, "agent", "client-kube-proxy.crt")
clientKubeProxyKey := filepath.Join(envInfo.DataDir, "agent", "client-kube-proxy.key")
// Ask the server to sign our kube-proxy client cert.
if err := getClientCert(clientKubeProxyCert, clientKubeProxyKey, info); err != nil {
return nil, errors.WithMessage(err, clientKubeProxyCert)
}
// Generate a kubeconfig for kube-proxy.
kubeconfigKubeproxy := filepath.Join(envInfo.DataDir, "agent", "kubeproxy.kubeconfig")
if err := deps.KubeConfig(kubeconfigKubeproxy, apiServerURL, serverCAFile, clientKubeProxyCert, clientKubeProxyKey); err != nil {
return nil, err
}
clientK3sControllerCert := filepath.Join(envInfo.DataDir, "agent", "client-"+version.Program+"-controller.crt")
clientK3sControllerKey := filepath.Join(envInfo.DataDir, "agent", "client-"+version.Program+"-controller.key")
// Ask the server to sign our agent controller client cert.
if err := getClientCert(clientK3sControllerCert, clientK3sControllerKey, info); err != nil {
return nil, errors.WithMessage(err, clientK3sControllerCert)
}
// Generate a kubeconfig for the agent controller.
kubeconfigK3sController := filepath.Join(envInfo.DataDir, "agent", version.Program+"controller.kubeconfig")
if err := deps.KubeConfig(kubeconfigK3sController, apiServerURL, serverCAFile, clientK3sControllerCert, clientK3sControllerKey); err != nil {
return nil, err
}
// Ensure kubelet config dir exists
kubeletConfigDir := filepath.Join(envInfo.DataDir, "agent", "etc", "kubelet.conf.d")
@ -769,6 +730,53 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
return nil, err
}
// allow executor to do additional configuration; this is the last chance to modify nodeConfig before certs are signed
if err := executor.Prepare(ctx, nodeConfig, *envInfo); err != nil {
err = errors.WithMessage(err, "failed to prepare configuration")
signals.RequestShutdown(err)
return nil, err
}
// Ensure that the kubelet's server certificate is valid for all configured node IPs. Note
// that in the case of an external CCM, additional IPs may be added by the infra provider
// that the cert will not be valid for, as they are not present in the list collected here.
nodeExternalAndInternalIPs := append(nodeConfig.AgentConfig.NodeIPs, nodeConfig.AgentConfig.NodeExternalIPs...)
// Ask the server to sign our kubelet server cert.
if err := getKubeletServingCert(nodeConfig.AgentConfig.NodeName, nodeExternalAndInternalIPs, servingKubeletCert, servingKubeletKey, newNodePasswordFile, info); err != nil {
return nil, errors.WithMessage(err, servingKubeletCert)
}
// Ask the server to sign our kubelet client cert.
if err := getKubeletClientCert(clientKubeletCert, clientKubeletKey, nodeConfig.AgentConfig.NodeName, nodeConfig.AgentConfig.NodeIPs, newNodePasswordFile, info); err != nil {
return nil, errors.WithMessage(err, clientKubeletCert)
}
// Generate a kubeconfig for the kubelet.
if err := deps.KubeConfig(kubeconfigKubelet, apiServerURL, serverCAFile, clientKubeletCert, clientKubeletKey); err != nil {
return nil, err
}
// Ask the server to sign our kube-proxy client cert.
if err := getClientCert(clientKubeProxyCert, clientKubeProxyKey, info); err != nil {
return nil, errors.WithMessage(err, clientKubeProxyCert)
}
// Generate a kubeconfig for kube-proxy.
if err := deps.KubeConfig(kubeconfigKubeproxy, apiServerURL, serverCAFile, clientKubeProxyCert, clientKubeProxyKey); err != nil {
return nil, err
}
// Ask the server to sign our agent controller client cert.
if err := getClientCert(clientK3sControllerCert, clientK3sControllerKey, info); err != nil {
return nil, errors.WithMessage(err, clientK3sControllerCert)
}
// Generate a kubeconfig for the agent controller.
if err := deps.KubeConfig(kubeconfigK3sController, apiServerURL, serverCAFile, clientK3sControllerCert, clientK3sControllerKey); err != nil {
return nil, err
}
return nodeConfig, nil
}

17
pkg/cli/agent/agent.go
View File

@ -23,7 +23,6 @@ import (
"github.com/k3s-io/k3s/pkg/util/mux"
"github.com/k3s-io/k3s/pkg/util/permissions"
"github.com/k3s-io/k3s/pkg/version"
"github.com/k3s-io/k3s/pkg/vpn"
"github.com/sirupsen/logrus"
"github.com/urfave/cli/v2"
"k8s.io/klog/v2"
@ -72,7 +71,7 @@ func Run(clx *cli.Context) (rerr error) {
}
if cmds.AgentConfig.TokenFile != "" {
token, err := util.ReadFile(cmds.AgentConfig.TokenFile)
token, err := util.ReadFile(ctx, cmds.AgentConfig.TokenFile)
if err != nil {
return err
}
@ -111,20 +110,6 @@ func Run(clx *cli.Context) (rerr error) {
cfg.DataDir = dataDir
go cmds.WriteCoverage(ctx)
if cfg.VPNAuthFile != "" {
cfg.VPNAuth, err = util.ReadFile(cfg.VPNAuthFile)
if err != nil {
return err
}
}
// Starts the VPN in the agent if config was set up
if cfg.VPNAuth != "" {
err := vpn.StartVPN(cfg.VPNAuth)
if err != nil {
return err
}
}
// Until the agent is run and retrieves config from the server, we won't know
// if the embedded registry is enabled. If it is not enabled, these are not

118
pkg/cli/server/server.go
View File

@ -117,21 +117,6 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
}
}
if cmds.AgentConfig.VPNAuthFile != "" {
cmds.AgentConfig.VPNAuth, err = util.ReadFile(cmds.AgentConfig.VPNAuthFile)
if err != nil {
return err
}
}
// Starts the VPN in the server if config was set up
if cmds.AgentConfig.VPNAuth != "" {
err := vpn.StartVPN(cmds.AgentConfig.VPNAuth)
if err != nil {
return err
}
}
serverConfig := server.Config{}
serverConfig.DisableAgent = cfg.DisableAgent
serverConfig.ControlConfig.Runtime = config.NewRuntime()
@ -139,13 +124,13 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
serverConfig.ControlConfig.AgentToken = cfg.AgentToken
serverConfig.ControlConfig.JoinURL = cfg.ServerURL
if cfg.AgentTokenFile != "" {
serverConfig.ControlConfig.AgentToken, err = util.ReadFile(cfg.AgentTokenFile)
serverConfig.ControlConfig.AgentToken, err = util.ReadFile(ctx, cfg.AgentTokenFile)
if err != nil {
return err
}
}
if cfg.TokenFile != "" {
serverConfig.ControlConfig.Token, err = util.ReadFile(cfg.TokenFile)
serverConfig.ControlConfig.Token, err = util.ReadFile(ctx, cfg.TokenFile)
if err != nil {
return err
}
@ -298,60 +283,14 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
}
serverConfig.ControlConfig.ServerNodeName = nodeName
serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, "127.0.0.1", "::1", "localhost", nodeName)
serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, cmds.AgentConfig.NodeExternalIP.Value()...)
if shortName := strings.SplitN(nodeName, ".", 2)[0]; shortName != nodeName {
serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, shortName)
}
for _, ip := range nodeIPs {
serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, ip.String())
}
// if not set, try setting advertise-ip from agent VPN
if cmds.AgentConfig.VPNAuth != "" {
vpnInfo, err := vpn.GetInfo(cmds.AgentConfig.VPNAuth)
if err != nil {
return err
}
// If we are in ipv6-only mode, we should pass the ipv6 address. Otherwise, ipv4
if utilsnet.IsIPv6(nodeIPs[0]) {
if vpnInfo.IPv6Address != nil {
logrus.Infof("Changed advertise-address to %v due to VPN", vpnInfo.IPv6Address)
if serverConfig.ControlConfig.AdvertiseIP != "" {
logrus.Warn("Conflict in the config detected. VPN integration overwrites advertise-address but the config is setting the advertise-address parameter")
}
serverConfig.ControlConfig.AdvertiseIP = vpnInfo.IPv6Address.String()
} else {
return errors.New("tailscale does not provide an ipv6 address")
}
} else {
// We are in dual-stack or ipv4-only mode
if vpnInfo.IPv4Address != nil {
logrus.Infof("Changed advertise-address to %v due to VPN", vpnInfo.IPv4Address)
if serverConfig.ControlConfig.AdvertiseIP != "" {
logrus.Warn("Conflict in the config detected. VPN integration overwrites advertise-address but the config is setting the advertise-address parameter")
}
serverConfig.ControlConfig.AdvertiseIP = vpnInfo.IPv4Address.String()
} else {
return errors.New("tailscale does not provide an ipv4 address")
}
}
logrus.Warn("Etcd IP (PrivateIP) remains the local IP. Running etcd traffic over VPN is not recommended due to performance issues")
} else {
// if not set, try setting advertise-ip from agent node-external-ip
if serverConfig.ControlConfig.AdvertiseIP == "" && len(cmds.AgentConfig.NodeExternalIP.Value()) != 0 {
serverConfig.ControlConfig.AdvertiseIP = util.GetFirstValidIPString(cmds.AgentConfig.NodeExternalIP.Value())
}
// if not set, try setting advertise-ip from agent node-ip
if serverConfig.ControlConfig.AdvertiseIP == "" && len(cmds.AgentConfig.NodeIP.Value()) != 0 {
serverConfig.ControlConfig.AdvertiseIP = util.GetFirstValidIPString(cmds.AgentConfig.NodeIP.Value())
}
}
// if we ended up with any advertise-ips, ensure they're added to the SAN list;
// note that kube-apiserver does not support dual-stack advertise-ip as of 1.21.0:
// https://github.com/kubernetes/kubeadm/issues/1612#issuecomment-772583989
if serverConfig.ControlConfig.AdvertiseIP != "" {
serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, serverConfig.ControlConfig.AdvertiseIP)
}
// configure ClusterIPRanges. Use default 10.42.0.0/16 or fd00:42::/56 if user did not set it
_, defaultClusterCIDR, defaultServiceCIDR, _ := util.GetDefaultAddresses(nodeIPs[0])
if len(cmds.ServerConfig.ClusterCIDR.Value()) == 0 {
@ -613,6 +552,51 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
}
}
// try setting advertise-ip from agent VPN
if vpnInfo, _ := vpn.GetInfoFromExecutor(); vpnInfo != nil {
// If we are in ipv6-only mode, we should pass the ipv6 address. Otherwise, ipv4
if utilsnet.IsIPv6(nodeIPs[0]) {
if vpnInfo.IPv6Address != nil {
logrus.Infof("Changed advertise-address to %v due to VPN", vpnInfo.IPv6Address)
if serverConfig.ControlConfig.AdvertiseIP != "" {
logrus.Warn("Conflict in the config detected. VPN integration overwrites advertise-address but the config is setting the advertise-address parameter")
}
serverConfig.ControlConfig.AdvertiseIP = vpnInfo.IPv6Address.String()
} else {
return errors.New("tailscale does not provide an ipv6 address")
}
} else {
// We are in dual-stack or ipv4-only mode
if vpnInfo.IPv4Address != nil {
logrus.Infof("Changed advertise-address to %v due to VPN", vpnInfo.IPv4Address)
if serverConfig.ControlConfig.AdvertiseIP != "" {
logrus.Warn("Conflict in the config detected. VPN integration overwrites advertise-address but the config is setting the advertise-address parameter")
}
serverConfig.ControlConfig.AdvertiseIP = vpnInfo.IPv4Address.String()
} else {
return errors.New("tailscale does not provide an ipv4 address")
}
}
logrus.Warn("Etcd IP (PrivateIP) remains the local IP. Running etcd traffic over VPN is not recommended due to performance issues")
} else {
// if not set, try setting advertise-ip from agent node-external-ip
if serverConfig.ControlConfig.AdvertiseIP == "" && len(cmds.AgentConfig.NodeExternalIP.Value()) != 0 {
serverConfig.ControlConfig.AdvertiseIP = util.GetFirstValidIPString(cmds.AgentConfig.NodeExternalIP.Value())
}
// if not set, try setting advertise-ip from agent node-ip
if serverConfig.ControlConfig.AdvertiseIP == "" && len(cmds.AgentConfig.NodeIP.Value()) != 0 {
serverConfig.ControlConfig.AdvertiseIP = util.GetFirstValidIPString(cmds.AgentConfig.NodeIP.Value())
}
}
// if we ended up with any advertise-ips, ensure they're added to the SAN list;
// note that kube-apiserver does not support dual-stack advertise-ip as of 1.21.0:
// https://github.com/kubernetes/kubeadm/issues/1612#issuecomment-772583989
if serverConfig.ControlConfig.AdvertiseIP != "" {
serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, serverConfig.ControlConfig.AdvertiseIP)
}
go cmds.WriteCoverage(ctx)
serverConfig.ControlConfig.Runtime.StartupHooksWg = &sync.WaitGroup{}

18
pkg/daemons/executor/executor.go
View File

@ -28,6 +28,7 @@ var (
// The enableMaintenance flag enables attempts to perform corrective maintenance during the test process.
type TestFunc func(ctx context.Context, enableMaintenance bool) error
// Executor is a set of functions for bootstrapping a node and starting the CRI, CNI, and Kubernetes components
type Executor interface {
Bootstrap(ctx context.Context, nodeConfig *daemonconfig.Node, cfg cmds.Agent) error
Kubelet(ctx context.Context, args []string) error
@ -49,6 +50,12 @@ type Executor interface {
IsSelfHosted() bool
}
// PreparingExecutor is an optional interface that Executors may implement to modify node configuration
// and CLI flags before config is retrieved from the server.
type PreparingExecutor interface {
Prepare(ctx context.Context, nodeConfig *daemonconfig.Node, cfg cmds.Agent) error
}
type ETCDSocketOpts struct {
ReuseAddress bool `json:"reuse-address,omitempty"`
ReusePort bool `json:"reuse-port,omitempty"`
@ -155,6 +162,10 @@ func Set(driver Executor) {
executor = driver
}
func Get() Executor {
return executor
}
func Bootstrap(ctx context.Context, nodeConfig *daemonconfig.Node, cfg cmds.Agent) error {
if executor == nil {
return ErrNotInitialized
@ -278,6 +289,13 @@ func IsSelfHosted() bool {
return executor.IsSelfHosted()
}
func Prepare(ctx context.Context, nodeConfig *daemonconfig.Node, cfg cmds.Agent) error {
if ex, ok := executor.(PreparingExecutor); ok {
return ex.Prepare(ctx, nodeConfig, cfg)
}
return nil
}
func CloseIfNilErr(err error, ch chan struct{}) error {
if err == nil {
close(ch)

119
pkg/executor/embed/embed.go
View File

@ -57,12 +57,76 @@ func init() {
// explicit type check
var _ executor.Executor = &Embedded{}
var _ executor.PreparingExecutor = &Embedded{}
var _ vpn.InfoProvider = &Embedded{}
type Embedded struct {
apiServerReady <-chan struct{}
etcdReady chan struct{}
criReady chan struct{}
nodeConfig *daemonconfig.Node
vpnInfo *vpn.Info
}
// Prepare modifies the node config prior to downloading client and server certificates from the server.
// If node IPs or names need to be modified, it should be done here so that the kubelet client and serving certs are valid.
func (e *Embedded) Prepare(ctx context.Context, nodeConfig *daemonconfig.Node, cfg cmds.Agent) error {
// If there is a VPN, we must start it early to overwrite NodeIP and flannel interface
var err error
if cfg.VPNAuthFile != "" {
cfg.VPNAuth, err = util.ReadFile(ctx, cfg.VPNAuthFile)
if err != nil {
return errors.WithMessage(err, "failed to read vpn-auth-file")
}
}
if cfg.VPNAuth != "" {
err = vpn.StartVPN(cfg.VPNAuth)
if err != nil {
return err
}
e.vpnInfo, err = vpn.GetInfo(cfg.VPNAuth)
if err != nil {
return err
}
// Pass ipv4, ipv6 or both depending on nodeIPs mode
nodeIPs := nodeConfig.AgentConfig.NodeIPs
var vpnIPs []net.IP
if utilsnet.IsIPv4(nodeIPs[0]) && e.vpnInfo.IPv4Address != nil {
vpnIPs = append(vpnIPs, e.vpnInfo.IPv4Address)
if e.vpnInfo.IPv6Address != nil {
vpnIPs = append(vpnIPs, e.vpnInfo.IPv6Address)
}
} else if utilsnet.IsIPv6(nodeIPs[0]) && e.vpnInfo.IPv6Address != nil {
vpnIPs = append(vpnIPs, e.vpnInfo.IPv6Address)
if e.vpnInfo.IPv4Address != nil {
vpnIPs = append(vpnIPs, e.vpnInfo.IPv4Address)
}
} else {
return fmt.Errorf("address family mismatch when assigning VPN addresses to node: node=%v, VPN ipv4=%v ipv6=%v", nodeIPs, e.vpnInfo.IPv4Address, e.vpnInfo.IPv6Address)
}
// Overwrite nodeip and flannel interface and throw a warning if user explicitly set those parameters
if len(vpnIPs) != 0 {
logrus.Infof("Node-ip changed to %v due to VPN", vpnIPs)
if len(cfg.NodeIP.Value()) != 0 {
logrus.Warn("VPN provider overrides configured node-ip parameter")
}
if len(cfg.NodeExternalIP.Value()) != 0 {
logrus.Warn("VPN provider overrides node-external-ip parameter")
}
nodeIPs = vpnIPs
nodeConfig.AgentConfig.NodeIPs = vpnIPs
nodeConfig.AgentConfig.NodeIP = vpnIPs[0].String()
nodeConfig.Flannel.Iface, err = net.InterfaceByName(e.vpnInfo.Interface)
if err != nil {
return errors.WithMessagef(err, "unable to find vpn interface: %s", e.vpnInfo.Interface)
}
}
}
return err
}
func (e *Embedded) Bootstrap(ctx context.Context, nodeConfig *daemonconfig.Node, cfg cmds.Agent) error {
@ -100,50 +164,6 @@ func (e *Embedded) Bootstrap(ctx context.Context, nodeConfig *daemonconfig.Node,
}
}
// If there is a VPN, we must overwrite NodeIP and flannel interface
var vpnInfo *vpn.Info
if cfg.VPNAuth != "" {
vpnInfo, err = vpn.GetInfo(cfg.VPNAuth)
if err != nil {
return err
}
// Pass ipv4, ipv6 or both depending on nodeIPs mode
nodeIPs := nodeConfig.AgentConfig.NodeIPs
var vpnIPs []net.IP
if utilsnet.IsIPv4(nodeIPs[0]) && vpnInfo.IPv4Address != nil {
vpnIPs = append(vpnIPs, vpnInfo.IPv4Address)
if vpnInfo.IPv6Address != nil {
vpnIPs = append(vpnIPs, vpnInfo.IPv6Address)
}
} else if utilsnet.IsIPv6(nodeIPs[0]) && vpnInfo.IPv6Address != nil {
vpnIPs = append(vpnIPs, vpnInfo.IPv6Address)
if vpnInfo.IPv4Address != nil {
vpnIPs = append(vpnIPs, vpnInfo.IPv4Address)
}
} else {
return fmt.Errorf("address family mismatch when assigning VPN addresses to node: node=%v, VPN ipv4=%v ipv6=%v", nodeIPs, vpnInfo.IPv4Address, vpnInfo.IPv6Address)
}
// Overwrite nodeip and flannel interface and throw a warning if user explicitly set those parameters
if len(vpnIPs) != 0 {
logrus.Infof("Node-ip changed to %v due to VPN", vpnIPs)
if len(cfg.NodeIP.Value()) != 0 {
logrus.Warn("VPN provider overrides configured node-ip parameter")
}
if len(cfg.NodeExternalIP.Value()) != 0 {
logrus.Warn("VPN provider overrides node-external-ip parameter")
}
nodeIPs = vpnIPs
nodeConfig.AgentConfig.NodeIPs = vpnIPs
nodeConfig.AgentConfig.NodeIP = vpnIPs[0].String()
nodeConfig.Flannel.Iface, err = net.InterfaceByName(vpnInfo.Interface)
if err != nil {
return errors.WithMessagef(err, "unable to find vpn interface: %s", vpnInfo.Interface)
}
}
}
// set paths for embedded flannel if enabled
hostLocal, err := exec.LookPath("host-local")
if err != nil {
@ -160,8 +180,8 @@ func (e *Embedded) Bootstrap(ctx context.Context, nodeConfig *daemonconfig.Node,
nodeConfig.AgentConfig.CNIConfDir = filepath.Join(cfg.DataDir, "agent", "etc", "cni", "net.d")
// It does not make sense to use VPN without its flannel backend
if cfg.VPNAuth != "" {
nodeConfig.Flannel.Backend = vpnInfo.ProviderName
if e.vpnInfo != nil {
nodeConfig.Flannel.Backend = e.vpnInfo.ProviderName
}
}
@ -419,3 +439,10 @@ func (e *Embedded) CRIReadyChan() <-chan struct{} {
func (e Embedded) IsSelfHosted() bool {
return false
}
func (e Embedded) GetVPNInfo() (*vpn.Info, error) {
if e.vpnInfo != nil {
return e.vpnInfo, nil
}
return nil, errors.New("VPN info not set")
}

27
pkg/util/file.go
View File

@ -1,7 +1,7 @@
package util
import (
"errors"
"context"
"os"
"os/user"
"path/filepath"
@ -10,6 +10,7 @@ import (
"time"
"github.com/sirupsen/logrus"
"k8s.io/apimachinery/pkg/util/wait"
)
func SetFileModeForPath(name string, mode os.FileMode) error {
@ -41,25 +42,23 @@ func SetFileModeForFile(file *os.File, mode os.FileMode) error {
return file.Chmod(mode)
}
// ReadFile reads from a file
func ReadFile(path string) (string, error) {
// ReadFile waits for a file to exist, then returns its trimmed contents as a string
func ReadFile(ctx context.Context, path string) (string, error) {
if path == "" {
return "", nil
}
for start := time.Now(); time.Since(start) < 4*time.Minute; {
vpnBytes, err := os.ReadFile(path)
var trimmed string
return trimmed, wait.PollUntilContextTimeout(ctx, 2*time.Second, 4*time.Minute, true, func(ctx context.Context) (bool, error) {
b, err := os.ReadFile(path)
if err == nil {
return strings.TrimSpace(string(vpnBytes)), nil
trimmed = strings.TrimSpace(string(b))
return true, nil
} else if os.IsNotExist(err) {
logrus.Infof("Waiting for %s to be available\n", path)
time.Sleep(2 * time.Second)
} else {
return "", err
logrus.Infof("Waiting for file %q to be created\n", path)
return false, nil
}
}
return "", errors.New("Timeout while trying to read the file")
return false, err
})
}
// AtomicWrite firsts writes data to a temp file, then renames to the destination file.

18
pkg/vpn/vpn.go
View File

@ -8,6 +8,7 @@ import (
"net/url"
"strings"
"github.com/k3s-io/k3s/pkg/daemons/executor"
"github.com/k3s-io/k3s/pkg/util"
"github.com/k3s-io/k3s/pkg/util/errors"
@ -18,6 +19,10 @@ const (
tailscaleIf = "tailscale0"
)
type InfoProvider interface {
GetVPNInfo() (*Info, error)
}
type TailscaleOutput struct {
TailscaleIPs []string `json:"TailscaleIPs"`
BackendState string `json:"BackendState"`
@ -72,7 +77,10 @@ func StartVPN(vpnAuthConfigFile string) error {
logrus.Debugf("Flags passed to tailscale up: %v", args)
output, err := util.ExecCommand("tailscale", args)
if err != nil {
return errors.WithMessage(err, "tailscale up failed: "+output)
if output != "" {
return errors.WithMessagef(err, "tailscale up failed (%q)", output)
}
return errors.WithMessage(err, "tailscale up failed")
}
logrus.Debugf("Output from tailscale up: %v", output)
return nil
@ -94,6 +102,14 @@ func GetInfo(vpnAuth string) (*Info, error) {
return nil, nil
}
func GetInfoFromExecutor() (*Info, error) {
ex := executor.Get()
if provider, ok := ex.(InfoProvider); ok {
return provider.GetVPNInfo()
}
return nil, errors.New("executor does not provide VPN info")
}
// getVPNAuthInfo returns the required authInfo object
func getVPNAuthInfo(vpnAuth string) (vpnCliAuthInfo, error) {
var authInfo vpnCliAuthInfo

2
tests/e2e/rootless/rootless_test.go
View File

@ -228,7 +228,7 @@ func SaveRootlessJournalLogs(nodes []e2e.VagrantNode) error {
return err
}
defer lf.Close()
cmd := "vagrant ssh --no-tty " + node.Name + " -c \"journalctl -u --user k3s-rootless --no-pager\""
cmd := "vagrant ssh --no-tty " + node.Name + " -c \"journalctl --user -u k3s-rootless --no-pager\""
logs, err := tests.RunCommand(cmd)
if err != nil {
return err