From 06c779c57dff534106402196bb3474afdae2ff98 Mon Sep 17 00:00:00 2001 From: Roberto Bonafiglia Date: Thu, 31 Mar 2022 11:49:30 +0200 Subject: [PATCH] Fixed loadbalancer in case of IPv6 addresses Signed-off-by: Roberto Bonafiglia --- pkg/agent/config/config.go | 3 ++- pkg/agent/loadbalancer/loadbalancer.go | 8 ++++++-- pkg/agent/loadbalancer/loadbalancer_test.go | 4 ++-- pkg/agent/proxy/apiproxy.go | 10 +++++----- pkg/agent/run.go | 3 ++- pkg/cluster/cluster.go | 4 +++- pkg/etcd/etcdproxy.go | 4 ++-- 7 files changed, 22 insertions(+), 14 deletions(-) diff --git a/pkg/agent/config/config.go b/pkg/agent/config/config.go index 4c56abbdc1..6aed8904e9 100644 --- a/pkg/agent/config/config.go +++ b/pkg/agent/config/config.go @@ -308,7 +308,8 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N // If the supervisor and externally-facing apiserver are not on the same port, tell the proxy where to find the apiserver. if controlConfig.SupervisorPort != controlConfig.HTTPSPort { - if err := proxy.SetAPIServerPort(ctx, controlConfig.HTTPSPort); err != nil { + _, isIPv6, _ := util.GetFirstString([]string{envInfo.NodeIP.String()}) + if err := proxy.SetAPIServerPort(ctx, controlConfig.HTTPSPort, isIPv6); err != nil { return nil, errors.Wrapf(err, "failed to setup access to API Server port %d on at %s", controlConfig.HTTPSPort, proxy.SupervisorURL()) } } diff --git a/pkg/agent/loadbalancer/loadbalancer.go b/pkg/agent/loadbalancer/loadbalancer.go index 4bf0a2eee4..063785531c 100644 --- a/pkg/agent/loadbalancer/loadbalancer.go +++ b/pkg/agent/loadbalancer/loadbalancer.go @@ -40,9 +40,13 @@ var ( ETCDServerServiceName = version.Program + "-etcd-server-load-balancer" ) -func New(ctx context.Context, dataDir, serviceName, serverURL string, lbServerPort int) (_lb *LoadBalancer, _err error) { +func New(ctx context.Context, dataDir, serviceName, serverURL string, lbServerPort int, isIPv6 bool) (_lb *LoadBalancer, _err error) { config := net.ListenConfig{Control: reusePort} - listener, err := config.Listen(ctx, "tcp", "127.0.0.1:"+strconv.Itoa(lbServerPort)) + localhostAddress := "127.0.0.1" + if isIPv6 { + localhostAddress = "[::1]" + } + listener, err := config.Listen(ctx, "tcp", localhostAddress+":"+strconv.Itoa(lbServerPort)) defer func() { if _err != nil { logrus.Warnf("Error starting load balancer: %s", _err) diff --git a/pkg/agent/loadbalancer/loadbalancer_test.go b/pkg/agent/loadbalancer/loadbalancer_test.go index 3a577e7693..9fe1f9fc26 100644 --- a/pkg/agent/loadbalancer/loadbalancer_test.go +++ b/pkg/agent/loadbalancer/loadbalancer_test.go @@ -106,7 +106,7 @@ func Test_UnitFailOver(t *testing.T) { DataDir: tmpDir, } - lb, err := New(context.TODO(), cfg.DataDir, SupervisorServiceName, cfg.ServerURL, RandomPort) + lb, err := New(context.TODO(), cfg.DataDir, SupervisorServiceName, cfg.ServerURL, RandomPort, false) if err != nil { assertEqual(t, err, nil) } @@ -157,7 +157,7 @@ func Test_UnitFailFast(t *testing.T) { DataDir: tmpDir, } - lb, err := New(context.TODO(), cfg.DataDir, SupervisorServiceName, cfg.ServerURL, RandomPort) + lb, err := New(context.TODO(), cfg.DataDir, SupervisorServiceName, cfg.ServerURL, RandomPort, false) if err != nil { assertEqual(t, err, nil) } diff --git a/pkg/agent/proxy/apiproxy.go b/pkg/agent/proxy/apiproxy.go index 9f7e0773c3..834bf1bffb 100644 --- a/pkg/agent/proxy/apiproxy.go +++ b/pkg/agent/proxy/apiproxy.go @@ -14,7 +14,7 @@ import ( type Proxy interface { Update(addresses []string) - SetAPIServerPort(ctx context.Context, port int) error + SetAPIServerPort(ctx context.Context, port int, isIPv6 bool) error SetSupervisorDefault(address string) SupervisorURL() string SupervisorAddresses() []string @@ -29,7 +29,7 @@ type Proxy interface { // NOTE: This is a proxy in the API sense - it returns either actual server URLs, or the URL of the // local load-balancer. It is not actually responsible for proxying requests at the network level; // this is handled by the load-balancers that the proxy optionally steers connections towards. -func NewSupervisorProxy(ctx context.Context, lbEnabled bool, dataDir, supervisorURL string, lbServerPort int) (Proxy, error) { +func NewSupervisorProxy(ctx context.Context, lbEnabled bool, dataDir, supervisorURL string, lbServerPort int, isIPv6 bool) (Proxy, error) { p := proxy{ lbEnabled: lbEnabled, dataDir: dataDir, @@ -40,7 +40,7 @@ func NewSupervisorProxy(ctx context.Context, lbEnabled bool, dataDir, supervisor } if lbEnabled { - lb, err := loadbalancer.New(ctx, dataDir, loadbalancer.SupervisorServiceName, supervisorURL, p.lbServerPort) + lb, err := loadbalancer.New(ctx, dataDir, loadbalancer.SupervisorServiceName, supervisorURL, p.lbServerPort, isIPv6) if err != nil { return nil, err } @@ -110,7 +110,7 @@ func (p *proxy) setSupervisorPort(addresses []string) []string { // load-balancing is enabled, another load-balancer is started on a port one below the supervisor // load-balancer, and the address of this load-balancer is returned instead of the actual apiserver // addresses. -func (p *proxy) SetAPIServerPort(ctx context.Context, port int) error { +func (p *proxy) SetAPIServerPort(ctx context.Context, port int, isIPv6 bool) error { u, err := url.Parse(p.initialSupervisorURL) if err != nil { return errors.Wrapf(err, "failed to parse server URL %s", p.initialSupervisorURL) @@ -125,7 +125,7 @@ func (p *proxy) SetAPIServerPort(ctx context.Context, port int) error { if lbServerPort != 0 { lbServerPort = lbServerPort - 1 } - lb, err := loadbalancer.New(ctx, p.dataDir, loadbalancer.APIServerServiceName, p.apiServerURL, lbServerPort) + lb, err := loadbalancer.New(ctx, p.dataDir, loadbalancer.APIServerServiceName, p.apiServerURL, lbServerPort, isIPv6) if err != nil { return err } diff --git a/pkg/agent/run.go b/pkg/agent/run.go index 160761e885..4c62b9fdab 100644 --- a/pkg/agent/run.go +++ b/pkg/agent/run.go @@ -239,8 +239,9 @@ func createProxyAndValidateToken(ctx context.Context, cfg *cmds.Agent) (proxy.Pr if err := os.MkdirAll(agentDir, 0700); err != nil { return nil, err } + _, isIPv6, _ := util.GetFirstString([]string{cfg.NodeIP.String()}) - proxy, err := proxy.NewSupervisorProxy(ctx, !cfg.DisableLoadBalancer, agentDir, cfg.ServerURL, cfg.LBServerPort) + proxy, err := proxy.NewSupervisorProxy(ctx, !cfg.DisableLoadBalancer, agentDir, cfg.ServerURL, cfg.LBServerPort, isIPv6) if err != nil { return nil, err } diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 84c94dcd9c..6e6d56cd72 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -10,6 +10,7 @@ import ( "github.com/k3s-io/k3s/pkg/cluster/managed" "github.com/k3s-io/k3s/pkg/daemons/config" "github.com/k3s-io/k3s/pkg/etcd" + "github.com/k3s-io/k3s/pkg/util" "github.com/k3s-io/kine/pkg/endpoint" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -52,7 +53,8 @@ func (c *Cluster) Start(ctx context.Context) (<-chan struct{}, error) { clientURL.Host = clientURL.Hostname() + ":2379" clientURLs = append(clientURLs, clientURL.String()) } - etcdProxy, err := etcd.NewETCDProxy(ctx, true, c.config.DataDir, clientURLs[0]) + IPv6OnlyService, _ := util.IsIPv6OnlyCIDRs(c.config.ServiceIPRanges) + etcdProxy, err := etcd.NewETCDProxy(ctx, true, c.config.DataDir, clientURLs[0], IPv6OnlyService) if err != nil { return nil, err } diff --git a/pkg/etcd/etcdproxy.go b/pkg/etcd/etcdproxy.go index 46ec6dcaf6..6b0cc966f5 100644 --- a/pkg/etcd/etcdproxy.go +++ b/pkg/etcd/etcdproxy.go @@ -17,7 +17,7 @@ type Proxy interface { // NewETCDProxy initializes a new proxy structure that contain a load balancer // which listens on port 2379 and proxy between etcd cluster members -func NewETCDProxy(ctx context.Context, enabled bool, dataDir, etcdURL string) (Proxy, error) { +func NewETCDProxy(ctx context.Context, enabled bool, dataDir, etcdURL string, isIPv6 bool) (Proxy, error) { u, err := url.Parse(etcdURL) if err != nil { return nil, errors.Wrap(err, "failed to parse etcd client URL") @@ -30,7 +30,7 @@ func NewETCDProxy(ctx context.Context, enabled bool, dataDir, etcdURL string) (P } if enabled { - lb, err := loadbalancer.New(ctx, dataDir, loadbalancer.ETCDServerServiceName, etcdURL, 2379) + lb, err := loadbalancer.New(ctx, dataDir, loadbalancer.ETCDServerServiceName, etcdURL, 2379, isIPv6) if err != nil { return nil, err }