diff --git a/pkg/agent/config/config.go b/pkg/agent/config/config.go index a7d6d7a19f..1c0ba40144 100644 --- a/pkg/agent/config/config.go +++ b/pkg/agent/config/config.go @@ -8,6 +8,7 @@ import ( "fmt" "io/ioutil" sysnet "net" + "net/http" "net/url" "os" "os/exec" @@ -43,16 +44,60 @@ func Get(ctx context.Context, agent cmds.Agent) *config.Node { } } -func getNodeCert(info *clientaccess.Info) (*tls.Certificate, error) { - nodeCert, err := clientaccess.Get("/v1-k3s/node.crt", info) +type HTTPRequester func(u string, client *http.Client, username, password string) ([]byte, error) + +func Request(path string, info *clientaccess.Info, requester HTTPRequester) ([]byte, error) { + u, err := url.Parse(info.URL) if err != nil { return nil, err } + u.Path = path + username, password, _ := clientaccess.ParseUsernamePassword(info.Token) + return requester(u.String(), clientaccess.GetHTTPClient(info.CACerts), username, password) +} + +func getNodeNamedCrt(nodeName string) HTTPRequester { + return func(u string, client *http.Client, username, password string) ([]byte, error) { + req, err := http.NewRequest(http.MethodGet, u, nil) + if err != nil { + return nil, err + } + + if username != "" { + req.SetBasicAuth(username, password) + } + + req.Header.Set("K3s-Node-Name", nodeName) + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("%s: %s", u, resp.Status) + } + + return ioutil.ReadAll(resp.Body) + } +} + +func getNodeCert(nodeName, nodeCertFile, nodeKeyFile string, info *clientaccess.Info) (*tls.Certificate, error) { + nodeCert, err := Request("/v1-k3s/node.crt", info, getNodeNamedCrt(nodeName)) + if err != nil { + return nil, err + } + if err := ioutil.WriteFile(nodeCertFile, nodeCert, 0600); err != nil { + return nil, errors.Wrapf(err, "failed to write node cert") + } nodeKey, err := clientaccess.Get("/v1-k3s/node.key", info) if err != nil { return nil, err } + if err := ioutil.WriteFile(nodeKeyFile, nodeKey, 0600); err != nil { + return nil, errors.Wrapf(err, "failed to write node key") + } cert, err := tls.X509KeyPair(nodeCert, nodeKey) if err != nil { @@ -181,7 +226,14 @@ func get(envInfo *cmds.Agent) (*config.Node, error) { return nil, err } - nodeCert, err := getNodeCert(info) + nodeName, nodeIP, err := getHostnameAndIP(*envInfo) + if err != nil { + return nil, err + } + + nodeCertFile := filepath.Join(envInfo.DataDir, "token-node.crt") + nodeKeyFile := filepath.Join(envInfo.DataDir, "token-node.key") + nodeCert, err := getNodeCert(nodeName, nodeCertFile, nodeKeyFile, info) if err != nil { return nil, err } @@ -191,11 +243,6 @@ func get(envInfo *cmds.Agent) (*config.Node, error) { return nil, err } - nodeName, nodeIP, err := getHostnameAndIP(*envInfo) - if err != nil { - return nil, err - } - kubeConfig, err := writeKubeConfig(envInfo, *info, controlConfig, nodeCert) if err != nil { return nil, err @@ -224,6 +271,8 @@ func get(envInfo *cmds.Agent) (*config.Node, error) { nodeConfig.Images = filepath.Join(envInfo.DataDir, "images") nodeConfig.AgentConfig.NodeIP = nodeIP nodeConfig.AgentConfig.NodeName = nodeName + nodeConfig.AgentConfig.NodeCertFile = nodeCertFile + nodeConfig.AgentConfig.NodeKeyFile = nodeKeyFile nodeConfig.AgentConfig.ClusterDNS = controlConfig.ClusterDNS nodeConfig.AgentConfig.ClusterDomain = controlConfig.ClusterDomain nodeConfig.AgentConfig.ResolvConf = locateOrGenerateResolvConf(envInfo) diff --git a/pkg/daemons/agent/agent.go b/pkg/daemons/agent/agent.go index 3417dc206a..9e8014beb2 100644 --- a/pkg/daemons/agent/agent.go +++ b/pkg/daemons/agent/agent.go @@ -94,6 +94,10 @@ func kubelet(cfg *config.Agent) { argsMap["anonymous-auth"] = "false" argsMap["client-ca-file"] = cfg.CACertPath } + if cfg.NodeCertFile != "" && cfg.NodeKeyFile != "" { + argsMap["tls-cert-file"] = cfg.NodeCertFile + argsMap["tls-private-key-file"] = cfg.NodeKeyFile + } if cfg.NodeName != "" { argsMap["hostname-override"] = cfg.NodeName } diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index 577f26eb3b..ddbe3474f5 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -36,6 +36,8 @@ type Containerd struct { type Agent struct { NodeName string + NodeCertFile string + NodeKeyFile string ClusterCIDR net.IPNet ClusterDNS net.IP ClusterDomain string diff --git a/pkg/server/router.go b/pkg/server/router.go index 0626cae997..7665e78693 100644 --- a/pkg/server/router.go +++ b/pkg/server/router.go @@ -1,6 +1,10 @@ package server import ( + "crypto/rsa" + "crypto/x509" + "io/ioutil" + "net" "net/http" "path/filepath" "strconv" @@ -8,7 +12,10 @@ import ( "github.com/gorilla/mux" "github.com/rancher/k3s/pkg/daemons/config" "github.com/rancher/k3s/pkg/openapi" + certutil "github.com/rancher/norman/pkg/cert" + "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/util/json" + "k8s.io/kubernetes/pkg/master" ) const ( @@ -59,7 +66,71 @@ func nodeCrt(server *config.Control) http.Handler { resp.WriteHeader(http.StatusNotFound) return } - http.ServeFile(resp, req, server.Runtime.NodeCert) + + var nodeName string + nodeNames := req.Header["K3s-Node-Name"] + if len(nodeNames) == 1 { + nodeName = nodeNames[0] + } + + nodeKey, err := ioutil.ReadFile(server.Runtime.NodeKey) + if err != nil { + sendError(err, resp) + return + } + + key, err := certutil.ParsePrivateKeyPEM(nodeKey) + if err != nil { + sendError(err, resp) + return + } + + caKeyBytes, err := ioutil.ReadFile(server.Runtime.TokenCAKey) + if err != nil { + sendError(err, resp) + return + } + + caBytes, err := ioutil.ReadFile(server.Runtime.TokenCA) + if err != nil { + sendError(err, resp) + return + } + + caKey, err := certutil.ParsePrivateKeyPEM(caKeyBytes) + if err != nil { + sendError(err, resp) + return + } + + caCert, err := certutil.ParseCertsPEM(caBytes) + if err != nil { + sendError(err, resp) + return + } + + _, apiServerServiceIP, err := master.DefaultServiceIPRange(*server.ServiceIPRange) + if err != nil { + sendError(err, resp) + return + } + + cfg := certutil.Config{ + CommonName: "kubernetes", + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + AltNames: certutil.AltNames{ + DNSNames: []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes", "localhost", nodeName}, + IPs: []net.IP{apiServerServiceIP, net.ParseIP("127.0.0.1")}, + }, + } + + cert, err := certutil.NewSignedCert(cfg, key.(*rsa.PrivateKey), caCert[0], caKey.(*rsa.PrivateKey)) + if err != nil { + sendError(err, resp) + return + } + + resp.Write(append(certutil.EncodeCertPEM(cert), certutil.EncodeCertPEM(caCert[0])...)) }) } @@ -118,3 +189,9 @@ func ping() http.Handler { func serveStatic(urlPrefix, staticDir string) http.Handler { return http.StripPrefix(urlPrefix, http.FileServer(http.Dir(staticDir))) } + +func sendError(err error, resp http.ResponseWriter) { + logrus.Error(err) + resp.WriteHeader(http.StatusInternalServerError) + resp.Write([]byte(err.Error())) +}