When bootstrapping a client cert, store it with other client certs

The kubelet uses two different locations to store certificates on
initial bootstrap and then on subsequent rotation:

* bootstrap: certDir/kubelet-client.(crt|key)
* rotation:  certDir/kubelet-client-(DATE|current).pem

Bootstrap also creates an initial node.kubeconfig that points to the
certs. Unfortunately, with short rotation the node.kubeconfig then
becomes out of date because it points to the initial cert/key, not the
rotated cert key.

Alter the bootstrap code to store client certs exactly as if they would
be rotated (using the same cert Store code), and reference the PEM file
containing cert/key from node.kubeconfig, which is supported by kubectl
and other Go tooling. This ensures that the node.kubeconfig continues to
be valid past the first expiration.

pkg/kubelet/certificate/bootstrap/BUILD

@@ -30,6 +30,7 @@ go_library(
         "//vendor/k8s.io/client-go/tools/clientcmd/api:go_default_library",
         "//vendor/k8s.io/client-go/transport:go_default_library",
         "//vendor/k8s.io/client-go/util/cert:go_default_library",
+        "//vendor/k8s.io/client-go/util/certificate:go_default_library",
         "//vendor/k8s.io/client-go/util/certificate/csr:go_default_library",
     ],
 )

pkg/kubelet/certificate/bootstrap/bootstrap.go

@@ -19,7 +19,6 @@ package bootstrap
 import (
 	"fmt"
 	"os"
-	"path/filepath"
 	"time"
 
 	"github.com/golang/glog"
@@ -32,14 +31,10 @@ import (
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 	"k8s.io/client-go/transport"
 	certutil "k8s.io/client-go/util/cert"
+	"k8s.io/client-go/util/certificate"
 	"k8s.io/client-go/util/certificate/csr"
 )
 
-const (
-	defaultKubeletClientCertificateFile = "kubelet-client.crt"
-	defaultKubeletClientKeyFile         = "kubelet-client.key"
-)
-
 // LoadClientCert requests a client cert for kubelet if the kubeconfigPath file does not exist.
 // The kubeconfig at bootstrapPath is used to request a client certificate from the API server.
 // On success, a kubeconfig file referencing the generated key and obtained certificate is written to kubeconfigPath.
@@ -66,49 +61,38 @@ func LoadClientCert(kubeconfigPath string, bootstrapPath string, certDir string,
 		return fmt.Errorf("unable to create certificates signing request client: %v", err)
 	}
 
-	success := false
-
-	// Get the private key.
-	keyPath, err := filepath.Abs(filepath.Join(certDir, defaultKubeletClientKeyFile))
+	store, err := certificate.NewFileStore("kubelet-client", certDir, certDir, "", "")
 	if err != nil {
-		return fmt.Errorf("unable to build bootstrap key path: %v", err)
-	}
-	// If we are unable to generate a CSR, we remove our key file and start fresh.
-	// This method is used before enabling client rotation and so we must ensure we
-	// can make forward progress if we crash and exit when a CSR exists but the cert
-	// it is signed for has expired.
-	defer func() {
-		if !success {
-			if err := os.Remove(keyPath); err != nil && !os.IsNotExist(err) {
-				glog.Warningf("Cannot clean up the key file %q: %v", keyPath, err)
-			}
-		}
-	}()
-	keyData, _, err := certutil.LoadOrGenerateKeyFile(keyPath)
-	if err != nil {
-		return err
+		return fmt.Errorf("unable to build bootstrap cert store")
 	}
 
-	// Get the cert.
-	certPath, err := filepath.Abs(filepath.Join(certDir, defaultKubeletClientCertificateFile))
-	if err != nil {
-		return fmt.Errorf("unable to build bootstrap client cert path: %v", err)
-	}
-	defer func() {
-		if !success {
-			if err := os.Remove(certPath); err != nil && !os.IsNotExist(err) {
-				glog.Warningf("Cannot clean up the cert file %q: %v", certPath, err)
+	var keyData []byte
+	if cert, err := store.Current(); err == nil {
+		if cert.PrivateKey != nil {
+			keyData, err = certutil.MarshalPrivateKeyToPEM(cert.PrivateKey)
+			if err != nil {
+				keyData = nil
 			}
 		}
-	}()
+	}
+	if !verifyKeyData(keyData) {
+		glog.V(2).Infof("No valid private key found for bootstrapping, creating a new one")
+		keyData, err = certutil.MakeEllipticPrivateKeyPEM()
+		if err != nil {
+			return err
+		}
+	}
+
 	certData, err := csr.RequestNodeCertificate(bootstrapClient.CertificateSigningRequests(), keyData, nodeName)
 	if err != nil {
 		return err
 	}
-	if err := certutil.WriteCert(certPath, certData); err != nil {
+	if _, err := store.Update(certData, keyData); err != nil {
 		return err
 	}
 
+	pemPath := store.CurrentPath()
+
 	// Get the CA data from the bootstrap client config.
 	caFile, caData := bootstrapClientConfig.CAFile, []byte{}
 	if len(caFile) == 0 {
@@ -126,8 +110,8 @@ func LoadClientCert(kubeconfigPath string, bootstrapPath string, certDir string,
 		}},
 		// Define auth based on the obtained client cert.
 		AuthInfos: map[string]*clientcmdapi.AuthInfo{"default-auth": {
-			ClientCertificate: certPath,
-			ClientKey:         keyPath,
+			ClientCertificate: pemPath,
+			ClientKey:         pemPath,
 		}},
 		// Define a context that connects the auth info and cluster, and set it as the default
 		Contexts: map[string]*clientcmdapi.Context{"default-context": {
@@ -142,8 +126,6 @@ func LoadClientCert(kubeconfigPath string, bootstrapPath string, certDir string,
 	if err := clientcmd.WriteToFile(kubeconfigData, kubeconfigPath); err != nil {
 		return err
 	}
-
-	success = true
 	return nil
 }
 
@@ -207,3 +189,14 @@ func verifyBootstrapClientConfig(kubeconfigPath string) (bool, error) {
 	}
 	return true, nil
 }
+
+// verifyKeyData returns true if the provided data appears to be a valid private key.
+func verifyKeyData(data []byte) bool {
+	if len(data) == 0 {
+		return false
+	}
+	if _, err := certutil.ParsePrivateKeyPEM(data); err != nil {
+		return false
+	}
+	return true
+}

staging/src/k8s.io/client-go/util/cert/io.go

@@ -17,7 +17,11 @@ limitations under the License.
 package cert
 
 import (
+	"crypto"
+	"crypto/ecdsa"
+	"crypto/rsa"
 	"crypto/x509"
+	"encoding/pem"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -101,6 +105,27 @@ func LoadOrGenerateKeyFile(keyPath string) (data []byte, wasGenerated bool, err
 	return generatedData, true, nil
 }
 
+// MarshalPrivateKeyToPEM converts a known private key type of RSA or ECDSA to
+// a PEM encoded block or returns an error.
+func MarshalPrivateKeyToPEM(privateKey crypto.PrivateKey) ([]byte, error) {
+	switch t := privateKey.(type) {
+	case *ecdsa.PrivateKey:
+		derBytes, err := x509.MarshalECPrivateKey(t)
+		if err != nil {
+			return nil, err
+		}
+		privateKeyPemBlock := &pem.Block{
+			Type:  ECPrivateKeyBlockType,
+			Bytes: derBytes,
+		}
+		return pem.EncodeToMemory(privateKeyPemBlock), nil
+	case *rsa.PrivateKey:
+		return EncodePrivateKeyPEM(t), nil
+	default:
+		return nil, fmt.Errorf("private key is not a recognized type: %T", privateKey)
+	}
+}
+
 // NewPool returns an x509.CertPool containing the certificates in the given PEM-encoded file.
 // Returns an error if the file could not be read, a certificate could not be parsed, or if the file does not contain any certificates
 func NewPool(filename string) (*x509.CertPool, error) {

staging/src/k8s.io/client-go/util/certificate/certificate_store.go

@@ -46,6 +46,15 @@ type fileStore struct {
 	keyFile        string
 }
 
+// FileStore is a store that provides certificate retrieval as well as
+// the path on disk of the current PEM.
+type FileStore interface {
+	Store
+	// CurrentPath returns the path on disk of the current certificate/key
+	// pair encoded as PEM files.
+	CurrentPath() string
+}
+
 // NewFileStore returns a concrete implementation of a Store that is based on
 // storing the cert/key pairs in a single file per pair on disk in the
 // designated directory. When starting up it will look for the currently
@@ -64,7 +73,7 @@ func NewFileStore(
 	certDirectory string,
 	keyDirectory string,
 	certFile string,
-	keyFile string) (Store, error) {
+	keyFile string) (FileStore, error) {
 
 	s := fileStore{
 		pairNamePrefix: pairNamePrefix,
@@ -79,6 +88,11 @@ func NewFileStore(
 	return &s, nil
 }
 
+// CurrentPath returns the path to the current version of these certificates.
+func (s *fileStore) CurrentPath() string {
+	return filepath.Join(s.certDirectory, s.filename(currentPair))
+}
+
 // recover checks if there is a certificate rotation that was interrupted while
 // progress, and if so, attempts to recover to a good state.
 func (s *fileStore) recover() error {