diff --git a/cmd/kubelet/app/BUILD b/cmd/kubelet/app/BUILD index ae004b5f48..64269fd555 100644 --- a/cmd/kubelet/app/BUILD +++ b/cmd/kubelet/app/BUILD @@ -121,6 +121,7 @@ go_library( "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library", "//vendor/k8s.io/client-go/tools/record:go_default_library", "//vendor/k8s.io/client-go/util/cert:go_default_library", + "//vendor/k8s.io/client-go/util/certificate:go_default_library", ] + select({ "@io_bazel_rules_go//go/platform:linux_amd64": [ "//vendor/golang.org/x/exp/inotify:go_default_library", diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 22b283613f..bd5731abb4 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -52,6 +52,7 @@ import ( "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/tools/record" certutil "k8s.io/client-go/util/cert" + "k8s.io/client-go/util/certificate" "k8s.io/kubernetes/cmd/kubelet/app/options" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/capabilities" @@ -64,7 +65,7 @@ import ( kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme" kubeletconfigv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/cadvisor" - "k8s.io/kubernetes/pkg/kubelet/certificate" + kubeletcertificate "k8s.io/kubernetes/pkg/kubelet/certificate" "k8s.io/kubernetes/pkg/kubelet/certificate/bootstrap" "k8s.io/kubernetes/pkg/kubelet/cm" "k8s.io/kubernetes/pkg/kubelet/config" @@ -336,11 +337,11 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies) (err error) { var clientCertificateManager certificate.Manager if err == nil { if s.RotateCertificates && utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletClientCertificate) { - clientCertificateManager, err = certificate.NewKubeletClientCertificateManager(s.CertDirectory, nodeName, clientConfig.CertData, clientConfig.KeyData, clientConfig.CertFile, clientConfig.KeyFile) + clientCertificateManager, err = kubeletcertificate.NewKubeletClientCertificateManager(s.CertDirectory, nodeName, clientConfig.CertData, clientConfig.KeyData, clientConfig.CertFile, clientConfig.KeyFile) if err != nil { return err } - if err := certificate.UpdateTransport(wait.NeverStop, clientConfig, clientCertificateManager); err != nil { + if err := kubeletcertificate.UpdateTransport(wait.NeverStop, clientConfig, clientCertificateManager); err != nil { return err } } diff --git a/hack/.golint_failures b/hack/.golint_failures index c30b9d4980..fab2d064f0 100644 --- a/hack/.golint_failures +++ b/hack/.golint_failures @@ -218,7 +218,6 @@ pkg/kubelet/apis/kubeletconfig pkg/kubelet/apis/kubeletconfig/v1alpha1 pkg/kubelet/cadvisor pkg/kubelet/cadvisor/testing -pkg/kubelet/certificate pkg/kubelet/client pkg/kubelet/cm pkg/kubelet/cm/util diff --git a/pkg/kubelet/BUILD b/pkg/kubelet/BUILD index 151492dbdf..fc9ed4fb23 100644 --- a/pkg/kubelet/BUILD +++ b/pkg/kubelet/BUILD @@ -136,6 +136,7 @@ go_library( "//vendor/k8s.io/client-go/tools/cache:go_default_library", "//vendor/k8s.io/client-go/tools/record:go_default_library", "//vendor/k8s.io/client-go/tools/remotecommand:go_default_library", + "//vendor/k8s.io/client-go/util/certificate:go_default_library", "//vendor/k8s.io/client-go/util/flowcontrol:go_default_library", "//vendor/k8s.io/client-go/util/integer:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", diff --git a/pkg/kubelet/certificate/BUILD b/pkg/kubelet/certificate/BUILD index 9dfc95f814..50223ce663 100644 --- a/pkg/kubelet/certificate/BUILD +++ b/pkg/kubelet/certificate/BUILD @@ -9,49 +9,34 @@ load( go_library( name = "go_default_library", srcs = [ - "certificate_manager.go", - "certificate_store.go", "kubelet.go", "transport.go", ], deps = [ "//pkg/kubelet/apis/kubeletconfig:go_default_library", "//pkg/kubelet/metrics:go_default_library", - "//pkg/util/file:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", "//vendor/k8s.io/api/certificates/v1beta1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", "//vendor/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library", "//vendor/k8s.io/client-go/rest:go_default_library", - "//vendor/k8s.io/client-go/util/cert:go_default_library", + "//vendor/k8s.io/client-go/util/certificate:go_default_library", ], ) go_test( name = "go_default_test", - srcs = [ - "certificate_manager_test.go", - "certificate_store_test.go", - "transport_test.go", - ], + srcs = ["transport_test.go"], library = ":go_default_library", deps = [ - "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", - "//vendor/k8s.io/api/certificates/v1beta1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library", "//vendor/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library", "//vendor/k8s.io/client-go/rest:go_default_library", - "//vendor/k8s.io/client-go/util/cert:go_default_library", ], ) diff --git a/pkg/kubelet/certificate/kubelet.go b/pkg/kubelet/certificate/kubelet.go index c346bcb4f8..097266c7f4 100644 --- a/pkg/kubelet/certificate/kubelet.go +++ b/pkg/kubelet/certificate/kubelet.go @@ -22,21 +22,25 @@ import ( "fmt" "net" + "github.com/prometheus/client_golang/prometheus" + certificates "k8s.io/api/certificates/v1beta1" "k8s.io/apimachinery/pkg/types" clientset "k8s.io/client-go/kubernetes" clientcertificates "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" + "k8s.io/client-go/util/certificate" "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig" + "k8s.io/kubernetes/pkg/kubelet/metrics" ) // NewKubeletServerCertificateManager creates a certificate manager for the kubelet when retrieving a server certificate // or returns an error. -func NewKubeletServerCertificateManager(kubeClient clientset.Interface, kubeCfg *kubeletconfig.KubeletConfiguration, nodeName types.NodeName, ips []net.IP, hostnames []string, certDirectory string) (Manager, error) { +func NewKubeletServerCertificateManager(kubeClient clientset.Interface, kubeCfg *kubeletconfig.KubeletConfiguration, nodeName types.NodeName, ips []net.IP, hostnames []string, certDirectory string) (certificate.Manager, error) { var certSigningRequestClient clientcertificates.CertificateSigningRequestInterface if kubeClient != nil && kubeClient.Certificates() != nil { certSigningRequestClient = kubeClient.Certificates().CertificateSigningRequests() } - certificateStore, err := NewFileStore( + certificateStore, err := certificate.NewFileStore( "kubelet-server", certDirectory, certDirectory, @@ -45,8 +49,17 @@ func NewKubeletServerCertificateManager(kubeClient clientset.Interface, kubeCfg if err != nil { return nil, fmt.Errorf("failed to initialize server certificate store: %v", err) } - m, err := NewManager(&Config{ - Name: "server", + var certificateExpiration = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: metrics.KubeletSubsystem, + Subsystem: "certificate_manager", + Name: "server_expiration_seconds", + Help: "Gauge of the lifetime of a certificate. The value is the date the certificate will expire in seconds since January 1, 1970 UTC.", + }, + ) + prometheus.MustRegister(certificateExpiration) + + m, err := certificate.NewManager(&certificate.Config{ CertificateSigningRequestClient: certSigningRequestClient, Template: &x509.CertificateRequest{ Subject: pkix.Name{ @@ -70,7 +83,8 @@ func NewKubeletServerCertificateManager(kubeClient clientset.Interface, kubeCfg // authenticate itself to a TLS client. certificates.UsageServerAuth, }, - CertificateStore: certificateStore, + CertificateStore: certificateStore, + CertificateExpiration: certificateExpiration, }) if err != nil { return nil, fmt.Errorf("failed to initialize server certificate manager: %v", err) @@ -82,8 +96,8 @@ func NewKubeletServerCertificateManager(kubeClient clientset.Interface, kubeCfg // client that can be used to sign new certificates (or rotate). It answers with // whatever certificate it is initialized with. If a CSR client is set later, it // may begin rotating/renewing the client cert -func NewKubeletClientCertificateManager(certDirectory string, nodeName types.NodeName, certData []byte, keyData []byte, certFile string, keyFile string) (Manager, error) { - certificateStore, err := NewFileStore( +func NewKubeletClientCertificateManager(certDirectory string, nodeName types.NodeName, certData []byte, keyData []byte, certFile string, keyFile string) (certificate.Manager, error) { + certificateStore, err := certificate.NewFileStore( "kubelet-client", certDirectory, certDirectory, @@ -92,8 +106,17 @@ func NewKubeletClientCertificateManager(certDirectory string, nodeName types.Nod if err != nil { return nil, fmt.Errorf("failed to initialize client certificate store: %v", err) } - m, err := NewManager(&Config{ - Name: "client", + var certificateExpiration = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: metrics.KubeletSubsystem, + Subsystem: "certificate_manager", + Name: "client_expiration_seconds", + Help: "Gauge of the lifetime of a certificate. The value is the date the certificate will expire in seconds since January 1, 1970 UTC.", + }, + ) + prometheus.MustRegister(certificateExpiration) + + m, err := certificate.NewManager(&certificate.Config{ Template: &x509.CertificateRequest{ Subject: pkix.Name{ CommonName: fmt.Sprintf("system:node:%s", nodeName), @@ -118,6 +141,7 @@ func NewKubeletClientCertificateManager(certDirectory string, nodeName types.Nod CertificateStore: certificateStore, BootstrapCertificatePEM: certData, BootstrapKeyPEM: keyData, + CertificateExpiration: certificateExpiration, }) if err != nil { return nil, fmt.Errorf("failed to initialize client certificate manager: %v", err) diff --git a/pkg/kubelet/certificate/transport.go b/pkg/kubelet/certificate/transport.go index bb472039f3..49c089b44a 100644 --- a/pkg/kubelet/certificate/transport.go +++ b/pkg/kubelet/certificate/transport.go @@ -30,6 +30,7 @@ import ( utilnet "k8s.io/apimachinery/pkg/util/net" "k8s.io/apimachinery/pkg/util/wait" restclient "k8s.io/client-go/rest" + "k8s.io/client-go/util/certificate" ) // UpdateTransport instruments a restconfig with a transport that dynamically uses @@ -44,13 +45,13 @@ import ( // // stopCh should be used to indicate when the transport is unused and doesn't need // to continue checking the manager. -func UpdateTransport(stopCh <-chan struct{}, clientConfig *restclient.Config, clientCertificateManager Manager) error { +func UpdateTransport(stopCh <-chan struct{}, clientConfig *restclient.Config, clientCertificateManager certificate.Manager) error { return updateTransport(stopCh, 10*time.Second, clientConfig, clientCertificateManager) } // updateTransport is an internal method that exposes how often this method checks that the // client cert has changed. Intended for testing. -func updateTransport(stopCh <-chan struct{}, period time.Duration, clientConfig *restclient.Config, clientCertificateManager Manager) error { +func updateTransport(stopCh <-chan struct{}, period time.Duration, clientConfig *restclient.Config, clientCertificateManager certificate.Manager) error { if clientConfig.Transport != nil { return fmt.Errorf("there is already a transport configured") } diff --git a/pkg/kubelet/certificate/transport_test.go b/pkg/kubelet/certificate/transport_test.go index d2d2f5286b..306c3c18a0 100644 --- a/pkg/kubelet/certificate/transport_test.go +++ b/pkg/kubelet/certificate/transport_test.go @@ -19,6 +19,7 @@ package certificate import ( "crypto/tls" "crypto/x509" + "fmt" "math/big" "net/http" "net/http/httptest" @@ -89,6 +90,29 @@ uC6Jo2eLcSV1sSdzTjaaWdM6XeYj6yHOAm8ZBIQs7m6V -----END RSA PRIVATE KEY-----`) ) +type certificateData struct { + keyPEM []byte + certificatePEM []byte + certificate *tls.Certificate +} + +func newCertificateData(certificatePEM string, keyPEM string) *certificateData { + certificate, err := tls.X509KeyPair([]byte(certificatePEM), []byte(keyPEM)) + if err != nil { + panic(fmt.Sprintf("Unable to initialize certificate: %v", err)) + } + certs, err := x509.ParseCertificates(certificate.Certificate[0]) + if err != nil { + panic(fmt.Sprintf("Unable to initialize certificate leaf: %v", err)) + } + certificate.Leaf = certs[0] + return &certificateData{ + keyPEM: []byte(keyPEM), + certificatePEM: []byte(certificatePEM), + certificate: &certificate, + } +} + type fakeManager struct { cert atomic.Value // Always a *tls.Certificate } diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index b6178ce296..f3d0e25f3d 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -54,6 +54,7 @@ import ( corelisters "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/certificate" "k8s.io/client-go/util/flowcontrol" "k8s.io/client-go/util/integer" "k8s.io/kubernetes/cmd/kubelet/app/options" @@ -64,7 +65,7 @@ import ( kubeletconfiginternal "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig" kubeletconfigv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/cadvisor" - "k8s.io/kubernetes/pkg/kubelet/certificate" + kubeletcertificate "k8s.io/kubernetes/pkg/kubelet/certificate" "k8s.io/kubernetes/pkg/kubelet/cm" "k8s.io/kubernetes/pkg/kubelet/config" "k8s.io/kubernetes/pkg/kubelet/configmap" @@ -743,7 +744,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, ips = append(ips, cloudIPs...) names := append([]string{klet.GetHostname(), hostnameOverride}, cloudNames...) - klet.serverCertificateManager, err = certificate.NewKubeletServerCertificateManager(klet.kubeClient, kubeCfg, klet.nodeName, ips, names, certDirectory) + klet.serverCertificateManager, err = kubeletcertificate.NewKubeletServerCertificateManager(klet.kubeClient, kubeCfg, klet.nodeName, ips, names, certDirectory) if err != nil { return nil, fmt.Errorf("failed to initialize certificate manager: %v", err) } diff --git a/staging/BUILD b/staging/BUILD index 70107f50af..74838f56b5 100644 --- a/staging/BUILD +++ b/staging/BUILD @@ -180,6 +180,7 @@ filegroup( "//staging/src/k8s.io/client-go/transport:all-srcs", "//staging/src/k8s.io/client-go/util/buffer:all-srcs", "//staging/src/k8s.io/client-go/util/cert:all-srcs", + "//staging/src/k8s.io/client-go/util/certificate:all-srcs", "//staging/src/k8s.io/client-go/util/exec:all-srcs", "//staging/src/k8s.io/client-go/util/flowcontrol:all-srcs", "//staging/src/k8s.io/client-go/util/homedir:all-srcs", diff --git a/staging/src/k8s.io/client-go/util/certificate/BUILD b/staging/src/k8s.io/client-go/util/certificate/BUILD new file mode 100644 index 0000000000..6743ff55f4 --- /dev/null +++ b/staging/src/k8s.io/client-go/util/certificate/BUILD @@ -0,0 +1,59 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", + "go_test", +) + +go_test( + name = "go_default_test", + srcs = [ + "certificate_manager_test.go", + "certificate_store_test.go", + ], + library = ":go_default_library", + tags = ["automanaged"], + deps = [ + "//vendor/k8s.io/api/certificates/v1beta1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library", + "//vendor/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library", + "//vendor/k8s.io/client-go/util/cert:go_default_library", + ], +) + +go_library( + name = "go_default_library", + srcs = [ + "certificate_manager.go", + "certificate_store.go", + ], + tags = ["automanaged"], + deps = [ + "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/api/certificates/v1beta1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library", + "//vendor/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library", + "//vendor/k8s.io/client-go/util/cert:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/staging/src/k8s.io/client-go/util/certificate/OWNERS b/staging/src/k8s.io/client-go/util/certificate/OWNERS new file mode 100644 index 0000000000..2dce803b34 --- /dev/null +++ b/staging/src/k8s.io/client-go/util/certificate/OWNERS @@ -0,0 +1,8 @@ +reviewers: +- mikedanese +- liggit +- smarterclayton +approvers: +- mikedanese +- liggit +- smarterclayton diff --git a/pkg/kubelet/certificate/certificate_manager.go b/staging/src/k8s.io/client-go/util/certificate/certificate_manager.go similarity index 89% rename from pkg/kubelet/certificate/certificate_manager.go rename to staging/src/k8s.io/client-go/util/certificate/certificate_manager.go index b262d73400..08363491b8 100644 --- a/pkg/kubelet/certificate/certificate_manager.go +++ b/staging/src/k8s.io/client-go/util/certificate/certificate_manager.go @@ -28,7 +28,6 @@ import ( "time" "github.com/golang/glog" - "github.com/prometheus/client_golang/prometheus" certificates "k8s.io/api/certificates/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -37,12 +36,6 @@ import ( "k8s.io/apimachinery/pkg/watch" certificatesclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" "k8s.io/client-go/util/cert" - "k8s.io/kubernetes/pkg/kubelet/metrics" -) - -const ( - certificateManagerSubsystem = "certificate_manager" - certificateExpirationKey = "expiration_seconds" ) // Manager maintains and updates the certificates in use by this certificate @@ -62,10 +55,6 @@ type Manager interface { // Config is the set of configuration parameters available for a new Manager. type Config struct { - // Name is a name describing the certificate being managed by this - // certificate manager. It will be used for recording metrics relevant to - // the certificate. - Name string // CertificateSigningRequestClient will be used for signing new certificate // requests generated when a key rotation occurs. It must be set either at // initialization or by using CertificateSigningRequestClient before @@ -103,6 +92,9 @@ type Config struct { // initialized using a generic, multi-use cert/key pair which will be // quickly replaced with a unique cert/key pair. BootstrapKeyPEM []byte + // CertificateExpiration will record a metric that shows the remaining + // lifetime of the certificate. + CertificateExpiration Gauge } // Store is responsible for getting and updating the current certificate. @@ -121,6 +113,12 @@ type Store interface { Update(cert, key []byte) (*tls.Certificate, error) } +// Gauge will record the remaining lifetime of the certificate each time it is +// updated. +type Gauge interface { + Set(float64) +} + // NoCertKeyError indicates there is no cert/key currently available. type NoCertKeyError string @@ -135,17 +133,13 @@ type manager struct { cert *tls.Certificate rotationDeadline time.Time forceRotation bool - certificateExpiration prometheus.Gauge + certificateExpiration Gauge } // NewManager returns a new certificate manager. A certificate manager is // responsible for being the authoritative source of certificates in the // Kubelet and handling updates due to rotation. func NewManager(config *Config) (Manager, error) { - if config.Name == "" { - return nil, fmt.Errorf("the 'Name' is required to disambiguate metric values of different certificate manager instances") - } - cert, forceRotation, err := getCurrentCertificateOrBootstrap( config.CertificateStore, config.BootstrapCertificatePEM, @@ -154,17 +148,6 @@ func NewManager(config *Config) (Manager, error) { return nil, err } - var certificateExpiration = prometheus.NewGauge( - prometheus.GaugeOpts{ - Namespace: metrics.KubeletSubsystem, - Subsystem: certificateManagerSubsystem, - Name: fmt.Sprintf("%s_%s", config.Name, certificateExpirationKey), - Help: "Gauge of the lifetime of a certificate. The value is the date the certificate will expire in seconds since January 1, 1970 UTC.", - }, - ) - - prometheus.MustRegister(certificateExpiration) - m := manager{ certSigningRequestClient: config.CertificateSigningRequestClient, template: config.Template, @@ -172,7 +155,7 @@ func NewManager(config *Config) (Manager, error) { certStore: config.CertificateStore, cert: cert, forceRotation: forceRotation, - certificateExpiration: certificateExpiration, + certificateExpiration: config.CertificateExpiration, } return &m, nil @@ -199,7 +182,7 @@ func (m *manager) SetCertificateSigningRequestClient(certSigningRequestClient ce m.certSigningRequestClient = certSigningRequestClient return nil } - return fmt.Errorf("CertificateSigningRequestClient is already set.") + return fmt.Errorf("property CertificateSigningRequestClient is already set") } // Start will start the background work of rotating the certificates. @@ -335,16 +318,23 @@ func (m *manager) setRotationDeadline() { notAfter := m.cert.Leaf.NotAfter totalDuration := float64(notAfter.Sub(m.cert.Leaf.NotBefore)) - // Use some jitter to set the rotation threshold so each node will rotate - // at approximately 70-90% of the total lifetime of the certificate. With - // jitter, if a number of nodes are added to a cluster at approximately the - // same time (such as cluster creation time), they won't all try to rotate - // certificates at the same time for the rest of the life of the cluster. - jitteryDuration := wait.Jitter(time.Duration(totalDuration), 0.2) - time.Duration(totalDuration*0.3) - - m.rotationDeadline = m.cert.Leaf.NotBefore.Add(jitteryDuration) + m.rotationDeadline = m.cert.Leaf.NotBefore.Add(jitteryDuration(totalDuration)) glog.V(2).Infof("Certificate expiration is %v, rotation deadline is %v", notAfter, m.rotationDeadline) - m.certificateExpiration.Set(float64(notAfter.Unix())) + if m.certificateExpiration != nil { + m.certificateExpiration.Set(float64(notAfter.Unix())) + } +} + +// jitteryDuration uses some jitter to set the rotation threshold so each node +// will rotate at approximately 70-90% of the total lifetime of the +// certificate. With jitter, if a number of nodes are added to a cluster at +// approximately the same time (such as cluster creation time), they won't all +// try to rotate certificates at the same time for the rest of the life of the +// cluster. +// +// This function is represented as a variable to allow replacement during testing. +var jitteryDuration = func(totalDuration float64) time.Duration { + return wait.Jitter(time.Duration(totalDuration), 0.2) - time.Duration(totalDuration*0.3) } func (m *manager) updateCached(cert *tls.Certificate) { diff --git a/pkg/kubelet/certificate/certificate_manager_test.go b/staging/src/k8s.io/client-go/util/certificate/certificate_manager_test.go similarity index 95% rename from pkg/kubelet/certificate/certificate_manager_test.go rename to staging/src/k8s.io/client-go/util/certificate/certificate_manager_test.go index e2f35706bd..6a77f99bf5 100644 --- a/pkg/kubelet/certificate/certificate_manager_test.go +++ b/staging/src/k8s.io/client-go/util/certificate/certificate_manager_test.go @@ -26,20 +26,12 @@ import ( "testing" "time" - "github.com/prometheus/client_golang/prometheus" - certificates "k8s.io/api/certificates/v1beta1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" watch "k8s.io/apimachinery/pkg/watch" certificatesclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" ) -type certificateData struct { - keyPEM []byte - certificatePEM []byte - certificate *tls.Certificate -} - var storeCertData = newCertificateData(`-----BEGIN CERTIFICATE----- MIICRzCCAfGgAwIBAgIJALMb7ecMIk3MMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNV BAYTAkdCMQ8wDQYDVQQIDAZMb25kb24xDzANBgNVBAcMBkxvbmRvbjEYMBYGA1UE @@ -115,6 +107,12 @@ bDQT1r8Q3Gx+h9LRqQeHgPBQ3F5ylqqBAiBaJ0hkYvrIdWxNlcLqD3065bJpHQ4S WQkuZUQN1M/Xvg== -----END RSA PRIVATE KEY-----`) +type certificateData struct { + keyPEM []byte + certificatePEM []byte + certificate *tls.Certificate +} + func newCertificateData(certificatePEM string, keyPEM string) *certificateData { certificate, err := tls.X509KeyPair([]byte(certificatePEM), []byte(keyPEM)) if err != nil { @@ -137,7 +135,6 @@ func TestNewManagerNoRotation(t *testing.T) { cert: storeCertData.certificate, } if _, err := NewManager(&Config{ - Name: "test_no_rotation", Template: &x509.CertificateRequest{}, Usages: []certificates.KeyUsage{}, CertificateStore: store, @@ -173,11 +170,6 @@ func TestShouldRotate(t *testing.T) { }, template: &x509.CertificateRequest{}, usages: []certificates.KeyUsage{}, - certificateExpiration: prometheus.NewGauge( - prometheus.GaugeOpts{ - Name: "test_gauge_name", - }, - ), } m.setRotationDeadline() if m.shouldRotate() != test.shouldRotate { @@ -191,7 +183,19 @@ func TestShouldRotate(t *testing.T) { } } +type gaugeMock struct { + calls int + lastValue float64 +} + +func (g *gaugeMock) Set(v float64) { + g.calls++ + g.lastValue = v +} + func TestSetRotationDeadline(t *testing.T) { + defer func(original func(float64) time.Duration) { jitteryDuration = original }(jitteryDuration) + now := time.Now() testCases := []struct { name string @@ -211,6 +215,7 @@ func TestSetRotationDeadline(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { + g := gaugeMock{} m := manager{ cert: &tls.Certificate{ Leaf: &x509.Certificate{ @@ -218,27 +223,27 @@ func TestSetRotationDeadline(t *testing.T) { NotAfter: tc.notAfter, }, }, - template: &x509.CertificateRequest{}, - usages: []certificates.KeyUsage{}, - certificateExpiration: prometheus.NewGauge( - prometheus.GaugeOpts{ - Name: "test_gauge_name", - }, - ), + template: &x509.CertificateRequest{}, + usages: []certificates.KeyUsage{}, + certificateExpiration: &g, } + jitteryDuration = func(float64) time.Duration { return time.Duration(float64(tc.notAfter.Sub(tc.notBefore)) * 0.7) } lowerBound := tc.notBefore.Add(time.Duration(float64(tc.notAfter.Sub(tc.notBefore)) * 0.7)) - upperBound := tc.notBefore.Add(time.Duration(float64(tc.notAfter.Sub(tc.notBefore)) * 0.9)) - for i := 0; i < 1000; i++ { - // setRotationDeadline includes jitter, so this needs to run many times for validation. - m.setRotationDeadline() - if m.rotationDeadline.Before(lowerBound) || m.rotationDeadline.After(upperBound) { - t.Errorf("For notBefore %v, notAfter %v, the rotationDeadline %v should be between %v and %v.", - tc.notBefore, - tc.notAfter, - m.rotationDeadline, - lowerBound, - upperBound) - } + + m.setRotationDeadline() + + if !m.rotationDeadline.Equal(lowerBound) { + t.Errorf("For notBefore %v, notAfter %v, the rotationDeadline %v should be %v.", + tc.notBefore, + tc.notAfter, + m.rotationDeadline, + lowerBound) + } + if g.calls != 1 { + t.Errorf("%d metrics were recorded, wanted %d", g.calls, 1) + } + if g.lastValue != float64(tc.notAfter.Unix()) { + t.Errorf("%d value for metric was recorded, wanted %d", g.lastValue, tc.notAfter.Unix()) } }) } @@ -295,7 +300,6 @@ func TestNewManagerBootstrap(t *testing.T) { var cm Manager cm, err := NewManager(&Config{ - Name: "test_bootstrap", Template: &x509.CertificateRequest{}, Usages: []certificates.KeyUsage{}, CertificateStore: store, @@ -333,7 +337,6 @@ func TestNewManagerNoBootstrap(t *testing.T) { } cm, err := NewManager(&Config{ - Name: "test_no_bootstrap", Template: &x509.CertificateRequest{}, Usages: []certificates.KeyUsage{}, CertificateStore: store, @@ -469,14 +472,13 @@ func TestInitializeCertificateSigningRequestClient(t *testing.T) { }, } - for i, tc := range testCases { + for _, tc := range testCases { t.Run(tc.description, func(t *testing.T) { certificateStore := &fakeStore{ cert: tc.storeCert.certificate, } certificateManager, err := NewManager(&Config{ - Name: fmt.Sprintf("test_initialize_client_%d", i), Template: &x509.CertificateRequest{ Subject: pkix.Name{ Organization: []string{"system:nodes"}, @@ -571,14 +573,13 @@ func TestInitializeOtherRESTClients(t *testing.T) { }, } - for i, tc := range testCases { + for _, tc := range testCases { t.Run(tc.description, func(t *testing.T) { certificateStore := &fakeStore{ cert: tc.storeCert.certificate, } certificateManager, err := NewManager(&Config{ - Name: fmt.Sprintf("test_initialize_other_rest_clients_%d", i), Template: &x509.CertificateRequest{ Subject: pkix.Name{ Organization: []string{"system:nodes"}, diff --git a/pkg/kubelet/certificate/certificate_store.go b/staging/src/k8s.io/client-go/util/certificate/certificate_store.go similarity index 93% rename from pkg/kubelet/certificate/certificate_store.go rename to staging/src/k8s.io/client-go/util/certificate/certificate_store.go index a98a903197..029f9916b7 100644 --- a/pkg/kubelet/certificate/certificate_store.go +++ b/staging/src/k8s.io/client-go/util/certificate/certificate_store.go @@ -28,8 +28,6 @@ import ( "time" "github.com/golang/glog" - - utilfile "k8s.io/kubernetes/pkg/util/file" ) const ( @@ -86,7 +84,7 @@ func NewFileStore( func (s *fileStore) recover() error { // If the 'current' file doesn't exist, continue on with the recovery process. currentPath := filepath.Join(s.certDirectory, s.filename(currentPair)) - if exists, err := utilfile.FileExists(currentPath); err != nil { + if exists, err := fileExists(currentPath); err != nil { return err } else if exists { return nil @@ -113,18 +111,18 @@ func (s *fileStore) recover() error { func (s *fileStore) Current() (*tls.Certificate, error) { pairFile := filepath.Join(s.certDirectory, s.filename(currentPair)) - if pairFileExists, err := utilfile.FileExists(pairFile); err != nil { + if pairFileExists, err := fileExists(pairFile); err != nil { return nil, err } else if pairFileExists { glog.Infof("Loading cert/key pair from %q.", pairFile) return loadFile(pairFile) } - certFileExists, err := utilfile.FileExists(s.certFile) + certFileExists, err := fileExists(s.certFile) if err != nil { return nil, err } - keyFileExists, err := utilfile.FileExists(s.keyFile) + keyFileExists, err := fileExists(s.keyFile) if err != nil { return nil, err } @@ -135,11 +133,11 @@ func (s *fileStore) Current() (*tls.Certificate, error) { c := filepath.Join(s.certDirectory, s.pairNamePrefix+certExtension) k := filepath.Join(s.keyDirectory, s.pairNamePrefix+keyExtension) - certFileExists, err = utilfile.FileExists(c) + certFileExists, err = fileExists(c) if err != nil { return nil, err } - keyFileExists, err = utilfile.FileExists(k) + keyFileExists, err = fileExists(k) if err != nil { return nil, err } @@ -240,7 +238,7 @@ func (s *fileStore) updateSymlink(filename string) error { return err } } else if fi.Mode()&os.ModeSymlink != os.ModeSymlink { - return fmt.Errorf("expected %q to be a symlink but it is a file.", currentPath) + return fmt.Errorf("expected %q to be a symlink but it is a file", currentPath) } else { currentPathExists = true } @@ -253,7 +251,7 @@ func (s *fileStore) updateSymlink(filename string) error { return err } } else if fi.Mode()&os.ModeSymlink != os.ModeSymlink { - return fmt.Errorf("expected %q to be a symlink but it is a file.", updatedPath) + return fmt.Errorf("expected %q to be a symlink but it is a file", updatedPath) } else { if err := os.Remove(updatedPath); err != nil { return fmt.Errorf("unable to remove %q: %v", updatedPath, err) @@ -262,7 +260,7 @@ func (s *fileStore) updateSymlink(filename string) error { // Check that the new cert/key pair file exists to avoid rotating to an // invalid cert/key. - if filenameExists, err := utilfile.FileExists(filename); err != nil { + if filenameExists, err := fileExists(filename); err != nil { return err } else if !filenameExists { return fmt.Errorf("file %q does not exist so it can not be used as the currently selected cert/key", filename) @@ -307,3 +305,13 @@ func loadX509KeyPair(certFile, keyFile string) (*tls.Certificate, error) { cert.Leaf = certs[0] return &cert, nil } + +// FileExists checks if specified file exists. +func fileExists(filename string) (bool, error) { + if _, err := os.Stat(filename); os.IsNotExist(err) { + return false, nil + } else if err != nil { + return false, err + } + return true, nil +} diff --git a/pkg/kubelet/certificate/certificate_store_test.go b/staging/src/k8s.io/client-go/util/certificate/certificate_store_test.go similarity index 100% rename from pkg/kubelet/certificate/certificate_store_test.go rename to staging/src/k8s.io/client-go/util/certificate/certificate_store_test.go