Merge pull request #50387 from jcbsmpsn/metric-certificate-expiration

Automatic merge from submit-queue Add metric for remaining lifetime of certificates authenticating requests fixes #50778 When incoming requests to the API server are authenticated by a certificate, the expiration of the certificate can affect the validity of the authentication. With auto rotation of certificates, which is starting with kubelet certificates, the goal is to use shorter lifetimes and let the kubelet renew the certificate as desired. Monitoring certificates which are approaching expiration and not renewing would be an early warning sign that nodes are about to stop participating in the cluster. **Release note**: ```release-note Add new Prometheus metric that monitors the remaining lifetime of certificates used to authenticate requests to the API server. ```

Merge pull request #50387 from jcbsmpsn/metric-certificate-expiration
6bc0b295 · Kubernetes Submit Queue · GitHub · e76ecd63 · 49a19c60 · 6bc0b295
Commit 6bc0b295 authored Aug 16, 2017 by Kubernetes Submit Queue Committed by GitHub Aug 16, 2017
Hide whitespace changes
Inline Side-by-side

Showing with 27 additions and 0 deletions

BUILD ...rc/k8s.io/apiserver/pkg/authentication/request/x509/BUILD +1 -0

x509.go .../k8s.io/apiserver/pkg/authentication/request/x509/x509.go +26 -0

No files found.
--- a/staging/src/k8s.io/apiserver/pkg/authentication/request/x509/BUILD
+++ b/staging/src/k8s.io/apiserver/pkg/authentication/request/x509/BUILD
@@ -31,6 +31,7 @@ go_library(
    ],
    deps = [
        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
        "//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library",
        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
        "//vendor/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library",

--- a/staging/src/k8s.io/apiserver/pkg/authentication/request/x509/x509.go
+++ b/staging/src/k8s.io/apiserver/pkg/authentication/request/x509/x509.go
@@ -22,8 +22,10 @@ import (
 	"encoding/asn1"
 	"fmt"
 	"net/http"
+	"time"

 	"github.com/golang/glog"
+	"github.com/prometheus/client_golang/prometheus"

 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/apimachinery/pkg/util/sets"
@@ -31,6 +33,28 @@ import (
 	"k8s.io/apiserver/pkg/authentication/user"
 )

+var clientCertificateExpirationHistogram = prometheus.NewHistogram(
+	prometheus.HistogramOpts{
+		Namespace: "apiserver",
+		Subsystem: "client",
+		Name:      "certificate_expiration_seconds",
+		Help:      "Distribution of the remaining lifetime on the certificate used to authenticate a request.",
+		Buckets: []float64{
+			0,
+			(6 * time.Hour).Seconds(),
+			(12 * time.Hour).Seconds(),
+			(24 * time.Hour).Seconds(),
+			(2 * 24 * time.Hour).Seconds(),
+			(4 * 24 * time.Hour).Seconds(),
+			(7 * 24 * time.Hour).Seconds(),
+		},
+	},
+)
+
+func init() {
+	prometheus.MustRegister(clientCertificateExpirationHistogram)
+}
+
 // UserConversion defines an interface for extracting user info from a client certificate chain
 type UserConversion interface {
 	User(chain []*x509.Certificate) (user.Info, bool, error)
@@ -71,6 +95,8 @@ func (a *Authenticator) AuthenticateRequest(req *http.Request) (user.Info, bool,
 		}
 	}

+	remaining := req.TLS.PeerCertificates[0].NotAfter.Sub(time.Now())
+	clientCertificateExpirationHistogram.Observe(remaining.Seconds())
 	chains, err := req.TLS.PeerCertificates[0].Verify(optsCopy)
 	if err != nil {
 		return nil, false, err